From 40b489c8d1ff825b573fdbbd2ce219c5f2c08292 Mon Sep 17 00:00:00 2001 From: Florian Schreiber Date: Sun, 8 Jun 2025 05:25:42 +0200 Subject: [PATCH 1/2] Remove manual creation of tokio runtime By leveraging the tokio feature of UniFFI we don't need to manually instantiate the runtime and thus can easily maintain the same code between the BTLB iOS app and the CLI. GH-6 --- Cargo.lock | 14 +++ Cargo.toml | 2 +- paper/src/api/api_client.rs | 22 ++++ .../src/authenticators/opac_authenticator.rs | 10 -- .../public_hamburg_authenticator.rs | 23 ++-- .../authenticators/scrape_authenticator.rs | 6 +- paper/src/lib.rs | 2 +- paper/src/scrapers/library_scraper.rs | 45 +++----- .../opc4v2_13vzg6_search_detail_scraper.rs | 9 +- .../hamburg_public_search_detail_scraper.rs | 41 +++---- .../hamburg_public_search_scraper.rs | 99 ++++++---------- .../public_hamburg_account_scraper.rs | 6 +- .../public_hamburg_loans_scraper.rs | 61 +++++----- paper/src/scrapers/renewal_service.rs | 106 +++++++++--------- paper/src/scrapers/search_detail_scraper.rs | 22 +--- paper/src/scrapers/search_scraper.rs | 24 +--- 16 files changed, 218 insertions(+), 274 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7999fd3..be082d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -192,6 +192,19 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "async-compat" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bab94bde396a3f7b4962e396fdad640e241ed797d4d8d77fc8c237d14c58fc0" +dependencies = [ + "futures-core", + "futures-io", + "once_cell", + "pin-project-lite", + "tokio", +] + [[package]] name = "async-executor" version = "1.11.0" @@ -2760,6 +2773,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a02e67ac9634b10da9e4aa63a29a7920b8f1395eafef1ea659b2dd76dda96906" dependencies = [ "anyhow", + "async-compat", "bytes", "camino", "log", diff --git a/Cargo.toml b/Cargo.toml index 07c8084..b5b44ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,5 +14,5 @@ debug = true # Enable debug symbols. For example, we can use `dwarfdump` to strip = "symbols" [workspace.dependencies] -uniffi = { version = "0.27" } +uniffi = { version = "0.27", features = ["tokio"] } uniffi_bindgen = { version = "0.27" } diff --git a/paper/src/api/api_client.rs b/paper/src/api/api_client.rs index e8e4acf..a6f4324 100644 --- a/paper/src/api/api_client.rs +++ b/paper/src/api/api_client.rs @@ -23,6 +23,28 @@ impl APIClient { Ok(response.status().as_u16()) } + /// Performs a HEAD request to the specified URLs to check its availability and returns the + /// first one that returns a status code 200 + /// + /// # Arguments + /// * `urls` - The URLs to ping + /// + /// # Returns + /// * `Option` - The first URL that returned 200 or None + pub(crate) async fn test_urls( + urls: impl IntoIterator>, + ) -> Option { + for url in urls { + if let Some(url) = url { + if matches!(APIClient::ping_url(&url).await, Ok(200)) { + return Some(url); + } + } + } + + None + } + pub fn new_with_network_client(network_client: reqwest::Client, base_url: String) -> APIClient { APIClient { network_client, diff --git a/paper/src/authenticators/opac_authenticator.rs b/paper/src/authenticators/opac_authenticator.rs index 3aa6f54..f4b1040 100644 --- a/paper/src/authenticators/opac_authenticator.rs +++ b/paper/src/authenticators/opac_authenticator.rs @@ -7,16 +7,6 @@ pub(crate) struct OpacAuthenticator { } impl OpacAuthenticator { - pub(crate) async fn verify_credentials_opc4vs2_13vzg6( - &self, - client: &Client, - ) -> Result { - return tokio::runtime::Builder::new_current_thread() - .enable_all() - .build()? - .block_on(async { self.authenticate(client).await }); - } - pub(crate) async fn authenticate(&self, client: &Client) -> Result { println!("`OpacAuthenticator::authenticate`"); let username = self.configuration.username.clone().unwrap(); diff --git a/paper/src/authenticators/public_hamburg_authenticator.rs b/paper/src/authenticators/public_hamburg_authenticator.rs index e66b08e..fe3b39a 100644 --- a/paper/src/authenticators/public_hamburg_authenticator.rs +++ b/paper/src/authenticators/public_hamburg_authenticator.rs @@ -17,22 +17,19 @@ pub(crate) struct PublicHamburgAuthenticator { impl PublicHamburgAuthenticator { pub(crate) async fn verify_credentials_public_hamburg(&self) -> Result { let client = reqwest::ClientBuilder::new().cookie_store(true).build()?; - return tokio::runtime::Builder::new_current_thread() - .enable_all() - .build()? - .block_on(async { - self.public_hamburg_authenticate_and_get_request_access_token(&client) - .await - }); + self.public_hamburg_authenticate_and_get_request_access_token(&client) + .await } async fn authenticate_public_hamburg_via_cookies( &self, client: &Client, request_token: String, - configuration: Configuration, ) -> Result { - if let (Some(username), Some(password)) = (configuration.username, configuration.password) { + if let (Some(username), Some(password)) = ( + self.configuration.username.clone(), + self.configuration.password.clone(), + ) { if username == "" || password == "" { return Err(PaperError::CredentialsBadInput); } @@ -77,7 +74,7 @@ impl PublicHamburgAuthenticator { let token = token_scraper.get_request_token(&client).await?; let login_result = self - .authenticate_public_hamburg_via_cookies(&client, token, self.configuration.clone()) + .authenticate_public_hamburg_via_cookies(&client, token) .await; return match login_result { @@ -98,11 +95,7 @@ impl PublicHamburgAuthenticator { let request_token = token_scraper.get_request_token(&client).await?; let result = self - .authenticate_public_hamburg_via_cookies( - &client, - request_token.clone(), - self.configuration.clone(), - ) + .authenticate_public_hamburg_via_cookies(&client, request_token.clone()) .await; return match result { diff --git a/paper/src/authenticators/scrape_authenticator.rs b/paper/src/authenticators/scrape_authenticator.rs index 46ee40f..7518d9e 100644 --- a/paper/src/authenticators/scrape_authenticator.rs +++ b/paper/src/authenticators/scrape_authenticator.rs @@ -8,7 +8,7 @@ pub struct Authenticator { pub(crate) configuration: Configuration, } -#[uniffi::export] +#[uniffi::export(async_runtime = "tokio")] impl Authenticator { #[uniffi::constructor] fn new(configuration: Configuration) -> Self { @@ -35,9 +35,7 @@ impl Authenticator { configuration: self.configuration.clone(), }; let client = reqwest::ClientBuilder::new().cookie_store(true).build()?; - let result = opac_authenticator - .verify_credentials_opc4vs2_13vzg6(&client) - .await; + let result = opac_authenticator.authenticate(&client).await; return match result { Ok(signed_in) => Ok(if signed_in { ValidationStatus::Valid diff --git a/paper/src/lib.rs b/paper/src/lib.rs index eaa64ca..e06ef93 100644 --- a/paper/src/lib.rs +++ b/paper/src/lib.rs @@ -104,7 +104,7 @@ impl Paper { let search = HamburgPublicSearchScraper {}; - match search.search_on_current_runtime(&input, None).await { + match search.search(&input, None).await { Ok(result) => Self::print_search_result(result), Err(err) => println!("Failed to search: {:?}", err), } diff --git a/paper/src/scrapers/library_scraper.rs b/paper/src/scrapers/library_scraper.rs index 1b832c4..79715ae 100644 --- a/paper/src/scrapers/library_scraper.rs +++ b/paper/src/scrapers/library_scraper.rs @@ -1,20 +1,17 @@ -use std::sync::Arc; - use crate::authenticators::{OpacAuthenticator, PublicHamburgAuthenticator}; use crate::{ configuration::Configuration, error::PaperError, scrapers::public_hamburg::PublicHamburgAccountScraper, scrapers::BalanceScraper, }; +use std::sync::Arc; +use super::opc4v2_13vzg6::Opc4v2_13Vzg6AccountScraper; +use super::SearchDetailScraper; use crate::model::Loan; use crate::model::Loans; use futures::future; use reqwest::cookie::Jar; use reqwest::ClientBuilder; -use tokio::runtime::Builder; - -use super::opc4v2_13vzg6::Opc4v2_13Vzg6AccountScraper; -use super::SearchDetailScraper; use crate::model::Account; @@ -27,7 +24,7 @@ pub struct LibraryScraper { configuration: Configuration, } -#[uniffi::export] +#[uniffi::export(async_runtime = "tokio")] impl LibraryScraper { #[uniffi::constructor] pub fn new(configuration: Configuration) -> Self { @@ -43,29 +40,21 @@ impl LibraryScraper { .cookie_store(true) .cookie_provider(cookie_store.clone()) .build()?; - let runtime = Builder::new_multi_thread() - .worker_threads(4) - .thread_name("fetch account") - .enable_io() - .enable_time() - .build()?; - return runtime.block_on(async { - match self.configuration.api_configuration.api { - crate::model::API::HamburgPublic => { - self.public_hamburg_fetch_on_current_runtime(&client).await - } - crate::model::API::Opc4v2_13Vzg6 => { - client - .get(self.configuration.session_url()) - .query(&[("USR", "1022"), ("LAN", "DU"), ("BES", "1")]) - .send() - .await?; - - self.opc4v2_13vzg6_fetch_on_current_runtime(&client).await - } + match self.configuration.api_configuration.api { + crate::model::API::HamburgPublic => { + self.public_hamburg_fetch_on_current_runtime(&client).await + } + crate::model::API::Opc4v2_13Vzg6 => { + client + .get(self.configuration.session_url()) + .query(&[("USR", "1022"), ("LAN", "DU"), ("BES", "1")]) + .send() + .await?; + + self.opc4v2_13vzg6_fetch_on_current_runtime(&client).await } - }); + } } } diff --git a/paper/src/scrapers/opc4v2_13vzg6/opc4v2_13vzg6_search_detail_scraper.rs b/paper/src/scrapers/opc4v2_13vzg6/opc4v2_13vzg6_search_detail_scraper.rs index 2897ee0..4e0b7b3 100644 --- a/paper/src/scrapers/opc4v2_13vzg6/opc4v2_13vzg6_search_detail_scraper.rs +++ b/paper/src/scrapers/opc4v2_13vzg6/opc4v2_13vzg6_search_detail_scraper.rs @@ -8,10 +8,7 @@ use crate::{error::PaperError, model::SearchResultDetail}; pub(crate) struct Opc4v2_13Vzg6SearchDetailScraper {} impl Opc4v2_13Vzg6SearchDetailScraper { - pub(crate) async fn search_detail_from( - &self, - html: Html, - ) -> Result { + pub(crate) fn search_detail_from(&self, html: Html) -> Result { let mut detail = SearchResultDetail::new(); let content_table_selector = scraper::Selector::parse(r#"body > table > tbody > tr:nth-child(7) > td.cnt > table > tbody > tr:nth-child(4) > td > table > tbody > tr:nth-child(1) > td:nth-child(2) > table > tbody > tr"#).unwrap(); let rows = html.select(&content_table_selector); @@ -63,7 +60,7 @@ mod tests { fs::read_to_string("src/fixtures/opc4v2_13Vzg6/search/katalog_detail_book_sbb.html") .expect("Something went wrong reading the file"); let html = scraper::Html::parse_document(html_string.as_str()); - let search_detail = sut.search_detail_from(html).await.unwrap(); + let search_detail = sut.search_detail_from(html).unwrap(); assert_eq!( search_detail.full_title, @@ -83,7 +80,7 @@ mod tests { fs::read_to_string("src/fixtures/opc4v2_13Vzg6/search/katalog_detail_book_gbv_hh.html") .expect("Something went wrong reading the file"); let html = scraper::Html::parse_document(html_string.as_str()); - let search_detail = sut.search_detail_from(html).await.unwrap(); + let search_detail = sut.search_detail_from(html).unwrap(); assert_eq!( search_detail.full_title, diff --git a/paper/src/scrapers/public_hamburg/hamburg_public_search_detail_scraper.rs b/paper/src/scrapers/public_hamburg/hamburg_public_search_detail_scraper.rs index deec5f3..de1b4ba 100644 --- a/paper/src/scrapers/public_hamburg/hamburg_public_search_detail_scraper.rs +++ b/paper/src/scrapers/public_hamburg/hamburg_public_search_detail_scraper.rs @@ -6,35 +6,24 @@ use crate::model::Location; use crate::model::SearchResultDetail; use crate::scrapers::text_provider::TextProvider; use scraper::Selector; +use std::future::Future; #[derive(uniffi::Object)] pub(crate) struct HamburgPublicSearchDetailScraper {} impl HamburgPublicSearchDetailScraper { - pub(crate) async fn search_result_detail_from( + pub(crate) fn search_result_detail_from( document: scraper::Html, - ) -> Option { + ) -> impl Future + Send { let mut detail = SearchResultDetail::new(); detail.medium_title = document.get_text(r#".medium-detail-title"#); detail.medium_author = document.get_text(r#".medium-detail-author > a"#); - if let Some(url1) = - document.get_attribute("data-src", r#"img[class="b-lazy img-lazyload"]"#) - { - match APIClient::ping_url(url1.as_str()).await { - Ok(status) => match status { - 200 => { - detail.small_image_url = Some(url1); - } - _ => { - detail.small_image_url = document - .get_attribute("data-alt-src", r#"img[class="b-lazy img-lazyload"]"#); - } - }, - Err(e) => println!("Error checking site: {}", e), - } - } + let image_url = APIClient::test_urls([ + document.get_attribute("data-src", r#"img[class="b-lazy img-lazyload"]"#), + document.get_attribute("data-alt-src", r#"img[class="b-lazy img-lazyload"]"#), + ]); if let Ok(detail_data_selector) = Selector::parse(r#"div[class="medium-detail-data medium-detail-data-cols"] > ul > li"#) @@ -81,7 +70,13 @@ impl HamburgPublicSearchDetailScraper { let availabilities = HamburgPublicSearchDetailScraper::parse_availabilities(document); detail.availability = ItemAvailability::with(availabilities); } - Some(detail) + + async move { + // run the network request last, such that document was already dropped since it is not Send + detail.small_image_url = image_url.await; + + detail + } } fn parse_availabilities(html: scraper::Html) -> Vec { @@ -134,9 +129,7 @@ mod tests { .expect("Something went wrong reading the file"); let document = scraper::Html::parse_document(html.as_str()); let search_result_detail = - HamburgPublicSearchDetailScraper::search_result_detail_from(document) - .await - .unwrap(); + HamburgPublicSearchDetailScraper::search_result_detail_from(document).await; assert_eq!( search_result_detail.medium_title, @@ -156,9 +149,7 @@ mod tests { .expect("Something went wrong reading the file"); let document = scraper::Html::parse_document(html.as_str()); let search_result_detail = - HamburgPublicSearchDetailScraper::search_result_detail_from(document) - .await - .unwrap(); + HamburgPublicSearchDetailScraper::search_result_detail_from(document).await; assert_eq!( search_result_detail.small_image_url, Some("https://www.hugendubel.info/annotstream/9783836961592/COP".to_string()) diff --git a/paper/src/scrapers/public_hamburg/hamburg_public_search_scraper.rs b/paper/src/scrapers/public_hamburg/hamburg_public_search_scraper.rs index 4569661..99ee251 100644 --- a/paper/src/scrapers/public_hamburg/hamburg_public_search_scraper.rs +++ b/paper/src/scrapers/public_hamburg/hamburg_public_search_scraper.rs @@ -6,7 +6,7 @@ use crate::scrapers::text_provider::TextProvider; use futures::future; use reqwest::Client; use scraper::Selector; -use tokio::runtime::Builder; +use std::future::Future; use uuid::Uuid; pub(crate) struct HamburgPublicSearchScraper {} @@ -17,22 +17,6 @@ impl HamburgPublicSearchScraper { &self, text: &str, next_page_url: Option, - ) -> Result { - let runtime = Builder::new_multi_thread() - .worker_threads(5) - .thread_name("search") - .enable_io() - .enable_time() - .build()?; - - return runtime - .block_on(async { self.search_on_current_runtime(text, next_page_url).await }); - } - - pub(crate) async fn search_on_current_runtime( - &self, - text: &str, - next_page_url: Option, ) -> Result { let next_page_url = next_page_url.or_else(|| { Some(format!( @@ -47,7 +31,7 @@ impl HamburgPublicSearchScraper { if let Some(next_page_url) = next_page_url { let html = api_client.get_html_at_path(next_page_url).await?; - return self.search_result_list_from(text.to_string(), html).await; + return Ok(self.search_result_list_from(text.to_string(), html).await); } return Err(PaperError::SearchFailed); @@ -59,15 +43,17 @@ impl HamburgPublicSearchScraper { /// /// https://www.buecherhallen.de/katalog-suchergebnisse.html?suchbegriff=extra+terrestrial&seite-m37=1 /// https://www.buecherhallen.de/katalog-suchergebnisse.html?suchbegriff=extra+terrestrial&seite-m37=2 - async fn search_result_list_from( + fn search_result_list_from( &self, text: String, document: scraper::Html, - ) -> Result { - if let Ok(result_selector) = Selector::parse(r#"li[class="search-results-item"]"#) { - let result_items = document.select(&result_selector); + ) -> impl Future + Send { + let result_selector = Selector::parse(r#"li[class="search-results-item"]"#).unwrap(); + let result_items = document.select(&result_selector); - let items = future::join_all(result_items.into_iter().map(|item| async move { + let items = result_items + .into_iter() + .map(move |item| { let mut list_item = SearchResultListItem::new(); list_item.title = item.get_text("div.search-results-text > h2"); @@ -84,25 +70,10 @@ impl HamburgPublicSearchScraper { list_item.subtitle = Some(cleaned_text); } - if let Some(url1) = item.get_attribute("data-src", "div.search-results-image > img") - { - match APIClient::ping_url(url1.as_str()).await { - Ok(status) => match status { - 200 => { - println!("source exists: {}", status); - list_item.cover_image_url = Some(url1); - } - _ => { - println!("source does not exist: {}", status); - list_item.cover_image_url = item.get_attribute( - "data-alt-src", - "div.search-results-image > img", - ); - } - }, - Err(e) => println!("Error checking site: {}", e), - } - } + let image_url = APIClient::test_urls([ + item.get_attribute("data-src", "div.search-results-image > img"), + item.get_attribute("data-alt-src", "div.search-results-image > img"), + ]); if let Some(item_number) = item.attr("id").map(|s| s.to_string()) { list_item.detail_url = Some( @@ -114,21 +85,29 @@ impl HamburgPublicSearchScraper { list_item.item_number = Some(item_number); } - list_item - })) + (list_item, image_url) + }) + .collect::>(); + + let next_page_url = document.get_attribute("href", r#"a[class="pagination-next"]"#); + let result_count = self.search_result_count(document); + + async move { + let items = future::join_all(items.into_iter().map( + |(mut list_item, image_url)| async move { + // run the network request last, such that document was already dropped since it is not Send + list_item.cover_image_url = image_url.await; + list_item + }, + )) .await; - let next_page_url = document.get_attribute("href", r#"a[class="pagination-next"]"#); - let result_count = self.search_result_count(document); - - Ok(SearchResultList { + SearchResultList { text, next_page_url, result_count, items, - }) - } else { - return Err(PaperError::SearchFailed); + } } } @@ -165,8 +144,7 @@ mod tests { let document = scraper::Html::parse_document(html.as_str()); let search_result_list = sut .search_result_list_from("abc".to_string(), document) - .await - .unwrap(); + .await; assert_eq!(search_result_list.items.len(), 7); assert_eq!(search_result_list.next_page_url, None); @@ -189,8 +167,7 @@ mod tests { let document = scraper::Html::parse_document(html.as_str()); let search_result_list = sut .search_result_list_from("abc".to_string(), document) - .await - .unwrap(); + .await; assert_eq!(search_result_list.items.len(), 10); @@ -219,8 +196,7 @@ mod tests { let document = scraper::Html::parse_document(html.as_str()); let search_result_list = sut .search_result_list_from("abc".to_string(), document) - .await - .unwrap(); + .await; assert_eq!(search_result_list.items.len(), 7); @@ -243,8 +219,7 @@ mod tests { let document = scraper::Html::parse_document(html.as_str()); let search_result_list = sut .search_result_list_from("abc".to_string(), document) - .await - .unwrap(); + .await; assert_eq!(search_result_list.result_count, 0); assert_eq!(search_result_list.next_page_url, None); @@ -260,8 +235,7 @@ mod tests { let document = scraper::Html::parse_document(html.as_str()); let search_result_list = sut .search_result_list_from("abc".to_string(), document) - .await - .unwrap(); + .await; assert_eq!(search_result_list.result_count, 57); assert_eq!( @@ -282,8 +256,7 @@ mod tests { let document = scraper::Html::parse_document(html.as_str()); let search_result_list = sut .search_result_list_from("abc".to_string(), document) - .await - .unwrap(); + .await; assert_eq!(search_result_list.result_count, 3893); assert_eq!( diff --git a/paper/src/scrapers/public_hamburg/public_hamburg_account_scraper.rs b/paper/src/scrapers/public_hamburg/public_hamburg_account_scraper.rs index 0ab3dae..6166cbf 100644 --- a/paper/src/scrapers/public_hamburg/public_hamburg_account_scraper.rs +++ b/paper/src/scrapers/public_hamburg/public_hamburg_account_scraper.rs @@ -20,10 +20,10 @@ impl PublicHamburgAccountScraper { let html = resource.load().await?; let document = scraper::Html::parse_document(html.as_str()); - self.scrape_document(document).await + self.scrape_document(document) } - async fn scrape_document(&self, document: Html) -> Result { + fn scrape_document(&self, document: Html) -> Result { let mut account = Account::new(); let info_notifications_selector = Selector::parse(r#"div[class="box box-info"]"#) @@ -130,7 +130,6 @@ mod tests { let document = scraper::Html::parse_document(html.as_str()); let account = PublicHamburgAccountScraper {} .scrape_document(document) - .await .unwrap(); assert_eq!(account.notifications.len(), 2); @@ -152,7 +151,6 @@ mod tests { let document = scraper::Html::parse_document(html.as_str()); let account = PublicHamburgAccountScraper {} .scrape_document(document) - .await .unwrap(); assert_eq!(account.notifications.len(), 2); diff --git a/paper/src/scrapers/public_hamburg/public_hamburg_loans_scraper.rs b/paper/src/scrapers/public_hamburg/public_hamburg_loans_scraper.rs index dd2ff3b..eb63a54 100644 --- a/paper/src/scrapers/public_hamburg/public_hamburg_loans_scraper.rs +++ b/paper/src/scrapers/public_hamburg/public_hamburg_loans_scraper.rs @@ -105,9 +105,9 @@ impl LoansScraper { #[cfg(test)] mod tests { - use std::fs; - use crate::model::{ItemAvailability, Loan, Loans, SearchResultDetail}; use super::LoansScraper; + use crate::model::{ItemAvailability, Loan, Loans, SearchResultDetail}; + use std::fs; #[test] fn it_parses_loans_from_login_success() { @@ -142,7 +142,9 @@ mod tests { availabilities: vec![], }, }, - search_result_detail_url: Some("suchergebnis-detail/medium/T020062902.html".to_string()), + search_result_detail_url: Some( + "suchergebnis-detail/medium/T020062902.html".to_string() + ), }, Loan { title: "Sternenschweif / 1 Sternenschweif - geheimnisvo".to_string(), @@ -166,7 +168,9 @@ mod tests { availabilities: vec![], }, }, - search_result_detail_url: Some("suchergebnis-detail/medium/T021001401.html".to_string()), + search_result_detail_url: Some( + "suchergebnis-detail/medium/T021001401.html".to_string() + ), }, Loan { title: "Der kleine Wassermann".to_string(), @@ -190,7 +194,9 @@ mod tests { availabilities: vec![], }, }, - search_result_detail_url: Some("suchergebnis-detail/medium/T019494523.html".to_string()), + search_result_detail_url: Some( + "suchergebnis-detail/medium/T019494523.html".to_string() + ), }, Loan { title: "Bambino-LÜK / [...] Tiere im Zoo : Alter 3 - 5".to_string(), @@ -214,38 +220,43 @@ mod tests { availabilities: vec![], }, }, - search_result_detail_url: Some("suchergebnis-detail/medium/T010693899.html".to_string()), + search_result_detail_url: Some( + "suchergebnis-detail/medium/T010693899.html".to_string() + ), }, ] - }); + } + ); } #[test] fn it_parses_loans_from_login_success_info_notice() { - let html = fs::read_to_string("src/fixtures/hamburg_public/login/login_success_info_notice.html") - .expect("Something went wrong reading the file"); + let html = + fs::read_to_string("src/fixtures/hamburg_public/login/login_success_info_notice.html") + .expect("Something went wrong reading the file"); let loans = LoansScraper::loans_from_html(html) .expect("Parsing loans should work with the given html"); assert_eq!( loans, Loans { - loans: vec![ - Loan { - title: "MiniLÜK / […] Lösungsgerät".to_string(), - author: "".to_string(), - can_renew: true, - renewal_token: None, - renewals_count: 0, - date_due: "10.05.2025".to_string(), - borrowed_at: "12.04.2025".to_string(), - item_number: "M58 385 945 2".to_string(), - locked_by_preorder: false, - details: SearchResultDetail::new(), - search_result_detail_url: Some("suchergebnis-detail/medium/T010694188.html".to_string()), - } - ] - }); + loans: vec![Loan { + title: "MiniLÜK / […] Lösungsgerät".to_string(), + author: "".to_string(), + can_renew: true, + renewal_token: None, + renewals_count: 0, + date_due: "10.05.2025".to_string(), + borrowed_at: "12.04.2025".to_string(), + item_number: "M58 385 945 2".to_string(), + locked_by_preorder: false, + details: SearchResultDetail::new(), + search_result_detail_url: Some( + "suchergebnis-detail/medium/T010694188.html".to_string() + ), + }] + } + ); } #[test] diff --git a/paper/src/scrapers/renewal_service.rs b/paper/src/scrapers/renewal_service.rs index e7e88cb..90bd26d 100644 --- a/paper/src/scrapers/renewal_service.rs +++ b/paper/src/scrapers/renewal_service.rs @@ -17,7 +17,7 @@ use super::{opc4v2_13vzg6::Opc4v2_13Vzg6RenewalParser, public_hamburg::RenewalLo #[derive(uniffi::Object)] pub struct RenewalService {} -#[uniffi::export] +#[uniffi::export(async_runtime = "tokio")] impl RenewalService { #[uniffi::constructor] pub fn new() -> Self { @@ -50,61 +50,55 @@ impl RenewalService { configuration: Configuration, ) -> Result { println!("opc4v2_13vzg6_renew start"); - return tokio::runtime::Builder::new_current_thread() - .enable_all() - .build()? - .block_on(async { - let cookie_store = Arc::new(Jar::default()); - let client_builder = ClientBuilder::new(); - let client = client_builder - .cookie_store(true) - .cookie_provider(cookie_store.clone()) - .build()?; - - let authenticator = OpacAuthenticator { - configuration: configuration.clone(), - }; - _ = authenticator.authenticate(&client).await; - - let username = configuration.username.clone().unwrap(); - let password = configuration.password.clone().unwrap(); - - client.get(configuration.base_url()).send().await?; - client.get(configuration.session_url()).send().await?; - - let url = format!( - "{}/LBS_WEB/borrower/loans.htm", - configuration.api_configuration.base_url - ); - - let mut headers = HeaderMap::new(); - headers.insert( - "Content-Type", - HeaderValue::from_str("application/x-www-form-urlencoded; charset=utf-8") - .unwrap(), - ); - - headers.insert( - "User-Agent", - HeaderValue::from_str("Flying Penguin").unwrap(), - ); - - let request = client - .post(url) - .headers(headers) - .timeout(Duration::from_secs(20)) - .query(&[ - ("volumeNumbersToRenew", renewal_token.as_str()), - ("LAN", "DU"), - ("username", username.as_str()), - ("password", password.as_str()), - ("renew", "Verlängern"), - ]); - - let html = request.send().await?.text().await?; - - Opc4v2_13Vzg6RenewalParser::loan_from(html) - }); + let cookie_store = Arc::new(Jar::default()); + let client_builder = ClientBuilder::new(); + let client = client_builder + .cookie_store(true) + .cookie_provider(cookie_store.clone()) + .build()?; + + let authenticator = OpacAuthenticator { + configuration: configuration.clone(), + }; + _ = authenticator.authenticate(&client).await; + + let username = configuration.username.clone().unwrap(); + let password = configuration.password.clone().unwrap(); + + client.get(configuration.base_url()).send().await?; + client.get(configuration.session_url()).send().await?; + + let url = format!( + "{}/LBS_WEB/borrower/loans.htm", + configuration.api_configuration.base_url + ); + + let mut headers = HeaderMap::new(); + headers.insert( + "Content-Type", + HeaderValue::from_str("application/x-www-form-urlencoded; charset=utf-8").unwrap(), + ); + + headers.insert( + "User-Agent", + HeaderValue::from_str("Flying Penguin").unwrap(), + ); + + let request = client + .post(url) + .headers(headers) + .timeout(Duration::from_secs(20)) + .query(&[ + ("volumeNumbersToRenew", renewal_token.as_str()), + ("LAN", "DU"), + ("username", username.as_str()), + ("password", password.as_str()), + ("renew", "Verlängern"), + ]); + + let html = request.send().await?.text().await?; + + Opc4v2_13Vzg6RenewalParser::loan_from(html) } async fn public_hamburg_renew( diff --git a/paper/src/scrapers/search_detail_scraper.rs b/paper/src/scrapers/search_detail_scraper.rs index ccd4329..860c913 100644 --- a/paper/src/scrapers/search_detail_scraper.rs +++ b/paper/src/scrapers/search_detail_scraper.rs @@ -5,7 +5,6 @@ use crate::model::Availability; use crate::model::AvailabilityStatus; use crate::model::Loan; use crate::model::SearchResultDetail; -use tokio::runtime::Builder; use super::opc4v2_13vzg6::Opc4v2_13Vzg6SearchDetailScraper; use super::public_hamburg::HamburgPublicSearchDetailScraper; @@ -15,7 +14,7 @@ pub(crate) struct SearchDetailScraper { configuration: APIConfiguration, } -#[uniffi::export] +#[uniffi::export(async_runtime = "tokio")] impl SearchDetailScraper { #[uniffi::constructor] pub fn new(configuration: APIConfiguration) -> Self { @@ -26,16 +25,7 @@ impl SearchDetailScraper { async fn details_for_url(&self, url: String) -> Result { let client = reqwest::ClientBuilder::new().cookie_store(true).build()?; - - let runtime = Builder::new_multi_thread() - .worker_threads(4) - .thread_name("fetch details") - .enable_io() - .enable_time() - .build()?; - - return runtime - .block_on(async { self.details_for_url_on_current_runtime(&client, url).await }); + self.details_for_url_on_current_runtime(&client, url).await } pub fn status(&self, availabilities: Vec) -> AvailabilityStatus { @@ -93,14 +83,10 @@ impl SearchDetailScraper { match self.configuration.api { crate::model::API::HamburgPublic => { - HamburgPublicSearchDetailScraper::search_result_detail_from(document) - .await - .ok_or_else(|| PaperError::ParseErrorSearchResultDetail) + Ok(HamburgPublicSearchDetailScraper::search_result_detail_from(document).await) } crate::model::API::Opc4v2_13Vzg6 => { - Opc4v2_13Vzg6SearchDetailScraper {} - .search_detail_from(document) - .await + Opc4v2_13Vzg6SearchDetailScraper {}.search_detail_from(document) } } } diff --git a/paper/src/scrapers/search_scraper.rs b/paper/src/scrapers/search_scraper.rs index 2624ead..ab24ad4 100644 --- a/paper/src/scrapers/search_scraper.rs +++ b/paper/src/scrapers/search_scraper.rs @@ -4,14 +4,13 @@ use crate::model::{APIConfiguration, SearchResultList, API}; use crate::scrapers::opc4v2_13vzg6::Opc4v2_13Vzg6SearchScraper; use crate::scrapers::public_hamburg::HamburgPublicSearchScraper; use reqwest::Client; -use tokio::runtime::Builder; #[derive(uniffi::Object)] pub struct SearchScraper { configuration: APIConfiguration, } -#[uniffi::export] +#[uniffi::export(async_runtime = "tokio")] impl SearchScraper { #[uniffi::constructor] pub fn new(configuration: APIConfiguration) -> Self { @@ -28,28 +27,17 @@ impl SearchScraper { match self.configuration.api { API::HamburgPublic => { let search_scraper = HamburgPublicSearchScraper {}; - let search_result = search_scraper.search(text, next_page_url).await; - return search_result; + search_scraper.search(text, next_page_url).await } API::Opc4v2_13Vzg6 => { let client = Client::new(); let api_client = APIClient::new_with_network_client(client, self.configuration.base_url.clone()); - let runtime = Builder::new_multi_thread() - .worker_threads(5) - .thread_name("search") - .enable_io() - .enable_time() - .build()?; - - return runtime.block_on(async { - let search_scraper = Opc4v2_13Vzg6SearchScraper {}; - let search_result = search_scraper - .search(text, next_page_url, &api_client) - .await; - return search_result; - }); + let search_scraper = Opc4v2_13Vzg6SearchScraper {}; + search_scraper + .search(text, next_page_url, &api_client) + .await } } } From 636d8c8e548d8af9e84bb55ba0f8d95282a3aab5 Mon Sep 17 00:00:00 2001 From: Florian Schreiber Date: Fri, 13 Jun 2025 20:01:21 +0200 Subject: [PATCH 2/2] Rename APIClient::test_urls to APIClient::validate_urls GH-6 --- paper/src/api/api_client.rs | 2 +- .../public_hamburg/hamburg_public_search_detail_scraper.rs | 2 +- .../scrapers/public_hamburg/hamburg_public_search_scraper.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/paper/src/api/api_client.rs b/paper/src/api/api_client.rs index a6f4324..1311c1e 100644 --- a/paper/src/api/api_client.rs +++ b/paper/src/api/api_client.rs @@ -31,7 +31,7 @@ impl APIClient { /// /// # Returns /// * `Option` - The first URL that returned 200 or None - pub(crate) async fn test_urls( + pub(crate) async fn validate_urls( urls: impl IntoIterator>, ) -> Option { for url in urls { diff --git a/paper/src/scrapers/public_hamburg/hamburg_public_search_detail_scraper.rs b/paper/src/scrapers/public_hamburg/hamburg_public_search_detail_scraper.rs index de1b4ba..9fcfb09 100644 --- a/paper/src/scrapers/public_hamburg/hamburg_public_search_detail_scraper.rs +++ b/paper/src/scrapers/public_hamburg/hamburg_public_search_detail_scraper.rs @@ -20,7 +20,7 @@ impl HamburgPublicSearchDetailScraper { detail.medium_title = document.get_text(r#".medium-detail-title"#); detail.medium_author = document.get_text(r#".medium-detail-author > a"#); - let image_url = APIClient::test_urls([ + let image_url = APIClient::validate_urls([ document.get_attribute("data-src", r#"img[class="b-lazy img-lazyload"]"#), document.get_attribute("data-alt-src", r#"img[class="b-lazy img-lazyload"]"#), ]); diff --git a/paper/src/scrapers/public_hamburg/hamburg_public_search_scraper.rs b/paper/src/scrapers/public_hamburg/hamburg_public_search_scraper.rs index 99ee251..1a859ab 100644 --- a/paper/src/scrapers/public_hamburg/hamburg_public_search_scraper.rs +++ b/paper/src/scrapers/public_hamburg/hamburg_public_search_scraper.rs @@ -70,7 +70,7 @@ impl HamburgPublicSearchScraper { list_item.subtitle = Some(cleaned_text); } - let image_url = APIClient::test_urls([ + let image_url = APIClient::validate_urls([ item.get_attribute("data-src", "div.search-results-image > img"), item.get_attribute("data-alt-src", "div.search-results-image > img"), ]);