Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -14,5 +14,5 @@ debug = true # Enable debug symbols. For example, we can use `dwarfdump` to
strip = "symbols"

[workspace.dependencies]
uniffi = { version = "0.27" }
uniffi = { version = "0.27", features = ["tokio"] }
uniffi_bindgen = { version = "0.27" }
22 changes: 22 additions & 0 deletions paper/src/api/api_client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,28 @@ impl APIClient {
Ok(response.status().as_u16())
}

/// Performs a HEAD request to the specified URLs to check its availability and returns the
/// first one that returns a status code 200
///
/// # Arguments
/// * `urls` - The URLs to ping
///
/// # Returns
/// * `Option<String>` - The first URL that returned 200 or None
pub(crate) async fn validate_urls(
urls: impl IntoIterator<Item = Option<String>>,
) -> Option<String> {
for url in urls {
if let Some(url) = url {
if matches!(APIClient::ping_url(&url).await, Ok(200)) {
return Some(url);
}
}
}

None
}

pub fn new_with_network_client(network_client: reqwest::Client, base_url: String) -> APIClient {
APIClient {
network_client,
Expand Down
10 changes: 0 additions & 10 deletions paper/src/authenticators/opac_authenticator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,6 @@ pub(crate) struct OpacAuthenticator {
}

impl OpacAuthenticator {
pub(crate) async fn verify_credentials_opc4vs2_13vzg6(
&self,
client: &Client,
) -> Result<bool, PaperError> {
return tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()?
.block_on(async { self.authenticate(client).await });
}

pub(crate) async fn authenticate(&self, client: &Client) -> Result<bool, PaperError> {
println!("`OpacAuthenticator::authenticate`");
let username = self.configuration.username.clone().unwrap();
Expand Down
23 changes: 8 additions & 15 deletions paper/src/authenticators/public_hamburg_authenticator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,22 +17,19 @@ pub(crate) struct PublicHamburgAuthenticator {
impl PublicHamburgAuthenticator {
pub(crate) async fn verify_credentials_public_hamburg(&self) -> Result<String, PaperError> {
let client = reqwest::ClientBuilder::new().cookie_store(true).build()?;
return tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()?
.block_on(async {
self.public_hamburg_authenticate_and_get_request_access_token(&client)
.await
});
self.public_hamburg_authenticate_and_get_request_access_token(&client)
.await
}

async fn authenticate_public_hamburg_via_cookies(
&self,
client: &Client,
request_token: String,
configuration: Configuration,
) -> Result<RawLoansPage, PaperError> {
if let (Some(username), Some(password)) = (configuration.username, configuration.password) {
if let (Some(username), Some(password)) = (
self.configuration.username.clone(),
self.configuration.password.clone(),
) {
if username == "" || password == "" {
return Err(PaperError::CredentialsBadInput);
}
Expand Down Expand Up @@ -77,7 +74,7 @@ impl PublicHamburgAuthenticator {

let token = token_scraper.get_request_token(&client).await?;
let login_result = self
.authenticate_public_hamburg_via_cookies(&client, token, self.configuration.clone())
.authenticate_public_hamburg_via_cookies(&client, token)
.await;

return match login_result {
Expand All @@ -98,11 +95,7 @@ impl PublicHamburgAuthenticator {

let request_token = token_scraper.get_request_token(&client).await?;
let result = self
.authenticate_public_hamburg_via_cookies(
&client,
request_token.clone(),
self.configuration.clone(),
)
.authenticate_public_hamburg_via_cookies(&client, request_token.clone())
.await;

return match result {
Expand Down
6 changes: 2 additions & 4 deletions paper/src/authenticators/scrape_authenticator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ pub struct Authenticator {
pub(crate) configuration: Configuration,
}

#[uniffi::export]
#[uniffi::export(async_runtime = "tokio")]

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, so this is where the magic happens ☝️

impl Authenticator {
#[uniffi::constructor]
fn new(configuration: Configuration) -> Self {
Expand All @@ -35,9 +35,7 @@ impl Authenticator {
configuration: self.configuration.clone(),
};
let client = reqwest::ClientBuilder::new().cookie_store(true).build()?;
let result = opac_authenticator
.verify_credentials_opc4vs2_13vzg6(&client)
.await;
let result = opac_authenticator.authenticate(&client).await;
return match result {
Ok(signed_in) => Ok(if signed_in {
ValidationStatus::Valid
Expand Down
2 changes: 1 addition & 1 deletion paper/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ impl Paper {

let search = HamburgPublicSearchScraper {};

match search.search_on_current_runtime(&input, None).await {
match search.search(&input, None).await {
Ok(result) => Self::print_search_result(result),
Err(err) => println!("Failed to search: {:?}", err),
}
Expand Down
45 changes: 17 additions & 28 deletions paper/src/scrapers/library_scraper.rs
Original file line number Diff line number Diff line change
@@ -1,20 +1,17 @@
use std::sync::Arc;

use crate::authenticators::{OpacAuthenticator, PublicHamburgAuthenticator};
use crate::{
configuration::Configuration, error::PaperError,
scrapers::public_hamburg::PublicHamburgAccountScraper, scrapers::BalanceScraper,
};
use std::sync::Arc;

use super::opc4v2_13vzg6::Opc4v2_13Vzg6AccountScraper;
use super::SearchDetailScraper;
use crate::model::Loan;
use crate::model::Loans;
use futures::future;
use reqwest::cookie::Jar;
use reqwest::ClientBuilder;
use tokio::runtime::Builder;

use super::opc4v2_13vzg6::Opc4v2_13Vzg6AccountScraper;
use super::SearchDetailScraper;

use crate::model::Account;

Expand All @@ -27,7 +24,7 @@ pub struct LibraryScraper {
configuration: Configuration,
}

#[uniffi::export]
#[uniffi::export(async_runtime = "tokio")]
impl LibraryScraper {
#[uniffi::constructor]
pub fn new(configuration: Configuration) -> Self {
Expand All @@ -43,29 +40,21 @@ impl LibraryScraper {
.cookie_store(true)
.cookie_provider(cookie_store.clone())
.build()?;
let runtime = Builder::new_multi_thread()
.worker_threads(4)
.thread_name("fetch account")
.enable_io()
.enable_time()
.build()?;

return runtime.block_on(async {
match self.configuration.api_configuration.api {
crate::model::API::HamburgPublic => {
self.public_hamburg_fetch_on_current_runtime(&client).await
}
crate::model::API::Opc4v2_13Vzg6 => {
client
.get(self.configuration.session_url())
.query(&[("USR", "1022"), ("LAN", "DU"), ("BES", "1")])
.send()
.await?;

self.opc4v2_13vzg6_fetch_on_current_runtime(&client).await
}
match self.configuration.api_configuration.api {
crate::model::API::HamburgPublic => {
self.public_hamburg_fetch_on_current_runtime(&client).await
}
crate::model::API::Opc4v2_13Vzg6 => {
client
.get(self.configuration.session_url())
.query(&[("USR", "1022"), ("LAN", "DU"), ("BES", "1")])
.send()
.await?;

self.opc4v2_13vzg6_fetch_on_current_runtime(&client).await
}
});
}
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,7 @@ use crate::{error::PaperError, model::SearchResultDetail};
pub(crate) struct Opc4v2_13Vzg6SearchDetailScraper {}

impl Opc4v2_13Vzg6SearchDetailScraper {
pub(crate) async fn search_detail_from(
&self,
html: Html,
) -> Result<SearchResultDetail, PaperError> {
pub(crate) fn search_detail_from(&self, html: Html) -> Result<SearchResultDetail, PaperError> {
let mut detail = SearchResultDetail::new();
let content_table_selector = scraper::Selector::parse(r#"body > table > tbody > tr:nth-child(7) > td.cnt > table > tbody > tr:nth-child(4) > td > table > tbody > tr:nth-child(1) > td:nth-child(2) > table > tbody > tr"#).unwrap();
let rows = html.select(&content_table_selector);
Expand Down Expand Up @@ -63,7 +60,7 @@ mod tests {
fs::read_to_string("src/fixtures/opc4v2_13Vzg6/search/katalog_detail_book_sbb.html")
.expect("Something went wrong reading the file");
let html = scraper::Html::parse_document(html_string.as_str());
let search_detail = sut.search_detail_from(html).await.unwrap();
let search_detail = sut.search_detail_from(html).unwrap();

assert_eq!(
search_detail.full_title,
Expand All @@ -83,7 +80,7 @@ mod tests {
fs::read_to_string("src/fixtures/opc4v2_13Vzg6/search/katalog_detail_book_gbv_hh.html")
.expect("Something went wrong reading the file");
let html = scraper::Html::parse_document(html_string.as_str());
let search_detail = sut.search_detail_from(html).await.unwrap();
let search_detail = sut.search_detail_from(html).unwrap();

assert_eq!(
search_detail.full_title,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,35 +6,24 @@ use crate::model::Location;
use crate::model::SearchResultDetail;
use crate::scrapers::text_provider::TextProvider;
use scraper::Selector;
use std::future::Future;

#[derive(uniffi::Object)]
pub(crate) struct HamburgPublicSearchDetailScraper {}

impl HamburgPublicSearchDetailScraper {
pub(crate) async fn search_result_detail_from(
pub(crate) fn search_result_detail_from(
document: scraper::Html,
) -> Option<SearchResultDetail> {
) -> impl Future<Output = SearchResultDetail> + Send {

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This confused be for a moment that the 2 scrapers Opc4v2_13Vzg6SearchDetailScraper/HamburgPublicSearchDetailScraper have similar API with different return types. Nothing you need to change, just something I stumbled upon.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good point, I could remove the Result from Opc4v2_13Vzg6SearchDetailScraper as well, but there would still be the difference of this function being async, while the other function is sync (and it doesn't really make sense to introduce async there for no reason). I'm currently thinking about some other ways to align the APIs, so let's keep it like this for now and followup at a later point.

let mut detail = SearchResultDetail::new();

detail.medium_title = document.get_text(r#".medium-detail-title"#);
detail.medium_author = document.get_text(r#".medium-detail-author > a"#);

if let Some(url1) =
document.get_attribute("data-src", r#"img[class="b-lazy img-lazyload"]"#)
{
match APIClient::ping_url(url1.as_str()).await {
Ok(status) => match status {
200 => {
detail.small_image_url = Some(url1);
}
_ => {
detail.small_image_url = document
.get_attribute("data-alt-src", r#"img[class="b-lazy img-lazyload"]"#);
}
},
Err(e) => println!("Error checking site: {}", e),
}
}
let image_url = APIClient::validate_urls([
document.get_attribute("data-src", r#"img[class="b-lazy img-lazyload"]"#),
document.get_attribute("data-alt-src", r#"img[class="b-lazy img-lazyload"]"#),
]);

if let Ok(detail_data_selector) =
Selector::parse(r#"div[class="medium-detail-data medium-detail-data-cols"] > ul > li"#)
Expand Down Expand Up @@ -81,7 +70,13 @@ impl HamburgPublicSearchDetailScraper {
let availabilities = HamburgPublicSearchDetailScraper::parse_availabilities(document);
detail.availability = ItemAvailability::with(availabilities);
}
Some(detail)

async move {
// run the network request last, such that document was already dropped since it is not Send
detail.small_image_url = image_url.await;

detail
}
}

fn parse_availabilities(html: scraper::Html) -> Vec<Availability> {
Expand Down Expand Up @@ -134,9 +129,7 @@ mod tests {
.expect("Something went wrong reading the file");
let document = scraper::Html::parse_document(html.as_str());
let search_result_detail =
HamburgPublicSearchDetailScraper::search_result_detail_from(document)
.await
.unwrap();
HamburgPublicSearchDetailScraper::search_result_detail_from(document).await;

assert_eq!(
search_result_detail.medium_title,
Expand All @@ -156,9 +149,7 @@ mod tests {
.expect("Something went wrong reading the file");
let document = scraper::Html::parse_document(html.as_str());
let search_result_detail =
HamburgPublicSearchDetailScraper::search_result_detail_from(document)
.await
.unwrap();
HamburgPublicSearchDetailScraper::search_result_detail_from(document).await;
assert_eq!(
search_result_detail.small_image_url,
Some("https://www.hugendubel.info/annotstream/9783836961592/COP".to_string())
Expand Down
Loading