From a4ee85ef844a922c4cbfbda16f8cc3ad4963dc5e Mon Sep 17 00:00:00 2001 From: zeynepaki Date: Mon, 19 May 2025 11:56:04 +0100 Subject: [PATCH 1/9] Fixes #25 and #24 --- frontend/index.html | 71 +++++++++++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 31 deletions(-) diff --git a/frontend/index.html b/frontend/index.html index e6e8b77..a81e3c1 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -1,46 +1,55 @@ - - - Historical Document Search - - - + + + Historical Document Search + + + -
+
-

Historical Document Search

-
-
- +

Historical Document Search

+
+
+ +
-
-
-
-
-
- +
+
+
+ +
+
+ +
+
+
+ +
+ +
+ +
+
-
-
- -
-
-
- + +
+
-
+
- + From d7cb7cae120142005d366cdc55ac448d4fc5743a Mon Sep 17 00:00:00 2001 From: zeynepaki Date: Mon, 19 May 2025 12:15:42 +0100 Subject: [PATCH 2/9] Fixes #12 --- frontend/script.js | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/frontend/script.js b/frontend/script.js index ce06370..d097d90 100644 --- a/frontend/script.js +++ b/frontend/script.js @@ -104,10 +104,13 @@ function displayResults(results) { title.className = 'title is-4'; title.textContent = `${result.document.title_id} - Page ${result.document.page_number}`; - const sourceTag = document.createElement('span'); + const sourceTag = document.createElement('a'); sourceTag.className = 'tag is-info source-tag'; + sourceTag.href = result.document.remote_path; + sourceTag.target = '_blank'; sourceTag.textContent = result.document.source; + titleContainer.appendChild(title); titleContainer.appendChild(sourceTag); @@ -146,10 +149,6 @@ function displayResults(results) { const actions = document.createElement('div'); actions.className = 'field is-grouped mt-4'; - const viewButton = document.createElement('p'); - viewButton.className = 'control'; - viewButton.innerHTML = `View Original`; - const viewFullTextButton = document.createElement('p'); viewFullTextButton.className = 'control'; viewFullTextButton.innerHTML = ``; @@ -180,9 +179,8 @@ function displayResults(results) { imageButton.className = 'control'; imageButton.innerHTML = `View Image`; - actions.appendChild(viewButton); - actions.appendChild(imageButton); actions.appendChild(viewFullTextButton); + actions.appendChild(imageButton); cardContent.appendChild(titleContainer); cardContent.appendChild(snippet); From bd2db878cf575b6fc8e073a786b640a4724a7162 Mon Sep 17 00:00:00 2001 From: zeynepaki Date: Mon, 19 May 2025 12:23:40 +0100 Subject: [PATCH 3/9] Fixes #33 --- frontend/script.js | 8 +++++++- frontend/style.css | 3 ++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/frontend/script.js b/frontend/script.js index d097d90..83ac478 100644 --- a/frontend/script.js +++ b/frontend/script.js @@ -74,10 +74,16 @@ closeButton.onclick = () => { const popupContent = document.createElement('div'); popupContent.id = 'popup-content'; -popup.appendChild(closeButton); popup.appendChild(popupContent); +popup.appendChild(closeButton); document.body.appendChild(popup); +popup.addEventListener('click', (event) => { + if (!popupContent.contains(event.target) && event.target !== closeButton) { + popup.style.display = 'none'; + document.body.style.overflow = 'auto'; + } +}); function displayResults(results) { const resultsContainer = document.getElementById('results'); diff --git a/frontend/style.css b/frontend/style.css index 8e09fce..b6779c5 100644 --- a/frontend/style.css +++ b/frontend/style.css @@ -53,7 +53,7 @@ } #text-popup .close-button { - position: absolute; + position: fixed; top: 10px; right: 15px; border: none; @@ -63,6 +63,7 @@ cursor: pointer; border-radius: 4px; font-size: 1rem; + z-index: 1001; } #text-popup .close-button:hover { From 3ee836fea0bf0280666488a7d0d212f9aba1fff9 Mon Sep 17 00:00:00 2001 From: Samantha Finnigan <1038320+sjmf@users.noreply.github.com> Date: Thu, 12 Jun 2025 20:01:36 +0100 Subject: [PATCH 4/9] Create scripts to create typesense read-only API keys. Provisioning these resolved #4. --- scripts/create-admin-key.sh | 5 +++++ scripts/create-readonly-key.sh | 5 +++++ scripts/readenv.sh | 3 +++ 3 files changed, 13 insertions(+) create mode 100755 scripts/create-admin-key.sh create mode 100755 scripts/create-readonly-key.sh create mode 100755 scripts/readenv.sh diff --git a/scripts/create-admin-key.sh b/scripts/create-admin-key.sh new file mode 100755 index 0000000..31352b7 --- /dev/null +++ b/scripts/create-admin-key.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +curl 'http://localhost:8108/keys' -X POST -H "X-TYPESENSE-API-KEY: ${TYPESENSE_API_KEY}" \ + -H 'Content-Type: application/json' \ + -d '{"description":"Admin key.","actions": ["*"], "collections": ["*"]}' + diff --git a/scripts/create-readonly-key.sh b/scripts/create-readonly-key.sh new file mode 100755 index 0000000..4e33761 --- /dev/null +++ b/scripts/create-readonly-key.sh @@ -0,0 +1,5 @@ +#/usr/bin/env bash +curl 'http://localhost:8108/keys' -X POST -H "X-TYPESENSE-API-KEY: ${TYPESENSE_API_KEY}" \ + -H 'Content-Type: application/json' \ + -d '{"description":"Search-only key.","actions": ["documents:search"], "collections": ["companies"]}' + diff --git a/scripts/readenv.sh b/scripts/readenv.sh new file mode 100755 index 0000000..0b2c91c --- /dev/null +++ b/scripts/readenv.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +# Source this script using "source scripts/readenv.sh" +export $(cat .env | awk '''!/^\s*#/''' | xargs) From a4e7d8c7eab39b54e26562c07f642ceb9bea2f03 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Jun 2025 09:24:18 +0000 Subject: [PATCH 5/9] Bump requests from 2.32.3 to 2.32.4 in /fetcher Bumps [requests](https://github.com/psf/requests) from 2.32.3 to 2.32.4. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.32.3...v2.32.4) --- updated-dependencies: - dependency-name: requests dependency-version: 2.32.4 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- fetcher/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fetcher/requirements.txt b/fetcher/requirements.txt index 4760476..a94e421 100644 --- a/fetcher/requirements.txt +++ b/fetcher/requirements.txt @@ -1,6 +1,6 @@ beautifulsoup4==4.13.3 mistune==3.1.1 -Requests==2.32.3 +Requests==2.32.4 requests_cache==1.2.1 tqdm==4.67.1 typesense==0.21.0 From 3e05ddda4a15e5503b1a0166d34cead4961e9460 Mon Sep 17 00:00:00 2001 From: Eamonn Bell Date: Tue, 24 Jun 2025 10:46:49 +0100 Subject: [PATCH 6/9] Update README with valid .env example for local testing. --- README.md | 29 +++++++---------------------- 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 8c0135c..0bb285a 100644 --- a/README.md +++ b/README.md @@ -59,36 +59,21 @@ Some useful commands It is important to keep track of when environment variables are being "injected" into the application. Sometimes it is done during the image build and other times it is done during runtime. -#### Sample `.env` file for local testing +#### `.env` file for local testing -The frontent application will be available at localhost:8100 +The frontend application will be available at localhost:8100 ```bash -TYPESENSE_API_KEY= +TYPESENSE_API_KEY=replace-me TYPESENSE_HOST=localhost +TYPESENSE_PORT=8100 TYPESENSE_UPSTREAM_HOST=typesense +TYPESENSE_PROTOCOL=http +TYPESENSE_PATH=/api TYPESENSE_FETCHER_HOST=nginx TYPESENSE_FETCHER_PORT=80 TYPESENSE_FETCHER_PROTOCOL=http -TYPESENSE_FETCHER_PATH="/api" -TYPESENSE_PORT=8100 -TYPESENSE_PROTOCOL=http -TYPESENSE_PATH="/api" - -``` - -#### Indicative `.env` file for deployment to AWH - -This respects the convention that AWH sidecar applications are all exposed as `localhost`, rather than at their own hostnames (as works for local Docker compose) - -```bash -TYPESENSE_API_KEY= -TYPESENSE_HOST= -TYPESENSE_UPSTREAM_HOST=localhost -TYPESENSE_FETCHER_HOST=localhost -TYPESENSE_PORT=80 -TYPESENSE_PROTOCOL=https -TYPESENSE_PATH="/api" +TYPESENSE_FETCHER_PATH=/api ``` ## Design From 01215f1f49786a2dd83d22bd70e8038b588efba8 Mon Sep 17 00:00:00 2001 From: Eamonn Bell Date: Wed, 25 Jun 2025 08:48:00 +0100 Subject: [PATCH 7/9] Increase number of items for test corpus to increase variety of metadata types available. --- fetcher/sources/items_test.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fetcher/sources/items_test.yaml b/fetcher/sources/items_test.yaml index 6e56f74..268906f 100644 --- a/fetcher/sources/items_test.yaml +++ b/fetcher/sources/items_test.yaml @@ -3,9 +3,11 @@ mdz: title_ids: - bsb10616989_00623_u001 + - bsb10527965 abo: title_ids: - Z251542006 + - Z259832605 anno: title_ids: sam: From 1a6879d67c4cf24a229deb57f7a2216795143cea Mon Sep 17 00:00:00 2001 From: Eamonn Bell Date: Wed, 25 Jun 2025 08:49:09 +0100 Subject: [PATCH 8/9] Adds support for dates where data allows, and standardises presentation of page numbers --- fetcher/abo.py | 3 ++- fetcher/anno.py | 3 ++- fetcher/mdz.py | 13 ++++++++++++- fetcher/utils.py | 3 +++ 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/fetcher/abo.py b/fetcher/abo.py index 75827ea..aacac46 100644 --- a/fetcher/abo.py +++ b/fetcher/abo.py @@ -138,7 +138,8 @@ def process(file_path, data_directory): "source": SOURCE_ID, "title_id": title_id, "title_full": title_full, - "page_number": label, + "date": "n.d.", + "page_number": "(" + label.replace("page", "") + ")", "remote_path": remote_path, "image_url": image_url, "ocr_text_original": ocr_text, diff --git a/fetcher/anno.py b/fetcher/anno.py index 92d3138..eb0b6ec 100644 --- a/fetcher/anno.py +++ b/fetcher/anno.py @@ -120,7 +120,8 @@ def process(file_path, data_directory): "title_id": title_id, "title_full": TITLE_MAP[title_id], "datum": datum, - "page_number": page_number, + "date": utils.datum_to_iso_8601(datum), + "page_number": "(" + page_number + ")", "remote_path": remote_path, "image_url": image_url, "ocr_text_original": ocr_text, diff --git a/fetcher/mdz.py b/fetcher/mdz.py index a71cfcd..a5ff913 100644 --- a/fetcher/mdz.py +++ b/fetcher/mdz.py @@ -52,7 +52,7 @@ def _download_hocr_files(self, item_id, output_dir): hocr_dir = os.path.join(output_dir, 'hocr') os.makedirs(hocr_dir, exist_ok=True) - + for sequence in manifest['sequences']: for canvas in tqdm.tqdm(sequence['canvases'], desc=f"Downloading {item_id}"): label = canvas['label'] @@ -79,11 +79,21 @@ def process(file_path, data_directory): title_full = manifest['label'] + structures = manifest.get("structures", None) + canvas_to_structure_label = {} + + if structures: + for structure in structures: + canvases = structure.get("canvases", []) + for canvas in canvases: + canvas_to_structure_label[canvas] = structure["label"] + remote_path = None image_url = None for sequence in manifest['sequences']: for canvas in sequence['canvases']: if label == canvas['label']: + date = canvas_to_structure_label.get(canvas['@id'], "n.d.") remote_path = canvas['seeAlso']['@id'] for image in canvas['images']: original_image_url = image['resource']['@id'] @@ -100,6 +110,7 @@ def process(file_path, data_directory): "source": SOURCE_ID, "title_id": title_id, "title_full": title_full, + "date": date, "page_number": label, "remote_path": remote_path, "image_url": image_url, diff --git a/fetcher/utils.py b/fetcher/utils.py index b77d291..fecc491 100644 --- a/fetcher/utils.py +++ b/fetcher/utils.py @@ -60,6 +60,9 @@ def get_query_value(url, key): value = query_params.get(key, [None])[0] return value +def datum_to_iso_8601(datum): + return "-".join([datum[0:4], datum[4:6], datum[6:]]) + def download_remote_file(url, path, session): os.makedirs(os.path.dirname(path), exist_ok=True) From 9439a9e457f7d68fcd7f2f79b2679e0d033bb0f8 Mon Sep 17 00:00:00 2001 From: Eamonn Bell Date: Mon, 30 Jun 2025 07:33:37 +0100 Subject: [PATCH 9/9] Add minimal frontend change to support date and full titles --- frontend/script.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/script.js b/frontend/script.js index 843af5d..525d482 100644 --- a/frontend/script.js +++ b/frontend/script.js @@ -143,7 +143,7 @@ function displayResults(results) { const title = document.createElement('p'); title.className = 'title is-4'; - title.textContent = `${result.document.title_id} - Page ${result.document.page_number}`; + title.textContent = `${result.document.title_full} - Date ${result.document.date} - Page ${result.document.page_number}`; const sourceTag = document.createElement('span'); sourceTag.className = 'tag is-info source-tag';