diff --git a/README.md b/README.md index 8c0135c..0bb285a 100644 --- a/README.md +++ b/README.md @@ -59,36 +59,21 @@ Some useful commands It is important to keep track of when environment variables are being "injected" into the application. Sometimes it is done during the image build and other times it is done during runtime. -#### Sample `.env` file for local testing +#### `.env` file for local testing -The frontent application will be available at localhost:8100 +The frontend application will be available at localhost:8100 ```bash -TYPESENSE_API_KEY= +TYPESENSE_API_KEY=replace-me TYPESENSE_HOST=localhost +TYPESENSE_PORT=8100 TYPESENSE_UPSTREAM_HOST=typesense +TYPESENSE_PROTOCOL=http +TYPESENSE_PATH=/api TYPESENSE_FETCHER_HOST=nginx TYPESENSE_FETCHER_PORT=80 TYPESENSE_FETCHER_PROTOCOL=http -TYPESENSE_FETCHER_PATH="/api" -TYPESENSE_PORT=8100 -TYPESENSE_PROTOCOL=http -TYPESENSE_PATH="/api" - -``` - -#### Indicative `.env` file for deployment to AWH - -This respects the convention that AWH sidecar applications are all exposed as `localhost`, rather than at their own hostnames (as works for local Docker compose) - -```bash -TYPESENSE_API_KEY= -TYPESENSE_HOST= -TYPESENSE_UPSTREAM_HOST=localhost -TYPESENSE_FETCHER_HOST=localhost -TYPESENSE_PORT=80 -TYPESENSE_PROTOCOL=https -TYPESENSE_PATH="/api" +TYPESENSE_FETCHER_PATH=/api ``` ## Design diff --git a/fetcher/abo.py b/fetcher/abo.py index 75827ea..aacac46 100644 --- a/fetcher/abo.py +++ b/fetcher/abo.py @@ -138,7 +138,8 @@ def process(file_path, data_directory): "source": SOURCE_ID, "title_id": title_id, "title_full": title_full, - "page_number": label, + "date": "n.d.", + "page_number": "(" + label.replace("page", "") + ")", "remote_path": remote_path, "image_url": image_url, "ocr_text_original": ocr_text, diff --git a/fetcher/anno.py b/fetcher/anno.py index 92d3138..eb0b6ec 100644 --- a/fetcher/anno.py +++ b/fetcher/anno.py @@ -120,7 +120,8 @@ def process(file_path, data_directory): "title_id": title_id, "title_full": TITLE_MAP[title_id], "datum": datum, - "page_number": page_number, + "date": utils.datum_to_iso_8601(datum), + "page_number": "(" + page_number + ")", "remote_path": remote_path, "image_url": image_url, "ocr_text_original": ocr_text, diff --git a/fetcher/mdz.py b/fetcher/mdz.py index a71cfcd..a5ff913 100644 --- a/fetcher/mdz.py +++ b/fetcher/mdz.py @@ -52,7 +52,7 @@ def _download_hocr_files(self, item_id, output_dir): hocr_dir = os.path.join(output_dir, 'hocr') os.makedirs(hocr_dir, exist_ok=True) - + for sequence in manifest['sequences']: for canvas in tqdm.tqdm(sequence['canvases'], desc=f"Downloading {item_id}"): label = canvas['label'] @@ -79,11 +79,21 @@ def process(file_path, data_directory): title_full = manifest['label'] + structures = manifest.get("structures", None) + canvas_to_structure_label = {} + + if structures: + for structure in structures: + canvases = structure.get("canvases", []) + for canvas in canvases: + canvas_to_structure_label[canvas] = structure["label"] + remote_path = None image_url = None for sequence in manifest['sequences']: for canvas in sequence['canvases']: if label == canvas['label']: + date = canvas_to_structure_label.get(canvas['@id'], "n.d.") remote_path = canvas['seeAlso']['@id'] for image in canvas['images']: original_image_url = image['resource']['@id'] @@ -100,6 +110,7 @@ def process(file_path, data_directory): "source": SOURCE_ID, "title_id": title_id, "title_full": title_full, + "date": date, "page_number": label, "remote_path": remote_path, "image_url": image_url, diff --git a/fetcher/requirements.txt b/fetcher/requirements.txt index 4760476..a94e421 100644 --- a/fetcher/requirements.txt +++ b/fetcher/requirements.txt @@ -1,6 +1,6 @@ beautifulsoup4==4.13.3 mistune==3.1.1 -Requests==2.32.3 +Requests==2.32.4 requests_cache==1.2.1 tqdm==4.67.1 typesense==0.21.0 diff --git a/fetcher/sources/items_test.yaml b/fetcher/sources/items_test.yaml index 6e56f74..268906f 100644 --- a/fetcher/sources/items_test.yaml +++ b/fetcher/sources/items_test.yaml @@ -3,9 +3,11 @@ mdz: title_ids: - bsb10616989_00623_u001 + - bsb10527965 abo: title_ids: - Z251542006 + - Z259832605 anno: title_ids: sam: diff --git a/fetcher/utils.py b/fetcher/utils.py index b77d291..fecc491 100644 --- a/fetcher/utils.py +++ b/fetcher/utils.py @@ -60,6 +60,9 @@ def get_query_value(url, key): value = query_params.get(key, [None])[0] return value +def datum_to_iso_8601(datum): + return "-".join([datum[0:4], datum[4:6], datum[6:]]) + def download_remote_file(url, path, session): os.makedirs(os.path.dirname(path), exist_ok=True) diff --git a/frontend/index.html b/frontend/index.html index 9959aa9..bd1b361 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -1,43 +1,52 @@ - - - Historical Document Search - - - + + + Historical Document Search + + + -
+
-

Historical Document Search

-
-
- +

Historical Document Search

+
+
+ +
-
-
-
-
-
- +
+
+
+ +
+
+ +
+
+
+ +
+ +
+ +
+
-
-
- -
-
-
- + +
+
diff --git a/frontend/script.js b/frontend/script.js index 843af5d..0f1eb6c 100644 --- a/frontend/script.js +++ b/frontend/script.js @@ -112,6 +112,12 @@ function createPopup() { document.body.appendChild(popup); } +popup.addEventListener('click', (event) => { + if (!popupContent.contains(event.target) && event.target !== closeButton) { + popup.style.display = 'none'; + document.body.style.overflow = 'auto'; + } +}); /** * Display the search results in the UI. @@ -143,12 +149,15 @@ function displayResults(results) { const title = document.createElement('p'); title.className = 'title is-4'; - title.textContent = `${result.document.title_id} - Page ${result.document.page_number}`; + title.textContent = `${result.document.title_full} - Date ${result.document.date} - Page ${result.document.page_number}`; - const sourceTag = document.createElement('span'); + const sourceTag = document.createElement('a'); sourceTag.className = 'tag is-info source-tag'; + sourceTag.href = result.document.remote_path; + sourceTag.target = '_blank'; sourceTag.textContent = result.document.source; + titleContainer.appendChild(title); titleContainer.appendChild(sourceTag); @@ -187,10 +196,6 @@ function displayResults(results) { const actions = document.createElement('div'); actions.className = 'field is-grouped mt-4'; - const viewButton = document.createElement('p'); - viewButton.className = 'control'; - viewButton.innerHTML = `View Original`; - const viewFullTextButton = document.createElement('p'); viewFullTextButton.className = 'control'; viewFullTextButton.innerHTML = ``; @@ -221,9 +226,8 @@ function displayResults(results) { imageButton.className = 'control'; imageButton.innerHTML = `View Image`; - actions.appendChild(viewButton); - actions.appendChild(imageButton); actions.appendChild(viewFullTextButton); + actions.appendChild(imageButton); cardContent.appendChild(titleContainer); cardContent.appendChild(snippet); diff --git a/frontend/style.css b/frontend/style.css index 8e09fce..b6779c5 100644 --- a/frontend/style.css +++ b/frontend/style.css @@ -53,7 +53,7 @@ } #text-popup .close-button { - position: absolute; + position: fixed; top: 10px; right: 15px; border: none; @@ -63,6 +63,7 @@ cursor: pointer; border-radius: 4px; font-size: 1rem; + z-index: 1001; } #text-popup .close-button:hover { diff --git a/scripts/create-admin-key.sh b/scripts/create-admin-key.sh new file mode 100755 index 0000000..31352b7 --- /dev/null +++ b/scripts/create-admin-key.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +curl 'http://localhost:8108/keys' -X POST -H "X-TYPESENSE-API-KEY: ${TYPESENSE_API_KEY}" \ + -H 'Content-Type: application/json' \ + -d '{"description":"Admin key.","actions": ["*"], "collections": ["*"]}' + diff --git a/scripts/create-readonly-key.sh b/scripts/create-readonly-key.sh new file mode 100755 index 0000000..4e33761 --- /dev/null +++ b/scripts/create-readonly-key.sh @@ -0,0 +1,5 @@ +#/usr/bin/env bash +curl 'http://localhost:8108/keys' -X POST -H "X-TYPESENSE-API-KEY: ${TYPESENSE_API_KEY}" \ + -H 'Content-Type: application/json' \ + -d '{"description":"Search-only key.","actions": ["documents:search"], "collections": ["companies"]}' + diff --git a/scripts/readenv.sh b/scripts/readenv.sh new file mode 100755 index 0000000..0b2c91c --- /dev/null +++ b/scripts/readenv.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +# Source this script using "source scripts/readenv.sh" +export $(cat .env | awk '''!/^\s*#/''' | xargs)