Release to Azure from a4ee85e

README.md

@@ -59,36 +59,21 @@ Some useful commands
 
 It is important to keep track of when environment variables are being "injected" into the application. Sometimes it is done during the image build and other times it is done during runtime.
 
-#### Sample `.env` file for local testing
+#### `.env` file for local testing
 
-The frontent application will be available at localhost:8100 
+The frontend application will be available at localhost:8100 
 
 ```bash
-TYPESENSE_API_KEY=
+TYPESENSE_API_KEY=replace-me
 TYPESENSE_HOST=localhost
+TYPESENSE_PORT=8100
 TYPESENSE_UPSTREAM_HOST=typesense
+TYPESENSE_PROTOCOL=http
+TYPESENSE_PATH=/api
 TYPESENSE_FETCHER_HOST=nginx
 TYPESENSE_FETCHER_PORT=80
 TYPESENSE_FETCHER_PROTOCOL=http
-TYPESENSE_FETCHER_PATH="/api"
-TYPESENSE_PORT=8100
-TYPESENSE_PROTOCOL=http
-TYPESENSE_PATH="/api"
-
-```
-
-#### Indicative `.env` file for deployment to AWH
-
-This respects the convention that AWH sidecar applications are all exposed as `localhost`, rather than at their own hostnames (as works for local Docker compose) 
-
-```bash
-TYPESENSE_API_KEY=
-TYPESENSE_HOST=
-TYPESENSE_UPSTREAM_HOST=localhost
-TYPESENSE_FETCHER_HOST=localhost
-TYPESENSE_PORT=80
-TYPESENSE_PROTOCOL=https
-TYPESENSE_PATH="/api"
+TYPESENSE_FETCHER_PATH=/api
 ```
 
 ## Design

fetcher/abo.py

@@ -138,7 +138,8 @@ def process(file_path, data_directory):
             "source": SOURCE_ID,
             "title_id": title_id,
             "title_full": title_full,
-            "page_number": label,
+            "date": "n.d.",
+            "page_number": "(" + label.replace("page", "") + ")",
             "remote_path": remote_path,
             "image_url": image_url,
             "ocr_text_original": ocr_text,

fetcher/anno.py

@@ -120,7 +120,8 @@ def process(file_path, data_directory):
             "title_id": title_id,
             "title_full": TITLE_MAP[title_id],
             "datum": datum,
-            "page_number": page_number,
+            "date": utils.datum_to_iso_8601(datum),
+            "page_number": "(" + page_number + ")",
             "remote_path": remote_path,
             "image_url": image_url,
             "ocr_text_original": ocr_text,

fetcher/mdz.py

@@ -52,7 +52,7 @@ def _download_hocr_files(self, item_id, output_dir):
 
         hocr_dir = os.path.join(output_dir, 'hocr')
         os.makedirs(hocr_dir, exist_ok=True)
-        
+
         for sequence in manifest['sequences']:
             for canvas in tqdm.tqdm(sequence['canvases'], desc=f"Downloading {item_id}"):
                 label = canvas['label']
@@ -79,11 +79,21 @@ def process(file_path, data_directory):
 
         title_full = manifest['label']
 
+        structures = manifest.get("structures", None)
+        canvas_to_structure_label = {}
+
+        if structures:
+            for structure in structures:
+                canvases = structure.get("canvases", [])
+                for canvas in canvases:
+                    canvas_to_structure_label[canvas] = structure["label"]
+
         remote_path = None
         image_url = None
         for sequence in manifest['sequences']:
             for canvas in sequence['canvases']:
                 if label == canvas['label']:
+                    date = canvas_to_structure_label.get(canvas['@id'], "n.d.")
                     remote_path = canvas['seeAlso']['@id']
                     for image in canvas['images']:
                         original_image_url = image['resource']['@id']
@@ -100,6 +110,7 @@ def process(file_path, data_directory):
             "source": SOURCE_ID,
             "title_id": title_id,
             "title_full": title_full,
+            "date": date,
             "page_number": label,
             "remote_path": remote_path,
             "image_url": image_url,

fetcher/requirements.txt

@@ -1,6 +1,6 @@
 beautifulsoup4==4.13.3
 mistune==3.1.1
-Requests==2.32.3
+Requests==2.32.4
 requests_cache==1.2.1
 tqdm==4.67.1
 typesense==0.21.0

fetcher/sources/items_test.yaml

@@ -3,9 +3,11 @@
 mdz:
   title_ids:
     - bsb10616989_00623_u001
+    - bsb10527965
 abo:
   title_ids:
     - Z251542006
+    - Z259832605
 anno:
   title_ids:
     sam:

fetcher/utils.py

@@ -60,6 +60,9 @@ def get_query_value(url, key):
     value = query_params.get(key, [None])[0]
     return value
 
+def datum_to_iso_8601(datum):
+    return "-".join([datum[0:4], datum[4:6], datum[6:]])
+
 def download_remote_file(url, path, session):
     os.makedirs(os.path.dirname(path), exist_ok=True)
 

frontend/index.html

@@ -1,43 +1,52 @@
 <!DOCTYPE html>
 <html lang="en">
 <head>
-  <meta charset="UTF-8">
-  <meta name="viewport" content="width=device-width, initial-scale=1.0">
-  <title>Historical Document Search</title>
-  <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/bulma/0.9.3/css/bulma.min.css">
-  <link rel="stylesheet" href="style.css">
-  <script src="https://cdn.jsdelivr.net/npm/typesense@1.8.2/dist/typesense.min.js"></script>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Historical Document Search</title>
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/bulma/0.9.3/css/bulma.min.css">
+    <link rel="stylesheet" href="style.css">
+    <script src="https://cdn.jsdelivr.net/npm/typesense@1.8.2/dist/typesense.min.js"></script>
 </head>
 <body>
-  <section class="section">
+<section class="section">
     <div class="container">
-      <h1 class="title">Historical Document Search</h1>
-      <div class="field">
-        <div class="control">
-          <input id="search-box" class="input is-large" type="text" placeholder="Search historical documents...">
+        <h1 class="title">Historical Document Search</h1>
+        <div class="field">
+            <div class="control">
+                <input id="search-box" class="input is-large" type="text" placeholder="Search historical documents...">
+            </div>
         </div>
-      </div>
 
-      <div class="filters">
-        <div class="field is-grouped">
-          <div class="control">
-            <div class="select">
-              <select id="source-filter">
-                <option value="">All Sources</option>
-                <option value="anno.onb.ac.at">ANNO</option>
-                <option value="api.digitale-sammlungen.de">MDZ</option>
-                <option value="iiif.onb.ac.at">ABO</option>
-              </select>
+        <div class="filters">
+            <div class="field is-grouped">
+                <div class="field">
+                    <label class="label">Filter by Platform</label>
+                    <div class="control">
+                        <div class="select">
+                            <select id="source-filter">
+                                <option value="">Select Platform</option>
+                                <option value="anno.onb.ac.at">ANNO</option>
+                                <option value="api.digitale-sammlungen.de">MDZ</option>
+                                <option value="iiif.onb.ac.at">ABO</option>
+                            </select>
+                        </div>
+                    </div>
+                </div>
+
+                <div class="field" style="margin-left: 1rem;">
+                    <label class="label">Results per Page</label>
+                    <div class="control">
+                        <input id="results-per-page" class="input" type="number" min="5" max="50" value="10"
+                               placeholder="Results per page">
+                    </div>
+                </div>
             </div>
-          </div>
-          <div class="control">
-            <input id="results-per-page" class="input" type="number" min="5" max="50" value="10" placeholder="Results per page">
-          </div>
         </div>
-      </div>
 
-      <div id="results"></div>
-      <div id="pagination" class="pagination is-centered" role="navigation"></div>
+
+        <div id="results"></div>
+        <div id="pagination" class="pagination is-centered" role="navigation"></div>
     </div>
   </section>
 

frontend/script.js

@@ -112,6 +112,12 @@ function createPopup() {
     document.body.appendChild(popup);
 }
 
+popup.addEventListener('click', (event) => {
+  if (!popupContent.contains(event.target) && event.target !== closeButton) {
+    popup.style.display = 'none';
+    document.body.style.overflow = 'auto';
+  }
+});
 
 /**
  * Display the search results in the UI.
@@ -143,12 +149,15 @@ function displayResults(results) {
 
         const title = document.createElement('p');
         title.className = 'title is-4';
-        title.textContent = `${result.document.title_id} - Page ${result.document.page_number}`;
+        title.textContent = `${result.document.title_full} - Date ${result.document.date} - Page ${result.document.page_number}`;
 
-        const sourceTag = document.createElement('span');
+        const sourceTag = document.createElement('a');
         sourceTag.className = 'tag is-info source-tag';
+        sourceTag.href = result.document.remote_path;
+        sourceTag.target = '_blank';
         sourceTag.textContent = result.document.source;
 
+
         titleContainer.appendChild(title);
         titleContainer.appendChild(sourceTag);
 
@@ -187,10 +196,6 @@ function displayResults(results) {
         const actions = document.createElement('div');
         actions.className = 'field is-grouped mt-4';
 
-        const viewButton = document.createElement('p');
-        viewButton.className = 'control';
-        viewButton.innerHTML = `<a href="${result.document.remote_path}" target="_blank" class="button is-link">View Original</a>`;
-
         const viewFullTextButton = document.createElement('p');
         viewFullTextButton.className = 'control';
         viewFullTextButton.innerHTML = `<button class="button is-link">View Full Text</button>`;
@@ -221,9 +226,8 @@ function displayResults(results) {
         imageButton.className = 'control';
         imageButton.innerHTML = `<a href="${result.document.image_url}" target="_blank" class="button is-link is-light">View Image</a>`;
 
-        actions.appendChild(viewButton);
-        actions.appendChild(imageButton);
         actions.appendChild(viewFullTextButton);
+        actions.appendChild(imageButton);
 
         cardContent.appendChild(titleContainer);
         cardContent.appendChild(snippet);

frontend/style.css

@@ -53,7 +53,7 @@
 }
 
 #text-popup .close-button {
-  position: absolute;
+  position: fixed;
   top: 10px;
   right: 15px;
   border: none;
@@ -63,6 +63,7 @@
   cursor: pointer;
   border-radius: 4px;
   font-size: 1rem;
+  z-index: 1001;
 }
 
 #text-popup .close-button:hover {

scripts/create-admin-key.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+curl 'http://localhost:8108/keys' -X POST -H "X-TYPESENSE-API-KEY: ${TYPESENSE_API_KEY}" \
+    -H 'Content-Type: application/json' \
+    -d '{"description":"Admin key.","actions": ["*"], "collections": ["*"]}'
+

scripts/create-readonly-key.sh

@@ -0,0 +1,5 @@
+#/usr/bin/env bash
+curl 'http://localhost:8108/keys' -X POST -H "X-TYPESENSE-API-KEY: ${TYPESENSE_API_KEY}" \
+    -H 'Content-Type: application/json' \
+    -d '{"description":"Search-only key.","actions": ["documents:search"], "collections": ["companies"]}'
+

scripts/readenv.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+# Source this script using "source scripts/readenv.sh"
+export $(cat .env | awk '''!/^\s*#/''' | xargs)