From 7505b3473c2cd6f53977dd13dca2307bd32f6c05 Mon Sep 17 00:00:00 2001 From: Julian Lopez Gordillo Date: Wed, 1 Jul 2026 11:15:06 +0200 Subject: [PATCH 1/5] Add .gitignore to ignore local Jekyll builds --- .gitignore | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7d0ff1f --- /dev/null +++ b/.gitignore @@ -0,0 +1,15 @@ +# Jekyll build output +_site/ +.jekyll-metadata +.jekyll-cache/ + +# Sass cache +.sass-cache/ + +# Local configuration overrides (if any) +_config.yml.tmp +_config.yml.dev + +# OS-specific files +.DS_Store +Thumbs.db From 4e57258c07887c7f093f332148590a4760b025ec Mon Sep 17 00:00:00 2001 From: Julian Lopez Gordillo Date: Wed, 1 Jul 2026 11:15:55 +0200 Subject: [PATCH 2/5] Add Gemfile.lock to lock gem dependencies --- Gemfile.lock | 255 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 255 insertions(+) create mode 100644 Gemfile.lock diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 0000000..79187a7 --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,255 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.9.0) + public_suffix (>= 2.0.2, < 8.0) + base64 (0.3.0) + bigdecimal (4.1.2) + colorator (1.1.0) + concurrent-ruby (1.3.7) + csv (3.3.5) + em-websocket (0.5.3) + eventmachine (>= 0.12.9) + http_parser.rb (~> 0) + eventmachine (1.2.7) + ffi (1.17.4) + ffi (1.17.4-aarch64-linux-gnu) + ffi (1.17.4-aarch64-linux-musl) + ffi (1.17.4-arm-linux-gnu) + ffi (1.17.4-arm-linux-musl) + ffi (1.17.4-arm64-darwin) + ffi (1.17.4-x86-linux-gnu) + ffi (1.17.4-x86-linux-musl) + ffi (1.17.4-x86_64-darwin) + ffi (1.17.4-x86_64-linux-gnu) + ffi (1.17.4-x86_64-linux-musl) + forwardable-extended (2.6.0) + google-protobuf (4.35.1) + bigdecimal + rake (~> 13.3) + google-protobuf (4.35.1-aarch64-linux-gnu) + bigdecimal + rake (~> 13.3) + google-protobuf (4.35.1-aarch64-linux-musl) + bigdecimal + rake (~> 13.3) + google-protobuf (4.35.1-arm64-darwin) + bigdecimal + rake (~> 13.3) + google-protobuf (4.35.1-x86-linux-gnu) + bigdecimal + rake (~> 13.3) + google-protobuf (4.35.1-x86-linux-musl) + bigdecimal + rake (~> 13.3) + google-protobuf (4.35.1-x86_64-darwin) + bigdecimal + rake (~> 13.3) + google-protobuf (4.35.1-x86_64-linux-gnu) + bigdecimal + rake (~> 13.3) + google-protobuf (4.35.1-x86_64-linux-musl) + bigdecimal + rake (~> 13.3) + http_parser.rb (0.8.1) + i18n (1.15.2) + concurrent-ruby (~> 1.0) + jekyll (4.4.1) + addressable (~> 2.4) + base64 (~> 0.2) + colorator (~> 1.0) + csv (~> 3.0) + em-websocket (~> 0.5) + i18n (~> 1.0) + jekyll-sass-converter (>= 2.0, < 4.0) + jekyll-watch (~> 2.0) + json (~> 2.6) + kramdown (~> 2.3, >= 2.3.1) + kramdown-parser-gfm (~> 1.0) + liquid (~> 4.0) + mercenary (~> 0.3, >= 0.3.6) + pathutil (~> 0.9) + rouge (>= 3.0, < 5.0) + safe_yaml (~> 1.0) + terminal-table (>= 1.8, < 4.0) + webrick (~> 1.7) + jekyll-default-layout (0.1.5) + jekyll (>= 3.0, < 5.0) + jekyll-feed (0.17.0) + jekyll (>= 3.7, < 5.0) + jekyll-include-cache (0.2.1) + jekyll (>= 3.7, < 5.0) + jekyll-sass-converter (3.1.0) + sass-embedded (~> 1.75) + jekyll-seo-tag (2.9.0) + jekyll (>= 3.8, < 5.0) + jekyll-watch (2.2.1) + listen (~> 3.0) + json (2.20.0) + just-the-docs (0.12.0) + jekyll (>= 3.8.5) + jekyll-include-cache + jekyll-seo-tag (>= 2.0) + rake (>= 12.3.1) + kramdown (2.5.2) + rexml (>= 3.4.4) + kramdown-parser-gfm (1.1.0) + kramdown (~> 2.0) + liquid (4.0.4) + listen (3.10.0) + logger + rb-fsevent (~> 0.10, >= 0.10.3) + rb-inotify (~> 0.9, >= 0.9.10) + logger (1.7.0) + mercenary (0.4.0) + pathutil (0.16.2) + forwardable-extended (~> 2.6) + public_suffix (7.0.5) + rake (13.4.2) + rb-fsevent (0.11.2) + rb-inotify (0.11.1) + ffi (~> 1.0) + rexml (3.4.4) + rouge (4.7.0) + safe_yaml (1.0.5) + sass-embedded (1.101.0) + google-protobuf (~> 4.31) + rake (>= 13) + sass-embedded (1.101.0-aarch64-linux-android) + google-protobuf (~> 4.31) + sass-embedded (1.101.0-aarch64-linux-gnu) + google-protobuf (~> 4.31) + sass-embedded (1.101.0-aarch64-linux-musl) + google-protobuf (~> 4.31) + sass-embedded (1.101.0-arm-linux-androideabi) + google-protobuf (~> 4.31) + sass-embedded (1.101.0-arm-linux-gnueabihf) + google-protobuf (~> 4.31) + sass-embedded (1.101.0-arm-linux-musleabihf) + google-protobuf (~> 4.31) + sass-embedded (1.101.0-arm64-darwin) + google-protobuf (~> 4.31) + sass-embedded (1.101.0-riscv64-linux-android) + google-protobuf (~> 4.31) + sass-embedded (1.101.0-riscv64-linux-gnu) + google-protobuf (~> 4.31) + sass-embedded (1.101.0-riscv64-linux-musl) + google-protobuf (~> 4.31) + sass-embedded (1.101.0-x86_64-darwin) + google-protobuf (~> 4.31) + sass-embedded (1.101.0-x86_64-linux-android) + google-protobuf (~> 4.31) + sass-embedded (1.101.0-x86_64-linux-gnu) + google-protobuf (~> 4.31) + sass-embedded (1.101.0-x86_64-linux-musl) + google-protobuf (~> 4.31) + terminal-table (3.0.2) + unicode-display_width (>= 1.1.1, < 3) + unicode-display_width (2.6.0) + webrick (1.9.2) + +PLATFORMS + aarch64-linux-android + aarch64-linux-gnu + aarch64-linux-musl + arm-linux-androideabi + arm-linux-gnu + arm-linux-gnueabihf + arm-linux-musl + arm-linux-musleabihf + arm64-darwin + riscv64-linux-android + riscv64-linux-gnu + riscv64-linux-musl + ruby + x86-linux-gnu + x86-linux-musl + x86_64-darwin + x86_64-linux-android + x86_64-linux-gnu + x86_64-linux-musl + +DEPENDENCIES + jekyll + jekyll-default-layout + jekyll-feed + jekyll-seo-tag + just-the-docs + +CHECKSUMS + addressable (2.9.0) sha256=7fdf6ac3660f7f4e867a0838be3f6cf722ace541dd97767fa42bc6cfa980c7af + base64 (0.3.0) sha256=27337aeabad6ffae05c265c450490628ef3ebd4b67be58257393227588f5a97b + bigdecimal (4.1.2) sha256=53d217666027eab4280346fba98e7d5b66baaae1b9c3c1c0ffe89d48188a3fbd + bundler (4.0.15) sha256=a4ceb882fe94a0e0ac63cd0813932bbfd631a14e5ac0b7975189b19a4d28d9e7 + colorator (1.1.0) sha256=e2f85daf57af47d740db2a32191d1bdfb0f6503a0dfbc8327d0c9154d5ddfc38 + concurrent-ruby (1.3.7) sha256=4412caec3a5ea2e5fdc52076724c071a81f2c0593d83b2ac8cbb8ca63b3151b0 + csv (3.3.5) sha256=6e5134ac3383ef728b7f02725d9872934f523cb40b961479f69cf3afa6c8e73f + em-websocket (0.5.3) sha256=f56a92bde4e6cb879256d58ee31f124181f68f8887bd14d53d5d9a292758c6a8 + eventmachine (1.2.7) sha256=994016e42aa041477ba9cff45cbe50de2047f25dd418eba003e84f0d16560972 + ffi (1.17.4) sha256=bcd1642e06f0d16fc9e09ac6d49c3a7298b9789bcb58127302f934e437d60acf + ffi (1.17.4-aarch64-linux-gnu) sha256=b208f06f91ffd8f5e1193da3cae3d2ccfc27fc36fba577baf698d26d91c080df + ffi (1.17.4-aarch64-linux-musl) sha256=9286b7a615f2676245283aef0a0a3b475ae3aae2bb5448baace630bb77b91f39 + ffi (1.17.4-arm-linux-gnu) sha256=d6dbddf7cb77bf955411af5f187a65b8cd378cb003c15c05697f5feee1cb1564 + ffi (1.17.4-arm-linux-musl) sha256=9d4838ded0465bef6e2426935f6bcc93134b6616785a84ffd2a3d82bc3cf6f95 + ffi (1.17.4-arm64-darwin) sha256=19071aaf1419251b0a46852abf960e77330a3b334d13a4ab51d58b31a937001b + ffi (1.17.4-x86-linux-gnu) sha256=38e150df5f4ca555e25beca4090823ae09657bceded154e3c52f8631c1ed72cf + ffi (1.17.4-x86-linux-musl) sha256=fbeec0fc7c795bcf86f623bb18d31ea1820f7bd580e1703a3d3740d527437809 + ffi (1.17.4-x86_64-darwin) sha256=aa70390523cf3235096cf64962b709b4cfbd5c082a2cb2ae714eb0fe2ccda496 + ffi (1.17.4-x86_64-linux-gnu) sha256=9d3db14c2eae074b382fa9c083fe95aec6e0a1451da249eab096c34002bc752d + ffi (1.17.4-x86_64-linux-musl) sha256=3fdf9888483de005f8ef8d1cf2d3b20d86626af206cbf780f6a6a12439a9c49e + forwardable-extended (2.6.0) sha256=1bec948c469bbddfadeb3bd90eb8c85f6e627a412a3e852acfd7eaedbac3ec97 + google-protobuf (4.35.1) sha256=a3a6471331d918f58dfa4d014a8f6286f0af2cf4840216bde52fcf2ea3fe3726 + google-protobuf (4.35.1-aarch64-linux-gnu) sha256=50ca44d0eeff3f8475e630a1accdd974256f3510694d574e2c9d6119ea8bc9e1 + google-protobuf (4.35.1-aarch64-linux-musl) sha256=d5c65cef6bd6498a9e5ed5f88cf6cf7e341c10b0a005e32137d5d1a2b6e8c18a + google-protobuf (4.35.1-arm64-darwin) sha256=d9c957df04fa89c749fa9a72a7b383eb4296efc9b2303dc6fd6fbe39c698ad6b + google-protobuf (4.35.1-x86-linux-gnu) sha256=cc7492566b27ad8b5dfa3a7b6f9b11e905050cc53c2fa8ff22de375a9e7a70fb + google-protobuf (4.35.1-x86-linux-musl) sha256=ab403790b59e4dc588ffbed1eaacf05d4ca2f0d12ac9c13d6c64e69380d8c99e + google-protobuf (4.35.1-x86_64-darwin) sha256=66b62b4df00931018a692806df66393efa960d6d2b7da69735187249f950d3ee + google-protobuf (4.35.1-x86_64-linux-gnu) sha256=c786439087512a3fbd199e9897d265b855f951d4027e218ea55e858d45969edd + google-protobuf (4.35.1-x86_64-linux-musl) sha256=91890eb0002934a339fdb7d77a147c46b7474b6799db27872b747b905837f744 + http_parser.rb (0.8.1) sha256=9ae8df145b39aa5398b2f90090d651c67bd8e2ebfe4507c966579f641e11097a + i18n (1.15.2) sha256=00f9eb62412fe593b2a65a97daa75300d37abb8f7202ec748e94b6d46a9dd1b5 + jekyll (4.4.1) sha256=4c1144d857a5b2b80d45b8cf5138289579a9f8136aadfa6dd684b31fe2bc18c1 + jekyll-default-layout (0.1.5) sha256=c626be4e4a5deafca123539da2cd22ff873be350cafd4da134039efdf24320af + jekyll-feed (0.17.0) sha256=689aab16c877949bb9e7a5c436de6278318a51ecb974792232fd94d8b3acfcc3 + jekyll-include-cache (0.2.1) sha256=c7d4b9e551732a27442cb2ce853ba36a2f69c66603694b8c1184c99ab1a1a205 + jekyll-sass-converter (3.1.0) sha256=83925d84f1d134410c11d0c6643b0093e82e3a3cf127e90757a85294a3862443 + jekyll-seo-tag (2.9.0) sha256=0260015a8e1df9bf195cdfb0c675b7b2883fd8cbf12556e1c1cbe36a831c6852 + jekyll-watch (2.2.1) sha256=bc44ed43f5e0a552836245a54dbff3ea7421ecc2856707e8a1ee203a8387a7e1 + json (2.20.0) sha256=9362bc6e55a952b056abf9167cf053358181c904cb70cd6eee0808ea830fc32b + just-the-docs (0.12.0) sha256=15f2839ac9082898d60f33b978aa6f8e46fc50ba8fac20ae7a7f0e1fb295523e + kramdown (2.5.2) sha256=1ba542204c66b6f9111ff00dcc26075b95b220b07f2905d8261740c82f7f02fa + kramdown-parser-gfm (1.1.0) sha256=fb39745516427d2988543bf01fc4cf0ab1149476382393e0e9c48592f6581729 + liquid (4.0.4) sha256=4fcfebb1a045e47918388dbb7a0925e7c3893e58d2bd6c3b3c73ec17a2d8fdb3 + listen (3.10.0) sha256=c6e182db62143aeccc2e1960033bebe7445309c7272061979bb098d03760c9d2 + logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203 + mercenary (0.4.0) sha256=b25a1e4a59adca88665e08e24acf0af30da5b5d859f7d8f38fba52c28f405138 + pathutil (0.16.2) sha256=e43b74365631cab4f6d5e4228f812927efc9cb2c71e62976edcb252ee948d589 + public_suffix (7.0.5) sha256=1a8bb08f1bbea19228d3bed6e5ed908d1cb4f7c2726d18bd9cadf60bc676f623 + rake (13.4.2) sha256=cb825b2bd5f1f8e91ca37bddb4b9aaf345551b4731da62949be002fa89283701 + rb-fsevent (0.11.2) sha256=43900b972e7301d6570f64b850a5aa67833ee7d87b458ee92805d56b7318aefe + rb-inotify (0.11.1) sha256=a0a700441239b0ff18eb65e3866236cd78613d6b9f78fea1f9ac47a85e47be6e + rexml (3.4.4) sha256=19e0a2c3425dfbf2d4fc1189747bdb2f849b6c5e74180401b15734bc97b5d142 + rouge (4.7.0) sha256=dba5896715c0325c362e895460a6d350803dbf6427454f49a47500f3193ea739 + safe_yaml (1.0.5) sha256=a6ac2d64b7eb027bdeeca1851fe7e7af0d668e133e8a88066a0c6f7087d9f848 + sass-embedded (1.101.0) sha256=57dbc3409e2c0a2c581a4c9945c2bd72ec88e71ec98017bd02dd1da8a76b22f4 + sass-embedded (1.101.0-aarch64-linux-android) sha256=ad0a35c6ff5cdc4e31c5d8261a768ef460c6c7d1458081183e42170e1474c4b5 + sass-embedded (1.101.0-aarch64-linux-gnu) sha256=8f6926349e880dbb4fda75fb182086fb60aa821b756bcbfc8e5b2b23b1f269d6 + sass-embedded (1.101.0-aarch64-linux-musl) sha256=0a02b4b75db305160db1b0512f040762242128a9b5c01b0fa7a504f2b24557f2 + sass-embedded (1.101.0-arm-linux-androideabi) sha256=88e762531f90d4a2eef55ffafeba337585e1bb24b22307ebd7fe5505de9a4476 + sass-embedded (1.101.0-arm-linux-gnueabihf) sha256=358df7f98d13ff53739ebc45f5c5acd4dd96833bf87f39c74d11f798081d4158 + sass-embedded (1.101.0-arm-linux-musleabihf) sha256=d993a3ad017250d46d1312bc10cfded5ab4585906d37a60a56494bd662182d3a + sass-embedded (1.101.0-arm64-darwin) sha256=9fed684380b49499dfc856aba0d026a0748f924bd78044ffff2bae1537aea73e + sass-embedded (1.101.0-riscv64-linux-android) sha256=1bd35527e1ca24af6aa2ba9db797bc9b72331ebbec9ff6e99ea02b24f83b3da1 + sass-embedded (1.101.0-riscv64-linux-gnu) sha256=7805a249ed4c31b58b2f7d054d14f0ed24c61a836f579cd76ed2367088142bcf + sass-embedded (1.101.0-riscv64-linux-musl) sha256=8bd60eb9ebe6f9f2740a4e9ddb792a3db1fbba282e3375103844b4979eb7c731 + sass-embedded (1.101.0-x86_64-darwin) sha256=20ff4afd7c052b3f8d7b4abe511e8114985a07dff7616750a43ee1fc2759fb28 + sass-embedded (1.101.0-x86_64-linux-android) sha256=472b72114ededa00a22e13553dd31f98d1a92d52bdbe2e738a4dbac2dd50c779 + sass-embedded (1.101.0-x86_64-linux-gnu) sha256=ff48452b2351eaaf4e6e02f59de0e9e35f6f163e8456d520db49b4d87cf73c27 + sass-embedded (1.101.0-x86_64-linux-musl) sha256=2286feef3c7a8d9bbb075aa1651e1a81942aca1213f7c2d30cc1d6f4eb5b4104 + terminal-table (3.0.2) sha256=f951b6af5f3e00203fb290a669e0a85c5dd5b051b3b023392ccfd67ba5abae91 + unicode-display_width (2.6.0) sha256=12279874bba6d5e4d2728cef814b19197dbb10d7a7837a869bab65da943b7f5a + webrick (1.9.2) sha256=beb4a15fc474defed24a3bda4ffd88a490d517c9e4e6118c3edce59e45864131 + +BUNDLED WITH + 4.0.15 From df89bfdf468aa92fd030756c2f6c379b1d0f7184 Mon Sep 17 00:00:00 2001 From: Julian Lopez Gordillo Date: Wed, 1 Jul 2026 13:49:36 +0200 Subject: [PATCH 3/5] docs: fix typos and spelling throughout the text --- architecture.md | 30 ++++++++--------- guide/data.md | 49 ++++++++++++++-------------- guide/development.md | 31 +++++++++--------- guide/index.md | 10 ++---- guide/registration.md | 76 +++++++++++++++++++++---------------------- guide/scheduling.md | 17 +++++----- index.md | 11 ++++--- resources/contact.md | 5 +-- resources/index.md | 20 +++++++----- 9 files changed, 124 insertions(+), 125 deletions(-) diff --git a/architecture.md b/architecture.md index 02eb0ed..c40cfe3 100644 --- a/architecture.md +++ b/architecture.md @@ -39,28 +39,25 @@ Other value services of DiSSCo: - Harmonising data to OpenDS - Minting Persistent Identifiers (PIDs) for resources - Versioning objects and maintaining a provenance record -- Extending data and linking +- Extending and linking data - Exposing data through APIs, including a robust search function # Machine Annotation Services -A Machine Annotation Service (MAS) is an automated service that annotates a target in DiSSCo. Most -often, MASs are scheduled by a user on a specific target - either a digital specimen or a media -object -- either through the DiSSCover platform or programmatically through the DiSSCo API. +A Machine Annotation Service (MAS) is an automated service that annotates a target in DiSSCo. Most +often, MASs are scheduled by a user on a specific target (a digital specimen or a media +object)—either through the DiSSCover platform or programmatically through the DiSSCo API. {: .note } + > When an existing service is adapted to work within DiSSCo, there are two components involved: > -> - **Value Service**: This is the original service being adapted to DiSSCo. It is deployed on - infrastructure separate from the core DiSSCo architecture, and should be accessible through - APIs. -> - **MAS Middleware**: This is a lightweight component containerized and deployed on the DiSSCo - core - architecture. +> - **Value Service**: This is the original service being adapted to DiSSCo. It is deployed on infrastructure separate from the DiSSCo Core Architecture, and should be accessible through APIs. +> - **MAS Middleware**: This is a lightweight component containerised and deployed on the DiSSCo Core Architecture. -## When a machine makes an annotation +## When a Machine Makes an Annotation -The flow of data can be summarised below: +The flow of data is summarised below: **1. The user requests a job** @@ -73,29 +70,32 @@ DiSSCo uses [Apache Kafka](https://kafka.apache.org/) as an asynchronous messagi machine annotation is requested, a "job" is created. A UUID is generated and associated with this process to track the state of the job. When it is first created, the job is marked as `SCHEDULED`. -The backend sends a message through Kafka. This message that includes the job ID and the target of +The backend sends a message through Kafka. This message includes the job ID and the target of the annotation. The message has a topic that identifies the message's destination. Each MAS is given its own unique topic by DiSSCo. **3. The job is received by the MAS middleware** -Listening for a Kafka message on its topic, the MAS receives the message. +After it started listening for a Kafka message on its topic, the MAS eventually receives the message. -**3a. (Recommended) The MAS middleware informs the backend the job was received.** +**3a. (Recommended) The MAS middleware informs the backend the job was received** It is strongly recommended that once the MAS receives the message, it informs the backend. This improves user experience. The MAS sends the job id to the `/running` endpoint. The backend, in turn, marks the job as `RUNNING` and informs the user. **4. The MAS middleware calls the value service** + The MAS middleware extracts relevant information from the target and sends it to the value service, usually through an API call. The results from the value service are formatted into an annotation event. **5. The results are sent to DiSSCo** + The MAS middelware sends the formatted annotation event through the `annotation` Kafka topic. **6. The annotation is processed** + DiSSCo's annotation processing service picks up the message in the `annotation` topic and processes it. The annotation is then available through the DiSSCo API and the DiSSCover interface. diff --git a/guide/data.md b/guide/data.md index a6446be..138d9ff 100644 --- a/guide/data.md +++ b/guide/data.md @@ -19,7 +19,7 @@ does your service add? Some questions to consider: - Does your service target digital specimens or media objects? -- If your service returns textual information, can its results be best mapped to terms or +- If your service returns textual information, can its results be mapped to terms or classes in OpenDS? Or does your service apply information to the entire target? - If your service returns information or predictions about a region, how can its region be mapped to the region of interest selector in the annotation data model? @@ -30,32 +30,33 @@ Some questions to consider: batching annotations. See [Batching Annotations](#batching-annotations) for more information. {: .note } -You can find more about openDS on the [OpenDS terms site](https://terms.dissco.tech/) +You can find more about openDS on the [OpenDS terms site](https://terms.dissco.tech/). ## What makes a good annotation? When an annotation is accepted, it updates the specimen with new, valuable information. A good annotation can be easily integrated into the target once it is accepted. -- `oa:value`: the value of the annotation ("**What** does the annotation say?") -- `ods:hasSelector`: the part of the target you're annotating ("**Where** is the annotation going?") -- `oa:motivation`: Motivation for what why the annotation was produced (**How** is the information - integrated) +- `oa:value`: The value of the annotation ("**What** does the annotation say?") +- `ods:hasSelector`: The part of the target you're annotating ("**Where** is the annotation going?") +- `oa:motivation`: Motivation for why the annotation was produced ("**How** is the information + integrated?") {: .note} + > The available motivations are: `ods:adding`, `ods:deleting`, `oa:assessing`, `oa:editing`, -`oa:commenting`. +> `oa:commenting`. > Only the `ods:adding` may reference a part of the target that doesn't exist yet because you are > adding information to the target. > Read more about [motivations](/mas-developers-documentation/#why-make-an-annotation) -> and [selectors](/mas-developers-documentation/#what-can-be-annotated) +> and [selectors](/mas-developers-documentation/#what-can-be-annotated). -### Example +### Examples Say you have a MAS that identifies taxonomy from an image. Your MAS should target the `ods:hasTaxonIdentifications` class of a specimen. -*Note: Some fields have been removed from the example annotations for brevity* +_Note: Some fields have been removed from the example annotations for brevity._ **Bad - Commenting on the specimen** @@ -234,8 +235,8 @@ The response from the MAS to the DiSSCo Architecture must follow the following s Filters are a valuable tool for ensuring that only relevant digital objects are processed by the MAS. Based on the OpenDS specification, filters define the criteria an object must meet for the MAS -to be applied. A MAS can filter objects on any term in OpenDs. Filters are provided in JSON Path -Block notation and support wildcards. +to be applied. A MAS can filter objects on any term in OpenDS. Filters are provided in JSONPath +block notation and support wildcards. When registering a MAS, providers can define filters to specify which objects their MAS will annotate. If a MAS has a filter applied, it will only be available for resources that meet the @@ -276,7 +277,7 @@ filters need to begin with `$['digitalSpecimen']`.** {: .note} To see which fields are in the media data model, and which fields are in the specimen data model, -see the [OpenDS terms page](https://terms.dissco.tech/) +see the [OpenDS terms page](https://terms.dissco.tech/). A filter for a MAS that targets media objects, but needs the subject to be a botany specimen, may have the following filters: @@ -302,7 +303,7 @@ discipline "Botany". ## ods:FDOType FDO (FAIR Digital Object) Types are blueprints for digital objects. In OpenDS, a unique, resolvable -identifier is used to specify Type of an object. Filtering by FDO Type is particularly useful, as it +identifier is used to specify the Type of an object. Filtering by FDO Type is particularly useful, as it helps distinguish between different kinds of objects in DiSSCo. The two key Types in DiSSCo are: @@ -330,7 +331,7 @@ resources, streamlining the process and conserving resources. For example, a MAS information from a specimen and return a set of georeferenced coordinates. If this service is batched, then all specimens with the same locality string will receive the same coordinate annotation, with the original calculation only made once. Batching reduces load on the -original service, as it does not need to redundant calculations. +original service, as it does not need to perform redundant calculations. To enable this process, MASs can include batchMetadata in their response. The information provided in the batchMetadata allows DiSSCo to generate search queries that identify resources matching the @@ -344,7 +345,7 @@ services. For instance, a service which finds identifiers in other infrastructur same result to different objects, as the result of the service is by nature unique to its original target. -The schema for batchMetadata can be found here, and it includes the following key fields: +The schema for `batchMetadata` can be found [here](https://schemas.dissco.tech/schemas/developer-schema/annotation/0.4.0/annotation-batch-metadata.json), and it includes the following key fields: ```json { @@ -392,20 +393,20 @@ The schema for batchMetadata can be found here, and it includes the following ke } ``` -`placeInBatch`: Integer that indicates which annotation this batch metadata corresponds to. There -MUST be a corresponding "placeInBatch" value in one annotation in the event. If more than one +`ods:placeInBatch`: Integer that indicates which annotation this batch metadata corresponds to. There +MUST be a corresponding `"ods:placeInBatch"` value in one annotation in the event. If more than one annotation -have the same placeInBatch value, only the first annotation will be used to create a base +have the same `ods:placeInBatch` value, only the first annotation will be used to create a base annotation. -`inputField`: The full JSONPath of the field used to generate MAS annotation, in JSONPath block +`inputField`: The full JSONPath of the field used to generate the MAS annotation, in JSONPath block notation, -e.g. ['ods:DigitalSpecimen']['ods:hasIdentifications'][*]['ods:hasTaxonIdentifications'][*]['dwc:taxonRank']. -Array indexes must be omitted - instead, use wildcards. +e.g. `['ods:DigitalSpecimen']['ods:hasIdentifications'][\*]['ods:hasTaxonIdentifications'][\*]['dwc:taxonRank']`. +Array indexes must be omitted—instead, use wildcards. -`inputValue`: value stored at the specified JSONPath. +`inputValue`: Value stored at the specified JSONPath. -Batching can only be done if the MAS sends annotations of one Type of object in one event - either +Batching can only be done if the MAS sends annotations of one Type of object in one event—either Digital Specimens OR Media Objects. # Moving Forward - Checklist diff --git a/guide/development.md b/guide/development.md index 581677b..611bb1a 100644 --- a/guide/development.md +++ b/guide/development.md @@ -10,11 +10,11 @@ nav_order: 2 {: .no_toc } -Once you know your inputs and outputs of your MAS middleware, you're ready to start developing. - - TOC {:toc} +Once you know the inputs and outputs of your MAS middleware, you're ready to start developing. + # Apache Kafka DiSSCo communicates with MASs through an Apache Kafka messaging queue. Kafka allows events, such as @@ -22,12 +22,12 @@ tasks or data updates, to be sent between systems in a highly reliable and scalable way. When a user schedules a MAS via the DiSSCover platform, DiSSCo dispatches a Kafka message to the designated MAS, initiating the annotation process. -Kafka topics are unique names used to organize messages. Kafka producers write data to topics, and -consumers read data from topics. You will need to set up a kafka topic provided by -the DiSSCo team. Your MAS will send its result as a kafka message of the same topic. It is best to +Kafka topics are unique names used to organise messages. Kafka producers write data to topics, while Kafka +consumers read data from topics. You will need to set up a Kafka topic provided by +the DiSSCo team. Your MAS will send its result as a Kafka message of the same topic. It is best to store this topic name in an environment variable. -In python, you can easily set up a producer like so: +In Python, you can easily set up a producer like so: ```python import os @@ -46,7 +46,7 @@ producer = KafkaProducer(bootstrap_servers=[os.environ.get('KAFKA_PRODUCER_HOST' The environmental variables (`KAFKA_CONSUMER_TOPIC`, `KAFKA_CONSUMER_GROUP`, `KAFKA_CONSUMER_HOST`, `KAFKA_PRODUCER_HOST`) are injected into the service by DiSSCo when the service is deployed. -You can capture incoming messages using the `consumer`. The following code will only run when a +You can capture incoming messages using the `consumer` variable. The following code will only run when a message is sent with the topic defined previously. ```python @@ -76,13 +76,13 @@ As an added value service to the user, DiSSCo tracks the progress of a job throu - FAILED When a MAS receives a message, it is strongly recommended to call the `/running` endpoint. This -indicates to DiSSCo the message has been received by the mas and the job is running. DiSSCo can then +indicates to DiSSCo the message has been received by the MAS and the job is running. DiSSCo can then inform the user of the development. The endpoint has no body, and is reached at `/api/mjr/v1/{JOB-ID}`. In deployment, DiSSCo automatically populates the `RUNNING_ENDPOINT` environmental variable with the -correct endpoint, depending on the environment is being run on. +correct endpoint, depending on what environment it is being run on. - Test: `https://dev.dissco.tech/api/mjr/v1/{JOB-ID}` - Acceptance: `https://sandbox.dissco.tech/api/mjr/v1/{JOB-ID}` @@ -97,7 +97,7 @@ handle multiple or no annotations. Your MAS may have multiple, distinct contributions to a target. There are two ways to handle this: -1. **In an array**: `oa:value` is a field in the Body of the annotation which contains the insights +1. **In an array**: `oa:value` is a field in the body of the annotation which contains the insights your MAS produces. This field is an array, so multiple values may be added. However, all values in the body are part of the same annotation, meaning they have the same motivation, target, selector (what part of the target does the annotation apply to), and other parameters. @@ -106,7 +106,7 @@ Your MAS may have multiple, distinct contributions to a target. There are two wa the same motivation. Example: An AI service that provides two different classifications on the same region of interest. -2. **Multiple annotations**: The kafka message sent by your MAS must adhere to the annotation +2. **Multiple annotations**: The Kafka message sent by your MAS must adhere to the annotation processing event ([schema](https://schemas.dissco.tech/schemas/developer-schema/annotation/latest/annotation-processing-event.json)). This event contains an array of annotations on the same target. @@ -123,7 +123,7 @@ annotation may provide useful insights into the target. If a plant organ detecti plant organs, a species recognition tool can not identify the specimen, or a locality can not be georeferenced, that information should still be captured in an annotation. -Qualities of a "no annotation" annotation +Qualities of a "no annotation" annotation: * **oa:motivation**: The motivation should be `oa:commenting` * **ods:hasSelector**: The selector determines which field(s) of the target are targeted. Note that @@ -140,7 +140,7 @@ If your MAS experiences an exception for whatever reason, that information shoul DiSSCo. That information is used to mark the job as `FAILED` and inform the user of any errors. {: .note } -Send the message to the kafka topic `mas-failed`, not the topic specific to your MAS. +Send the message to the Kafka topic `mas-failed`, not to the topic specific to your MAS. The failure message has the following structure: @@ -160,7 +160,7 @@ Before your MAS is integrated into the DiSSCo architecture, you may test it loca is to run your MAS on a target from DiSSCover, and compare the results against the [annotation event schema](https://schemas.dissco.tech/schemas/developer-schema/annotation/latest/annotation-processing-request.json). You can see `run_local()` methods in the demo enrichment services on GitHub, or use the following -example code as an example: +code as an example: ```python import requests @@ -183,8 +183,7 @@ def run_local(): # Moving Forward - Checklist - You know what selector to use, and what part of the target you're annotating -- You have a python script that accepts a target and outputs a valid annotation event +- You have a Python script that accepts a target and outputs a valid annotation event - Your MAS wrapper still captures "no result" information appropriately - Your MAS sends an error message if an exception is raised - You've tested your MAS locally, and it validates against the relevant schemas - diff --git a/guide/index.md b/guide/index.md index 44a13d0..021b1c6 100644 --- a/guide/index.md +++ b/guide/index.md @@ -22,19 +22,15 @@ service. The process for developing a MAS middleware can be summarized as follow {: .note } > When an existing service is adapted to work within DiSSCo, there are two components involved: > -> - **Value Service**: This is the original service being adapted to DiSSCo. It is deployed on - infrastructure separate from the core DiSSCo architecture, and should be accessible through - APIs. -> - **MAS Middleware**: This is a lightweight component containerized and deployed on the DiSSCo - core - architecture. +> - **Value Service**: This is the original service being adapted to DiSSCo. It is deployed on infrastructure separate from the core DiSSCo architecture, and should be accessible through APIs. +> - **MAS Middleware**: This is a lightweight component containerised and deployed on the DiSSCo core architecture. # Thank you! Thank you for your commitment to enhancing the DiSSCo community through the development and deployment of MASs. By following the guidelines outlined in this guide, you are playing a crucial role in advancing biodiversity research. Whether through novel machine learning approaches, -georeferencing tools, or automated data checks, the contribution of help improve natural science +georeferencing tools, or automated data checks, the contribution helps improve natural science collection data quality across Europe. Machine Annotation Services not only support ongoing research, but also empower the scientific community to engage in meaningful post-publication curation. This collaborative approach enhances the value of digitized collections, ensuring they diff --git a/guide/registration.md b/guide/registration.md index 5023a0a..4790c37 100644 --- a/guide/registration.md +++ b/guide/registration.md @@ -10,37 +10,37 @@ nav_order: 3 {: .no_toc } +- TOC +{:toc} + When the MAS middleware is finished, the DiSSCo team will help you register your Machine Annotation Service. The Orchestration Service is a DiSSCo service that creates deployment files for the MAS middleware. Once a MAS is registered, it will be available in the DiSSCo test environment. To register a MAS, the DiSSCo team needs the following information. -- TOC -{:toc} - -# Create a pull request +# Create a Pull Request MASs are all available on our -MAS [GitHub repository](https://github.com/DiSSCo/demo-enrichment-service-image). When your service +[MAS GitHub repository](https://github.com/DiSSCo/demo-enrichment-service-image). When your service is ready, you should create a pull request on this repo with your new service. Your pull request should meet the following requirements: -- **Code Quality** : Code should be run through a code quality checker ("linter"). By default, the +- **Code Quality**: Code should be run through a code quality checker ("linter"). By default, the DiSSCo pipeline runs code through the linter SonarQube and requires 0 code smells to move forward. -- **Requirements.txt**: Please include a `requirements.txt` file -- **Valid Dockerfile**: Your service must include a valid Dockerfile -- **Workflow file**: Include a GitHub workflow yaml file for your service in the `.github/workflows` - directory. It shoud be based - on [existing workflow files](https://github.com/DiSSCo/demo-enrichment-service-image/blob/main/.github/workflows/osm-georeferencing-pipeline.yml) +- **Requirements.txt**: Please include a `requirements.txt` file. +- **Valid Dockerfile**: Your service must include a valid Dockerfile. +- **Workflow file**: Include a GitHub workflow YAML file for your service in the `.github/workflows` + directory. It should be based + on [existing workflow files](https://github.com/DiSSCo/demo-enrichment-service-image/blob/main/.github/workflows/osm-georeferencing-pipeline.yml). ## Deploying with Docker When you've completed developing your MAS middleware, you're ready to containerize it. These services are deployed as a container image in DiSSCo. As such, make sure -there is a valid Docker file in the source code. +there is a valid Dockerfile in the source code. Pushing to the main branch triggers GitHub actions. During this pipeline, the image is built and -deployed to the DiSSCo AWS container registry. This image is then managed by kubernetes and is +deployed to the DiSSCo AWS container registry. This image is then managed by Kubernetes and is launched when a user schedules your service. {: .warning} @@ -53,17 +53,17 @@ If you use environmental variables or secrets in your MAS middleware, you need t DiSSCo so they can be injected into the application. {: .note} -> -> **Enviromental variables** can be used to set parameters for an algorithm or feature toggle -> specifics + +> **Enviromental variables** can be used to set parameters for an algorithm or feature-toggle +> specific > parts of the MAS middleware. They are not encrypted and can be read by anyone once the MAS is > registered. > -> **Secret variables** are often used to authenticate with a service or are otherwise values you -> don't want exposed. They are encrypted and - naturally - are not publicly available. +> **Secret variables** are often used to authenticate with a service or otherwise and are values you +> don't want exposed. They are encrypted and, naturally, are not publicly available. For environmental variables, you can send the DiSSCo team the value of the variable and the name of -the variable as it is used in code. +the variable as it is used in the code. For secret variables, we encrypt the values and inject them into the application. For this, the DiSSCo development team adds the secrets to the DiSSCo Secret Store on AWS. This is a @@ -71,13 +71,13 @@ manual action which hasn't been automated (yet). The secret should first be securely provided to the DiSSCo development team, **along with the name of the variable as it is used in the code**. For the transfer of the secrets, several options are -available, all of which should include a two-factor authentication. +available, all of which should include two-factor authentication. Examples include: -- A zip-file can be secured with a password and send via email. The password should then come +- A zip-file can be secured with a password and sent via email. The password should then come through a different medium such as a text message or a chat message. -- It is also possible to use a website as https://onetimesecret.com/ preferably with a passphrase +- It is also possible to use a website such as [https://onetimesecret.com/](https://onetimesecret.com/), preferably with a passphrase which is provided through a different medium. @@ -93,26 +93,24 @@ API_KEY=2bbd58fe-2654-4049-be60-4f1302a76c53 API_ENDPOINT=https://api.code.com/specimens ``` -Alice sends the value of `API_KEY` through https://onetimesecret.com/, and includes the password in +Alice sends the value of `API_KEY` through [https://onetimesecret.com/](https://onetimesecret.com/), and includes the password in another email. Because it is not sensitive information, she can send the value of `API_ENDPOINT` by email. At the same time, she also informs the DiSSCo of the name of the variables. -The DiSSCo team inserts the `API_KEY` in the secret store. Registering the MAS, the DiSSCo team -includes the environmental variables and secrets, which - -The resulting kubernetes file will thus have all the information needed to access the secret and -inject it into the running service. The following yaml will be generated and used in the deployment +The DiSSCo team inserts the `API_KEY` into the Secret Store. When registering the MAS, the DiSSCo team +includes the environmental variables and secrets. The resulting Kubernetes file will thus have all the information needed to access the secret and +inject it into the running service. The following YAML will be generated and used in the deployment of the MAS. ```yaml env: - - name: API_ENDPOINT - value: https://api.code.com/specimens - - name: API_KEY - valueFrom: - secretKeyRef: - key: alice-api-key - name: mas-secrets + - name: API_ENDPOINT + value: https://api.code.com/specimens + - name: API_KEY + valueFrom: + secretKeyRef: + key: alice-api-key + name: mas-secrets ``` Where `secretKeyRef` references the name of the secret (not the value) and the name of the secret @@ -182,14 +180,14 @@ for more information on the MAS data model. # Moving to production - Service Level Agreements (SLA) In order to move your MAS to the production environment, an SLA must be established between DiSSCo -and your organisation. This procedure is still in development - check back soon for more updates. +and your organisation. This procedure is still in development; check back soon for more updates. # Moving Forward - Checklist -- You've successfully containerized your MAS using Docker -- You've sent the DiSSCo team your environmental variables and (securely!) sent your secrets +- You've successfully containerised your MAS using Docker +- You've sent the DiSSCo team your environmental variables and (securely!) your secrets - You've sent the DiSSCo team the filters associated with your MAS - You've sent the DiSSco team any additional metadata about your MAS All set! The DiSSCo team will deploy your MAS middleware, and you should be able to use your MAS in -the sandbox environment! \ No newline at end of file +the sandbox environment! diff --git a/guide/scheduling.md b/guide/scheduling.md index 9aadb83..3c59678 100644 --- a/guide/scheduling.md +++ b/guide/scheduling.md @@ -8,7 +8,7 @@ nav_order: 4 # Scheduling a MAS -You can schedule a MAS in 2 ways: programmatically, through the API, and manually through the +You can schedule a MAS in 2 ways: programmatically (through the API), and manually through the DiSSCover User interface. {: .note} @@ -27,21 +27,21 @@ ORCID? [Register for one here](https://orcid.org/register)! **2. Select a Target** Search for a specimen through the DiSSCover interface and find a suitable target. You may filter on -discipline, taxonomy, organisation, if the specimen has media, and many other parameters. +discipline, taxonomy, organisation, whether the specimen has media, and many other parameters. **3a. Annotate a Specimen** -When you find specimen you want to annotate, you will come to the specimen overview page: +When you find a specimen you want to annotate, you will come to the specimen overview page: ![DiSSCover specimen overview page](../assets/disscover_specimen_1.png) -To open the annotation menu, select the "Annotate" button (circled red in above image). This will +To open the annotation menu, select the "Annotate" button (circled in red in the image above). This will open the annotation menu. ![Annotation overview page](../assets/disscover_specimen_2.png) Select the "Machine Annotation Services" button at the bottom, then "Schedule a MAS" on the top -left. +right. ![MAS overview page](../assets/disscover_specimen_3.png) @@ -53,8 +53,9 @@ Click "Schedule" to schedule the selected MASs. The MAS may take a few minutes to run. When complete, it will have a green check mark next to it. **3b. Annotate a Digital Media Object** + If the MAS you want to schedule is designed for media objects, it won't come up in the digital -specimen overview. Instead, navigate to the "digital media" tab and click on the media preview to be +specimen overview. Instead, navigate to the "Digital Media" tab and click on the media preview to be taken to the digital media page: ![Digital media overview page](../assets/disscover_media_1.png) @@ -71,5 +72,5 @@ and [/digital-media/{mas}](https://dev.dissco.tech/api/docs/swagger-ui/index.htm endpoints for more information. {: .note} -in order to use the API to schedule MASs, you will need an authenticated token. Reach out to -the DiSSCo team for more information. \ No newline at end of file +In order to use the API to schedule MASs, you will need an authenticated token. Reach out to +the DiSSCo team for more information. diff --git a/index.md b/index.md index 8f271d1..19f0618 100644 --- a/index.md +++ b/index.md @@ -4,6 +4,7 @@ title: Home permalink: / nav_order: 1 --- + # Welcome {: .no_toc } @@ -34,14 +35,14 @@ may reject or accept the annotation. An accepted annotation may then change the object in the source system. This approach allows for widespread community curation of the entire digitized European collection. -*Also see -the [annotation JSON Schema](https://schemas.dissco.tech/schemas/fdo-type/annotation/latest/annotation.json)* +_Also see +the [annotation JSON Schema](https://schemas.dissco.tech/schemas/fdo-type/annotation/latest/annotation.json)_. ## What is a Machine Annotation Service? A Machine Annotation Service (MAS) is an automated service that annotates a target in DiSSCo. These -services are scheduled by users on individual specimen or media in DiSSCo. What a MAS offers is -broad. From sophisticated AI services to taxonomic services to linking to other infrastructures, +services are scheduled by users on individual specimens or media in DiSSCo. What a MAS offers is +broad: From sophisticated AI services and taxonomic services to linking with other infrastructures, MASs add value to natural science collections data in all sorts of ways. ## What can be annotated? @@ -94,7 +95,7 @@ If you've developed a service that analyzes specimen data, you may be wondering publishing it on DiSSCo. Haven't you done enough work? -- **One data model to rule them all**: one service can be applied to data from hundreds of +- **One data model to rule them all**: One service can be applied to data from hundreds of institutions - **Modular design**: The DiSSCo architecture is designed to allow existing services to “plug in” easily, meaning anyone, not just those directly tied to DiSSCo, can develop a MAS diff --git a/resources/contact.md b/resources/contact.md index 2804aff..a515507 100644 --- a/resources/contact.md +++ b/resources/contact.md @@ -6,12 +6,13 @@ parent: Resources --- # Contact + Have a general question? [Create an issue on this repository](https://github.com/DiSSCo/mas-developers-documentation/issues/new?template=Blank+issue)! -if you're interested in developing a MAS with DiSSCo, please reach out! +If you're interested in developing a MAS with DiSSCo, please reach out! > **Soulaine Theocharides** > > Software Developer and MAS Coordinator, DiSSCo > -> [soulaine.theocharides@naturalis.nl](mailto:soulaine.theocharides@naturalis.nl) \ No newline at end of file +> [soulaine.theocharides@naturalis.nl](mailto:soulaine.theocharides@naturalis.nl) diff --git a/resources/index.md b/resources/index.md index 7ecf424..eedb220 100644 --- a/resources/index.md +++ b/resources/index.md @@ -5,39 +5,41 @@ permalink: /resources nav_order: 4 has_toc: false --- -# Your handy toolkit, all, in one page + +# Your handy toolkit, all in one page ## JSON Schemas + - [Schema Site](https://schemas.dissco.tech/schemas/) - [Latest Digital Media Schema](https://schemas.dissco.tech/schemas/fdo-type/digital-media/latest/digital-media.json) - [Latest Digital Specimen Schema](https://schemas.dissco.tech/schemas/fdo-type/digital-specimen/latest/digital-specimen.json) -- [Latest Annotation Schema - Request](https://schemas.dissco.tech/schemas/developer-schema/annotation/latest/annotation-processing-request.json) - Schema a request from the MAS to DISSCo must adhere to -- [Latest Annotation Schema - Full](https://schemas.dissco.tech/schemas/fdo-type/annotation/latest/annotation.json) Schema for a full, processed annotation +- [Latest Annotation Schema - Request](https://schemas.dissco.tech/schemas/developer-schema/annotation/latest/annotation-processing-request.json) - Schema that a request from the MAS to DiSSCo must adhere to. +- [Latest Annotation Schema - Full](https://schemas.dissco.tech/schemas/fdo-type/annotation/latest/annotation.json) - Schema for a full, processed annotation. ## OpenDS + - [Terms Site](https://terms.dissco.tech/) - [Digital Media Terms](https://terms.dissco.tech/digital-media-terms) - [Digital Specimen Terms](https://terms.dissco.tech/digital-specimen-terms) - [Annotation Terms](https://terms.dissco.tech/annotation-terms) ## Repositories + - [Template Repository](https://github.com/diSSCo/machine-annotation-service-template) - [Example MASs](https://github.com/DiSSCo/demo-enrichment-service-image/) ## DiSSCo + - [DiSSCover - Acceptance Environment](https://sandbox.dissco.tech/) - - [Acceptance Swagger Endpoint](https://sandbox.dissco.tech/api/docs/swagger-ui/index.html) + - [Acceptance Swagger Endpoint](https://sandbox.dissco.tech/api/docs/swagger-ui/index.html) - [DiSSCover - Production Environment](https://disscover.dissco.eu/) - [DiSSCo Program Site](https://www.dissco.eu/) - [Technical Blog](https://dissco.tech/) - ## Glossary -- **Annotation**: An additional piece of information associated with a specimen or media object -- **DiSSCo**: Distributed System of Scientific Collections, a European research infrastructure for digitized natural science collections +- **Annotation**: An additional piece of information associated with a specimen or media object. +- **DiSSCo**: Distributed System of Scientific Collections, a European research infrastructure for digitised natural science collections. - **MAS**: Machine Annotation Service, an automated service that produces annotations on DiSSCo data. - **MAS middleware**: A lightweight program used to adapt an existing service into a MAS. - **Value Service**: An existing service being adapted to DiSSCo. - - From 8aa134ddd8853378d912e964bf7f4ec39a2828cf Mon Sep 17 00:00:00 2001 From: Julian Lopez Gordillo Date: Wed, 1 Jul 2026 13:59:36 +0200 Subject: [PATCH 4/5] docs: fix inconsistencies between American and British spelling (switch to English) --- architecture.md | 4 ++-- guide/index.md | 6 +++--- guide/registration.md | 2 +- index.md | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/architecture.md b/architecture.md index c40cfe3..6b980b9 100644 --- a/architecture.md +++ b/architecture.md @@ -15,8 +15,8 @@ nav_order: 3 Before we can delve into how MASs connect to the DiSSCo architecture, it is useful to know how the DiSSCo architecture operates. -As a platform, DiSSCo sits in between data providers and data consumers. The core architecture adds -value by harmonizing data to openDS, minting unique identifiers for specimens and media objects, +As a platform, DiSSCo sits in-between data providers and data consumers. The Core Architecture adds +value by harmonising data to OpenDS, minting unique identifiers for specimens and media objects, capturing provenance, and, of course, facilitating annotations. As DiSSCo improves the data, DiSSCo will send this enhanced data upstream, back to the data providers, and downstream, to data aggregators (though this functionality is currently in development). diff --git a/guide/index.md b/guide/index.md index 021b1c6..1ec24d2 100644 --- a/guide/index.md +++ b/guide/index.md @@ -8,7 +8,7 @@ nav_order: 2 # Let's get started! This guide will walk you through the steps of developing MAS middleware for your existing value -service. The process for developing a MAS middleware can be summarized as follows: +service. The process for developing a MAS middleware can be summarised as follows: | Step | Section |:------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------| @@ -33,5 +33,5 @@ role in advancing biodiversity research. Whether through novel machine learning georeferencing tools, or automated data checks, the contribution helps improve natural science collection data quality across Europe. Machine Annotation Services not only support ongoing research, but also empower the scientific community to engage in meaningful post-publication -curation. This collaborative approach enhances the value of digitized collections, ensuring they -remain relevant and up to-date long after their initial publication. \ No newline at end of file +curation. This collaborative approach enhances the value of digitised collections, ensuring they +remain relevant and up-to-date long after their initial publication. diff --git a/guide/registration.md b/guide/registration.md index 4790c37..665aba7 100644 --- a/guide/registration.md +++ b/guide/registration.md @@ -35,7 +35,7 @@ should meet the following requirements: ## Deploying with Docker -When you've completed developing your MAS middleware, you're ready to containerize it. These +When you've completed developing your MAS middleware, you're ready to containerise it. These services are deployed as a container image in DiSSCo. As such, make sure there is a valid Dockerfile in the source code. diff --git a/index.md b/index.md index 19f0618..73f3a82 100644 --- a/index.md +++ b/index.md @@ -17,7 +17,7 @@ effort. # About DiSSCo -DiSSCo is a research infrastructure supporting **digitized natural science** collections. With over +DiSSCo is a research infrastructure supporting **digitised natural science** collections. With over 200 partners in 23 countries, DiSSCo aims to digitally unify all European natural science assets. It provides digital specimens and digital media objects from diverse collections in a **single harmonised data model**, openDS. These harmonised data are made available through a user-friendly @@ -33,7 +33,7 @@ An annotation is an additional piece of information associated with a specimen o annotation is initially separate from its target. The annotation is then evaluated by experts, who may reject or accept the annotation. An accepted annotation may then change the specimen or media object in the source system. This approach allows for widespread community curation of the entire -digitized European collection. +digitised European collection. _Also see the [annotation JSON Schema](https://schemas.dissco.tech/schemas/fdo-type/annotation/latest/annotation.json)_. @@ -91,7 +91,7 @@ user in DiSSCover. MASs often use external APIs or AI to produce their annotatio # Why develop a MAS? -If you've developed a service that analyzes specimen data, you may be wondering why bother with +If you've developed a service that analyses specimen data, you may be wondering why bother with publishing it on DiSSCo. Haven't you done enough work? From 5d24414286c4d9ac5090bfb90df9fbdcd3bf27e8 Mon Sep 17 00:00:00 2001 From: Julian Lopez Gordillo Date: Thu, 2 Jul 2026 09:33:39 +0200 Subject: [PATCH 5/5] docs: fix formatting and style inconsistencies --- architecture.md | 17 ++++++++--------- guide/data.md | 32 ++++++++++++++++---------------- guide/development.md | 42 +++++++++++++++++++++--------------------- guide/index.md | 11 ++++++----- guide/registration.md | 20 ++++++++++---------- index.md | 8 ++++---- 6 files changed, 65 insertions(+), 65 deletions(-) diff --git a/architecture.md b/architecture.md index 6b980b9..abb2e41 100644 --- a/architecture.md +++ b/architecture.md @@ -12,8 +12,8 @@ nav_order: 3 - TOC {:toc} -Before we can delve into how MASs connect to the DiSSCo architecture, it is useful to know how the -DiSSCo architecture operates. +Before we can delve into how MASs connect to the DiSSCo Architecture, it is useful to know how the +DiSSCo Architecture operates. As a platform, DiSSCo sits in-between data providers and data consumers. The Core Architecture adds value by harmonising data to OpenDS, minting unique identifiers for specimens and media objects, @@ -21,15 +21,15 @@ capturing provenance, and, of course, facilitating annotations. As DiSSCo improv will send this enhanced data upstream, back to the data providers, and downstream, to data aggregators (though this functionality is currently in development). -Due to its flexible nature, additional services can be built on top of the DiSSCo core architecture. +Due to its flexible nature, additional services can be built on top of the DiSSCo Core Architecture. A MAS is one such service. What a MAS does is essentially a "black box" from the DiSSCo -Architecture's perspective. A MAS receives a message as input and produces an annotation as an +Architecture's perspective. A MAS receives a message as input and produces an annotation as output. The DiSSCo position in the biodiversity data landscape is illustrated in the following diagram. On the left, data providers send data to the core infrastructure. In the middle, machine and human -agents rest on top of the -core infrastructure, adding additional value. On the right, the enriched data is sent to data +agents provide additional value, resting on top of the +core infrastructure. On the right, the enriched data is sent to data consumers. ![DiSSCo's position in the biodiversity data landscape](assets/dissco_arch.png) @@ -103,11 +103,10 @@ it. The annotation is then available through the DiSSCo API and the DiSSCover in ## MAS Deployment -The MAS middleware application is deployed as a container on the DiSSCo core architecture. The value +The MAS middleware application is deployed as a container on the DiSSCo Core Architecture. The value service will remain deployed where it was originally. {: .note} The MAS middleware is deployed on the DiSSCo network. If your value service requires whitelisting a specific IP address, let the DiSSCo team know so they can provide your MAS middleware with a static -IP address. That way, the middleware may communicate with the value service. - +IP address. That way, the middleware may communicate with the value service. diff --git a/guide/data.md b/guide/data.md index 138d9ff..531a020 100644 --- a/guide/data.md +++ b/guide/data.md @@ -64,10 +64,10 @@ The following annotation comments on a specimen's taxonomy. - This annotation targets the entire first instance of the `TaxonIdentifications` class, instead of a specific field -- The comment is not machine actionable - this information can not be integrated directly into the +- The comment is not machine actionable—this information cannot be integrated directly into the specimen -While it does provide useful information, this annotation can not be integrated into the specimen +While it does provide useful information, this annotation cannot be integrated into the specimen without additional work. ```json @@ -115,7 +115,7 @@ taxonomic field: } ``` -The following annotation adds a full TaxonIdentification: +The following annotation adds a full `TaxonIdentification`: ```json @@ -182,7 +182,7 @@ Requests sent to the MAS will follow the following structure: - `jobId` is a persistent identifier (a Handle) generated by DiSSCo to track job state (e.g. running, failed, scheduled) for the - user. **It must be returned unaltered to the DiSSCo architecture.** + user. **It must be returned unaltered to the DiSSCo Architecture.** - `object` contains the full digital specimen or media object, the target of the annotation. - `batchingRequested`: While [batching](#batching-annotations) may reduce computational load, a MAS must be specifically designed with it in mind, as it requires additional metadata to support this @@ -262,8 +262,8 @@ relevance of annotations within the DiSSCo ecosystem. In the example above, the MAS will only process objects that meet the following conditions: -- The FDO type is https://doi.org/21.T11148/894b1e6cad57e921764e (Digital Specimen). -- The basis of record is MaterialEntity. +- The FDO Type is `https://doi.org/21.T11148/894b1e6cad57e921764e` (Digital Specimen). +- The basis of record is `MaterialEntity`. - The object contains at least one identifier (non-null). ## Including Both Specimen and Media Filters @@ -296,11 +296,11 @@ have the following filters: } ``` -This indicates the target fdo type is `https://doi.org/21.T11148/bbad8c4e101e8af01115` (media -object), the target has an access URI, and it is linked to a digital specimen with a topic -discipline "Botany". +This indicates the target FDO Type is `https://doi.org/21.T11148/bbad8c4e101e8af01115` (Media +Object), the target has an access URI, and it is linked to a digital specimen with a topic +discipline `Botany`. -## ods:FDOType +## `ods:FDOType` FDO (FAIR Digital Object) Types are blueprints for digital objects. In OpenDS, a unique, resolvable identifier is used to specify the Type of an object. Filtering by FDO Type is particularly useful, as it @@ -333,15 +333,15 @@ batched, then all specimens with the same locality string will receive the same annotation, with the original calculation only made once. Batching reduces load on the original service, as it does not need to perform redundant calculations. -To enable this process, MASs can include batchMetadata in their response. The information provided -in the batchMetadata allows DiSSCo to generate search queries that identify resources matching the +To enable this process, MASs can include `batchMetadata` in their response. The information provided +in `batchMetadata` allows DiSSCo to generate search queries that identify resources matching the criteria originally used to produce the annotation. This allows the system to apply the same annotation across multiple resources without needing to re-run the MAS for each individual object. {: .warning } Batching is NOT suited for services which may produce different responses for the same input, e.g. -AI services. It is also not suited for services for which it doesn't make sense to apply to multiple -services. For instance, a service which finds identifiers in other infrastructures can not apply the +AI services. It is also not suited for services whose results do not logically apply to multiple +objects. For instance, a service which finds identifiers in other infrastructures cannot apply the same result to different objects, as the result of the service is by nature unique to its original target. @@ -412,6 +412,6 @@ Digital Specimens OR Media Objects. # Moving Forward - Checklist - You've determined the most appropriate motivation for your service -- You've determined which openDS fields (which selector) should be used as input for your value service -- You've mapped the output of your value service to openDS terms +- You've determined which OpenDS fields (which selector) should be used as input for your value service +- You've mapped the output of your value service to OpenDS terms - You've established filters on what kind of objects your MAS will accept diff --git a/guide/development.md b/guide/development.md index 611bb1a..a39b1b1 100644 --- a/guide/development.md +++ b/guide/development.md @@ -60,7 +60,7 @@ for msg in consumer: To get started on development, you can fork the [MAS Template](https://github.com/DiSSCo/machine-annotation-service-template) on GitHub. The -`annotation` package contains code that will format a result forom an API to the openDS annotation +`annotation` package contains code that will format a result from an API to the OpenDS annotation model. Two templates are provided: a default template and a batch template. There are also some functional MASs available @@ -70,10 +70,10 @@ on [GitHub](https://github.com/diSSCo/demo-enrichment-service-image/) you may us As an added value service to the user, DiSSCo tracks the progress of a job through states: -- SCHEDULED -- RUNNING -- COMPLETED -- FAILED +- `SCHEDULED` +- `RUNNING` +- `COMPLETED` +- `FAILED` When a MAS receives a message, it is strongly recommended to call the `/running` endpoint. This indicates to DiSSCo the message has been received by the MAS and the job is running. DiSSCo can then @@ -102,37 +102,37 @@ Your MAS may have multiple, distinct contributions to a target. There are two wa in the body are part of the same annotation, meaning they have the same motivation, target, selector (what part of the target does the annotation apply to), and other parameters. - **Use multiple bodies when**: Your MAS has multiple insights on the same part of the target, with - the same motivation. Example: An AI service that provides two different classifications on the - same region of interest. + **Use multiple bodies when**: Your MAS has multiple insights on the same part of the target, with + the same motivation. Example: An AI service that provides two different classifications on the + same region of interest. 2. **Multiple annotations**: The Kafka message sent by your MAS must adhere to the annotation processing event ([schema](https://schemas.dissco.tech/schemas/developer-schema/annotation/latest/annotation-processing-event.json)). This event contains an array of annotations on the same target. - **Use a list annotations when**: Your MAS has multiple insights on different parts of the target, - or produces annotations with different motivations. For example, an AI service that classifies - different segments of an image, or a taxonomic service that assesses different taxonomic fields ( - e.g. dwc:genus and dwc:species). + **Use a list annotations when**: Your MAS has multiple insights on different parts of the target, + or produces annotations with different motivations. Example: An AI service that classifies + different segments of an image, or a taxonomic service that assesses different taxonomic fields ( + e.g. `dwc:genus` and `dwc:species`). ## If Your MAS has No Insights If your MAS finds no results, that is still useful information for the user. A "no annotation" annotation may provide useful insights into the target. If a plant organ detection tool finds no -plant organs, a species recognition tool can not identify the specimen, or a locality can not be +plant organs, a species recognition tool cannot identify the specimen, or a locality cannot be georeferenced, that information should still be captured in an annotation. Qualities of a "no annotation" annotation: -* **oa:motivation**: The motivation should be `oa:commenting` -* **ods:hasSelector**: The selector determines which field(s) of the target are targeted. Note that +- **`oa:motivation`**: The motivation should be `oa:commenting` +- **`ods:hasSelector`**: The selector determines which field(s) of the target are targeted. Note that a `commenting` annotation may not be on a field that doesn't exist in the target. - * The selector type may either be `ods:ClassSelector` or `ods:TermSelector` - * Which field or class you target in this kind of annotation depends on your MAS, but -* **oa:value**: A simple message for the user indicating this job has no results: Examples: - * "Unable to find a match" - * "Too many potential matches" + - The selector type may either be `ods:ClassSelector` or `ods:TermSelector` + - Which field or class you target in this kind of annotation depends on your MAS +- **`oa:value`**: A simple message for the user indicating this job has no results. Examples: + - "Unable to find a match" + - "Too many potential matches" ## If your MAS Fails (Exception Handling) @@ -156,7 +156,7 @@ message you want to pass to DiSSCo. # Testing -Before your MAS is integrated into the DiSSCo architecture, you may test it locally. The easiest way +Before your MAS is integrated into the DiSSCo Architecture, you may test it locally. The easiest way is to run your MAS on a target from DiSSCover, and compare the results against the [annotation event schema](https://schemas.dissco.tech/schemas/developer-schema/annotation/latest/annotation-processing-request.json). You can see `run_local()` methods in the demo enrichment services on GitHub, or use the following diff --git a/guide/index.md b/guide/index.md index 1ec24d2..2dace8b 100644 --- a/guide/index.md +++ b/guide/index.md @@ -10,16 +10,17 @@ nav_order: 2 This guide will walk you through the steps of developing MAS middleware for your existing value service. The process for developing a MAS middleware can be summarised as follows: -| Step | Section -|:------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------| +| Step | Section | +| :---------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------ | | 1. Receive the target as JSON | [Data Modelling](/mas-developers-documentation/guide/data-model) | -| 2. Extract values from the target you need as input for your service, e.g. some OpenDS fields | [Data Modelling](/mas-developers-documentation/guide/data-model) | -| 3. Create a function that adds value, usually calling an external API that is the value service | [Development](/mas-developers-documentation/guide/development) | +| 2. Extract values from the target you need as input for your service, e.g. some OpenDS fields | [Data Modelling](/mas-developers-documentation/guide/data-model) | +| 3. Create a function that adds value, usually calling an external API that is the value service | [Development](/mas-developers-documentation/guide/development) | | 4. Publish the output as an annotation event | [Development](/mas-developers-documentation/guide/development) | | 5. Package your code into a Docker container | [Registration and Deployment](/mas-developers-documentation/guide/registration) | -| 6. Test out your MAS on DiSSCover | [Try it out!](/mas-developers-documentation/guide/scheduling) +| 6. Test out your MAS on DiSSCover | [Try it out!](/mas-developers-documentation/guide/scheduling) | {: .note } + > When an existing service is adapted to work within DiSSCo, there are two components involved: > > - **Value Service**: This is the original service being adapted to DiSSCo. It is deployed on infrastructure separate from the core DiSSCo architecture, and should be accessible through APIs. diff --git a/guide/registration.md b/guide/registration.md index 665aba7..981ed7b 100644 --- a/guide/registration.md +++ b/guide/registration.md @@ -160,16 +160,16 @@ modelling section. Additional metadata is useful for providing users with more information about the service. Please provide as many of the following terms as possible. -- name of the service (required) -- description -- creativeWorkStatus - The current status of the service -- codeRepository - Link to code base of MAS -- programmingLanguage -- serviceAvailability - Availability commitment of the service provider as described in the SLA -- maintainer (Follows Agent data model) -- license -- Contact point (Description, email, url, phone) -- SLA Documentation +- `name` of the service (required) +- `description` +- `creativeWorkStatus` - The current status of the service +- `codeRepository` - Link to code base of MAS +- `programmingLanguage` +- `serviceAvailability` - Availability commitment of the service provider as described in the SLA +- `maintainer` (Follows Agent data model) +- `license` +- Contact point (Description, email, URL, phone) +- Service Level Agreement (SLA) Documentation {: .note} See diff --git a/index.md b/index.md index 73f3a82..9f8793c 100644 --- a/index.md +++ b/index.md @@ -55,11 +55,11 @@ annotation may be as broad, or as narrow, as necessary. This is called the "sele specifically to the data according to the OpenDS specification. - **Whole object**: The whole specimen or media is being annotated -- **Class**: A whole class (e.g. TaxonIdentification or Event) is being annotated -- **Term**: A specific, individual property (e.g. dwc:genus or dwc:locality) +- **Class**: A whole class (e.g. `TaxonIdentification` or `Event`) is being annotated +- **Term**: A specific, individual property (e.g. `dwc:genus` or `dwc:locality`) - **Region of Interest**: (Media only) A specific area of the image is being annotated -*What terms can be annotated?* The [DiSSCo Terms Site](https://terms.dissco.tech/) has the most up +_What terms can be annotated?_ The [DiSSCo Terms Site](https://terms.dissco.tech/) has the most up to-date information on openDS terms. ## Why make an annotation? @@ -103,4 +103,4 @@ DiSSCo. Haven't you done enough work? sophisticated machine services on their data These three factors mean that a single service can have a much wider reach through DiSSCo. Through -collaboration, we can further our shared mission of fighting the biodiversity crisis. \ No newline at end of file +collaboration, we can further our shared mission of fighting the biodiversity crisis.