From 4e9355ccf09f30fa01e688e67a543babbcd69916 Mon Sep 17 00:00:00 2001 From: Chelsea Kelly-Reif <3114598+ChelseaKR@users.noreply.github.com> Date: Tue, 30 Jun 2026 11:13:59 -0700 Subject: [PATCH 1/2] Add user-research panel + research roadmap; implement top items MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds two research deliverables and implements the highest-leverage, self-contained items from the roadmap: - docs/USER-RESEARCH.md — synthetic stakeholder panel (clearly labelled synthetic; not evidence of demand; every "values today" maps to a real feature) - docs/RESEARCH-ROADMAP.md — cited, triaged roadmap (remediation + expansion backlogs, sequencing, persona traceability, external evidence) See the "Implementation status - 2026-06-30" record in docs/RESEARCH-ROADMAP.md (or the inline "Implemented" annotations) for exactly what shipped and the local verification result. Human/key/infra-gated items were deferred, not faked. Co-Authored-By: Claude Opus 4.8 (1M context) --- app/diversity.py | 128 ++++++++- app/render.py | 42 ++- app/server.py | 13 +- app/view.py | 5 +- docs/RESEARCH-ROADMAP.md | 175 ++++++++++++ docs/USER-RESEARCH.md | 431 ++++++++++++++++++++++++++++++ docs/audits/dashboard.html | 2 +- docs/ethical-book-data-sources.md | 65 ++++- ingest/config.py | 2 + recommender/catalogs.py | 24 +- recommender/sources.py | 139 +++++++++- tests/test_config.py | 7 + tests/test_diversity.py | 80 +++++- tests/test_render_view.py | 34 ++- tests/test_sources_registry.py | 44 ++- 15 files changed, 1159 insertions(+), 32 deletions(-) create mode 100644 docs/RESEARCH-ROADMAP.md create mode 100644 docs/USER-RESEARCH.md diff --git a/app/diversity.py b/app/diversity.py index f568c4d..57a605f 100644 --- a/app/diversity.py +++ b/app/diversity.py @@ -66,6 +66,21 @@ ("Historical", frozenset({"historical", "history"})), ) +#: The lenses whose descriptors are identity-adjacent — the ones a reading +#: history could be used to *out* someone by (EV-PRIVACY). When the privacy toggle +#: is on, the granular descriptors behind these lenses are aggregated/hidden in the +#: diverse-shelf view; the coarse lens *counts* stay, so the picture isn't lost. +SENSITIVE_DIMENSIONS: frozenset[str] = frozenset({"Trans & nonbinary", "Queer / LGBTQ+"}) + +#: The concrete sourced descriptors that fall under a sensitive lens. +SENSITIVE_DESCRIPTORS: frozenset[str] = frozenset( + label for name, labels in DIMENSIONS if name in SENSITIVE_DIMENSIONS for label in labels +) + +#: Stand-in labels used when the privacy toggle redacts granular sensitive tags. +REDACTED_LABEL = "(hidden for privacy)" +AGGREGATED_LABEL = "(sensitive descriptors — aggregated for privacy)" + @dataclass(frozen=True) class DimensionStat: @@ -82,6 +97,43 @@ def pct(self) -> float: return self.books / self.described_total if self.described_total > 0 else 0.0 +@dataclass(frozen=True) +class SourceRef: + """One distinct citation that asserted a descriptor: kind + where + when.""" + + kind: str # str(SourceKind), e.g. "calibre-tag", "openlibrary-subject" + citation: str # the stable reference the source carried + retrieved_at: str # ISO-8601 date the value was fetched + + +@dataclass(frozen=True) +class DescriptorProvenance: + """A single diverse-shelf descriptor with its source(s) + fetch date (R4). + + Surfaces, for every diverse-shelf tag, the :class:`~ingest.models.Source` + kind(s), citation, and ``retrieved_at`` already stored on the tag — so the + reader can see *who asserted* each theme/identity descriptor and when. An + ``aggregated`` row stands in for the hidden sensitive descriptors when the + privacy toggle is on. + """ + + label: str + books: int # how many considered books carry this descriptor + sources: tuple[SourceRef, ...] + sensitive: bool # identity-adjacent (could out a reader) + aggregated: bool = False # True when this row redacts hidden sensitive labels + + @property + def latest_retrieved_at(self) -> str: + """The freshest fetch date across this descriptor's sources.""" + return max((s.retrieved_at for s in self.sources), default="") + + @property + def source_kinds(self) -> tuple[str, ...]: + """The distinct source kinds that asserted this descriptor, sorted.""" + return tuple(sorted({s.kind for s in self.sources})) + + @dataclass(frozen=True) class DiversityReport: """The committed shape of the diverse-shelf analytics view.""" @@ -91,6 +143,8 @@ class DiversityReport: theme_breakdown: tuple[tuple[str, int], ...] # (sourced label, books), desc dimensions: tuple[DimensionStat, ...] source_provenance: tuple[tuple[str, int], ...] # (source-kind, descriptor count), desc + descriptor_provenance: tuple[DescriptorProvenance, ...] = () # per-tag Source + retrieved_at + hide_sensitive: bool = False # privacy toggle: sensitive descriptors aggregated/hidden @property def undescribed_books(self) -> int: @@ -103,19 +157,30 @@ def coverage_pct(self) -> float: return self.described_books / self.total_books if self.total_books > 0 else 0.0 -def compute_diversity(states: list[ReadingState]) -> DiversityReport: +def compute_diversity( + states: list[ReadingState], *, hide_sensitive: bool = False +) -> DiversityReport: """Compute the diverse-shelf report from sourced book descriptors only. Considers books you've actually engaged with (reading + finished), matching the stats theme-mix; unread owned books are excluded so the picture reflects your reading, not your shelf's backlog. + + Every descriptor carries its full provenance — the :class:`SourceRef`\\ s that + assert it, each with a citation and ``retrieved_at`` (R4). With + ``hide_sensitive=True`` the *granular* identity-adjacent descriptors + (:data:`SENSITIVE_DESCRIPTORS`) are aggregated into a single redacted row and + the matching lens labels are masked — a privacy posture for screen-sharing a + queer/trans reading history (EV-PRIVACY) — while the coarse lens counts stay. """ considered = [s for s in states if s.status is not ReadingStatus.UNREAD] total = len(considered) theme_counter: Counter[str] = Counter() provenance: Counter[str] = Counter() + desc_sources: dict[str, set[SourceRef]] = {} described = 0 + sensitive_books = 0 # distinct considered books carrying any sensitive descriptor # Per-dimension book counts + the concrete labels that matched (transparency). dim_books: dict[str, int] = {name: 0 for name, _ in DIMENSIONS} dim_labels: dict[str, set[str]] = {name: set() for name, _ in DIMENSIONS} @@ -124,11 +189,15 @@ def compute_diversity(states: list[ReadingState]) -> DiversityReport: labels = {t.normalized for t in state.theme_tags} if labels: described += 1 + if labels & SENSITIVE_DESCRIPTORS: + sensitive_books += 1 for label in labels: theme_counter[label] += 1 # Provenance is counted per descriptor (a book can be described by several). for tag in state.theme_tags: provenance[str(tag.source.kind)] += 1 + ref = SourceRef(str(tag.source.kind), tag.source.citation, tag.source.retrieved_at) + desc_sources.setdefault(tag.normalized, set()).add(ref) for name, descriptors in DIMENSIONS: hit = labels & descriptors if hit: @@ -140,7 +209,11 @@ def compute_diversity(states: list[ReadingState]) -> DiversityReport: name=name, books=dim_books[name], described_total=described, - matched_labels=tuple(sorted(dim_labels[name])), + matched_labels=( + (REDACTED_LABEL,) + if hide_sensitive and name in SENSITIVE_DIMENSIONS + else tuple(sorted(dim_labels[name])) + ), ) for name, _ in DIMENSIONS if dim_books[name] > 0 # only surface lenses your shelf actually populates @@ -149,7 +222,56 @@ def compute_diversity(states: list[ReadingState]) -> DiversityReport: return DiversityReport( total_books=total, described_books=described, - theme_breakdown=tuple(theme_counter.most_common()), + theme_breakdown=_theme_breakdown(theme_counter, hide_sensitive, sensitive_books), dimensions=dimensions, source_provenance=tuple(provenance.most_common()), + descriptor_provenance=_descriptor_provenance( + theme_counter, desc_sources, hide_sensitive, sensitive_books + ), + hide_sensitive=hide_sensitive, ) + + +def _sort_ref(ref: SourceRef) -> tuple[str, str, str]: + return (ref.kind, ref.citation, ref.retrieved_at) + + +def _theme_breakdown( + theme_counter: Counter[str], hide_sensitive: bool, sensitive_books: int +) -> tuple[tuple[str, int], ...]: + """The (label, books) breakdown, redacting sensitive labels when asked.""" + if not hide_sensitive: + return tuple(theme_counter.most_common()) + visible = [(lbl, n) for lbl, n in theme_counter.items() if lbl not in SENSITIVE_DESCRIPTORS] + if sensitive_books: + visible.append((AGGREGATED_LABEL, sensitive_books)) + return tuple(sorted(visible, key=lambda item: (-item[1], item[0]))) + + +def _descriptor_provenance( + theme_counter: Counter[str], + desc_sources: dict[str, set[SourceRef]], + hide_sensitive: bool, + sensitive_books: int, +) -> tuple[DescriptorProvenance, ...]: + """Build per-descriptor provenance (R4), aggregating sensitive tags if hidden.""" + rows: list[DescriptorProvenance] = [] + aggregated_refs: set[SourceRef] = set() + for label, books in theme_counter.items(): + refs = tuple(sorted(desc_sources.get(label, set()), key=_sort_ref)) + sensitive = label in SENSITIVE_DESCRIPTORS + if hide_sensitive and sensitive: + aggregated_refs |= set(refs) + continue + rows.append(DescriptorProvenance(label, books, refs, sensitive)) + if hide_sensitive and sensitive_books: + rows.append( + DescriptorProvenance( + AGGREGATED_LABEL, + sensitive_books, + tuple(sorted(aggregated_refs, key=_sort_ref)), + sensitive=True, + aggregated=True, + ) + ) + return tuple(sorted(rows, key=lambda d: (-d.books, d.label))) diff --git a/app/render.py b/app/render.py index baf7f3b..127b25a 100644 --- a/app/render.py +++ b/app/render.py @@ -290,13 +290,49 @@ def _diversity_section(report: Optional[DiversityReport]) -> str: for kind, count in report.source_provenance ) provenance = ( - "' + "
Where these descriptors came from (provenance of every " - 'sourced tag)
Source
' f'{prov_rows}
Where these descriptors came from (count of sourced tags " + 'by source)
SourceDescriptors
' if prov_rows else "

No descriptor provenance recorded yet.

" ) + # R4: every diverse-shelf descriptor with the source that asserted it + when. + # The sensitive marker is text (never colour-only) for the a11y contract. + if report.descriptor_provenance: + desc_rows = "".join( + f'{escape(d.label)}' + f"{' (sensitive)' if d.sensitive else ''}" + f"{d.books}" + f"{escape(', '.join(d.source_kinds) or '—')}" + f"{escape(d.latest_retrieved_at or '—')}" + for d in report.descriptor_provenance + ) + descriptor_table = ( + "' + '' + '' + f"{desc_rows}
Per-descriptor provenance — every diverse-shelf tag, the " + "source that asserted it, and when it was fetched (sourced, never inferred)" + '
DescriptorBooksSource(s)Retrieved
" + ) + else: + descriptor_table = "

No per-descriptor provenance recorded yet.

" + + if report.hide_sensitive: + privacy_note = ( + "

Privacy: identity-adjacent descriptors are aggregated " + "and hidden in this view (the coarse lens counts remain). Unset the privacy " + "toggle to see every sourced descriptor individually.

" + ) + else: + privacy_note = ( + "

Every sourced descriptor is shown individually below. To aggregate the " + "identity-adjacent ones (handy when screen-sharing a queer/trans reading " + "history), set STACKS_HIDE_SENSITIVE=1 or load " + "?hide_sensitive=1.

" + ) + return ( "

Reading diversity

" "

Built only from sourced descriptors of the books " @@ -304,7 +340,7 @@ def _diversity_section(report: Optional[DiversityReport]) -> str: "never infer an author's identity and never auto-label a person; a book " "with no sourced descriptor is reported as unknown, not as “not " "diverse”.

" - f"{coverage}{dimensions}{provenance}" + f"{privacy_note}{coverage}{dimensions}{provenance}{descriptor_table}" ) diff --git a/app/server.py b/app/server.py index bd28166..a277360 100644 --- a/app/server.py +++ b/app/server.py @@ -42,8 +42,12 @@ def require_auth(authorization: Optional[str] = Header(default=None)) -> None: ) -def _load_view() -> DashboardView: - """Load the dashboard from the persisted store, refreshing on first run.""" +def _load_view(*, hide_sensitive: bool = False) -> DashboardView: + """Load the dashboard from the persisted store, refreshing on first run. + + ``hide_sensitive`` is a per-request privacy override; it can only *add* + aggregation on top of the configured default (you can always hide more). + """ config = load_config() store = Store(config.store_path) try: @@ -59,6 +63,7 @@ def _load_view() -> DashboardView: goal_pages=config.goal_pages, goal_hours=config.goal_hours, goal_streak_days=config.goal_streak_days, + hide_sensitive_descriptors=config.hide_sensitive_descriptors or hide_sensitive, ) finally: store.close() @@ -116,8 +121,8 @@ def _readyz() -> Response: return JSONResponse(status_code=200, content={"status": "ok", "checks": checks}) -def _dashboard() -> HTMLResponse: - return HTMLResponse(content=render_view(_load_view())) +def _dashboard(hide_sensitive: bool = False) -> HTMLResponse: + return HTMLResponse(content=render_view(_load_view(hide_sensitive=hide_sensitive))) def _browse( diff --git a/app/view.py b/app/view.py index db3f57a..7cfa709 100644 --- a/app/view.py +++ b/app/view.py @@ -67,6 +67,7 @@ def build_view( goal_pages: int = 0, goal_hours: int = 0, goal_streak_days: int = 0, + hide_sensitive_descriptors: bool = False, ) -> DashboardView: """Build the dashboard view from unified state + candidates (pure).""" today_ordinal, year = _infer_today_and_year(states, daily_activity) @@ -80,7 +81,7 @@ def build_view( hours_target=goal_hours, streak_target=goal_streak_days, ) - diversity = compute_diversity(states) + diversity = compute_diversity(states, hide_sensitive=hide_sensitive_descriptors) candidate_books = tuple(c.book for c in candidates) # type: ignore[attr-defined] recs = recommend_hybrid( states, @@ -137,6 +138,7 @@ def view_from_store( goal_pages: int = 0, goal_hours: int = 0, goal_streak_days: int = 0, + hide_sensitive_descriptors: bool = False, ) -> DashboardView: """Build the dashboard view from persisted derived state in the store. @@ -162,6 +164,7 @@ def view_from_store( goal_pages=goal_pages, goal_hours=goal_hours, goal_streak_days=goal_streak_days, + hide_sensitive_descriptors=hide_sensitive_descriptors, ) diff --git a/docs/RESEARCH-ROADMAP.md b/docs/RESEARCH-ROADMAP.md new file mode 100644 index 0000000..cc2f654 --- /dev/null +++ b/docs/RESEARCH-ROADMAP.md @@ -0,0 +1,175 @@ +# Research-Backed Roadmap — Queer the Stacks + +> **Companion to [`ROADMAP.md`](./ROADMAP.md) (shipped M0–M6) and +> [`ROADMAP-FUTURE.md`](./ROADMAP-FUTURE.md) (shipped N1–N6).** This document does +> **not** replace either; it triages the [synthetic persona panel](./USER-RESEARCH.md) +> and recent public research into a backlog and a sequence that *complement* the +> existing plans. Where an item restates an existing roadmap line it is tagged +> **[corroborates …]** (independent triangulation is signal, not noise); where it +> surfaced only from this exercise it is tagged **[NET-NEW]**. +> **Last assembled: 2026-06-30.** + +Every item must hold the four hard guardrails or it does not ship (verbatim from +`ROADMAP-FUTURE.md`): **(1)** read-only / snapshot-first source access; **(2)** +reading data never leaves the instance; **(3)** no Goodreads / no gatekept catalogs; +**(4)** describe books via sourced tags, never auto-label authors. + +--- + +## 1. Framing: what this adds to the existing roadmaps + +The shipped roadmaps are feature- and phase-complete (N1–N6 green). The persona +panel doesn't find missing *features* so much as missing **proof and provenance**: + +- The **auto-gated** guardrails are strong and tested. The **review-gated** human + sign-offs the audits depend on (privacy, representation, screen-reader) are still + *"pending first release"* — a gate with nothing committed behind it yet. +- The invariants are **enforced in code** (`assert_allowed` default-deny; + `ThemeTag` requires a `Source`; `Author` has no identity field) but **not surfaced** + to the people who care: the reader, the catalogs, the authors. +- The fairness win is **real but thinly evidenced** (Precision/Recall/MAP@5 on a small + fixture) against a literature that demands diversity-aware, temporally honest evaluation. + +So this roadmap is weighted toward **assurance, provenance, and the demo→real-library +transition** — and it slots the rest of the persona wishes onto the *existing* N-phase +sequence rather than inventing a parallel one. + +## 2. Research basis / evidence (cited; accessed 2026-06-30) + +| Tag | Claim used | Sources | +|---|---|---| +| **EV-GOODREADS** | Goodreads (Amazon-owned) stopped issuing API keys 2020-12-08 and retired developer access, breaking OSS apps → justifies the "no Goodreads" guardrail | [Slashdot](https://developers.slashdot.org/story/20/12/17/1522242/goodreads-is-retiring-its-current-api-and-book-loving-developers-arent-happy) · [Goodreads help](https://help.goodreads.com/s/article/Why-did-my-API-key-stop-working) · [dev post-mortem](https://stephanieawilkinson.com/2020/12/10/yonderbook-and-goodreads/) | +| **EV-ETHICAL-ALT** | StoryGraph (independent, Black-owned), Hardcover (no-VC, reader-funded), Bookwyrm (federated) are the recognized non-gatekept options | [The Nod Mag](https://thenodmag.com/content/storygraph-reading-app-goodreads-alternative-woman-founded) · [Bookwise/Hardcover](https://bookwiseapp.com/blog/hardcover-app) · [joinbookwyrm](https://joinbookwyrm.com/) · [Good Good Good](https://www.goodgoodgood.co/articles/goodreads-alternatives) | +| **EV-LICENSE** | Three different obligations: OL = CC0 (attribution appreciated); Hardcover = token-gated GraphQL, "in flux," localhost/server-only; Bookwyrm = per-instance ToS, federation | [CC / Open Library](https://wiki.creativecommons.org/wiki/Case_Studies/Open_Library) · [Hardcover API docs](https://docs.hardcover.app/api/getting-started/) · [Bookwyrm FEDERATION.md](https://github.com/bookwyrm-social/bookwyrm/blob/main/FEDERATION.md) | +| **EV-SELFHOST** | Calibre-Web (GPL, OPDS) is the standard self-host frontend; KOReader sync keys progress by per-doc MD5; server-side **stats** sync is still a developing upstream feature | [Calibre-Web](https://github.com/janeczku/calibre-web) · [koreader-sync-server](https://github.com/koreader/koreader-sync-server) · [stats-sync FR #15182](https://github.com/koreader/koreader/issues/15182) | +| **EV-POPBIAS** | Book recommenders measurably under-serve niche/diverse tastes (popularity bias); thematic/genre bias specifically disadvantages long-tail interests | [Naghiaei et al. 2022](https://arxiv.org/abs/2202.13446) · [MDPI 2025](https://www.mdpi.com/2078-2489/16/2/151) · [Kalra & Daniil, RecSys 2025](https://arxiv.org/abs/2508.15643) | +| **EV-QUEER-ALGO** | Algorithmic systems under-surface / mis-handle LGBTQ+ cultural content; discoverability privileges large-corp, English content | [GLAAD 2026](https://glaad.org/2026-ai-report-build-for-everyone/lgbtq-impacts/) · [Paquette 2025](https://journals.sagepub.com/doi/10.1177/27018466251366274) | +| **EV-LABELS** | Identity labels on books harm authors; WNDB retired #OwnVoices (2021) for invading privacy / outing pressure → "describe books, not authors" | [Book Riot](https://bookriot.com/what-happened-to-the-own-voices-label/) · [Bitch Media](https://www.bitchmedia.org/article/own-voices-forcing-lgbtq-authors-out-of-closet) | +| **EV-PRIVACY** | A queer/trans reading history is sensitive: ALA logged 4,235 titles challenged in 2025 (~40% LGBTQIA+/PoC; 92% by pressure groups) → local-only/no-egress threat model | [ALA 2026](https://www.ala.org/news/2026/04/american-library-association-releases-2025-most-challenged-books-list-national-library) · [ALA data](https://www.ala.org/bbooks/book-ban-data) · [NYPL](https://www.nypl.org/blog/2024/05/28/stand-against-book-banning-lgbtq-titles-targeted-censorship) | +| **EV-A11Y** | Complex charts need a text summary + structured data-table equivalent; color must not be the only channel | [TPGi](https://www.tpgi.com/making-data-visualizations-accessible/) · [USWDS](https://designsystem.digital.gov/components/data-visualizations/) | + +## 3. Remediation backlog (close gaps in what exists) + +Priority: **P0** now · **P1** next · **P2** soon · **P3** opportunistic. Effort: S/M/L. + +| ID | Remediation | Personas | Pri | Effort | Evidence / notes | +|---|---|---|---|---|---| +| R1 | **First real-library run** — point `stacks doctor`/`refresh` at the actual Calibre `metadata.db` + KOReader `statistics.sqlite`; verify read-only, surface "data as of …" freshness | A1, B2, B3 | P0 | M | EV-SELFHOST · **[corroborates ROADMAP-FUTURE §0 N1]** (built; first run is the untrodden path) | +| R2 | **Commit the manual a11y / screen-reader sign-off** — dated VoiceOver+NVDA walkthrough, 320px reflow, forced-colors; publish the live a11y report | A3, D2 | P0 | M | EV-A11Y · **[corroborates ROADMAP-FUTURE §0 + `accessibility-2026-06-05.md` pending items]** | +| R3 | **Commit privacy + representation review sign-offs** as dated artifacts under `docs/audits/`, each with a "what wasn't reviewed" note | D2, A2, B1 | P0 | S | EV-PRIVACY, EV-LABELS · **[NET-NEW packaging; corroborates RESPONSIBLE-TECH-AUDITS review-gated rows]** | +| R4 | **Surface descriptor provenance in the UI** — every diverse-shelf tag shows its `Source` + `retrieved_at`; option to aggregate/hide sensitive descriptors | A2, C3, D2 | P1 | S | EV-LABELS · **[NET-NEW UI; corroborates source-ethics `ThemeTag`-requires-`Source`]** · ✅ Implemented 2026-06-30 (working tree, uncommitted) — per-descriptor provenance table (kind + citation + retrieved date) in the diversity section; `STACKS_HIDE_SENSITIVE` / `?hide_sensitive=1` aggregates identity-adjacent descriptors while keeping coarse lens counts | +| R5 | **Per-source compliance card** — expand `ethical-book-data-sources.md`: CC0 vs per-instance ToS vs Hardcover token/localhost/"in-flux"; attribution posture; cache + rate-limit + robots policy; contact | C1, C2 | P1 | S | EV-LICENSE · **[corroborates ROADMAP-FUTURE C2; NET-NEW licensing nuance]** · ✅ Implemented 2026-06-30 (working tree, uncommitted) — `EthicalSource` carries license/attribution/auth/rate-limit/contact/terms; `to_markdown()` renders per-source compliance cards; doc regenerated | +| R6 | **Richer, diversity-aware eval** — add nDCG, catalog coverage, intra-list diversity, a temporal hold-out on real finishes; track metric drift; publish a fairness/diversity card | D1 | P1 | M | EV-POPBIAS · **[corroborates ROADMAP-FUTURE B5]** | +| R7 | **Live-path contract cassettes** — recorded-response tests for `KosyncClient` / `OpenLibraryClient` / Hardcover / Bookwyrm so parsers are exercised vs real shapes (today `pragma: no cover`); add TestClient route coverage | B3, D1, E1 | P1 | M | EV-SELFHOST · **[corroborates ROADMAP-FUTURE N2 + coverage-honesty]** | +| R8 | **Real (still-local) embedding model** behind the existing off-by-default flag, replacing the placeholder; strictly no egress | A1, D1 | P2 | M | **[corroborates ROADMAP-FUTURE B2]** | +| R9 | **Accessible share cards** — bake alt text + a text/plain-text equivalent into every generated Wrapped/finish card so it survives leaving the app | A3, E2 | P2 | S | EV-A11Y · **[NET-NEW]** | +| R10 | **Promote perf + reliability to merge-blocking** — k6/Locust (p95 < 500 ms), Lighthouse-CI, restart-recovery, kosync-down degrade | B2, B3 | P2 | M | **[corroborates ROADMAP-FUTURE §deferred gates / N5]** | +| R11 | **Federation etiquette, documented + enforced** — User-Agent, robots respect, backoff, aggressive caching, per-instance opt-out for Bookwyrm reads | C2, B1 | P2 | S | EV-LICENSE · **[NET-NEW; corroborates source-ethics allowlist]** · ✅ Implemented 2026-06-30 (working tree, uncommitted) — policy documented in the compliance card (`FETCH_ETIQUETTE`); descriptive User-Agent enforced on the live OpenLibrary/Bookwyrm clients via `catalogs.etiquette_headers`. Backoff/robots remain documented (live clients are `pragma: no cover` pending R7 cassettes) | +| R12 | **Egress legibility** — a one-page data-flow/egress diagram + `security.txt` + auth rate-limiting, so the no-egress property is readable outside the test suite | B1 | P3 | S | EV-PRIVACY · **[NET-NEW]** | + +## 4. Expansion backlog (new capability) + +| ID | Expansion | Personas | Pri | Effort | Evidence / notes | +|---|---|---|---|---|---| +| E1 | **More read-only sources, same join** — Readest progress, Kobo `KoboReader.sqlite`, Calibre-Web read-state, sideloaded EPUB/PDF | A1, B2 | P1 | M | **[corroborates ROADMAP-FUTURE A1]** | +| E2 | **Series/TBR intelligence + search/browse** by sourced theme/genre/series/status | A1 | P1 | M | **[corroborates ROADMAP-FUTURE A3, D3]** | +| E3 | **Aperture/diversity boost-only slider** — lean into small-press/own-voices/translated/underread; "unknown" stays first-class, never penalized | A2, C3, D1 | P1 | S | EV-POPBIAS, EV-QUEER-ALGO · **[corroborates ROADMAP-FUTURE B3]** | +| E4 | **Curated-list ingestion pipeline** — import named community/queer-canon lists with citation + `retrieved_at`; refresh job flags rotted links | C1, A2 | P1 | M | EV-ETHICAL-ALT · **[corroborates ROADMAP-FUTURE C3]** | +| E5 | **Adapter/plugin contract + conformance suite** — so a contributor can add an ethical catalog that *cannot* break the allowlist/provenance/sourced-tags/no-egress invariants | E1, C1 | P2 | M | **[NET-NEW]** | +| E6 | **Container one-command compose + auth upgrade + backups drill + schema-drift CI matrix** | B2, B3 | P1 | M | **[corroborates ROADMAP-FUTURE E1, E2, E3, E5]** | +| E7 | **Expanded Wrapped** — monthly timelines, pace, theme evolution across years, opt-in local PNG/PDF export (nothing auto-published) | A1, E2 | P2 | S | **[corroborates ROADMAP-FUTURE D2]** | +| E8 | **Author feedback / correction path** — a lightweight channel for a surfaced author to confirm provenance, correct a tag, or opt out; never auto-assigns identity | C3 | P2 | S | EV-LABELS · **[NET-NEW]** | +| E9 | **Bookwyrm activity import** — your own shelves/reviews over ActivityPub, read-only, behind the allowlist, cached | E2, A1 | P2 | M | EV-LICENSE · **[corroborates ROADMAP.md §3 "Could: federated Bookwyrm activity import"]** | +| E10 | **KOReader server-side statistics sync** — ingest stats sync when upstream ships it, still read-only/local | A1, B2 | P3 | M | EV-SELFHOST (tracks upstream FR #15182) · **[NET-NEW]** | +| E11 | **Private annotations / commonplace book** + sidecar highlight-*text* import from KOReader; never synced anywhere | A1 | P3 | M | **[corroborates ROADMAP-FUTURE A2]** | +| E12 | **Gentle negative signals** — opt-in DNF / low-dwell down-weighting, explained | A1, D1 | P3 | S | **[corroborates ROADMAP-FUTURE B4]** | + +## 5. Sequenced roadmap (slots onto the existing N-phases) + +| Window | Theme | Items | Why now | +|---|---|---|---| +| **Now (assurance sprint)** | Close the review-gated gap + go real | R1, R2, R3 | The audits already gate the release on these; they're the highest-trust, lowest-effort work and unblock "first real run" | +| **Next (provenance & sourcing)** | Make invariants visible; serve the stewards | R4, R5, R7, R11, E4 | Provenance serves River/Tomás/Lior/Petra at once; cassettes make sourcing honest in CI | +| **Then (fairness & depth)** | Prove the recommender fairly; daily-driver depth | R6, R8, E1, E2, E3 | Diversity-aware eval answers EV-POPBIAS; more sources + browse make it a daily tool | +| **Later (production & polish)** | Ship-on-a-stranger's-box; expand | R9, R10, E5, E6, E7, E9 | Perf/reliability gates + accessible share cards + contributor on-ramp | +| **Opportunistic** | Nice-to-have | R12, E8, E10, E11, E12 | Bounded by upstream (E10) or low demand-signal | + +## 6. Recommended first sprint (highest-leverage, mostly already-built infra) + +The triage and the existing roadmaps converge on the same starting line: **the +features are built; the *assurance* isn't.** Ship these five: + +1. **R1 — first real-library run.** Turns the demo into the tool it's meant to be; + unblocks Chelsea (A1) and de-risks every downstream item. (`stacks doctor`/`refresh` + already exist — this is the first run + freshness stamp.) +2. **R2 — commit the screen-reader / manual a11y sign-off.** The audit lists it + "pending"; a real VoiceOver+NVDA walkthrough + published live report converts + "0 axe violations" into lived-experience proof (A3, D2). +3. **R3 — commit the privacy + representation sign-offs.** Same shape: dated artifacts + behind gates that currently have nothing behind them (D2, A2, B1). EV-PRIVACY, + EV-LABELS make these the most consequential reviews in the project. +4. **R4 — surface descriptor provenance in the UI.** One change that serves four + personas (River, Tomás, Lior, Petra): show each tag's `Source` + `retrieved_at`, + and let sensitive descriptors be aggregated/hidden. Leans entirely on the existing + `ThemeTag`→`Source` invariant. +5. **R5 — per-source compliance card.** Expand `ethical-book-data-sources.md` into the + three real obligations (CC0 / per-instance ToS / Hardcover token-localhost). Cheap, + and it's what the stewards the tool depends on actually want. + +Bundle the afternoon-sized wins alongside: **R9** (accessible share cards) and the +first half of **R11** (documented federation etiquette). + +## 7. Traceability matrix (persona → findings) + +| Persona | Remediations | Expansions | +|---|---|---| +| A1 Chelsea (reader/owner) | R1, R8 | E1, E2, E7, E9, E10, E11, E12 | +| A2 River (queer/trans reader) | R3, R4 | E3, E4 | +| A3 Mara (a11y user) | R2, R9 | — | +| B1 Sam (privacy self-hoster) | R3, R11, R12 | — | +| B2 Devon (tinkerer/deployer) | R1, R7, R10 | E1, E6, E10 | +| B3 Chelsea (ops/maintainer) | R1, R7, R10 | E6 | +| C1 Lior (Open Library steward) | R5, R7 | E4, E5 | +| C2 Petra (Bookwyrm admin) | R5, R11 | E9 | +| C3 Tomás (indie author) | R4 | E3, E8 | +| D1 Dr. Ada (ML reviewer) | R6, R7, R8 | E3, E12 | +| D2 Noor (responsible-tech reviewer) | R2, R3, R4 | — | +| E1 Robin (OSS contributor) | R7 | E5 | +| E2 Jules (Fediverse mutual) | R9 | E7, E9 | + +## 8. Validate with real users / risks + +Because the product is single-user, "validation" means talking to the **stakeholders +the tool depends on**, not running a usability lab: + +- **Data stewards (highest stakes).** Before any wider sourcing, confirm with an Open + Library / Internet Archive contact and a representative Bookwyrm instance admin that + the cache/rate-limit/robots posture (R5, R11) is genuinely welcome. *Risk:* a + well-meaning federated reader can still be an unwelcome load — verify EV-LICENSE + assumptions with a real admin, not docs alone. +- **Assistive-tech walkthrough (R2).** The manual SR pass must be done by — or with — + an actual screen-reader user; "0 axe violations" is necessary, not sufficient (EV-A11Y). +- **An indie/small-press author (R4, E8).** Show one real surfaced author their + provenance view and ask whether anything reads as a mislabel. *Risk:* the worst + failure (EV-LABELS) is invisible to the author who can't see what's attached to them. +- **Recommender fairness (R6).** *Risk:* a perfect 1.00 on a tiny synthetic fixture is + not evidence the model beats popularity on real, temporally-split reads — the + literature (EV-POPBIAS) predicts the hard case is exactly the niche/diverse tail this + tool targets. Treat R6 as falsification, not confirmation. +- **A stranger's deploy (R10, E6).** Watch one homelabber stand it up cold; the + demo→real and reverse-proxy/auth paths are where it'll break. + +## 9. Honest limits + +This roadmap is derived from a **synthetic** panel plus public research, not from real +discovery. It can sequence work and flag obligations, but it cannot tell you which +matter most to the real people behind the catalogs, whether anyone but Chelsea would +deploy it, or how a steward/author/SR-user would actually react. It over-weights what +is legible from the repo (the guardrails, the audits) and under-weights what only a +real conversation would surface. The **[corroborates …]** tags show most of this isn't +new — it's the existing N-phase plan re-prioritized toward *assurance and provenance*; +the genuinely **[NET-NEW]** items (R3 packaging, R4 UI provenance, R9 accessible cards, +R11 federation etiquette, E5 adapter contract, E8 author feedback, E10 stats sync) are +the parts most in need of real-world validation before they're treated as commitments. +Re-run against `ROADMAP.md` / `ROADMAP-FUTURE.md` whenever those change, and per +Calibre/KOReader schema or Open Library / Hardcover / Bookwyrm API change. diff --git a/docs/USER-RESEARCH.md b/docs/USER-RESEARCH.md new file mode 100644 index 0000000..1a8e525 --- /dev/null +++ b/docs/USER-RESEARCH.md @@ -0,0 +1,431 @@ +# User Research — Synthetic Personas & Simulated Interviews + +> [!WARNING] +> **These personas and interviews are synthetic.** They were generated as a +> structured brainstorming device — *not* conducted with real people. No real +> reader, contributor, data steward, or author said any of this. The panel exists +> to pressure-test the product from many angles at once; it is **not** evidence of +> demand and does **not** substitute for real discovery. Treat every "quote" as a +> hypothesis to validate, not a finding. (Consistent with how this project labels +> its synthetic eval fixtures — see [`audits/source-ethics.md`](./audits/source-ethics.md).) +> +> The honest next step is real conversations with people in ≥5 of these roles — +> especially the **data stewards** (Open Library / Bookwyrm) whose terms this tool +> depends on, and the **review-gated sign-offs** (privacy, representation, screen +> reader) the audits list as *pending first release*. +> **Last assembled: 2026-06-30.** + +## Why do this at all +Queer the Stacks is a single-user, self-hosted app, so the "user" is mostly one +person (Chelsea). But the *stakeholders* are not: the tool reads from other +people's catalogs (Open Library, Bookwyrm, Hardcover), surfaces other people's +books (small-press and indie authors), and would be deployed and audited by people +who are not Chelsea. Role-playing the full cast surfaces obligations a single +author misses — especially the ones that are ethical or legal, not just features. +Findings at the end are tagged so this doesn't become a wishlist: + +- **[shipped]** — already exists in the M0–M6 + N1–N6 build. +- **[pending]** — a review-gated sign-off the audits already name but haven't signed. +- **[roadmap]** — already in [`ROADMAP.md`](./ROADMAP.md) / [`ROADMAP-FUTURE.md`](./ROADMAP-FUTURE.md). +- **[NET-NEW]** — genuinely surfaced here. + +## How to read a persona +Each card is the simulated interview compressed to five lines — **Goal · What +they'd value today** (mapped to *real, shipped* features) **· Where they'd get +stuck · What they'd want next · The one thing that makes them adopt (or walk).** + +--- + +## Method + +- **Sampling frame.** Not "users" but *stakeholders around a private reading tool*: + people who **read & discover** (the owner; a queer/trans reader; an + assistive-tech user), people who **self-host & operate** (a privacy-conscious + self-hoster; a homelab tinkerer who'd deploy it; the maintainer), people whose + catalogs the tool **sources from** and whose terms it must respect (an Open Library / + open-data steward; a Bookwyrm instance admin; a small-press author whose book gets + surfaced), people who **assure & audit** (an ML/recommender-fairness reviewer; a + responsible-tech reviewer doing the human sign-offs), and people who'd **extend or + share** it (an OSS contributor; a Fediverse/Bookwyrm mutual on the receiving end of + a share card). +- **Protocol.** For each: a goal, a walkthrough of the surfaces or guarantees they'd + touch, what holds up against the *current* build, where they'd stall, and an open + "what would make this a 10/10" prompt. Frictions become **R**emediations; wishes + become **E**xpansions in [`RESEARCH-ROADMAP.md`](./RESEARCH-ROADMAP.md). +- **Effort scale (used in the roadmap).** S ≈ an afternoon · M ≈ a day or two · + L ≈ a week+. + +### Research basis + +The panel is grounded in the public record on the ebook self-hosting ecosystem, +the Goodreads-alternative landscape, book-data licensing, recommender bias, and +reading privacy. High-stakes claims are cross-checked against ≥2 sources. +Accessed **2026-06-30**. + +- **Goodreads' enclosure is real, not rhetoric.** Goodreads (Amazon-owned since + 2013) stopped issuing new API keys on **2020-12-08** and disabled many existing + ones, retiring developer access entirely and breaking open-source apps built on + it — corroborated by [Slashdot](https://developers.slashdot.org/story/20/12/17/1522242/goodreads-is-retiring-its-current-api-and-book-loving-developers-arent-happy), + Goodreads' own [help article](https://help.goodreads.com/s/article/Why-did-my-API-key-stop-working), + and a [developer post-mortem](https://stephanieawilkinson.com/2020/12/10/yonderbook-and-goodreads/). + This is the factual backbone of the project's "no Goodreads" guardrail. +- **The ethical-alternative landscape exists and is values-aligned.** + [The StoryGraph](https://thenodmag.com/content/storygraph-reading-app-goodreads-alternative-woman-founded) + (independent, Black-owned, Amazon-free), [Hardcover](https://bookwiseapp.com/blog/hardcover-app) + (small team, no venture capital, ad-free, reader-funded), and + [Bookwyrm](https://joinbookwyrm.com/) (federated, community-run) are the + recognized non-gatekept options; see also a [survey of ethical trackers](https://www.goodgoodgood.co/articles/goodreads-alternatives). + This tool sources from Open Library / Hardcover / Bookwyrm — not from these as + competitors, but as the same values commitment. +- **Licensing differs by source and must be honored per source.** Open Library data + is dedicated to the public domain under **CC0** (attribution appreciated, not + required) per [Creative Commons](https://wiki.creativecommons.org/wiki/Case_Studies/Open_Library); + [Hardcover's API](https://docs.hardcover.app/api/getting-started/) is a free + GraphQL endpoint that is "heavily in flux," gated by a personal token, and + restricted to localhost/server use; [Bookwyrm](https://github.com/bookwyrm-social/bookwyrm/blob/main/FEDERATION.md) + is ActivityPub-federated with **per-instance** terms. A single "ethical sources" + line under-describes three different obligations. +- **The self-hosting stack is real and read-only-friendly.** + [Calibre-Web](https://github.com/janeczku/calibre-web) (GPL-3.0, OPDS) is the + established self-hosted Calibre frontend; the + [KOReader sync server](https://github.com/koreader/koreader-sync-server) is + self-hostable and keys progress by a per-document MD5 — matching ADR-2's md5 + progress key. Server-side **statistics** sync is still a + [developing upstream feature](https://github.com/koreader/koreader/issues/15182), + which bounds what cross-device stats can promise today. +- **Mainstream book recommenders are measurably biased against the long tail.** + Naghiaei et al. find most state-of-the-art algorithms "suffer from popularity + bias in the book domain, and fail to meet users' expectations with Niche and + Diverse tastes" while "Bestseller-focused users… receive high-quality + recommendations" ([arXiv 2202.13446](https://arxiv.org/abs/2202.13446)); a 2025 + survey reaches the same conclusion about long-tail fairness + ([MDPI Information](https://www.mdpi.com/2078-2489/16/2/151)); and a RecSys-2025 + study shows *thematic/genre* bias specifically disadvantages "users with niche and + long-tail interests" ([arXiv 2508.15643](https://arxiv.org/abs/2508.15643)). This + is exactly the gap the content-beats-popularity eval targets. +- **Algorithmic systems under-surface and mis-handle LGBTQ+ cultural content.** GLAAD + documents recommendation/moderation harms to LGBTQ+ content + ([2026 AI report](https://glaad.org/2026-ai-report-build-for-everyone/lgbtq-impacts/)); + scholarship on discoverability finds algorithms privilege "English-language… + content produced by large global corporations" + ([Paquette, 2025](https://journals.sagepub.com/doi/10.1177/27018466251366274)). +- **Identity labels on books can harm the people behind them.** We Need Diverse + Books retired **#OwnVoices** in June 2021 because it was "vague and has been + misused to gate-keep identities and invade authors' privacy," and now uses "the + specific descriptions that authors use for themselves" + ([Book Riot](https://bookriot.com/what-happened-to-the-own-voices-label/)); the + label was warped into pressure to out authors + ([Bitch Media](https://www.bitchmedia.org/article/own-voices-forcing-lgbtq-authors-out-of-closet)). + This is the evidence behind "describe books via *sourced* tags; never auto-label + authors." +- **A queer/trans reading history is genuinely sensitive.** In 2025 the ALA logged + 4,235 unique titles challenged (its second-highest ever), ~40% representing the + lived experiences of LGBTQIA+ people and people of color, with 92% of challenges + driven by pressure groups and officials — *Gender Queer* and *Last Night at the + Telegraph Club* among the most targeted + ([ALA, 2026](https://www.ala.org/news/2026/04/american-library-association-releases-2025-most-challenged-books-list-national-library); + [ALA data](https://www.ala.org/bbooks/book-ban-data); + [NYPL](https://www.nypl.org/blog/2024/05/28/stand-against-book-banning-lgbtq-titles-targeted-censorship)). + This is why local-only, no-egress, behind-auth isn't paranoia — it's the threat model. +- **Charts must ship table equivalents to be accessible.** Complex visualizations + need a text summary plus a structured data table; color must never be the only + channel ([TPGi](https://www.tpgi.com/making-data-visualizations-accessible/); + [USWDS](https://designsystem.digital.gov/components/data-visualizations/)) — + exactly the contract `make a11y` already enforces. + +--- + +## Persona roster + +| # | Persona | Group | Primary goal | Top friction | +|---|---|---|---|---| +| A1 | **Chelsea** — primary reader & owner | Read & Discover | See all my reading in one private place; get recs that fit my canon | Build is demo-driven; not yet pointed at her real library | +| A2 | **River** — queer/trans reader, anti-essentialist | Read & Discover | Representation analytics that describe *books*, never label *me* or authors | Wants to see *where* each descriptor came from, and hide sensitive ones | +| A3 | **Mara** — low-vision dashboard user (magnifier + screen reader) | Read & Discover | Read the dashboard, stats, and Wrapped non-visually | Automated a11y passes, but no *human* SR walkthrough is signed off | +| B1 | **Sam** — privacy-conscious self-hoster | Self-Host & Operate | Prove reading data never leaves the box | Wants the no-egress claim demonstrable, not just asserted | +| B2 | **Devon** — homelab tinkerer / would-be deployer | Self-Host & Operate | Stand it up next to Calibre-Web on a seedbox in an evening | First-run is demo-shaped; real-library config path is new (N1) | +| B3 | **Chelsea (ops hat)** — owner / maintainer | Self-Host & Operate | Keep it running, backed up, and resilient to schema drift | Perf/reliability gates are deferred, not yet merge-blocking | +| C1 | **Lior** — Open Library / open-data steward | Source Ethically | See the project honor CC0 + rate limits + provenance | One "ethical sources" line flattens three different obligations | +| C2 | **Petra** — Bookwyrm instance admin | Source Ethically | Confirm federation etiquette: robots, rate limits, per-instance ToS | No public statement of how the tool treats her instance's terms | +| C3 | **Tomás** — small-press / indie author | Source Ethically | Be surfaced fairly, with no misattributed identity label | No path to confirm provenance or correct/opt-out | +| D1 | **Dr. Ada** — recommender / ML-fairness reviewer | Assure & Audit | Verify it beats popularity *fairly*, not just on one metric | Eval is Precision/Recall@5 only; no nDCG/diversity/temporal split | +| D2 | **Noor** — responsible-tech reviewer (human sign-off) | Assure & Audit | Sign the privacy + representation + SR reviews honestly | Those sign-offs are listed "pending," with no committed artifact | +| E1 | **Robin** — OSS contributor | Community / Expand | Add a new ethical catalog adapter and get it merged | No documented adapter contract / conformance test | +| E2 | **Jules** — Fediverse / Bookwyrm mutual | Community / Expand | Enjoy Chelsea's shared Wrapped/finish cards without surveillance | Share cards may lack alt text; wants the "no auto-egress" guarantee visible | + +--- + +## Group A — Read & Discover (the reader at the dashboard) + +### A1. Chelsea — primary reader & owner +- **Goal:** one private place that shows what she's reading across Calibre, KOReader, + Calibre-Web, and Readest — plus recs that fit Plett/Peters/Thom/Butler/Atwood. +- **Values today:** the unified cross-device "currently reading" + history; reading + stats, streaks, and the self-hosted **Wrapped**; the hybrid recommender with + **why + source** on every card; **diverse-shelf analytics built only from sourced + descriptors**; local **goals**; locally-composed **share cards** that egress + nothing until she copies them. *(this is the whole product thesis)* +- **Gets stuck:** the shipped build is demo-driven — it proves the gates but isn't + yet pointed at her real `metadata.db` / `statistics.sqlite`; `stacks doctor`/ + `refresh` (N1) exist but the real-library first run hasn't happened. +- **Wants next:** point it at the real library with a "data as of …" freshness stamp; + a real (still-local) embedding model behind the existing flag; more read-only + sources (Readest, Kobo, Calibre-Web read-state); series/TBR depth. +- **Adopts if:** the real library lights up the dashboard without touching the + sources. **Walks if:** it stays a demo she has to babysit. + +### A2. River — queer/trans reader seeking representation without essentialism +- **Goal:** understand the shape of their reading ("how queer/trans/spec-fic is my + shelf?") **without** the tool putting an identity label on them or on any author. +- **Values today:** diverse-shelf analytics that are, by construction, **sourced**: + `ThemeTag` cannot be built without a `Source` (a Calibre tag, OL subject, or + curated list), and `Author` has **no** gender/sexuality/identity field — so there + is literally nowhere to auto-assign one. Chart + **table** equivalents; tags shown + as text, never color-only. *(directly answers the #OwnVoices outing harm)* +- **Gets stuck:** can see *that* a book is tagged "trans" but not *where the tag came + from* or how confident it is; worries a sensitive descriptor could be visible on a + shared card. +- **Wants next:** provenance on every descriptor ("from Open Library subject / curated + list X, retrieved 2026-…"); the option to aggregate or hide sensitive descriptors; + an aperture lens that boosts (never penalizes) underread/own-voices/translated work + with "unknown" staying first-class. +- **Adopts if:** the analytics describe books, cite sources, and never essentialize. + **Walks if:** it ever infers a person's identity from a name, cover, or model. + +### A3. Mara — low-vision reader using magnification + a screen reader +- **Goal:** read the dashboard, stats, and Wrapped entirely non-visually / at 200% zoom. +- **Values today:** the *mechanically verified* a11y contract — one `

`, landmarks + + skip link, every chart shipped with a real `` equivalent, theme/progress + conveyed as text (a `#` glyph + label) not color, `prefers-reduced-motion` + respected, 0 axe violations via `make a11y`. *(matches WCAG complex-image guidance)* +- **Gets stuck:** the audit itself flags that the **manual** passes are *not yet + signed off* — no real VoiceOver/NVDA walkthrough, no 320px-reflow or + forced-colors check, and streamed/interactive updates aren't SR-audited. +- **Wants next:** a committed, dated SR-walkthrough artifact; a published live a11y + report; and — when share cards exist — **alt text baked into the card image** so + it's accessible after it leaves the app. +- **Adopts if:** a human like her has actually driven it end-to-end and signed it. + **Walks if:** "0 violations" is the *only* evidence and the lived path breaks. + +--- + +## Group B — Self-Host & Operate + +### B1. Sam — privacy-conscious self-hoster +- **Goal:** be convinced a queer/trans reading history can't leak off the box. +- **Values today:** auth fails closed (401 without a valid token; even demo needs + one); network is confined to two clients (the user's own kosync server + GET-only + public catalog metadata) and **never POSTs reading history**; no analytics/telemetry + SDK in core; a log-safety lint keeps reading content out of logs — all + *merge-blocking tests*, not promises. *(the threat model the book-ban data justifies)* +- **Gets stuck:** wants the guarantee legible from outside the test suite — a plain + data-flow he can read, and assurance the *catalog* lookups can't fingerprint him. +- **Wants next:** a committed dated **privacy review** sign-off (the audit lists it + "pending"); a one-page egress diagram; per-source notes on what each catalog call + reveals; `security.txt` + auth rate-limiting. +- **Adopts if:** "nothing sensitive leaves" is demonstrable. **Walks if:** any + reading data, even derived, touches a third party. + +### B2. Devon — homelab tinkerer who'd actually deploy it +- **Goal:** run it next to Calibre-Web on a seedbox in one evening, read-only against + the real libraries. +- **Values today:** the container + `docker-compose.yml` that mounts the real + libraries **`:ro`** (kernel-level belt-and-braces over the in-code snapshot); + `stacks doctor` to validate paths and confirm read-only access; config via env or + `stacks.toml`; the documented quickstart. *(mirrors the established Calibre-Web/ + KOReader-sync self-hosting pattern)* +- **Gets stuck:** the build is demo-shaped, so the real first run (paths, kosync key + from env, storage dir, detected schema versions) is the newest, least-trodden path; + reverse-proxy/auth wiring is documented but unverified by a stranger. +- **Wants next:** a one-command compose + reverse-proxy/auth recipe; OIDC/forward-auth + upgrade path; backups/restore drill; a schema-drift CI matrix so a different + Calibre/KOReader version still ingests. +- **Adopts if:** a clean box reaches a working, authed dashboard in <30 min. + **Walks if:** first-run requires reverse-engineering env vars. + +### B3. Chelsea (ops hat) — owner / maintainer / operator +- **Goal:** keep it running, backed up, and resilient as Calibre/KOReader schemas drift. +- **Values today:** persisted derived state with `stacks refresh` (ingest only if + source mtime changed) + freshness stamp; `stacks backup`/`restore`; restart-recovery + and "kosync down → degrade to KOReader-only" reliability tests; versioned, + fixture-tested parsers; `pip-audit` = 0 vulns on the Python 3.14 floor. +- **Gets stuck:** performance (k6/Lighthouse p95) and some reliability gates are + **deferred**, not yet merge-blocking; live-path clients (`KosyncClient`, + `OpenLibraryClient`) are `pragma: no cover`, so real response-shape drift isn't + caught in CI. +- **Wants next:** promote perf + reliability to merge-blocking; recorded-cassette + contract tests for the live clients; TestClient route coverage; a dated schema-drift + matrix. +- **Adopts if:** an unattended seedbox stays correct across restarts and upgrades. + **Walks if:** a silent schema change corrupts ingest. + +--- + +## Group C — Source Ethically (whose terms this tool depends on) + +### C1. Lior — Open Library / open-data steward +- **Goal:** see a downstream project use Open Library data *respectfully* — within + CC0, with rate-limit discipline and provenance. +- **Values today:** Open Library is on the hard allowlist; `assert_allowed()` is + default-deny; the recommender's single network choke point only GETs public + metadata; `docs/ethical-book-data-sources.md` is generated from + `recommender/sources.py` (one source of truth) and documents *why* each catalog is + used. *(CC0 = public-domain dedication, attribution appreciated)* +- **Gets stuck:** the sources doc is terse — it doesn't spell out CC0 vs ODbL nuance, + caching/rate-limit behavior, or robots respect, so a steward can't tell at a glance + that the project is a *good* consumer. +- **Wants next:** a per-source **compliance card**: license (CC0 here), attribution + posture, cache + rate-limit policy, `retrieved_at` provenance on every field, and + a contact. Credit Open Library/Internet Archive even though CC0 doesn't require it. +- **Adopts (endorses) if:** the project is visibly a careful, low-impact consumer. + **Walks if:** it hammers the API or strips provenance. + +### C2. Petra — Bookwyrm instance admin / Fediverse data steward +- **Goal:** make sure federated reads honor **her instance's** terms, robots, and rate + limits — Bookwyrm terms are per-instance, not global. +- **Values today:** Bookwyrm is allowlisted as *federated* with "per-instance terms; + honor robots + rate limits" already written into the sources table; the tool is + read-only and posts nothing back; share cards are composed locally and only posted + when Chelsea manually copies them (no auto-federation). *(matches ActivityPub + federation norms)* +- **Gets stuck:** there's no explicit, public statement of *how* the tool decides an + instance is fair game, how it backs off, or how it caches — so an admin has to take + it on faith. +- **Wants next:** documented federation etiquette (User-Agent, robots, backoff, + per-instance opt-out); pull only public list/metadata, never private activity; + cache aggressively to minimize hits. +- **Adopts if:** her instance is treated as a guest treats a host. **Walks if:** it + scrapes federated data as if it were a central API. + +### C3. Tomás — small-press / indie author whose book gets surfaced +- **Goal:** have his work discovered fairly, with **no** identity label pinned on him + that he didn't choose. +- **Values today:** the whole design is built so recommendations *widen* past + bestseller bias toward small-press/own-voices/translated work (the eval deliberately + hides on-canon picks among more-popular distractors and recovers them); books are + described by **sourced** tags, and `Author` carries only a name + sort key — so the + tool *cannot* auto-assign him a gender/sexuality. *(directly answers the #OwnVoices + outing harm and long-tail under-surfacing)* +- **Gets stuck:** he has no way to see *what* provenance/tags attach to his book, or to + correct a wrong tag or ask not to be surfaced. +- **Wants next:** a lightweight author feedback/correction path; visible provenance per + book; a guarantee that any descriptor traces to a source he or a catalog set, never + an inference. +- **Adopts if:** he's surfaced accurately and never mislabeled. **Walks if:** a model + ever guesses his identity from his name or cover. + +--- + +## Group D — Assure & Audit + +### D1. Dr. Ada — recommender / ML-fairness reviewer +- **Goal:** confirm the recommender beats the popularity baseline **fairly**, not by + cherry-picked metrics or leakage. +- **Values today:** a deterministic, seeded recommender; an eval that shows the content + model (themes + authors + curated lists) at Precision@5 / Recall@5 / MAP@5 = 1.00 vs + a popularity baseline at 0.40 / 0.40 / 0.13, regenerated into `eval-report.json`; + 100% of recs carry why + source; the aperture lens is **boost-only** and "unknown" + is never penalized. *(targets the documented popularity/thematic bias)* +- **Gets stuck:** the eval is small and Precision/Recall/MAP@5-only — no nDCG, catalog + coverage, intra-list diversity, or *temporal* hold-out on real finishes; a perfect + 1.00 on a tiny fixture invites "it's a toy benchmark." +- **Wants next:** richer ranking + diversity metrics, a temporal split on actual reads, + metric-drift tracking across runs, and a published fairness/diversity card so the + "beats popularity *fairly*" claim is legible. +- **Adopts if:** the win survives diversity-aware, temporally honest evaluation. + **Walks if:** the headline is one inflated number on synthetic data. + +### D2. Noor — responsible-tech reviewer doing the human sign-offs +- **Goal:** honestly sign the **privacy**, **representation**, and **screen-reader** + reviews that the audits gate the first release on. +- **Values today:** the auto-gated half is genuinely strong — no-egress, sourced-tags, + auth, and axe checks are all merge-blocking with named tests; the responsible-tech + framework cleanly separates *auto-gated* from *review-gated*. +- **Gets stuck:** all three human sign-offs are explicitly **"pending first release"** + with no committed artifact — so right now there's a gate with nothing behind it. +- **Wants next:** dated, committed sign-off artifacts under `docs/audits/` for the + privacy review, the representation review, and the SR walkthrough; a short + "what wasn't reviewed" note in each. +- **Adopts if:** every review-gated claim has a signed, dated artifact. **Walks if:** + "pending" silently becomes "done" at release without the walkthroughs. + +--- + +## Group E — Community / Expand + +### E1. Robin — OSS contributor +- **Goal:** add a new ethical catalog adapter (or a curated-list importer) and get it + merged without weakening the guardrails. +- **Values today:** clear architecture (`ingest/` · `recommender/` · `app/`); a single + network choke point with a default-deny allowlist; a generated sources doc; green + `make verify` (lint, `mypy --strict`, ~167 tests @ ~96%, a11y, eval) as a contract. +- **Gets stuck:** there's no documented *adapter contract* or conformance test, so a + new source could drift from the provenance/allowlist/sourced-tags invariants; unsure + how to add a source without tripping the no-egress test. +- **Wants next:** a published adapter/plugin contract + conformance suite (allowlist + honored, provenance attached, `ThemeTag` sourced, no egress of reading data); a + "add an ethical source" guide; good-first-issues. +- **Adopts if:** their first adapter PR is mergeable in a day and *can't* violate an + invariant. **Walks if:** contributing risks silently breaking a guardrail. + +### E2. Jules — Fediverse / Bookwyrm mutual (the sharing community) +- **Goal:** enjoy Chelsea's shared "year in books" / finished-book cards on Bookwyrm or + Mastodon — without being surveilled or fed an ad. +- **Values today:** share cards are **composed locally** and posted **only when Chelsea + copies and shares them** — no auto-egress, no tracking pixel, no follower-graph + scraping; nothing about Jules is ingested. *(values-aligned with federated, no-ad + Bookwyrm norms)* +- **Gets stuck:** a card posted as an image may lack alt text (excludes blind mutuals); + and the "this didn't phone home" guarantee isn't visible to the *recipient*. +- **Wants next:** alt text + a text equivalent baked into every generated card; an + optional plain-text version; a one-line "composed locally, shared manually" footer so + the no-surveillance property is legible to followers. +- **Adopts (engages) if:** the cards are accessible and demonstrably egress-free. + **Walks if:** a "share" quietly turns into auto-posting or tracking. + +--- + +## Cross-cutting themes (what the cast agrees on) + +1. **The guardrails are the product, and they hold — but the *human* sign-offs are + empty.** Sam (privacy), Mara (SR), Noor (all three), and River (representation) all + independently land on the same thing: the auto-gated tests are excellent, yet the + three **review-gated sign-offs the audits themselves name are still "pending."** + That's the single highest-trust, lowest-effort gap. **[pending]** +2. **"Demo works" ≠ "runs on the real library."** Chelsea, Devon, and Chelsea-ops all + hit the demo→real gap. N1 (`stacks doctor`/`refresh`, freshness stamp) is built but + the first real run, and a stranger's first deploy, are the least-trodden paths. + **[roadmap N1]** +3. **Provenance wants to be *visible*, not just *enforced*.** River, Tomás, Lior, and + Petra each ask the same question from a different seat: *where did this come from?* + The invariant exists in code (`ThemeTag` needs a `Source`; `assert_allowed` is + default-deny) — but it isn't surfaced in the UI or to the catalogs/authors involved. + Surfacing provenance is cheap and serves four personas at once. **[NET-NEW UI; + corroborates source-ethics]** +4. **Sources are three obligations, not one.** Lior (CC0, attribution-appreciated), + Petra (per-instance ToS + federation etiquette), and the Hardcover constraint + (token-gated, localhost-only, "in flux") mean the single "ethical sources" line + under-serves the stewards the tool depends on. A per-source compliance card fixes it. + **[NET-NEW; corroborates ROADMAP-FUTURE C2]** +5. **The fairness claim is real but thinly evidenced.** Ada wants the popularity-beating + result shown *fairly* — nDCG, diversity, a temporal split — which is exactly the + "richer eval" already on the future roadmap. The research backs the need: book + recommenders measurably under-serve niche/diverse/thematic tastes. **[roadmap B5]** +6. **Accessibility has to survive leaving the app.** Mara and Jules note the dashboard's + a11y contract is strong *inside* the app, but a shared card is an image that escapes + it — alt text must travel with the card. **[NET-NEW]** + +## Honest limits of this exercise +This is simulated. It can generate plausible obligations and obvious gaps, but it +cannot tell you **which** matter most to the real people behind the catalogs, which +self-hosters would actually deploy it, or how an Open Library / Bookwyrm steward would +*really* feel about being consumed. Because the product is single-user, the panel +leans heavily on the author's own mental model and on stakeholders who would never +file an issue — so it over-weights what is *legible from the repo* and under-weights +what only a real steward, author, or assistive-tech user would surprise you with. +**Do not prioritize off this alone.** Use it to design the questions for — and lower +the cost of — real conversations, especially with the data stewards and the +review-gated sign-offs. + +The triaged remediation/expansion backlog, sequencing, first sprint, and traceability +matrix live in **[`RESEARCH-ROADMAP.md`](./RESEARCH-ROADMAP.md)**. diff --git a/docs/audits/dashboard.html b/docs/audits/dashboard.html index 13f7535..90982aa 100644 --- a/docs/audits/dashboard.html +++ b/docs/audits/dashboard.html @@ -24,4 +24,4 @@ @media (prefers-reduced-motion: reduce) { * { animation: none !important; transition: none !important; } } -

Queer the Stacks

Your private reading dashboard, demo — unified read-only from Calibre and KOReader, with recommendations from ethical, non-gatekept catalogs. Reading data never leaves this instance.

Currently reading

  • Stone Butch Blues

    by Leslie Feinberg

    Progress: 47% · last on Kobo

    Themes: historical literary queer trans

Reading stats

Reading totals (data-table equivalent of the stats panel)
MetricValue
Books finished7
Currently reading1
Pages read2281
Time read (hours)105.8
Current streak (days)1
Longest streak (days)3
Active reading days16
Highlights130
Theme & genre mix, from sourced tags only
ThemeBooks
speculative5
queer4
trans4
literary3
science fiction3
dystopia3
short stories1
fabulist1
time travel1
historical1
feminist1

Reading diversity

Built only from sourced descriptors of the books themselves — Calibre tags, OpenLibrary subjects, and curated lists. We never infer an author's identity and never auto-label a person; a book with no sourced descriptor is reported as unknown, not as “not diverse”.

Coverage — how much of the shelf carries a sourced descriptor
MeasureValue
Books considered (reading + finished)8
With a sourced descriptor8
No sourced descriptor (unknown — not 'none')0
Descriptor coverage100%
Representation lenses, as a share of your described books (a grouping of sourced descriptors — never an author's identity)
LensBooks% of describedSourced descriptors seen
Trans & nonbinary450%trans
Queer / LGBTQ+450%queer
Speculative / SFF562%dystopia, fabulist, science fiction, speculative, time travel
Feminist112%feminist
Literary338%literary, short stories
Historical112%historical
Where these descriptors came from (provenance of every sourced tag)
SourceDescriptors
calibre-tag27

Reading Wrapped 2024

7 books · 37.6 hours · 16 reading days — computed locally, shared with no one.

Standout reads of 2024 by time spent
TitleAuthorHours
Oryx and CrakeMargaret Atwood16.9
Detransition, BabyTorrey Peters16.7
Parable of the SowerOctavia E. Butler16.1
KindredOctavia E. Butler14.4
The Handmaid's TaleMargaret Atwood13.9
Monthly reading in 2024 (pace: 141.1 pages per reading day)
MonthPagesHoursDays
Mar3746.23
Apr92615.47
May80813.55
Jun1502.51

Make a share card for Bookwyrm or Mastodon — composed locally; nothing is posted until you copy and share it yourself.

Up next in your series

Unread books in series you've started
TitleSeriesAuthor
Parable of the TalentsEarthseedOctavia E. Butler

To-read shelf

Recommended for you

Every pick shows why it surfaced and the source it came from.

Recommendation fit scores (data-table equivalent of the cards)
RankTitleAuthorFit
1An Unkindness of GhostsRivers Solomon1.053
2The Fifth SeasonN. K. Jemisin0.972
3DawnOctavia E. Butler0.843
4Confessions of the FoxJordy Rosenberg0.755
5NevadaImogen Binnie0.668

1. An Unkindness of Ghosts

by Rivers Solomon

Fit score: 1.053

Why recommended

Sources

Recommended because it shares your themes: queer, science fiction, speculative.

2. The Fifth Season

by N. K. Jemisin

Fit score: 0.972

Why recommended

Sources

Recommended because it shares your themes: dystopia, science fiction, speculative.

3. Dawn

by Octavia E. Butler

Fit score: 0.843

Why recommended

Sources

Recommended because it shares your themes: science fiction, speculative.

4. Confessions of the Fox

by Jordy Rosenberg

Fit score: 0.755

Why recommended

Sources

Recommended because it shares your themes: historical, queer, speculative, trans.

5. Nevada

by Imogen Binnie

Fit score: 0.668

Why recommended

Sources

Recommended because it shares your themes: literary, queer, trans.

Recently finished

Browse your library

Your library — browse by reading the rows, or filter via the box above (or the /browse route)
TitleAuthorStatusThemes (sourced)
A Safe Girl to LoveCasey Plettfinishedliterary, queer, short stories, trans
Detransition, BabyTorrey Petersfinishedliterary, queer, trans
Fierce Femmes and Notorious LiarsKai Cheng Thomfinishedfabulist, queer, speculative, trans
KindredOctavia E. Butlerfinishedscience fiction, speculative, time travel
Oryx and CrakeMargaret Atwoodfinisheddystopia, science fiction, speculative
Parable of the SowerOctavia E. Butlerfinisheddystopia, science fiction, speculative
Parable of the TalentsOctavia E. Butlerunreaddystopia, science fiction, speculative
Stone Butch BluesLeslie Feinbergreadinghistorical, literary, queer, trans
The Handmaid's TaleMargaret Atwoodfinisheddystopia, feminist, speculative
\ No newline at end of file +

Queer the Stacks

Your private reading dashboard, demo — unified read-only from Calibre and KOReader, with recommendations from ethical, non-gatekept catalogs. Reading data never leaves this instance.

Currently reading

Reading stats

Reading totals (data-table equivalent of the stats panel)
MetricValue
Books finished7
Currently reading1
Pages read2281
Time read (hours)105.8
Current streak (days)1
Longest streak (days)3
Active reading days16
Highlights130
Theme & genre mix, from sourced tags only
ThemeBooks
speculative5
queer4
trans4
literary3
science fiction3
dystopia3
short stories1
fabulist1
time travel1
historical1
feminist1

Reading diversity

Built only from sourced descriptors of the books themselves — Calibre tags, OpenLibrary subjects, and curated lists. We never infer an author's identity and never auto-label a person; a book with no sourced descriptor is reported as unknown, not as “not diverse”.

Every sourced descriptor is shown individually below. To aggregate the identity-adjacent ones (handy when screen-sharing a queer/trans reading history), set STACKS_HIDE_SENSITIVE=1 or load ?hide_sensitive=1.

Coverage — how much of the shelf carries a sourced descriptor
MeasureValue
Books considered (reading + finished)8
With a sourced descriptor8
No sourced descriptor (unknown — not 'none')0
Descriptor coverage100%
Representation lenses, as a share of your described books (a grouping of sourced descriptors — never an author's identity)
LensBooks% of describedSourced descriptors seen
Trans & nonbinary450%trans
Queer / LGBTQ+450%queer
Speculative / SFF562%dystopia, fabulist, science fiction, speculative, time travel
Feminist112%feminist
Literary338%literary, short stories
Historical112%historical
Where these descriptors came from (count of sourced tags by source)
SourceDescriptors
calibre-tag27
Per-descriptor provenance — every diverse-shelf tag, the source that asserted it, and when it was fetched (sourced, never inferred)
DescriptorBooksSource(s)Retrieved
speculative5calibre-tag2026-06-05
queer (sensitive)4calibre-tag2026-06-05
trans (sensitive)4calibre-tag2026-06-05
dystopia3calibre-tag2026-06-05
literary3calibre-tag2026-06-05
science fiction3calibre-tag2026-06-05
fabulist1calibre-tag2026-06-05
feminist1calibre-tag2026-06-05
historical1calibre-tag2026-06-05
short stories1calibre-tag2026-06-05
time travel1calibre-tag2026-06-05

Reading Wrapped 2024

7 books · 37.6 hours · 16 reading days — computed locally, shared with no one.

Standout reads of 2024 by time spent
TitleAuthorHours
Oryx and CrakeMargaret Atwood16.9
Detransition, BabyTorrey Peters16.7
Parable of the SowerOctavia E. Butler16.1
KindredOctavia E. Butler14.4
The Handmaid's TaleMargaret Atwood13.9
Monthly reading in 2024 (pace: 141.1 pages per reading day)
MonthPagesHoursDays
Mar3746.23
Apr92615.47
May80813.55
Jun1502.51

Make a share card for Bookwyrm or Mastodon — composed locally; nothing is posted until you copy and share it yourself.

Up next in your series

Unread books in series you've started
TitleSeriesAuthor
Parable of the TalentsEarthseedOctavia E. Butler

To-read shelf

Recommended for you

Every pick shows why it surfaced and the source it came from.

Recommendation fit scores (data-table equivalent of the cards)
RankTitleAuthorFit
1An Unkindness of GhostsRivers Solomon1.053
2The Fifth SeasonN. K. Jemisin0.972
3DawnOctavia E. Butler0.843
4Confessions of the FoxJordy Rosenberg0.755
5NevadaImogen Binnie0.668

1. An Unkindness of Ghosts

by Rivers Solomon

Fit score: 1.053

Why recommended

  • theme: shares your themes: queer, science fiction, speculative
  • collaborative: listed alongside Octavia E. Butler, whom you've finished, on “Speculative Feminist Classics”
  • list: on the curated list “Trans & Spec-Fic Canon”
  • list: on the curated list “Speculative Feminist Classics”

Sources

Recommended because it shares your themes: queer, science fiction, speculative.

2. The Fifth Season

by N. K. Jemisin

Fit score: 0.972

Why recommended

  • theme: shares your themes: dystopia, science fiction, speculative
  • collaborative: listed alongside Octavia E. Butler, whom you've finished, on “Speculative Feminist Classics”
  • list: on the curated list “Speculative Feminist Classics”

Sources

Recommended because it shares your themes: dystopia, science fiction, speculative.

3. Dawn

by Octavia E. Butler

Fit score: 0.843

Why recommended

  • theme: shares your themes: science fiction, speculative
  • author: by Octavia E. Butler, whom you've finished
  • list: on the curated list “Speculative Feminist Classics”

Sources

Recommended because it shares your themes: science fiction, speculative.

4. Confessions of the Fox

by Jordy Rosenberg

Fit score: 0.755

Why recommended

  • theme: shares your themes: historical, queer, speculative, trans
  • list: on the curated list “Trans & Spec-Fic Canon”

Sources

Recommended because it shares your themes: historical, queer, speculative, trans.

5. Nevada

by Imogen Binnie

Fit score: 0.668

Why recommended

  • theme: shares your themes: literary, queer, trans
  • list: on the curated list “Trans & Spec-Fic Canon”

Sources

Recommended because it shares your themes: literary, queer, trans.

Recently finished

Browse your library

Your library — browse by reading the rows, or filter via the box above (or the /browse route)
TitleAuthorStatusThemes (sourced)
A Safe Girl to LoveCasey Plettfinishedliterary, queer, short stories, trans
Detransition, BabyTorrey Petersfinishedliterary, queer, trans
Fierce Femmes and Notorious LiarsKai Cheng Thomfinishedfabulist, queer, speculative, trans
KindredOctavia E. Butlerfinishedscience fiction, speculative, time travel
Oryx and CrakeMargaret Atwoodfinisheddystopia, science fiction, speculative
Parable of the SowerOctavia E. Butlerfinisheddystopia, science fiction, speculative
Parable of the TalentsOctavia E. Butlerunreaddystopia, science fiction, speculative
Stone Butch BluesLeslie Feinbergreadinghistorical, literary, queer, trans
The Handmaid's TaleMargaret Atwoodfinisheddystopia, feminist, speculative
\ No newline at end of file diff --git a/docs/ethical-book-data-sources.md b/docs/ethical-book-data-sources.md index 907ddc2..d3ad89d 100644 --- a/docs/ethical-book-data-sources.md +++ b/docs/ethical-book-data-sources.md @@ -2,13 +2,70 @@ _Generated from `recommender/sources.py` — the single source of truth._ -## Used +## Used (summary) | Source | Host | Kind | License / terms | Why | |--------|------|------|-----------------|-----| -| Open Library | `openlibrary.org` | open-data | Open data (CC0 for data); attribution appreciated. | Non-profit (Internet Archive), open bibliographic data + subject headings. | -| Hardcover | `api.hardcover.app` | api | Public GraphQL API; respect rate limits + ToS. | Independent, reader-run alternative with community tags; not surveillance-funded. | -| Bookwyrm | `bookwyrm.social` | federated | ActivityPub; per-instance terms; honor robots + rate limits. | Federated, community-run reading lists — no central gatekeeper or ad model. | +| Open Library | `openlibrary.org` | open-data | Bibliographic data is CC0 (public-domain dedication); no key required, public JSON endpoints. | Non-profit (Internet Archive), open bibliographic data + subject headings. | +| Hardcover | `api.hardcover.app` | api | Token-gated GraphQL API; terms are explicitly in flux — treat the schema and policy as unstable and re-check before relying on a field. | Independent, reader-run alternative with community tags; not surveillance-funded. | +| Bookwyrm | `bookwyrm.social` | federated | Per-instance Terms of Service over ActivityPub; content licenses vary by instance and by user — there is no single global term. | Federated, community-run reading lists — no central gatekeeper or ad model. | + +## Per-source compliance cards + +The three sources impose **materially different** obligations — they are not interchangeable. Each card below states the headline license, the attribution posture, the auth/token handling, and the cache/rate-limit/robots policy we honour. + +### Open Library (`openlibrary.org`) + +_Non-profit (Internet Archive), open bibliographic data + subject headings._ + +| Obligation | Posture | +|------------|---------| +| Kind | open-data | +| License / terms | Bibliographic data is CC0 (public-domain dedication); no key required, public JSON endpoints. | +| Attribution | Attribution appreciated but not required — credit 'Open Library / Internet Archive' where practical. | +| Auth / token | No API key. Public, unauthenticated JSON (e.g. /subjects/.json). | +| Cache · rate-limit · robots | Cache on disk; honour robots.txt; back off on 429/5xx; send a descriptive User-Agent; keep volume modest. | +| Contact | Internet Archive / Open Library — https://openlibrary.org/help | +| Terms / API docs | https://openlibrary.org/developers/api | + +### Hardcover (`api.hardcover.app`) + +_Independent, reader-run alternative with community tags; not surveillance-funded._ + +| Obligation | Posture | +|------------|---------| +| Kind | api | +| License / terms | Token-gated GraphQL API; terms are explicitly in flux — treat the schema and policy as unstable and re-check before relying on a field. | +| Attribution | Credit Hardcover for community tags; respect contributor data. | +| Auth / token | Requires a personal API token. Keep it in the environment and use it localhost/server-side only — never ship the token to a browser. | +| Cache · rate-limit · robots | Respect published GraphQL rate limits; cache; back off on 429/5xx; identify via User-Agent. | +| Contact | Hardcover — https://docs.hardcover.app (community support via their Discord) | +| Terms / API docs | https://docs.hardcover.app/api/getting-started/ | + +### Bookwyrm (`bookwyrm.social`) + +_Federated, community-run reading lists — no central gatekeeper or ad model._ + +| Obligation | Posture | +|------------|---------| +| Kind | federated | +| License / terms | Per-instance Terms of Service over ActivityPub; content licenses vary by instance and by user — there is no single global term. | +| Attribution | Attribute the specific instance and author; honour each instance's license. | +| Auth / token | Public ActivityPub / JSON; no central key. Each instance is an independent host with its own rules and admins. | +| Cache · rate-limit · robots | Honour a per-instance opt-out for reads; cache aggressively; respect robots.txt and rate limits; back off on 429/5xx; descriptive User-Agent. | +| Contact | The individual instance admin (e.g. bookwyrm.social admins) — ask before any bulk/automated read. | +| Terms / API docs | https://github.com/bookwyrm-social/bookwyrm/blob/main/FEDERATION.md | + +## Federation & fetch etiquette + +Every catalog/federation request follows this policy (enforced by `recommender.catalogs.etiquette_headers` + `ResponseCache`): + +- Identify every request with a descriptive User-Agent (app + read-only intent). +- Fetch only public catalog metadata — the reader's reading history is never sent. +- Cache responses on disk (recommender.catalogs.ResponseCache) so we do not re-hit APIs. +- Honour robots.txt and any published rate limits; keep request volume low. +- Back off (exponentially) on HTTP 429 / 5xx instead of hammering a host. +- Treat each Bookwyrm instance as independent; honour a per-instance opt-out for reads. ## Excluded (on purpose) diff --git a/ingest/config.py b/ingest/config.py index c65376c..33afc49 100644 --- a/ingest/config.py +++ b/ingest/config.py @@ -42,6 +42,7 @@ class Config: goal_pages: int = 0 # yearly page goal (0 = unset) goal_hours: int = 0 # yearly reading-time goal in hours (0 = unset) goal_streak_days: int = 0 # streak goal in days (0 = unset) + hide_sensitive_descriptors: bool = False # privacy: aggregate identity-adjacent tags @property def store_path(self) -> Path: @@ -132,4 +133,5 @@ def pick_int(env_key: str, key: str) -> int: goal_pages=pick_int("STACKS_GOAL_PAGES", "pages"), goal_hours=pick_int("STACKS_GOAL_HOURS", "hours"), goal_streak_days=pick_int("STACKS_GOAL_STREAK", "streak_days"), + hide_sensitive_descriptors=resolved_env.get("STACKS_HIDE_SENSITIVE") == "1", ) diff --git a/recommender/catalogs.py b/recommender/catalogs.py index 0d85b60..2cf2a79 100644 --- a/recommender/catalogs.py +++ b/recommender/catalogs.py @@ -49,6 +49,26 @@ class SourceNotAllowed(Exception): """Raised when a catalog request targets a blocked or non-allowlisted host.""" +#: A descriptive, identifying User-Agent for every outbound catalog/federation +#: request. Federation etiquette (EV-LICENSE): a host can see exactly who we are +#: and that we are a read-only, caching, self-hosted consumer of *public* metadata. +USER_AGENT: str = ( + "QueerTheStacks/1.0 (self-hosted reading dashboard; read-only public-metadata " + "fetch; caches responses; see docs/ethical-book-data-sources.md)" +) + + +def etiquette_headers(accept: str = "application/json") -> dict[str, str]: + """Polite, identifying HTTP headers for every catalog/federation fetch. + + Pairs with the on-disk :class:`ResponseCache` (don't re-hit APIs), robots / + rate-limit respect, and backoff in the live clients — the documented + federation etiquette in ``docs/ethical-book-data-sources.md``. Only public + catalog metadata is ever requested; reading data is never sent. + """ + return {"User-Agent": USER_AGENT, "Accept": accept} + + def assert_allowed(url: str) -> str: """Return ``url`` iff its host is allowlisted; otherwise raise. @@ -275,7 +295,7 @@ def _fetch(self, url: str) -> str: cached = self.cache.get(url) if cached is not None: return cached - resp = requests.get(url, timeout=self.timeout) + resp = requests.get(url, timeout=self.timeout, headers=etiquette_headers()) resp.raise_for_status() if self.cache is not None: self.cache.put(url, resp.text) @@ -295,6 +315,6 @@ def fetch_list(self, list_url: str) -> tuple[Book, ...]: import requests url = assert_allowed(list_url) - resp = requests.get(url, timeout=self.timeout, headers={"accept": "application/json"}) + resp = requests.get(url, timeout=self.timeout, headers=etiquette_headers()) resp.raise_for_status() return parse_bookwyrm_list(json.loads(resp.text), url, time.strftime("%Y-%m-%d")) diff --git a/recommender/sources.py b/recommender/sources.py index ab4cdd0..f64ea82 100644 --- a/recommender/sources.py +++ b/recommender/sources.py @@ -2,12 +2,15 @@ This is the GTM-promised "ethical book-data sources" list, in code so it stays versioned and testable. Each entry documents a catalog the recommender is allowed -to draw from, its access kind, license/terms note, and why it qualifies — and the +to draw from, its access kind, the **three distinct license/compliance +obligations** (CC0 vs per-instance ToS vs token-gated/"in-flux" API), its +attribution posture, its cache/rate-limit/robots policy, and a contact — plus the deliberate exclusions (Goodreads/Amazon) with the reason. Every allowed host here must also be on :data:`recommender.catalogs.ALLOWED_HOSTS` (asserted by the source-ethics test), so this human-readable registry and the -machine-enforced allowlist can never drift apart. +machine-enforced allowlist can never drift apart. :func:`to_markdown` renders the +committed per-source *compliance card* in ``docs/ethical-book-data-sources.md``. """ from __future__ import annotations @@ -17,13 +20,24 @@ @dataclass(frozen=True) class EthicalSource: - """One vetted book-data source, with provenance and rationale.""" + """One vetted book-data source, with provenance, obligations, and rationale. + + The fields capture the three *different* obligations the EV-LICENSE research + surfaced — they are deliberately not collapsed into a single "terms" blob, + because Open Library (CC0), Hardcover (token-gated, "in flux"), and Bookwyrm + (per-instance ToS, federated) impose materially different duties on us. + """ name: str host: str kind: str # "open-data" | "api" | "federated" - license_note: str - why: str + license_note: str # the headline license / terms obligation + attribution: str # how we credit the source + auth: str # key/token posture (e.g. token-gated, localhost/server-only) + rate_limit: str # cache + rate-limit + robots + backoff policy we honour + contact: str # who to talk to before bulk/automated access + terms_url: str # canonical link to the source's API/terms documentation + why: str # why this source qualifies as ethical / non-gatekept @dataclass(frozen=True) @@ -35,26 +49,84 @@ class ExcludedSource: reason: str +#: A descriptive, identifying User-Agent string and the federation/fetch etiquette +#: policy we hold ourselves to. Surfaced in the compliance card and enforced by +#: :func:`recommender.catalogs.etiquette_headers`. +FETCH_ETIQUETTE: tuple[str, ...] = ( + "Identify every request with a descriptive User-Agent (app + read-only intent).", + "Fetch only public catalog metadata — the reader's reading history is never sent.", + "Cache responses on disk (recommender.catalogs.ResponseCache) so we do not re-hit APIs.", + "Honour robots.txt and any published rate limits; keep request volume low.", + "Back off (exponentially) on HTTP 429 / 5xx instead of hammering a host.", + "Treat each Bookwyrm instance as independent; honour a per-instance opt-out for reads.", +) + + ETHICAL_SOURCES: tuple[EthicalSource, ...] = ( EthicalSource( name="Open Library", host="openlibrary.org", kind="open-data", - license_note="Open data (CC0 for data); attribution appreciated.", + license_note=( + "Bibliographic data is CC0 (public-domain dedication); no key required, " + "public JSON endpoints." + ), + attribution=( + "Attribution appreciated but not required — credit " + "'Open Library / Internet Archive' where practical." + ), + auth="No API key. Public, unauthenticated JSON (e.g. /subjects/.json).", + rate_limit=( + "Cache on disk; honour robots.txt; back off on 429/5xx; send a descriptive " + "User-Agent; keep volume modest." + ), + contact="Internet Archive / Open Library — https://openlibrary.org/help", + terms_url="https://openlibrary.org/developers/api", why="Non-profit (Internet Archive), open bibliographic data + subject headings.", ), EthicalSource( name="Hardcover", host="api.hardcover.app", kind="api", - license_note="Public GraphQL API; respect rate limits + ToS.", + license_note=( + "Token-gated GraphQL API; terms are explicitly in flux — treat the schema " + "and policy as unstable and re-check before relying on a field." + ), + attribution="Credit Hardcover for community tags; respect contributor data.", + auth=( + "Requires a personal API token. Keep it in the environment and use it " + "localhost/server-side only — never ship the token to a browser." + ), + rate_limit=( + "Respect published GraphQL rate limits; cache; back off on 429/5xx; identify " + "via User-Agent." + ), + contact="Hardcover — https://docs.hardcover.app (community support via their Discord)", + terms_url="https://docs.hardcover.app/api/getting-started/", why="Independent, reader-run alternative with community tags; not surveillance-funded.", ), EthicalSource( name="Bookwyrm", host="bookwyrm.social", kind="federated", - license_note="ActivityPub; per-instance terms; honor robots + rate limits.", + license_note=( + "Per-instance Terms of Service over ActivityPub; content licenses vary by " + "instance and by user — there is no single global term." + ), + attribution="Attribute the specific instance and author; honour each instance's license.", + auth=( + "Public ActivityPub / JSON; no central key. Each instance is an independent " + "host with its own rules and admins." + ), + rate_limit=( + "Honour a per-instance opt-out for reads; cache aggressively; respect robots.txt " + "and rate limits; back off on 429/5xx; descriptive User-Agent." + ), + contact=( + "The individual instance admin (e.g. bookwyrm.social admins) — ask before " + "any bulk/automated read." + ), + terms_url="https://github.com/bookwyrm-social/bookwyrm/blob/main/FEDERATION.md", why="Federated, community-run reading lists — no central gatekeeper or ad model.", ), ) @@ -78,14 +150,40 @@ def allowed_hosts() -> frozenset[str]: return frozenset(s.host for s in ETHICAL_SOURCES) +def _compliance_card(s: EthicalSource) -> list[str]: + """Render one source as a per-source compliance card (heading + obligations).""" + return [ + f"### {s.name} (`{s.host}`)", + "", + f"_{s.why}_", + "", + "| Obligation | Posture |", + "|------------|---------|", + f"| Kind | {s.kind} |", + f"| License / terms | {s.license_note} |", + f"| Attribution | {s.attribution} |", + f"| Auth / token | {s.auth} |", + f"| Cache · rate-limit · robots | {s.rate_limit} |", + f"| Contact | {s.contact} |", + f"| Terms / API docs | {s.terms_url} |", + "", + ] + + def to_markdown() -> str: - """Render the registry as a committable Markdown document.""" + """Render the registry as a committable Markdown document. + + Emits a quick "Used" summary table, then a per-source **compliance card** + spelling out the three distinct obligations (CC0 / per-instance ToS / + token-gated, "in-flux" API), the federation/fetch etiquette we enforce, and + the deliberate exclusions. + """ lines = [ "# Ethical Book-Data Sources", "", "_Generated from `recommender/sources.py` — the single source of truth._", "", - "## Used", + "## Used (summary)", "", "| Source | Host | Kind | License / terms | Why |", "|--------|------|------|-----------------|-----|", @@ -94,6 +192,27 @@ def to_markdown() -> str: f"| {s.name} | `{s.host}` | {s.kind} | {s.license_note} | {s.why} |" for s in ETHICAL_SOURCES ] + lines += [ + "", + "## Per-source compliance cards", + "", + "The three sources impose **materially different** obligations — they are not " + "interchangeable. Each card below states the headline license, the attribution " + "posture, the auth/token handling, and the cache/rate-limit/robots policy we honour.", + "", + ] + for s in ETHICAL_SOURCES: + lines += _compliance_card(s) + + lines += [ + "## Federation & fetch etiquette", + "", + "Every catalog/federation request follows this policy (enforced by " + "`recommender.catalogs.etiquette_headers` + `ResponseCache`):", + "", + ] + lines += [f"- {rule}" for rule in FETCH_ETIQUETTE] + lines += [ "", "## Excluded (on purpose)", diff --git a/tests/test_config.py b/tests/test_config.py index e6a3827..c4459c7 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -71,6 +71,13 @@ def test_env_beats_toml(tmp_path: Path) -> None: assert cfg.calibre_db == Path("/from/env.db") +def test_hide_sensitive_descriptors_flag(tmp_path: Path) -> None: + absent = tmp_path / "absent.toml" + assert load_config(env={}, config_path=absent).hide_sensitive_descriptors is False + cfg = load_config(env={"STACKS_HIDE_SENSITIVE": "1"}, config_path=absent) + assert cfg.hide_sensitive_descriptors is True + + def test_key_only_from_env_never_file(tmp_path: Path) -> None: toml = tmp_path / "stacks.toml" # Even if someone puts a key in the file, it is ignored — secrets are env-only. diff --git a/tests/test_diversity.py b/tests/test_diversity.py index ef27da8..c9fefa6 100644 --- a/tests/test_diversity.py +++ b/tests/test_diversity.py @@ -2,7 +2,12 @@ from __future__ import annotations -from app.diversity import DIMENSIONS, compute_diversity +from app.diversity import ( + DIMENSIONS, + SENSITIVE_DESCRIPTORS, + SENSITIVE_DIMENSIONS, + compute_diversity, +) from ingest.models import ( Author, Book, @@ -102,3 +107,76 @@ def test_demo_diversity_reflects_the_canon(states: list) -> None: by_name = {d.name: d for d in report.dimensions} assert by_name["Trans & nonbinary"].books >= 3 assert by_name["Speculative / SFF"].books >= 3 + + +# --- R4: per-descriptor provenance + the privacy (hide-sensitive) toggle ------- + + +def test_descriptor_provenance_carries_source_and_retrieved_at() -> None: + """Every diverse-shelf tag exposes its Source kind, citation, and fetch date.""" + states = [_state("A", ReadingStatus.FINISHED, (_tag("literary"), _tag("trans")))] + report = compute_diversity(states) + by_label = {d.label: d for d in report.descriptor_provenance} + lit = by_label["literary"] + assert lit.source_kinds == ("calibre-tag",) + assert lit.latest_retrieved_at == "2026-06-05" + assert lit.sources[0].citation == "calibre:local" + assert lit.sensitive is False + # "trans" is identity-adjacent and flagged sensitive (but still shown by default). + assert by_label["trans"].sensitive is True + assert report.hide_sensitive is False + + +def test_descriptor_provenance_unions_multiple_sources() -> None: + states = [ + _state("A", ReadingStatus.FINISHED, (_tag("queer", SourceKind.CALIBRE_TAG),)), + _state("B", ReadingStatus.FINISHED, (_tag("queer", SourceKind.OPENLIBRARY_SUBJECT),)), + ] + report = compute_diversity(states) + queer = next(d for d in report.descriptor_provenance if d.label == "queer") + assert queer.books == 2 + assert queer.source_kinds == ("calibre-tag", "openlibrary-subject") + + +def test_hide_sensitive_aggregates_identity_descriptors() -> None: + states = [ + _state("A", ReadingStatus.FINISHED, (_tag("trans"), _tag("literary"))), + _state("B", ReadingStatus.FINISHED, (_tag("queer"),)), + ] + report = compute_diversity(states, hide_sensitive=True) + labels = {d.label for d in report.descriptor_provenance} + # Granular identity labels are gone; the non-sensitive one stays. + assert "trans" not in labels and "queer" not in labels + assert "literary" in labels + # Exactly one aggregated stand-in row, counting distinct books, keeping provenance. + agg = [d for d in report.descriptor_provenance if d.aggregated] + assert len(agg) == 1 + assert agg[0].sensitive and agg[0].books == 2 + assert agg[0].source_kinds == ("calibre-tag",) + # Coarse lens counts remain, but their concrete labels are masked. + by_name = {d.name: d for d in report.dimensions} + assert by_name["Trans & nonbinary"].books == 1 + assert by_name["Trans & nonbinary"].matched_labels == ("(hidden for privacy)",) + # The flat theme breakdown also redacts the granular sensitive labels. + tb = dict(report.theme_breakdown) + assert "trans" not in tb and "queer" not in tb + assert report.hide_sensitive is True + + +def test_hide_sensitive_keeps_nonsensitive_detail() -> None: + states = [_state("A", ReadingStatus.FINISHED, (_tag("speculative"), _tag("literary")))] + report = compute_diversity(states, hide_sensitive=True) + labels = {d.label for d in report.descriptor_provenance} + assert {"speculative", "literary"} <= labels + # No sensitive descriptors present, so no aggregated row is synthesised. + assert not any(d.aggregated for d in report.descriptor_provenance) + + +def test_sensitive_descriptors_are_identity_adjacent() -> None: + assert {"trans", "queer"} <= SENSITIVE_DESCRIPTORS + # Descriptors of works (not outing identity labels) are never sensitive. + assert "speculative" not in SENSITIVE_DESCRIPTORS + assert "literary" not in SENSITIVE_DESCRIPTORS + # The sensitive lenses are a subset of the published, auditable dimensions. + dimension_names = {name for name, _ in DIMENSIONS} + assert SENSITIVE_DIMENSIONS.issubset(dimension_names) diff --git a/tests/test_render_view.py b/tests/test_render_view.py index 70c5438..9226028 100644 --- a/tests/test_render_view.py +++ b/tests/test_render_view.py @@ -4,8 +4,9 @@ from pathlib import Path +from app.a11y_check import check_html from app.render import render_dashboard -from app.view import build_view, demo_view +from app.view import build_view, demo_view, render_view from ingest.models import ( Author, Book, @@ -93,3 +94,34 @@ def test_render_handles_empty_view() -> None: ) assert "Nothing in progress" in html assert "No recommendations yet" in html + + +# --- R4: descriptor provenance surfaced in the diversity section -------------- + + +def _diversity_section(html: str) -> str: + start = html.index("Reading diversity") + return html[start : html.index("Reading Wrapped", start)] + + +def test_diversity_provenance_shows_source_and_date(tmp_path: Path) -> None: + html = render_view(demo_view(tmp_path)) + section = _diversity_section(html) + # Every diverse-shelf tag shows the source that asserted it and when (R4). + assert "Per-descriptor provenance" in section + assert "calibre-tag" in section + assert "2026-06-05" in section + # Sensitive descriptors are flagged in text (never colour-only). + assert "(sensitive)" in section + + +def test_hide_sensitive_redacts_diversity_section(states: list, candidates: tuple) -> None: + view = build_view(states, [], candidates, hide_sensitive_descriptors=True) + html = render_view(view) + section = _diversity_section(html) + # Identity-adjacent labels are aggregated out of the diverse-shelf view. + assert "trans" not in section and "queer" not in section + assert "aggregated for privacy" in section + assert "Privacy:" in section + # The redacted page is still fully accessible (the a11y contract holds). + assert check_html(html) == [] diff --git a/tests/test_sources_registry.py b/tests/test_sources_registry.py index f78112a..2d27cc3 100644 --- a/tests/test_sources_registry.py +++ b/tests/test_sources_registry.py @@ -2,10 +2,11 @@ from __future__ import annotations -from recommender.catalogs import ALLOWED_HOSTS, BLOCKED_HOSTS +from recommender.catalogs import ALLOWED_HOSTS, BLOCKED_HOSTS, USER_AGENT, etiquette_headers from recommender.sources import ( ETHICAL_SOURCES, EXCLUDED_SOURCES, + FETCH_ETIQUETTE, allowed_hosts, to_markdown, ) @@ -29,8 +30,47 @@ def test_every_source_has_rationale_and_license() -> None: assert s.kind in {"open-data", "api", "federated"} -def test_markdown_renders_used_and_excluded() -> None: +def test_every_source_has_full_compliance_card() -> None: + # R5: each source spells out its distinct obligations, not a single blob. + for s in ETHICAL_SOURCES: + assert s.attribution.strip() + assert s.auth.strip() + assert s.rate_limit.strip() + assert s.contact.strip() + assert s.terms_url.startswith("https://") + + +def test_three_distinct_license_obligations_are_captured() -> None: + # The EV-LICENSE finding: CC0 vs token-gated/"in flux" vs per-instance ToS. + by_name = {s.name: s for s in ETHICAL_SOURCES} + assert "CC0" in by_name["Open Library"].license_note + assert "in flux" in by_name["Hardcover"].license_note.lower() + assert "token" in by_name["Hardcover"].auth.lower() + assert "localhost" in by_name["Hardcover"].auth.lower() + assert "per-instance" in by_name["Bookwyrm"].license_note.lower() + + +def test_etiquette_headers_identify_a_polite_read_only_client() -> None: + # R11: every fetch identifies the client and declares read-only intent. + headers = etiquette_headers() + assert "QueerTheStacks" in headers["User-Agent"] + assert headers["Accept"] == "application/json" + assert etiquette_headers("text/html")["Accept"] == "text/html" + assert "read-only" in USER_AGENT.lower() + # The documented policy never sends reading data and always caches. + joined = " ".join(FETCH_ETIQUETTE).lower() + assert "reading history is never sent" in joined + assert "cache" in joined + + +def test_markdown_renders_cards_etiquette_and_excluded() -> None: md = to_markdown() assert "Open Library" in md assert "Goodreads" in md assert "Excluded" in md + # R5: per-source compliance cards with the obligations. + assert "Per-source compliance cards" in md + assert "Attribution" in md + assert "Auth / token" in md + # R11: the documented federation/fetch etiquette policy. + assert "Federation & fetch etiquette" in md From 7018e0006fcc03975b30b3c4313c835eff699ee5 Mon Sep 17 00:00:00 2001 From: Chelsea Kelly-Reif <3114598+ChelseaKR@users.noreply.github.com> Date: Thu, 9 Jul 2026 13:41:04 -0700 Subject: [PATCH 2/2] docs(sources): state honestly which etiquette rules are code-enforced MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fetch-etiquette section claimed the whole policy was 'enforced by etiquette_headers + ResponseCache'. True for User-Agent, public-metadata- only, caching, and the host allowlist — but robots.txt honouring and 429/5xx backoff are not implemented in any client yet (the live clients are pragma: no cover network paths; backoff/robots are planned with the FIX-01 candidate pipeline + PR #27 cassettes). The compliance doc now distinguishes enforced-in-code rules from committed-policy-pending-code, matching the repo's no-overclaiming posture. docs/ethical-book-data- sources.md regenerated from to_markdown(); coverage.xml regenerated. make verify green: 210 tests, 96.67% branch coverage, pa11y 0, eval green. Co-Authored-By: Claude Fable 5 --- docs/ethical-book-data-sources.md | 2 +- recommender/sources.py | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/docs/ethical-book-data-sources.md b/docs/ethical-book-data-sources.md index d3ad89d..4416920 100644 --- a/docs/ethical-book-data-sources.md +++ b/docs/ethical-book-data-sources.md @@ -58,7 +58,7 @@ _Federated, community-run reading lists — no central gatekeeper or ad model._ ## Federation & fetch etiquette -Every catalog/federation request follows this policy (enforced by `recommender.catalogs.etiquette_headers` + `ResponseCache`): +Every catalog/federation request follows this policy. The User-Agent, public-metadata-only, caching, and host-allowlist rules are enforced in code (`recommender.catalogs.etiquette_headers`, `ResponseCache`, `assert_allowed`); robots.txt honouring and 429/5xx backoff are committed policy, enforced by review until the live candidate pipeline lands (FIX-01 / cassette tests): - Identify every request with a descriptive User-Agent (app + read-only intent). - Fetch only public catalog metadata — the reader's reading history is never sent. diff --git a/recommender/sources.py b/recommender/sources.py index f64ea82..ddf6258 100644 --- a/recommender/sources.py +++ b/recommender/sources.py @@ -50,8 +50,13 @@ class ExcludedSource: #: A descriptive, identifying User-Agent string and the federation/fetch etiquette -#: policy we hold ourselves to. Surfaced in the compliance card and enforced by -#: :func:`recommender.catalogs.etiquette_headers`. +#: policy we hold ourselves to. Honesty note on enforcement: the User-Agent, +#: no-reading-data, caching, and allowlist rules are enforced *in code* +#: (:func:`recommender.catalogs.etiquette_headers`, ``ResponseCache``, +#: ``assert_allowed``); robots.txt honouring and 429/5xx backoff are committed +#: policy the live clients must implement when the real candidate pipeline +#: lands (ideation FIX-01, cassette tests PR #27) — until then they are +#: enforced by review, not by code. FETCH_ETIQUETTE: tuple[str, ...] = ( "Identify every request with a descriptive User-Agent (app + read-only intent).", "Fetch only public catalog metadata — the reader's reading history is never sent.", @@ -207,8 +212,11 @@ def to_markdown() -> str: lines += [ "## Federation & fetch etiquette", "", - "Every catalog/federation request follows this policy (enforced by " - "`recommender.catalogs.etiquette_headers` + `ResponseCache`):", + "Every catalog/federation request follows this policy. The User-Agent, " + "public-metadata-only, caching, and host-allowlist rules are enforced in code " + "(`recommender.catalogs.etiquette_headers`, `ResponseCache`, `assert_allowed`); " + "robots.txt honouring and 429/5xx backoff are committed policy, enforced by " + "review until the live candidate pipeline lands (FIX-01 / cassette tests):", "", ] lines += [f"- {rule}" for rule in FETCH_ETIQUETTE]