Research panel + roadmap; implement top items

app/diversity.py

@@ -66,6 +66,21 @@
     ("Historical", frozenset({"historical", "history"})),
 )
 
+#: The lenses whose descriptors are identity-adjacent — the ones a reading
+#: history could be used to *out* someone by (EV-PRIVACY). When the privacy toggle
+#: is on, the granular descriptors behind these lenses are aggregated/hidden in the
+#: diverse-shelf view; the coarse lens *counts* stay, so the picture isn't lost.
+SENSITIVE_DIMENSIONS: frozenset[str] = frozenset({"Trans & nonbinary", "Queer / LGBTQ+"})
+
+#: The concrete sourced descriptors that fall under a sensitive lens.
+SENSITIVE_DESCRIPTORS: frozenset[str] = frozenset(
+    label for name, labels in DIMENSIONS if name in SENSITIVE_DIMENSIONS for label in labels
+)
+
+#: Stand-in labels used when the privacy toggle redacts granular sensitive tags.
+REDACTED_LABEL = "(hidden for privacy)"
+AGGREGATED_LABEL = "(sensitive descriptors — aggregated for privacy)"
+
 
 @dataclass(frozen=True)
 class DimensionStat:
@@ -82,6 +97,43 @@ def pct(self) -> float:
         return self.books / self.described_total if self.described_total > 0 else 0.0
 
 
+@dataclass(frozen=True)
+class SourceRef:
+    """One distinct citation that asserted a descriptor: kind + where + when."""
+
+    kind: str  # str(SourceKind), e.g. "calibre-tag", "openlibrary-subject"
+    citation: str  # the stable reference the source carried
+    retrieved_at: str  # ISO-8601 date the value was fetched
+
+
+@dataclass(frozen=True)
+class DescriptorProvenance:
+    """A single diverse-shelf descriptor with its source(s) + fetch date (R4).
+
+    Surfaces, for every diverse-shelf tag, the :class:`~ingest.models.Source`
+    kind(s), citation, and ``retrieved_at`` already stored on the tag — so the
+    reader can see *who asserted* each theme/identity descriptor and when. An
+    ``aggregated`` row stands in for the hidden sensitive descriptors when the
+    privacy toggle is on.
+    """
+
+    label: str
+    books: int  # how many considered books carry this descriptor
+    sources: tuple[SourceRef, ...]
+    sensitive: bool  # identity-adjacent (could out a reader)
+    aggregated: bool = False  # True when this row redacts hidden sensitive labels
+
+    @property
+    def latest_retrieved_at(self) -> str:
+        """The freshest fetch date across this descriptor's sources."""
+        return max((s.retrieved_at for s in self.sources), default="")
+
+    @property
+    def source_kinds(self) -> tuple[str, ...]:
+        """The distinct source kinds that asserted this descriptor, sorted."""
+        return tuple(sorted({s.kind for s in self.sources}))
+
+
 @dataclass(frozen=True)
 class DiversityReport:
     """The committed shape of the diverse-shelf analytics view."""
@@ -91,6 +143,8 @@ class DiversityReport:
     theme_breakdown: tuple[tuple[str, int], ...]  # (sourced label, books), desc
     dimensions: tuple[DimensionStat, ...]
     source_provenance: tuple[tuple[str, int], ...]  # (source-kind, descriptor count), desc
+    descriptor_provenance: tuple[DescriptorProvenance, ...] = ()  # per-tag Source + retrieved_at
+    hide_sensitive: bool = False  # privacy toggle: sensitive descriptors aggregated/hidden
 
     @property
     def undescribed_books(self) -> int:
@@ -103,19 +157,30 @@ def coverage_pct(self) -> float:
         return self.described_books / self.total_books if self.total_books > 0 else 0.0
 
 
-def compute_diversity(states: list[ReadingState]) -> DiversityReport:
+def compute_diversity(
+    states: list[ReadingState], *, hide_sensitive: bool = False
+) -> DiversityReport:
     """Compute the diverse-shelf report from sourced book descriptors only.
 
     Considers books you've actually engaged with (reading + finished), matching
     the stats theme-mix; unread owned books are excluded so the picture reflects
     your reading, not your shelf's backlog.
+
+    Every descriptor carries its full provenance — the :class:`SourceRef`\\ s that
+    assert it, each with a citation and ``retrieved_at`` (R4). With
+    ``hide_sensitive=True`` the *granular* identity-adjacent descriptors
+    (:data:`SENSITIVE_DESCRIPTORS`) are aggregated into a single redacted row and
+    the matching lens labels are masked — a privacy posture for screen-sharing a
+    queer/trans reading history (EV-PRIVACY) — while the coarse lens counts stay.
     """
     considered = [s for s in states if s.status is not ReadingStatus.UNREAD]
     total = len(considered)
 
     theme_counter: Counter[str] = Counter()
     provenance: Counter[str] = Counter()
+    desc_sources: dict[str, set[SourceRef]] = {}
     described = 0
+    sensitive_books = 0  # distinct considered books carrying any sensitive descriptor
     # Per-dimension book counts + the concrete labels that matched (transparency).
     dim_books: dict[str, int] = {name: 0 for name, _ in DIMENSIONS}
     dim_labels: dict[str, set[str]] = {name: set() for name, _ in DIMENSIONS}
@@ -124,11 +189,15 @@ def compute_diversity(states: list[ReadingState]) -> DiversityReport:
         labels = {t.normalized for t in state.theme_tags}
         if labels:
             described += 1
+        if labels & SENSITIVE_DESCRIPTORS:
+            sensitive_books += 1
         for label in labels:
             theme_counter[label] += 1
         # Provenance is counted per descriptor (a book can be described by several).
         for tag in state.theme_tags:
             provenance[str(tag.source.kind)] += 1
+            ref = SourceRef(str(tag.source.kind), tag.source.citation, tag.source.retrieved_at)
+            desc_sources.setdefault(tag.normalized, set()).add(ref)
         for name, descriptors in DIMENSIONS:
             hit = labels & descriptors
             if hit:
@@ -140,7 +209,11 @@ def compute_diversity(states: list[ReadingState]) -> DiversityReport:
             name=name,
             books=dim_books[name],
             described_total=described,
-            matched_labels=tuple(sorted(dim_labels[name])),
+            matched_labels=(
+                (REDACTED_LABEL,)
+                if hide_sensitive and name in SENSITIVE_DIMENSIONS
+                else tuple(sorted(dim_labels[name]))
+            ),
         )
         for name, _ in DIMENSIONS
         if dim_books[name] > 0  # only surface lenses your shelf actually populates
@@ -149,7 +222,56 @@ def compute_diversity(states: list[ReadingState]) -> DiversityReport:
     return DiversityReport(
         total_books=total,
         described_books=described,
-        theme_breakdown=tuple(theme_counter.most_common()),
+        theme_breakdown=_theme_breakdown(theme_counter, hide_sensitive, sensitive_books),
         dimensions=dimensions,
         source_provenance=tuple(provenance.most_common()),
+        descriptor_provenance=_descriptor_provenance(
+            theme_counter, desc_sources, hide_sensitive, sensitive_books
+        ),
+        hide_sensitive=hide_sensitive,
     )
+
+
+def _sort_ref(ref: SourceRef) -> tuple[str, str, str]:
+    return (ref.kind, ref.citation, ref.retrieved_at)
+
+
+def _theme_breakdown(
+    theme_counter: Counter[str], hide_sensitive: bool, sensitive_books: int
+) -> tuple[tuple[str, int], ...]:
+    """The (label, books) breakdown, redacting sensitive labels when asked."""
+    if not hide_sensitive:
+        return tuple(theme_counter.most_common())
+    visible = [(lbl, n) for lbl, n in theme_counter.items() if lbl not in SENSITIVE_DESCRIPTORS]
+    if sensitive_books:
+        visible.append((AGGREGATED_LABEL, sensitive_books))
+    return tuple(sorted(visible, key=lambda item: (-item[1], item[0])))
+
+
+def _descriptor_provenance(
+    theme_counter: Counter[str],
+    desc_sources: dict[str, set[SourceRef]],
+    hide_sensitive: bool,
+    sensitive_books: int,
+) -> tuple[DescriptorProvenance, ...]:
+    """Build per-descriptor provenance (R4), aggregating sensitive tags if hidden."""
+    rows: list[DescriptorProvenance] = []
+    aggregated_refs: set[SourceRef] = set()
+    for label, books in theme_counter.items():
+        refs = tuple(sorted(desc_sources.get(label, set()), key=_sort_ref))
+        sensitive = label in SENSITIVE_DESCRIPTORS
+        if hide_sensitive and sensitive:
+            aggregated_refs |= set(refs)
+            continue
+        rows.append(DescriptorProvenance(label, books, refs, sensitive))
+    if hide_sensitive and sensitive_books:
+        rows.append(
+            DescriptorProvenance(
+                AGGREGATED_LABEL,
+                sensitive_books,
+                tuple(sorted(aggregated_refs, key=_sort_ref)),
+                sensitive=True,
+                aggregated=True,
+            )
+        )
+    return tuple(sorted(rows, key=lambda d: (-d.books, d.label)))

app/render.py

@@ -290,21 +290,57 @@ def _diversity_section(report: Optional[DiversityReport]) -> str:
         for kind, count in report.source_provenance
     )
     provenance = (
-        "<table><caption>Where these descriptors came from (provenance of every "
-        'sourced tag)</caption><thead><tr><th scope="col">Source</th>'
+        "<table><caption>Where these descriptors came from (count of sourced tags "
+        'by source)</caption><thead><tr><th scope="col">Source</th>'
         f'<th scope="col">Descriptors</th></tr></thead><tbody>{prov_rows}</tbody></table>'
         if prov_rows
         else "<p>No descriptor provenance recorded yet.</p>"
     )
 
+    # R4: every diverse-shelf descriptor with the source that asserted it + when.
+    # The sensitive marker is text (never colour-only) for the a11y contract.
+    if report.descriptor_provenance:
+        desc_rows = "".join(
+            f'<tr><th scope="row">{escape(d.label)}'
+            f"{' (sensitive)' if d.sensitive else ''}</th>"
+            f"<td>{d.books}</td>"
+            f"<td>{escape(', '.join(d.source_kinds) or '—')}</td>"
+            f"<td>{escape(d.latest_retrieved_at or '—')}</td></tr>"
+            for d in report.descriptor_provenance
+        )
+        descriptor_table = (
+            "<table><caption>Per-descriptor provenance — every diverse-shelf tag, the "
+            "source that asserted it, and when it was fetched (sourced, never inferred)"
+            '</caption><thead><tr><th scope="col">Descriptor</th>'
+            '<th scope="col">Books</th><th scope="col">Source(s)</th>'
+            '<th scope="col">Retrieved</th></tr></thead>'
+            f"<tbody>{desc_rows}</tbody></table>"
+        )
+    else:
+        descriptor_table = "<p>No per-descriptor provenance recorded yet.</p>"
+
+    if report.hide_sensitive:
+        privacy_note = (
+            "<p><strong>Privacy:</strong> identity-adjacent descriptors are aggregated "
+            "and hidden in this view (the coarse lens counts remain). Unset the privacy "
+            "toggle to see every sourced descriptor individually.</p>"
+        )
+    else:
+        privacy_note = (
+            "<p>Every sourced descriptor is shown individually below. To aggregate the "
+            "identity-adjacent ones (handy when screen-sharing a queer/trans reading "
+            "history), set <code>STACKS_HIDE_SENSITIVE=1</code> or load "
+            "<code>?hide_sensitive=1</code>.</p>"
+        )
+
     return (
         "<h2>Reading diversity</h2>"
         "<p>Built <strong>only</strong> from sourced descriptors of the books "
         "themselves — Calibre tags, OpenLibrary subjects, and curated lists. We "
         "never infer an author's identity and never auto-label a person; a book "
         "with no sourced descriptor is reported as unknown, not as &ldquo;not "
         "diverse&rdquo;.</p>"
-        f"{coverage}{dimensions}{provenance}"
+        f"{privacy_note}{coverage}{dimensions}{provenance}{descriptor_table}"
     )
 
 

app/server.py

@@ -42,8 +42,12 @@ def require_auth(authorization: Optional[str] = Header(default=None)) -> None:
         )
 
 
-def _load_view() -> DashboardView:
-    """Load the dashboard from the persisted store, refreshing on first run."""
+def _load_view(*, hide_sensitive: bool = False) -> DashboardView:
+    """Load the dashboard from the persisted store, refreshing on first run.
+
+    ``hide_sensitive`` is a per-request privacy override; it can only *add*
+    aggregation on top of the configured default (you can always hide more).
+    """
     config = load_config()
     store = Store(config.store_path)
     try:
@@ -59,6 +63,7 @@ def _load_view() -> DashboardView:
             goal_pages=config.goal_pages,
             goal_hours=config.goal_hours,
             goal_streak_days=config.goal_streak_days,
+            hide_sensitive_descriptors=config.hide_sensitive_descriptors or hide_sensitive,
         )
     finally:
         store.close()
@@ -116,8 +121,8 @@ def _readyz() -> Response:
     return JSONResponse(status_code=200, content={"status": "ok", "checks": checks})
 
 
-def _dashboard() -> HTMLResponse:
-    return HTMLResponse(content=render_view(_load_view()))
+def _dashboard(hide_sensitive: bool = False) -> HTMLResponse:
+    return HTMLResponse(content=render_view(_load_view(hide_sensitive=hide_sensitive)))
 
 
 def _browse(

app/view.py

@@ -67,6 +67,7 @@ def build_view(
     goal_pages: int = 0,
     goal_hours: int = 0,
     goal_streak_days: int = 0,
+    hide_sensitive_descriptors: bool = False,
 ) -> DashboardView:
     """Build the dashboard view from unified state + candidates (pure)."""
     today_ordinal, year = _infer_today_and_year(states, daily_activity)
@@ -80,7 +81,7 @@ def build_view(
         hours_target=goal_hours,
         streak_target=goal_streak_days,
     )
-    diversity = compute_diversity(states)
+    diversity = compute_diversity(states, hide_sensitive=hide_sensitive_descriptors)
     candidate_books = tuple(c.book for c in candidates)  # type: ignore[attr-defined]
     recs = recommend_hybrid(
         states,
@@ -137,6 +138,7 @@ def view_from_store(
     goal_pages: int = 0,
     goal_hours: int = 0,
     goal_streak_days: int = 0,
+    hide_sensitive_descriptors: bool = False,
 ) -> DashboardView:
     """Build the dashboard view from persisted derived state in the store.
 
@@ -162,6 +164,7 @@ def view_from_store(
         goal_pages=goal_pages,
         goal_hours=goal_hours,
         goal_streak_days=goal_streak_days,
+        hide_sensitive_descriptors=hide_sensitive_descriptors,
     )