From 8480bbb974773f4a023af7dd26b263661b2a11ae Mon Sep 17 00:00:00 2001 From: Steve Geinitz Date: Wed, 6 May 2026 19:25:52 -0600 Subject: [PATCH 1/3] Add prep-class-digest task for pre-class cohort gap synthesis Synthesizes a 1-page Markdown brief from recent quiz misses, follow-up reply themes, and media-recording transcripts in a configurable lookback window (default 7 days), ending with 2-3 in-class discussion questions targeted at the highest-priority gaps. Local Gemma 4 handles all student-derived content; an opt-in --cloud-questions routes only the final question-drafting step to cloud Gemini 3 with a redacted prompt that contains tag names + miss counts but no transcripts or theme text. Co-Authored-By: Claude Opus 4.7 --- README.md | 54 ++- canvigator.py | 66 +++- canvigator_digest.py | 776 +++++++++++++++++++++++++++++++++++++++++++ canvigator_utils.py | 25 ++ test_canvigator.py | 480 ++++++++++++++++++++++++++ 5 files changed, 1386 insertions(+), 15 deletions(-) create mode 100644 canvigator_digest.py diff --git a/README.md b/README.md index b0cbc53..11da196 100644 --- a/README.md +++ b/README.md @@ -117,13 +117,15 @@ python canvigator.py --tag get-quiz-questions # add LLM-generated topic tags python canvigator.py --reply-window-days N # set the days-after-send window for assess-replies (default: 5) python canvigator.py --months N delete-old-conversations # cutoff age in months for delete-old-conversations (default: 6) python canvigator.py --auto-grade get-media-recordings # skip per-student review and auto-grade every submission at full credit +python canvigator.py --days N prep-class-digest # synthesize a 1-page brief on cohort gaps from the last N days (default 7) +python canvigator.py --cloud-questions prep-class-digest # opt the discussion-question step into cloud Gemini 3 with a redacted prompt (default: local Gemma 4) ``` The `--crn` option selects a course by its CRN (the last 5 digits of the Canvas course code), bypassing the interactive course selection prompt. This is useful for automated/scheduled runs, e.g. `python canvigator.py --crn 12345 get-activity`. -Available tasks: `analyze-media-recordings`, `assess-replies`, `award-bonus`, `award-bonus-partner-only`, `award-bonus-retake-only`, `create-media-recording-assignment`, `create-pairs`, `create-quiz`, `delete-old-conversations`, `export-anon-data`, `generate-follow-up-questions`, `get-activity`, `get-conversations`, `get-gradebook`, `get-media-recordings`, `get-quiz-questions`, `get-quiz-submission-events`, `get-roster`, `send-follow-up-assessments`, `send-follow-up-question`, `send-quiz-reminder` +Available tasks: `analyze-media-recordings`, `assess-replies`, `award-bonus`, `award-bonus-partner-only`, `award-bonus-retake-only`, `create-media-recording-assignment`, `create-pairs`, `create-quiz`, `delete-old-conversations`, `export-anon-data`, `generate-follow-up-questions`, `get-activity`, `get-conversations`, `get-gradebook`, `get-media-recordings`, `get-quiz-questions`, `get-quiz-submission-events`, `get-roster`, `prep-class-digest`, `send-follow-up-assessments`, `send-follow-up-question`, `send-quiz-reminder` All tasks begin by prompting you to select a course. Output files are written to `data//` and `figures//`, where `` is derived from the @@ -813,3 +815,53 @@ and `get-quiz-questions --tag` (to produce the tags CSV) first. | **Input** | `data//assignment_recordings_YYYYMMDD.csv` (from `get-media-recordings`) | | | `data//__questions_w_tags_YYYYMMDD.csv` (from `get-quiz-questions --tag`) | | **Output** | `data//assignment_analysis_YYYYMMDD.md` — Markdown report | + +--- + +#### `prep-class-digest` — Pre-class "where the cohort actually is" Markdown brief + +Synthesizes a 1-page brief from three signal sources collected over a +configurable lookback window (default 7 days, override with `--days N`): + +- **Recent quiz misses** — joined to the topic tags from + `*_questions_w_tags_*.csv` so misses are aggregated by concept rather than + by individual question. +- **Follow-up reply themes** — failing/borderline rows in + `*_followup_assessments.csv` (within the window) are summarized by a local + Gemma 4 call into "where students went wrong" bullets, one block per + quiz/question. +- **Media-recording transcripts** — every + `assignment_recordings_*.csv` in the window is run through the existing + tag-classification + theme-extraction Gemma 4 pipeline (the same one used + by `analyze-media-recordings`). + +The brief ends with **2–3 suggested in-class discussion questions** targeted +at the highest-priority gaps. By default these are drafted by the local +Gemma 4 model with a full-fidelity prompt that includes the derived theme +bullets and evidence snippets. With `--cloud-questions` (`-q`) the step +routes to cloud Gemini 3 instead, but only with a *redacted* prompt that +contains tag names + integer miss counts + integer theme-cluster counts — +no transcripts, no `criteria_evaluations`, no theme text. Student-derived +content stays local in either configuration. + +Empty windows (no quiz/follow-up/recording activity in the lookback +period) short-circuit with a friendly message and write no file. Same-day +re-runs overwrite. + +| | Files | +|---|---| +| **Input** | `data//__all_subs_by_question_YYYYMMDD.csv` files in window (from `get-quiz-submission-events`) | +| | `data//__questions_w_tags_YYYYMMDD.csv` per quiz (from `get-quiz-questions --tag`) | +| | `data//__followup_assessments.csv` (from `assess-replies`, persistent) | +| | `data//assignment_recordings_YYYYMMDD.csv` files in window (from `get-media-recordings`) | +| **Output** | `data//class_digest_YYYYMMDD.md` — Markdown report (5 sections: header, quiz performance table, follow-up reply themes, media-recording themes, suggested discussion questions) | + +**Typical workflow:** +1. Mid-week, after a quiz has been returned and any follow-ups assessed: + ```bash + python canvigator.py prep-class-digest # default 7-day window, Gemma-only + python canvigator.py --days 14 prep-class-digest # widen the window + python canvigator.py --cloud-questions prep-class-digest # opt the question step into Gemini + ``` +2. Open the generated `class_digest_*.md`, scan the priority gaps, and use + the suggested questions to seed a 5–10 minute in-class discussion. diff --git a/canvigator.py b/canvigator.py index e362472..fb2f6ea 100755 --- a/canvigator.py +++ b/canvigator.py @@ -34,6 +34,7 @@ ('get-gradebook', 'Export course gradebook'), ('get-roster', 'Export the full course roster (name, id, sis_id, enrollment_type)'), ('send-quiz-reminder', 'Send quiz reminder messages to students'), + ('prep-class-digest', 'Synthesize a 1-page Markdown brief on cohort gaps from the last N days (default 7; override with --days/-n)'), ]), ] task_descriptions = {name: desc for _, items in task_groups for name, desc in items} @@ -52,6 +53,8 @@ def print_help(): print(" -m, --months Age threshold in months for delete-old-conversations (default: 6)") print(" -w, --reply-window-days Days to accept replies after follow-up sent (default: 5, assess-replies only)") print(" -g, --auto-grade Skip per-student review prompt and auto-award points_possible (get-media-recordings only)") + print(" -n, --days Lookback window in days for prep-class-digest (default: 7)") + print(" -q, --cloud-questions Route the discussion-question step to cloud Gemini 3 with a redacted prompt (prep-class-digest only)") max_name = max(len(t) for t in tasks) for header, items in task_groups: print(f"\n{header}") @@ -79,6 +82,31 @@ def _run_assignment_task(task, course, canvas, canv_config, dry_run, auto_grade_ cassign.getMediaRecordings(auto_grade=auto_grade_flag, dry_run=dry_run) +# Course-level tasks that need only `course` + a few primitives. Kept in a +# helper to keep the main if/elif chain under flake8's max-complexity limit. +COURSE_TASKS = frozenset({ + 'get-activity', 'create-quiz', 'get-gradebook', 'get-roster', + 'get-conversations', 'prep-class-digest', +}) + + +def _run_course_task(task, course, canv_config, n_days, cloud_questions): + """Dispatch a course-level task that doesn't need quiz/assignment selection.""" + if task == 'get-activity': + course.saveStudentActivity(canv_config.data_path) + elif task == 'create-quiz': + course.createQuiz() + elif task == 'get-gradebook': + course.exportGradebook(canv_config.data_path) + elif task == 'get-roster': + course.exportRoster(canv_config.data_path) + elif task == 'get-conversations': + course.exportConversations(canv_config.data_path) + elif task == 'prep-class-digest': + import canvigator_digest as cd + cd.prepClassDigest(course, days=n_days, cloud_questions=cloud_questions) + + def _run_quiz_task(task, quiz, dry_run, tag, reply_window_days): """Dispatch a quiz-level task to the appropriate method.""" if task == 'get-quiz-questions': @@ -122,6 +150,8 @@ def _run_quiz_task(task, quiz, dry_run, tag, reply_window_days): '-m': '--months', '-w': '--reply-window-days', '-g': '--auto-grade', + '-n': '--days', + '-q': '--cloud-questions', } args = [_SHORT_TO_LONG.get(a, a) for a in sys.argv[1:]] @@ -141,6 +171,10 @@ def _run_quiz_task(task, quiz, dry_run, tag, reply_window_days): if auto_grade_flag: args.remove('--auto-grade') +cloud_questions_flag = '--cloud-questions' in args +if cloud_questions_flag: + args.remove('--cloud-questions') + crn = None if '--crn' in args: crn_idx = args.index('--crn') @@ -186,6 +220,22 @@ def _run_quiz_task(task, quiz, dry_run, tag, reply_window_days): args.pop(rw_idx) # remove '--reply-window-days' args.pop(rw_idx) # remove the value +n_days = 7 +if '--days' in args: + nd_idx = args.index('--days') + if nd_idx + 1 >= len(args): + print("Error: --days/-n requires a numeric value (number of days)") + sys.exit(1) + try: + n_days = int(args[nd_idx + 1]) + if n_days < 1: + raise ValueError + except ValueError: + print(f"Error: --days/-n must be a positive integer, got '{args[nd_idx + 1]}'") + sys.exit(1) + args.pop(nd_idx) # remove '--days' + args.pop(nd_idx) # remove the value + if len(args) < 1: print_help() sys.exit(1) @@ -293,8 +343,8 @@ def _run_quiz_task(task, quiz, dry_run, tag, reply_window_days): course = cc.CanvigatorCourse(canvas, course_choice, canv_config, verbose=False) -if task == 'get-activity': - course.saveStudentActivity(canv_config.data_path) +if task in COURSE_TASKS: + _run_course_task(task, course, canv_config, n_days, cloud_questions_flag) elif task == 'get-quiz-submission-events' and all_quizzes_flag: course.getAllQuizzesAndSubmissions() @@ -302,21 +352,9 @@ def _run_quiz_task(task, quiz, dry_run, tag, reply_window_days): elif task == 'send-quiz-reminder' and all_quizzes_flag: course.sendAllQuizReminders(dry_run=dry_run) -elif task == 'create-quiz': - course.createQuiz() - elif task in ('create-media-recording-assignment', 'get-media-recordings', 'analyze-media-recordings'): _run_assignment_task(task, course, canvas, canv_config, dry_run, auto_grade_flag) -elif task == 'get-gradebook': - course.exportGradebook(canv_config.data_path) - -elif task == 'get-roster': - course.exportRoster(canv_config.data_path) - -elif task == 'get-conversations': - course.exportConversations(canv_config.data_path) - elif task == 'generate-follow-up-questions': # Driven by a pre-selected tagged-questions CSV, not a Canvas quiz tagged_csv = cu.selectCSVFromList( diff --git a/canvigator_digest.py b/canvigator_digest.py new file mode 100644 index 0000000..fbcb464 --- /dev/null +++ b/canvigator_digest.py @@ -0,0 +1,776 @@ +"""Pre-class cohort digest. + +Synthesizes a 1-page Markdown brief on cohort gaps over a configurable window +(default 7 days) from three signal sources: + +* Recent quiz misses (joined to topic tags from ``*_questions_w_tags_*.csv``). +* Follow-up reply themes (failing/borderline rows in + ``*_followup_assessments.csv``). +* Media-recording transcripts in window + (``assignment_recordings_*.csv``). + +The brief ends with 2-3 suggested in-class discussion questions targeted at the +top-ranked gaps. Local Gemma 4 handles all student-derived content; an +instructor opt-in (``--cloud-questions``) routes only the final +question-drafting step to cloud Gemini 3 with a redacted prompt that contains +no transcripts or derived themes — just tag names and miss counts. +""" +import json +import logging +import os +import re +from collections import Counter +from datetime import datetime, timedelta, timezone + +import pandas as pd + +from canvigator_utils import find_csvs_in_window, find_latest_csv, today_str +from canvigator_assignment import _collectUniqueTags +from canvigator_llm import ( + DEFAULT_MODEL, + DEFAULT_TEXT_MODEL, + OLLAMA_CLOUD_HOST, + _chat_with_retry, + _make_client, + analyze_recording_tags, + extract_recording_themes, +) + +logger = logging.getLogger(__name__) + + +_MAX_FOLLOWUP_ROWS_PER_PROMPT = 12 +_TOP_PRIORITIES = 5 + +_FOLLOWUP_THEMES_SYSTEM_PROMPT = ( + "You are an instructor analyzing student responses to an open-ended follow-up " + "question to surface where the cohort is struggling. The students who replied " + "either failed the rubric or scored borderline; their transcripts and the " + "rubric-criterion evaluations from an automated grader are provided.\n\n" + "Identify 2-4 distinct concept-level struggles that show up across multiple " + "students. For each:\n" + "- A short title (3-6 words, in bold).\n" + "- One sentence describing the misunderstanding.\n" + "- A list of student indices (1-based) who exhibited it.\n\n" + "Format as Markdown bullets. No preamble, no headers, no summary — just " + "the themed bullets. If fewer than 2 distinct themes are evident, return " + "only what is genuinely present." +) + +_DISCUSSION_LOCAL_SYSTEM_PROMPT = ( + "You are helping an instructor plan the next class session. You will receive " + "a ranked list of cohort gaps drawn from recent quiz misses, follow-up reply " + "themes, and media-recording themes. Draft 2-3 short discussion questions " + "the instructor can ask in class to surface and resolve the highest-priority " + "gaps.\n\n" + "Each question should:\n" + "- Be open-ended (not yes/no, not single-word).\n" + "- Target a specific gap from the list — name the topic in the question.\n" + "- Be answerable in 2-5 minutes of class discussion.\n" + "- Invite student-to-student exchange, not just instructor-to-student Q&A.\n\n" + "Return ONLY the questions as Markdown bullets — one per line, prefixed with " + "``- ``. No preamble, no headers, no numbering." +) + +_DISCUSSION_CLOUD_SYSTEM_PROMPT = ( + "You are helping an instructor plan the next class session. You will receive " + "a ranked list of quiz topics, each with the number of students who missed " + "questions on it recently. Draft 2-3 short open-ended discussion questions " + "the instructor can ask in class to surface and resolve student gaps on the " + "highest-priority topics.\n\n" + "Each question should:\n" + "- Be open-ended (not yes/no, not single-word).\n" + "- Target a specific topic from the list — name the topic in the question.\n" + "- Be answerable in 2-5 minutes of class discussion.\n" + "- Invite student-to-student exchange, not just instructor-to-student Q&A.\n\n" + "Return ONLY the questions as Markdown bullets — one per line, prefixed with " + "``- ``. No preamble, no headers, no numbering." +) + + +# --------------------------------------------------------------------------- +# Phase A — Load (no LLM) +# --------------------------------------------------------------------------- + + +_QUIZ_PREFIX_RE = re.compile(r'^(.*?_\d+)_') + + +def _extractQuizPrefix(filename, marker): + """Return the ``__`` prefix from a filename built with ``marker``. + + For example, ``quiz1_547889_all_subs_by_question_20260424.csv`` with + ``marker='all_subs_by_question'`` returns ``'quiz1_547889_'``. + """ + parts = filename.split(f"_{marker}_") + if len(parts) < 2: + return None + return parts[0] + "_" + + +def _quizMetaFromPrefix(prefix): + """Split a ``__`` prefix into (quiz_name, quiz_id) strings.""" + stripped = prefix.rstrip('_') + m = re.match(r'^(.*)_(\d+)$', stripped) + if not m: + return stripped, '' + return m.group(1), m.group(2) + + +def _loadQuizMisses(data_path, since_date): + """Compute per-tag miss counts for every quiz with subs-by-question data in the window. + + For each ``*_all_subs_by_question_*.csv`` whose embedded date is in window, + locate that quiz's latest (static) ``*_questions_w_tags_*.csv``, join on + ``question_id``, and tally how many (student, question) rows had + ``points < points_possible``. Tags from each missed question's ``keywords`` + cell are credited once per missed row. + + Returns a list of dicts ``{quiz_id, quiz_name, total_attempts, n_missed_per_tag}``, + one per quiz that had usable data. Quizzes with no tagged-questions CSV are + skipped with a warning so a single missing artifact doesn't block the digest. + """ + sub_files = find_csvs_in_window(data_path, 'all_subs_by_question_', since_date) + by_prefix = {} + for f in sub_files: + prefix = _extractQuizPrefix(f.name, 'all_subs_by_question') + if prefix is None: + continue + # Keep the newest file per quiz (find_csvs_in_window already sorts ascending). + by_prefix[prefix] = f + + out = [] + for prefix, sub_path in by_prefix.items(): + quiz_name, quiz_id = _quizMetaFromPrefix(prefix) + tags_path = find_latest_csv(data_path, prefix + 'questions_w_tags_') + if tags_path is None: + logger.warning(f"No questions_w_tags CSV for {prefix}; skipping miss tally.") + continue + try: + subs_df = pd.read_csv(sub_path) + tags_df = pd.read_csv(tags_path) + except Exception as e: + logger.warning(f"Failed to read miss data for {prefix}: {e}") + continue + if 'keywords' not in tags_df.columns or 'question_id' not in tags_df.columns: + logger.warning(f"{tags_path.name}: missing keywords/question_id; skipping.") + continue + + kw_by_qid = {} + for _, row in tags_df.iterrows(): + qid = row.get('question_id') + if pd.isna(qid): + continue + kw_by_qid[int(qid)] = str(row.get('keywords') or '') + + n_missed_per_tag = Counter() + for _, row in subs_df.iterrows(): + pts = row.get('points') + pp = row.get('points_possible') + qid = row.get('question_id') + if pd.isna(pts) or pd.isna(pp) or pd.isna(qid): + continue + if float(pts) >= float(pp): + continue + kw = kw_by_qid.get(int(qid), '') + for tag in _splitKeywords(kw): + n_missed_per_tag[tag] += 1 + + out.append({ + 'quiz_id': quiz_id, + 'quiz_name': quiz_name, + 'total_attempts': int(subs_df['id'].nunique()) if 'id' in subs_df.columns else 0, + 'n_missed_per_tag': n_missed_per_tag, + }) + return out + + +def _splitKeywords(raw): + """Split a comma-separated keywords cell into a clean lowercase tag list.""" + if not raw or pd.isna(raw): + return [] + out = [] + seen = set() + for piece in str(raw).split(','): + t = piece.strip().lower() + if t and t not in seen: + seen.add(t) + out.append(t) + return out + + +def _loadFollowupAssessments(data_path, since_date): + """Load follow-up assessments per quiz, filtered to ``assessed_at >= since_date``. + + Returns a list of ``{quiz_id, quiz_name, rows_by_question, n_dropped_nat}`` + where ``rows_by_question`` is ``{question_id: list[dict]}`` containing only + rows whose ``result`` is ``fail`` or whose ``confidence`` is ``borderline`` + — i.e. the rows where students still need help. Pass + high-confidence rows + are not interesting for the digest's "where the cohort is struggling" framing. + """ + if not os.path.isdir(data_path): + return [] + assessments_files = sorted( + f for f in os.listdir(data_path) if f.endswith('_followup_assessments.csv') + ) + cutoff = pd.Timestamp(since_date.year, since_date.month, since_date.day, tz='UTC') + out = [] + for fname in assessments_files: + prefix = fname[:-len('followup_assessments.csv')] + quiz_name, quiz_id = _quizMetaFromPrefix(prefix) + path = os.path.join(data_path, fname) + try: + df = pd.read_csv(path) + except Exception as e: + logger.warning(f"Failed to read {fname}: {e}") + continue + if 'assessed_at' not in df.columns: + continue + parsed = pd.to_datetime(df['assessed_at'], errors='coerce', utc=True, format='ISO8601') + n_dropped = int(parsed.isna().sum()) + df = df.assign(_assessed_dt=parsed) + df = df[df['_assessed_dt'].notna() & (df['_assessed_dt'] >= cutoff)] + if df.empty: + continue + + is_fail = df['result'].astype(str).str.lower() == 'fail' + is_borderline = df.get('confidence', pd.Series(dtype=str)).astype(str).str.lower() == 'borderline' + focus = df[is_fail | is_borderline] + if focus.empty: + continue + + rows_by_question = {} + for _, row in focus.iterrows(): + qid = row.get('question_id') + if pd.isna(qid): + continue + rows_by_question.setdefault(int(qid), []).append(row.to_dict()) + + out.append({ + 'quiz_id': quiz_id, + 'quiz_name': quiz_name, + 'rows_by_question': rows_by_question, + 'n_dropped_nat': n_dropped, + }) + return out + + +def _loadRecordingsInWindow(data_path, since_date): + """Discover media-recording CSVs in window, deduped to the newest per assignment id. + + Each entry attaches the most recent ``*_questions_w_tags_*.csv`` from the same + course directory (the tag vocabulary used by ``analyze-media-recordings``). + Entries with no tags CSV available are still returned but with + ``tags_path=None``; the analyzer then falls back to per-recording theme + extraction without tag classification. + """ + rec_files = find_csvs_in_window(data_path, '_recordings_', since_date) + rec_files = [f for f in rec_files if re.match(r'^assignment\d+_recordings_', f.name)] + by_assignment = {} + for f in rec_files: + m = re.match(r'^assignment(\d+)_recordings_(\d{8})\.csv$', f.name) + if not m: + continue + # Latest wins (find_csvs_in_window sorted ascending → last write of the day). + by_assignment[m.group(1)] = f + + if not by_assignment: + return [] + + tags_path = find_latest_csv(data_path, '_questions_w_tags_') + out = [] + for assignment_id, rec_path in by_assignment.items(): + try: + rec_df = pd.read_csv(rec_path) + except Exception as e: + logger.warning(f"Failed to read {rec_path.name}: {e}") + continue + out.append({ + 'assignment_id': assignment_id, + 'recordings_df': rec_df, + 'tags_path': tags_path, + }) + return out + + +# --------------------------------------------------------------------------- +# Phase B — Per-quiz follow-up theme summarization (Gemma 4) +# --------------------------------------------------------------------------- + + +def _summarizeCriteriaEvaluations(criteria_json): + """Render the criteria_evaluations JSON blob as a compact bullet block, or empty.""" + if not criteria_json or pd.isna(criteria_json): + return '' + try: + data = json.loads(criteria_json) + except (ValueError, TypeError): + return '' + lines = [] + for crit in data.get('pass_criteria_evaluations', []) or []: + status = crit.get('status', '') + text = crit.get('criterion', '') + if text: + lines.append(f" - [{status}] {text}") + for err in data.get('fatal_errors_evaluations', []) or []: + status = err.get('status', '') + text = err.get('error', '') + if status == 'present' and text: + lines.append(f" - [FATAL] {text}") + return "\n".join(lines) + + +def _buildFollowupThemePrompt(quiz_name, question_id, focus_rows, mode): + """Build the user-side prompt for cohort-level follow-up theme summarization. + + ``focus_rows`` is the combined fail + borderline list (newest first); the + builder caps it at ``_MAX_FOLLOWUP_ROWS_PER_PROMPT`` to keep Gemma's context + bounded on long windows. + """ + capped = list(focus_rows)[:_MAX_FOLLOWUP_ROWS_PER_PROMPT] + parts = [ + f"Quiz: {quiz_name}", + f"Question id: {question_id}", + f"Mode: {mode}", + f"Number of struggling responses: {len(capped)}", + "", + "Responses (newest first):", + ] + for i, row in enumerate(capped, start=1): + result = row.get('result', '') + confidence = row.get('confidence', '') + parts.append(f"\nStudent {i} (result={result}, confidence={confidence}):") + if mode == 'explain': + transcript = str(row.get('transcript') or '').strip() + if transcript: + parts.append(f" Transcript: {transcript}") + feedback = str(row.get('feedback') or '').strip() + if feedback: + parts.append(f" Grader feedback: {feedback}") + else: + feedback = str(row.get('feedback') or '').strip() + if feedback: + parts.append(f" Grader feedback: {feedback}") + crit_block = _summarizeCriteriaEvaluations(row.get('criteria_evaluations')) + if crit_block: + parts.append(" Criteria:") + parts.append(crit_block) + parts.append("") + parts.append("Recurring concept-level struggles across these students (Markdown bullets):") + return "\n".join(parts) + + +def _summarizeFollowupThemes(quiz_blocks, client, model): + """Run one Gemma 4 call per quiz/question and return ``{(quiz_id, question_id): bullets}``. + + ``quiz_blocks`` is the output of ``_loadFollowupAssessments``. The model and + host are logged explicitly so the operator can verify locality. + """ + logger.info(f"Follow-up theme summarization: model={model} host={os.environ.get('OLLAMA_HOST', 'http://localhost:11434')}") + out = {} + for entry in quiz_blocks: + for question_id, rows in entry['rows_by_question'].items(): + mode = (rows[0].get('question_mode') or 'explain').lower() + prompt = _buildFollowupThemePrompt( + entry['quiz_name'], question_id, rows, mode, + ) + try: + resp = _chat_with_retry( + client, + model=model, + messages=[ + {"role": "system", "content": _FOLLOWUP_THEMES_SYSTEM_PROMPT}, + {"role": "user", "content": prompt}, + ], + options={"temperature": 0.3}, + ) + bullets = resp["message"]["content"].strip() + except Exception as e: + logger.warning(f"Follow-up theme summarization failed for quiz {entry['quiz_id']} q{question_id}: {e}") + bullets = f"_(theme extraction failed: {e})_" + out[(entry['quiz_id'], int(question_id))] = bullets + return out + + +# --------------------------------------------------------------------------- +# Phase C — Per-recording-assignment analysis (Gemma 4 — direct reuse) +# --------------------------------------------------------------------------- + + +def _analyzeOneRecordingsBatch(rec_entry, client, model): + """Run ``analyze_recording_tags`` and ``extract_recording_themes`` for one recordings CSV.""" + rec_df = rec_entry['recordings_df'].copy() + rec_df['transcript'] = rec_df.get('transcript', pd.Series(dtype=str)).fillna('').astype(str) + non_empty = rec_df[rec_df['transcript'].str.strip() != ''] + if non_empty.empty: + return { + 'assignment_id': rec_entry['assignment_id'], + 'transcripts_with_names': [], + 'tags': [], + 'tag_to_indices': {}, + 'themes_md': '_(no non-empty transcripts)_', + } + + transcripts_with_names = [ + (str(row.get('student_name') or f"id={row.get('student_id')}"), row['transcript']) + for _, row in non_empty.iterrows() + ] + transcripts_only = [t for _, t in transcripts_with_names] + + tags = [] + if rec_entry.get('tags_path') is not None: + try: + tags_df = pd.read_csv(rec_entry['tags_path']) + if 'keywords' in tags_df.columns: + tags = _collectUniqueTags(tags_df['keywords']) + except Exception as e: + logger.warning(f"Failed to load tags for assignment {rec_entry['assignment_id']}: {e}") + + tag_to_indices = analyze_recording_tags(transcripts_only, tags, client, model) if tags else {} + themes_md = extract_recording_themes(transcripts_only, tags, client, model) + + return { + 'assignment_id': rec_entry['assignment_id'], + 'transcripts_with_names': transcripts_with_names, + 'tags': tags, + 'tag_to_indices': tag_to_indices, + 'themes_md': themes_md, + } + + +# --------------------------------------------------------------------------- +# Phase D — Discussion-question synthesis +# --------------------------------------------------------------------------- + + +def _buildDigestPriorities(quiz_misses, followup_themes, recording_results, top_n=_TOP_PRIORITIES): + """Rank cohort gaps across all three signal sources, returning the top ``top_n``. + + Each priority is ``{tag, miss_count, sources, evidence_snippets}`` where + ``sources`` is the subset of ``{"quiz", "followup", "recording"}`` that + surfaced the tag and ``evidence_snippets`` is a small list of theme bullet + strings used in the local discussion-question prompt (not in the cloud one). + """ + counts = Counter() + sources_by_tag = {} + snippets_by_tag = {} + + for entry in quiz_misses: + for tag, n in entry['n_missed_per_tag'].items(): + counts[tag] += n + sources_by_tag.setdefault(tag, set()).add('quiz') + + # Follow-up themes don't carry a clean per-tag count; we credit each unique + # quiz that surfaced a theme so a struggle that shows up across multiple + # quizzes ranks higher than one isolated to a single question. Snippets are + # stored verbatim only for the local prompt. + for (quiz_id, question_id), bullets in followup_themes.items(): + if not bullets or bullets.startswith('_('): + continue + # Without a tag attribution, we route the snippet under a synthetic tag + # named after the quiz so it's still rankable; the snippet text is what + # actually feeds the local prompt. + synthetic = f"quiz {quiz_id}: q{question_id} struggles" + counts[synthetic] += 1 + sources_by_tag.setdefault(synthetic, set()).add('followup') + snippets_by_tag.setdefault(synthetic, []).append(bullets) + + for entry in recording_results: + for tag, indices in (entry.get('tag_to_indices') or {}).items(): + if not indices: + continue + counts[tag] += len(indices) + sources_by_tag.setdefault(tag, set()).add('recording') + themes_md = entry.get('themes_md') or '' + if themes_md and not themes_md.startswith('_('): + synthetic = f"recording themes (assignment {entry['assignment_id']})" + counts[synthetic] += 1 + sources_by_tag.setdefault(synthetic, set()).add('recording') + snippets_by_tag.setdefault(synthetic, []).append(themes_md) + + ranked = sorted(counts.items(), key=lambda kv: (-kv[1], kv[0])) + out = [] + for tag, n in ranked[:top_n]: + out.append({ + 'tag': tag, + 'miss_count': int(n), + 'sources': sorted(sources_by_tag.get(tag, set())), + 'evidence_snippets': list(snippets_by_tag.get(tag, [])), + }) + return out + + +def _buildDiscussionPromptLocal(priorities): + """Build the full-fidelity (local-only) discussion-question prompt.""" + parts = ["Top cohort gaps (ranked):"] + for i, p in enumerate(priorities, start=1): + parts.append( + f"\n{i}. {p['tag']} — {p['miss_count']} signal(s) " + f"[sources: {', '.join(p['sources'])}]" + ) + for snippet in p['evidence_snippets']: + parts.append(" Evidence:") + for line in snippet.splitlines(): + parts.append(f" {line}") + parts.append("") + parts.append( + "Draft 2-3 in-class discussion questions targeted at the highest-priority gaps. " + "Return ONLY the bullet list." + ) + return "\n".join(parts) + + +def _buildDiscussionPromptCloud(priorities): + """Build the redacted (cloud-safe) discussion-question prompt. + + Contains only ``tag`` strings + integer ``miss_count`` + integer + ``n_themes`` (the count of evidence snippets, not their content). No + transcripts, no criteria_evaluations, no theme bullet text — those would + be derived from student-submitted content. Unit-tested as a privacy guardrail. + """ + parts = ["Top cohort topics (ranked by recent miss count):"] + for i, p in enumerate(priorities, start=1): + parts.append( + f"{i}. {p['tag']} — {p['miss_count']} miss(es) " + f"({len(p['evidence_snippets'])} related theme cluster(s))" + ) + parts.append("") + parts.append( + "Draft 2-3 in-class discussion questions targeted at the highest-priority topics. " + "Return ONLY the bullet list." + ) + return "\n".join(parts) + + +def _suggestDiscussionQuestions(priorities, *, cloud_questions): + """Call the appropriate model to draft the discussion-question bullets. + + Default (``cloud_questions=False``): local Gemma 4 + full-fidelity prompt. + Opt-in (``cloud_questions=True``): cloud Gemini 3 + redacted prompt. Logs + the chosen model + host + prompt variant so the operator can verify which + path ran. + """ + if not priorities: + logger.info("Discussion-question step skipped: no priorities (all-pass cohort).") + return "_(no significant gaps detected — cohort is on track)_" + + if cloud_questions: + client = _make_client(cloud=True) + model = DEFAULT_TEXT_MODEL + host = OLLAMA_CLOUD_HOST + system_prompt = _DISCUSSION_CLOUD_SYSTEM_PROMPT + user_prompt = _buildDiscussionPromptCloud(priorities) + variant = 'cloud' + else: + client = _make_client(cloud=False) + model = DEFAULT_MODEL + host = os.environ.get('OLLAMA_HOST', 'http://localhost:11434') + system_prompt = _DISCUSSION_LOCAL_SYSTEM_PROMPT + user_prompt = _buildDiscussionPromptLocal(priorities) + variant = 'local' + + logger.info(f"Discussion-question generation: model={model} host={host} prompt_variant={variant}") + try: + resp = _chat_with_retry( + client, + model=model, + messages=[ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt}, + ], + options={"temperature": 0.4}, + ) + return resp["message"]["content"].strip() + except Exception as e: + logger.warning(f"Discussion-question generation failed: {e}") + return f"_(discussion-question generation failed: {e})_" + + +# --------------------------------------------------------------------------- +# Phase E — Render and orchestrate +# --------------------------------------------------------------------------- + + +def _renderQuizPerformanceSection(quiz_misses): + """Render the quiz-performance Markdown table sorted by descending miss count.""" + lines = ["## Quiz performance", ""] + if not quiz_misses: + lines.append("_(no quiz miss data in this window)_") + lines.append("") + return lines + + aggregated = Counter() + for entry in quiz_misses: + for tag, n in entry['n_missed_per_tag'].items(): + aggregated[tag] += n + + if not aggregated: + lines.append("_(no missed questions in this window)_") + lines.append("") + return lines + + lines.append("| Tag | Total misses | Contributing quizzes |") + lines.append("|---|---:|---|") + contrib_by_tag = {} + for entry in quiz_misses: + for tag in entry['n_missed_per_tag']: + contrib_by_tag.setdefault(tag, []).append(entry['quiz_name']) + for tag, n in sorted(aggregated.items(), key=lambda kv: (-kv[1], kv[0])): + contributors = ', '.join(sorted(set(contrib_by_tag.get(tag, [])))) + lines.append(f"| {tag} | {n} | {contributors} |") + lines.append("") + return lines + + +def _renderFollowupSection(followup_themes, followup_blocks): + """Render the follow-up reply themes section, one block per (quiz, question).""" + lines = ["## Follow-up reply themes", ""] + if not followup_themes: + lines.append("_(no follow-up replies in this window)_") + lines.append("") + return lines + + name_by_quiz = {entry['quiz_id']: entry['quiz_name'] for entry in followup_blocks} + for (quiz_id, question_id), bullets in followup_themes.items(): + quiz_name = name_by_quiz.get(quiz_id, quiz_id) + lines.append(f"### {quiz_name} — question {question_id}") + lines.append("") + lines.append(bullets) + lines.append("") + return lines + + +def _renderRecordingSection(recording_results): + """Render the per-assignment media-recording themes + tag-coverage section.""" + lines = ["## Media-recording themes", ""] + if not recording_results: + lines.append("_(no media-recording assignments in this window)_") + lines.append("") + return lines + + for entry in recording_results: + lines.append(f"### Assignment {entry['assignment_id']}") + lines.append("") + ranked = sorted( + (entry.get('tag_to_indices') or {}).items(), + key=lambda kv: (-len(kv[1]), kv[0]), + ) + nonzero = [(t, idxs) for t, idxs in ranked if idxs] + if nonzero: + lines.append("| Tag | Students |") + lines.append("|---|---:|") + for tag, idxs in nonzero: + lines.append(f"| {tag} | {len(idxs)} |") + lines.append("") + lines.append(entry.get('themes_md') or '') + lines.append("") + return lines + + +def _renderDigest(course_label, since_date, today, quiz_misses, followup_themes, + followup_blocks, recording_results, discussion_md, days): + """Assemble the five-section Markdown digest.""" + lines = [] + lines.append(f"# Pre-class digest — {course_label}") + lines.append("") + lines.append( + f"_Generated {today.isoformat()} covering the last {days} day(s) " + f"({since_date.isoformat()} — {today.isoformat()})._" + ) + lines.append("") + + lines.extend(_renderQuizPerformanceSection(quiz_misses)) + lines.extend(_renderFollowupSection(followup_themes, followup_blocks)) + lines.extend(_renderRecordingSection(recording_results)) + + lines.append("## Suggested in-class discussion questions") + lines.append("") + lines.append(discussion_md or "_(no discussion questions generated)_") + lines.append("") + + return "\n".join(lines) + + +def _haveAnySignal(quiz_misses, followup_blocks, recording_results): + """Return True if any of the three signal sources produced data in window.""" + if quiz_misses: + return True + if followup_blocks: + return True + if recording_results: + return True + return False + + +def prepClassDigest(course, days=7, cloud_questions=False): + """Top-level orchestrator: load → analyze → render → write. + + Returns the path of the written digest, or ``None`` when the window had no + signal (in which case nothing is written). + """ + today = datetime.now(timezone.utc).date() + since_date = today - timedelta(days=days) + data_path = course.config.data_path + + print(f"\nBuilding digest for last {days} day(s) ({since_date.isoformat()} → {today.isoformat()})") + + print(" Loading quiz misses...") + quiz_misses = _loadQuizMisses(data_path, since_date) + print(f" {len(quiz_misses)} quiz(zes) with miss data in window") + + print(" Loading follow-up assessments...") + followup_blocks = _loadFollowupAssessments(data_path, since_date) + n_followup_questions = sum(len(b['rows_by_question']) for b in followup_blocks) + print(f" {len(followup_blocks)} quiz(zes), {n_followup_questions} question(s) with struggling rows") + + print(" Loading media-recording transcripts...") + rec_entries = _loadRecordingsInWindow(data_path, since_date) + print(f" {len(rec_entries)} recording assignment(s) in window") + + if not _haveAnySignal(quiz_misses, followup_blocks, rec_entries): + print(f"\nNo quiz/follow-up/recording activity in the last {days} day(s) — nothing to digest.") + return None + + needs_local = bool(followup_blocks) or bool(rec_entries) or not cloud_questions + local_client = None + if needs_local: + local_client = _make_client(cloud=False) + try: + local_client.list() + except Exception as e: + raise RuntimeError( + f"Could not reach Ollama at its configured host " + f"({os.environ.get('OLLAMA_HOST', 'http://localhost:11434')}). " + f"Is the Ollama server running? Original error: {e}" + ) from e + + followup_themes = {} + if followup_blocks: + print(" Summarizing follow-up reply themes (Gemma 4)...") + followup_themes = _summarizeFollowupThemes(followup_blocks, local_client, DEFAULT_MODEL) + + recording_results = [] + if rec_entries: + print(" Analyzing media-recording transcripts (Gemma 4)...") + for entry in rec_entries: + recording_results.append(_analyzeOneRecordingsBatch(entry, local_client, DEFAULT_MODEL)) + + priorities = _buildDigestPriorities(quiz_misses, followup_themes, recording_results) + print(f" Top {len(priorities)} priority gap(s) identified.") + + print(" Drafting in-class discussion questions...") + discussion_md = _suggestDiscussionQuestions(priorities, cloud_questions=cloud_questions) + + course_code = getattr(course.canvas_course, 'course_code', '') or course.canvas_course.name + report = _renderDigest( + course_code, since_date, today, quiz_misses, followup_themes, + followup_blocks, recording_results, discussion_md, days, + ) + out_path = data_path / f"class_digest_{today_str()}.md" + out_path.write_text(report) + print(f"\nSaved digest to {out_path.name}") + logger.info( + f"prepClassDigest: wrote {out_path} " + f"(days={days}, cloud_questions={cloud_questions}, priorities={len(priorities)})" + ) + return out_path diff --git a/canvigator_utils.py b/canvigator_utils.py index cd4308d..302dec4 100644 --- a/canvigator_utils.py +++ b/canvigator_utils.py @@ -95,6 +95,31 @@ def find_latest_csv(data_path, pattern, exclude_substr=None): return Path(data_path) / matches[-1][1] +def find_csvs_in_window(data_path, pattern, since_date, exclude_substr=None): + """Return all CSVs in data_path matching `pattern` whose embedded YYYYMMDD is >= since_date. + + Companion to ``find_latest_csv``. ``since_date`` is a ``datetime.date`` (or + anything with a ``strftime('%Y%m%d')`` method); files are matched on the + same ``_YYYYMMDD.csv`` suffix and the same ``exclude_substr`` filter. Result + is sorted ascending by date so callers can reason about chronological order. + """ + if not os.path.isdir(data_path): + return [] + cutoff = since_date.strftime('%Y%m%d') + matches = [] + for f in os.listdir(data_path): + if exclude_substr and exclude_substr in f: + continue + m = re.search(r'(\d{8})\.csv$', f) + if not m or pattern not in f: + continue + if m.group(1) >= cutoff: + matches.append((m.group(1), f)) + + matches.sort(key=lambda t: t[0]) + return [Path(data_path) / fname for _, fname in matches] + + def selectCSVFromList(directory, keyword, prompt_msg, verbose=False): """ List CSV files in directory matching keyword, prompt user to select one, diff --git a/test_canvigator.py b/test_canvigator.py index 7385eb5..5584fee 100644 --- a/test_canvigator.py +++ b/test_canvigator.py @@ -2956,3 +2956,483 @@ def test_build_quiz_question_prompt_with_prior_drafts(self): assert "SUBSTANTIVELY DIFFERENT" in prompt assert "What is the time complexity of binary search?" in prompt assert "Rejected draft 1" in prompt + + +# --------------------------------------------------------------------------- +# canvigator_digest tests +# --------------------------------------------------------------------------- + + +class TestFindCsvsInWindow: + """Tests for canvigator_utils.find_csvs_in_window.""" + + def _write(self, tmp_path, name): + (tmp_path / name).write_text('x') + + def test_returns_only_files_at_or_after_cutoff(self, tmp_path): + """Files dated >= since_date are returned; older are filtered out.""" + from canvigator_utils import find_csvs_in_window + from datetime import date + self._write(tmp_path, 'foo_20260420.csv') + self._write(tmp_path, 'foo_20260424.csv') + self._write(tmp_path, 'foo_20260501.csv') + result = find_csvs_in_window(tmp_path, 'foo', date(2026, 4, 24)) + names = [p.name for p in result] + assert names == ['foo_20260424.csv', 'foo_20260501.csv'] + + def test_pattern_substring_filter(self, tmp_path): + """Only files whose name contains the pattern substring are returned.""" + from canvigator_utils import find_csvs_in_window + from datetime import date + self._write(tmp_path, 'recordings_20260501.csv') + self._write(tmp_path, 'gradebook_20260501.csv') + result = find_csvs_in_window(tmp_path, 'recordings', date(2026, 1, 1)) + assert [p.name for p in result] == ['recordings_20260501.csv'] + + def test_exclude_substr_skips_matches(self, tmp_path): + """Files containing exclude_substr are dropped even if they match the pattern.""" + from canvigator_utils import find_csvs_in_window + from datetime import date + self._write(tmp_path, 'foo_20260501.csv') + self._write(tmp_path, 'foo_dryrun_20260501.csv') + result = find_csvs_in_window(tmp_path, 'foo', date(2026, 1, 1), exclude_substr='dryrun') + assert [p.name for p in result] == ['foo_20260501.csv'] + + def test_missing_directory_returns_empty(self, tmp_path): + """A non-existent data_path returns an empty list rather than raising.""" + from canvigator_utils import find_csvs_in_window + from datetime import date + result = find_csvs_in_window(tmp_path / 'does_not_exist', 'foo', date(2026, 1, 1)) + assert result == [] + + def test_results_sorted_ascending_by_date(self, tmp_path): + """Output is sorted ascending so callers can reason chronologically.""" + from canvigator_utils import find_csvs_in_window + from datetime import date + self._write(tmp_path, 'foo_20260501.csv') + self._write(tmp_path, 'foo_20260424.csv') + self._write(tmp_path, 'foo_20260427.csv') + result = find_csvs_in_window(tmp_path, 'foo', date(2026, 4, 1)) + assert [p.name for p in result] == ['foo_20260424.csv', 'foo_20260427.csv', 'foo_20260501.csv'] + + +class TestLoadQuizMisses: + """Tests for canvigator_digest._loadQuizMisses.""" + + def _write_subs(self, tmp_path, prefix, date_str, rows): + df = pd.DataFrame(rows) + df.to_csv(tmp_path / f"{prefix}all_subs_by_question_{date_str}.csv", index=False) + + def _write_tags(self, tmp_path, prefix, date_str, rows): + df = pd.DataFrame(rows) + df.to_csv(tmp_path / f"{prefix}questions_w_tags_{date_str}.csv", index=False) + + def test_per_tag_miss_count_basic(self, tmp_path): + """A miss row credits each of its question's keywords once.""" + from canvigator_digest import _loadQuizMisses + from datetime import date + self._write_subs(tmp_path, 'quiz1_111_', '20260501', [ + {'id': 7, 'question_id': 1, 'points': 0, 'points_possible': 1}, + {'id': 7, 'question_id': 2, 'points': 1, 'points_possible': 1}, + {'id': 8, 'question_id': 1, 'points': 0, 'points_possible': 1}, + ]) + self._write_tags(tmp_path, 'quiz1_111_', '20260501', [ + {'question_id': 1, 'keywords': 'recursion, base case'}, + {'question_id': 2, 'keywords': 'sorting'}, + ]) + result = _loadQuizMisses(tmp_path, date(2026, 4, 1)) + assert len(result) == 1 + entry = result[0] + assert entry['quiz_id'] == '111' + assert entry['quiz_name'] == 'quiz1' + assert entry['n_missed_per_tag'] == {'recursion': 2, 'base case': 2} + + def test_partial_credit_counts_as_miss(self, tmp_path): + """`points < points_possible` is the miss rule — partial credit counts.""" + from canvigator_digest import _loadQuizMisses + from datetime import date + self._write_subs(tmp_path, 'quiz1_111_', '20260501', [ + {'id': 7, 'question_id': 1, 'points': 0.5, 'points_possible': 1}, + ]) + self._write_tags(tmp_path, 'quiz1_111_', '20260501', [ + {'question_id': 1, 'keywords': 'recursion'}, + ]) + result = _loadQuizMisses(tmp_path, date(2026, 4, 1)) + assert result[0]['n_missed_per_tag']['recursion'] == 1 + + def test_quiz_without_tags_csv_is_skipped(self, tmp_path): + """A quiz with subs in window but no questions_w_tags CSV is skipped with a warning.""" + from canvigator_digest import _loadQuizMisses + from datetime import date + self._write_subs(tmp_path, 'quiz1_111_', '20260501', [ + {'id': 7, 'question_id': 1, 'points': 0, 'points_possible': 1}, + ]) + result = _loadQuizMisses(tmp_path, date(2026, 4, 1)) + assert result == [] + + def test_subs_outside_window_excluded(self, tmp_path): + """Subs files dated before since_date are not loaded.""" + from canvigator_digest import _loadQuizMisses + from datetime import date + self._write_subs(tmp_path, 'quiz1_111_', '20260301', [ + {'id': 7, 'question_id': 1, 'points': 0, 'points_possible': 1}, + ]) + self._write_tags(tmp_path, 'quiz1_111_', '20260301', [ + {'question_id': 1, 'keywords': 'recursion'}, + ]) + result = _loadQuizMisses(tmp_path, date(2026, 4, 1)) + assert result == [] + + +class TestLoadFollowupAssessments: + """Tests for canvigator_digest._loadFollowupAssessments.""" + + def _write_assessments(self, tmp_path, prefix, rows): + df = pd.DataFrame(rows) + df.to_csv(tmp_path / f"{prefix}followup_assessments.csv", index=False) + + def test_filters_to_window_and_struggling_rows(self, tmp_path): + """Only fail or borderline rows whose assessed_at >= since_date are kept.""" + from canvigator_digest import _loadFollowupAssessments + from datetime import date + self._write_assessments(tmp_path, 'quiz1_111_', [ + {'student_id': 1, 'question_id': 5, 'question_mode': 'explain', + 'result': 'fail', 'confidence': 'high', + 'transcript': 'I missed it', 'criteria_evaluations': '', + 'feedback': 'wrong on X', 'assessed_at': '2026-05-01T12:00:00+00:00'}, + {'student_id': 2, 'question_id': 5, 'question_mode': 'explain', + 'result': 'pass', 'confidence': 'high', + 'transcript': 'good answer', 'criteria_evaluations': '', + 'feedback': 'great', 'assessed_at': '2026-05-01T12:00:00+00:00'}, + {'student_id': 3, 'question_id': 5, 'question_mode': 'explain', + 'result': 'pass', 'confidence': 'borderline', + 'transcript': 'shaky', 'criteria_evaluations': '', + 'feedback': 'almost', 'assessed_at': '2026-05-01T12:00:00+00:00'}, + {'student_id': 4, 'question_id': 5, 'question_mode': 'explain', + 'result': 'fail', 'confidence': 'high', + 'transcript': 'old miss', 'criteria_evaluations': '', + 'feedback': 'wrong', 'assessed_at': '2026-03-01T12:00:00+00:00'}, + ]) + result = _loadFollowupAssessments(tmp_path, date(2026, 4, 1)) + assert len(result) == 1 + entry = result[0] + assert entry['quiz_id'] == '111' + assert entry['quiz_name'] == 'quiz1' + # Pass+high is dropped; fail+high and pass+borderline are kept. + kept_ids = {row['student_id'] for row in entry['rows_by_question'][5]} + assert kept_ids == {1, 3} + + def test_drops_nat_rows_and_records_count(self, tmp_path): + """Rows whose assessed_at is unparseable are dropped and counted via n_dropped_nat.""" + from canvigator_digest import _loadFollowupAssessments + from datetime import date + self._write_assessments(tmp_path, 'quiz1_111_', [ + {'student_id': 1, 'question_id': 5, 'question_mode': 'explain', + 'result': 'fail', 'confidence': 'high', 'transcript': '', + 'criteria_evaluations': '', 'feedback': '', 'assessed_at': 'not-a-date'}, + {'student_id': 2, 'question_id': 5, 'question_mode': 'explain', + 'result': 'fail', 'confidence': 'high', 'transcript': '', + 'criteria_evaluations': '', 'feedback': '', 'assessed_at': '2026-05-01T12:00:00+00:00'}, + ]) + result = _loadFollowupAssessments(tmp_path, date(2026, 4, 1)) + assert len(result) == 1 + assert result[0]['n_dropped_nat'] == 1 + + +class TestBuildFollowupThemePrompt: + """Tests for canvigator_digest._buildFollowupThemePrompt.""" + + def test_explain_mode_includes_transcript(self): + """Explain-mode prompts include the transcript and the grader feedback.""" + from canvigator_digest import _buildFollowupThemePrompt + rows = [ + {'result': 'fail', 'confidence': 'high', + 'transcript': 'I confused base case with recursive case', + 'feedback': 'missed key idea', 'criteria_evaluations': ''}, + ] + prompt = _buildFollowupThemePrompt('quiz1', 5, rows, 'explain') + assert 'Mode: explain' in prompt + assert 'I confused base case' in prompt + assert 'missed key idea' in prompt + + def test_draw_mode_omits_transcript_includes_feedback(self): + """Draw-mode prompts skip transcript (always empty for draw) but keep feedback.""" + from canvigator_digest import _buildFollowupThemePrompt + rows = [ + {'result': 'fail', 'confidence': 'high', + 'transcript': '', 'feedback': 'BST drawn without root', + 'criteria_evaluations': ''}, + ] + prompt = _buildFollowupThemePrompt('quiz1', 5, rows, 'draw') + assert 'Mode: draw' in prompt + assert 'BST drawn without root' in prompt + + def test_caps_rows_at_max(self): + """Row count is capped at _MAX_FOLLOWUP_ROWS_PER_PROMPT to bound context size.""" + from canvigator_digest import _buildFollowupThemePrompt, _MAX_FOLLOWUP_ROWS_PER_PROMPT + rows = [ + {'result': 'fail', 'confidence': 'high', + 'transcript': f'transcript {i}', 'feedback': '', 'criteria_evaluations': ''} + for i in range(50) + ] + prompt = _buildFollowupThemePrompt('quiz1', 5, rows, 'explain') + assert f"Number of struggling responses: {_MAX_FOLLOWUP_ROWS_PER_PROMPT}" in prompt + # Student 13+ should not appear (we keep only the first 12). + assert "Student 13" not in prompt + + def test_criteria_evaluations_rendered_as_bullets(self): + """JSON criteria_evaluations are summarized as compact bullets in the prompt.""" + from canvigator_digest import _buildFollowupThemePrompt + import json as _json + crit_blob = _json.dumps({ + 'pass_criteria_evaluations': [ + {'criterion': 'mentions base case', 'status': 'missing'}, + {'criterion': 'explains stack frames', 'status': 'partial'}, + ], + 'fatal_errors_evaluations': [ + {'error': 'asserts recursion is just a loop', 'status': 'present'}, + ], + }) + rows = [{'result': 'fail', 'confidence': 'high', 'transcript': '', + 'feedback': '', 'criteria_evaluations': crit_blob}] + prompt = _buildFollowupThemePrompt('quiz1', 5, rows, 'explain') + assert 'mentions base case' in prompt + assert 'asserts recursion is just a loop' in prompt + assert '[FATAL]' in prompt + + +class TestBuildDigestPriorities: + """Tests for canvigator_digest._buildDigestPriorities.""" + + def test_ranks_by_descending_miss_count(self): + """Higher miss counts come first; tied tags break alphabetically.""" + from canvigator_digest import _buildDigestPriorities + quiz_misses = [{ + 'quiz_id': '111', 'quiz_name': 'quiz1', 'total_attempts': 10, + 'n_missed_per_tag': {'recursion': 5, 'sorting': 2, 'arrays': 5}, + }] + result = _buildDigestPriorities(quiz_misses, {}, [], top_n=10) + tags = [p['tag'] for p in result] + assert tags[:3] == ['arrays', 'recursion', 'sorting'] + + def test_three_sources_attributed(self): + """A tag that appears via quiz misses + recordings credits both sources.""" + from canvigator_digest import _buildDigestPriorities + quiz_misses = [{ + 'quiz_id': '111', 'quiz_name': 'quiz1', 'total_attempts': 10, + 'n_missed_per_tag': {'recursion': 3}, + }] + recording_results = [{ + 'assignment_id': '999', + 'transcripts_with_names': [], + 'tags': ['recursion'], + 'tag_to_indices': {'recursion': [1, 2]}, + 'themes_md': '', + }] + result = _buildDigestPriorities(quiz_misses, {}, recording_results, top_n=10) + recursion_entry = next(p for p in result if p['tag'] == 'recursion') + assert set(recursion_entry['sources']) == {'quiz', 'recording'} + # 3 quiz misses + 2 recording mentions = 5 + assert recursion_entry['miss_count'] == 5 + + def test_followup_themes_become_synthetic_tags(self): + """Follow-up themes get a synthetic tag attribution since they aren't tag-keyed.""" + from canvigator_digest import _buildDigestPriorities + followup_themes = {('111', 5): '- **Base case confusion**: students forget terminator'} + result = _buildDigestPriorities([], followup_themes, [], top_n=10) + assert len(result) == 1 + entry = result[0] + assert 'quiz 111' in entry['tag'] + assert entry['sources'] == ['followup'] + assert entry['evidence_snippets'] == ['- **Base case confusion**: students forget terminator'] + + def test_top_n_caps_results(self): + """top_n trims the priorities list to that many entries.""" + from canvigator_digest import _buildDigestPriorities + quiz_misses = [{ + 'quiz_id': '111', 'quiz_name': 'quiz1', 'total_attempts': 10, + 'n_missed_per_tag': {f't{i}': i for i in range(20)}, + }] + result = _buildDigestPriorities(quiz_misses, {}, [], top_n=3) + assert len(result) == 3 + + +class TestBuildDiscussionPromptCloud: + """Privacy guardrail: cloud prompt must not leak transcripts or theme content.""" + + def test_cloud_prompt_excludes_evidence_snippets(self): + """The redacted prompt embeds tag names + counts but no snippet text.""" + from canvigator_digest import _buildDiscussionPromptCloud + priorities = [{ + 'tag': 'recursion', + 'miss_count': 7, + 'sources': ['quiz', 'followup'], + 'evidence_snippets': [ + 'STUDENT_TRANSCRIPT: I confused base case with recursive call', + 'CRITERIA: missing(mentions base case)', + ], + }] + prompt = _buildDiscussionPromptCloud(priorities) + assert 'recursion' in prompt + assert '7' in prompt + # Privacy: no snippet text leaks through the redacted prompt. + assert 'STUDENT_TRANSCRIPT' not in prompt + assert 'CRITERIA' not in prompt + assert 'base case' not in prompt + + def test_cloud_prompt_includes_theme_count_only(self): + """The cloud prompt mentions how many evidence snippets exist, not their content.""" + from canvigator_digest import _buildDiscussionPromptCloud + priorities = [{ + 'tag': 'sorting', + 'miss_count': 4, + 'sources': ['quiz'], + 'evidence_snippets': ['snip A', 'snip B'], + }] + prompt = _buildDiscussionPromptCloud(priorities) + assert '2 related theme cluster(s)' in prompt + assert 'snip A' not in prompt + assert 'snip B' not in prompt + + def test_local_prompt_includes_evidence(self): + """By contrast, the local prompt does include the evidence snippets verbatim.""" + from canvigator_digest import _buildDiscussionPromptLocal + priorities = [{ + 'tag': 'recursion', + 'miss_count': 7, + 'sources': ['quiz'], + 'evidence_snippets': ['STUDENT_TRANSCRIPT: I confused base case'], + }] + prompt = _buildDiscussionPromptLocal(priorities) + assert 'STUDENT_TRANSCRIPT' in prompt + assert 'base case' in prompt + + +class TestSuggestDiscussionQuestions: + """Tests for canvigator_digest._suggestDiscussionQuestions.""" + + def test_empty_priorities_skips_llm_call(self): + """An empty priorities list returns the no-gaps string and never builds a client.""" + from canvigator_digest import _suggestDiscussionQuestions + with patch('canvigator_digest._make_client') as mk: + result = _suggestDiscussionQuestions([], cloud_questions=False) + mk.assert_not_called() + assert 'no significant gaps' in result + + def test_local_path_uses_local_client(self): + """cloud_questions=False routes through _make_client(cloud=False).""" + from canvigator_digest import _suggestDiscussionQuestions + priorities = [{'tag': 'recursion', 'miss_count': 3, 'sources': ['quiz'], 'evidence_snippets': []}] + fake_client = MagicMock() + with patch('canvigator_digest._make_client', return_value=fake_client) as mk, \ + patch('canvigator_digest._chat_with_retry', + return_value={'message': {'content': '- discuss recursion'}}): + result = _suggestDiscussionQuestions(priorities, cloud_questions=False) + mk.assert_called_once_with(cloud=False) + assert '- discuss recursion' in result + + def test_cloud_path_uses_cloud_client(self): + """cloud_questions=True routes through _make_client(cloud=True).""" + from canvigator_digest import _suggestDiscussionQuestions + priorities = [{'tag': 'recursion', 'miss_count': 3, 'sources': ['quiz'], 'evidence_snippets': ['secret']}] + fake_client = MagicMock() + captured = {} + + def _capture(client, **kwargs): + captured['messages'] = kwargs.get('messages') + return {'message': {'content': '- discuss recursion'}} + + with patch('canvigator_digest._make_client', return_value=fake_client) as mk, \ + patch('canvigator_digest._chat_with_retry', side_effect=_capture): + _suggestDiscussionQuestions(priorities, cloud_questions=True) + mk.assert_called_once_with(cloud=True) + # Privacy guardrail: the secret evidence snippet must not appear in the user message. + user_msg = captured['messages'][1]['content'] + assert 'secret' not in user_msg + + +class TestRenderDigest: + """Tests for canvigator_digest._renderDigest.""" + + def test_five_sections_present_in_order(self): + """The rendered Markdown contains all five sections in the documented order.""" + from canvigator_digest import _renderDigest + from datetime import date + out = _renderDigest( + course_label='CSI-3300', + since_date=date(2026, 4, 24), + today=date(2026, 5, 1), + quiz_misses=[], + followup_themes={}, + followup_blocks=[], + recording_results=[], + discussion_md='- example question', + days=7, + ) + # All four data-source sections plus the discussion section are present. + for header in ['## Quiz performance', '## Follow-up reply themes', + '## Media-recording themes', '## Suggested in-class discussion questions']: + assert header in out + # And they appear in order. + idx_quiz = out.index('## Quiz performance') + idx_followup = out.index('## Follow-up reply themes') + idx_recording = out.index('## Media-recording themes') + idx_discussion = out.index('## Suggested in-class discussion questions') + assert idx_quiz < idx_followup < idx_recording < idx_discussion + + def test_header_surfaces_window_dates(self): + """The header line includes the window range and day count.""" + from canvigator_digest import _renderDigest + from datetime import date + out = _renderDigest( + course_label='CSI-3300', + since_date=date(2026, 4, 24), + today=date(2026, 5, 1), + quiz_misses=[], + followup_themes={}, + followup_blocks=[], + recording_results=[], + discussion_md='', + days=7, + ) + assert '2026-04-24' in out + assert '2026-05-01' in out + assert '7 day' in out + + def test_quiz_performance_table_renders_with_data(self): + """When quiz misses exist, the section renders a Markdown table sorted by descending miss count.""" + from canvigator_digest import _renderDigest + from datetime import date + from collections import Counter + quiz_misses = [{ + 'quiz_id': '111', 'quiz_name': 'quiz1', 'total_attempts': 10, + 'n_missed_per_tag': Counter({'recursion': 5, 'sorting': 1}), + }] + out = _renderDigest( + 'CSI-3300', date(2026, 4, 24), date(2026, 5, 1), + quiz_misses, {}, [], [], 'discussion', 7, + ) + assert '| Tag | Total misses | Contributing quizzes |' in out + # recursion (5) listed before sorting (1) + idx_r = out.index('| recursion |') + idx_s = out.index('| sorting |') + assert idx_r < idx_s + + +class TestPrepClassDigestEmptyWindow: + """Tests for the orchestrator's empty-window guardrail.""" + + def test_empty_window_writes_no_file(self, tmp_path, capsys): + """With no CSVs in the data directory, the orchestrator returns None and writes nothing.""" + from canvigator_digest import prepClassDigest + course = SimpleNamespace( + config=SimpleNamespace(data_path=tmp_path), + canvas_course=SimpleNamespace(course_code='CSI-3300', name='Test'), + ) + result = prepClassDigest(course, days=7, cloud_questions=False) + assert result is None + # No output file created. + assert list(tmp_path.glob('class_digest_*.md')) == [] + # User-facing message confirms the no-op. + captured = capsys.readouterr() + assert 'nothing to digest' in captured.out From 338155243bad8b0d6bc8e254b426d9e8345295e5 Mon Sep 17 00:00:00 2001 From: Steve Geinitz Date: Fri, 8 May 2026 12:49:51 -0600 Subject: [PATCH 2/3] Document --all support for send-quiz-reminder in --help and README The -a/--all flag has worked with send-quiz-reminder for a while but the task-level help description and the README's send-quiz-reminder section didn't mention it, so users had to read the global flag description (or CLAUDE.md) to discover it. Co-Authored-By: Claude Opus 4.7 --- README.md | 10 ++++++++++ canvigator.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 11da196..9d72821 100644 --- a/README.md +++ b/README.md @@ -714,6 +714,16 @@ every send is reviewed before going to Canvas. Use `--dry-run` to preview all messages (recipient, subject, body, and reason) without sending anything to Canvas and without the interactive prompt. +Pass `--all` (or `-a`) to skip the interactive quiz picker and instead +iterate every published quiz with a future `due_at`. Each student's state is +aggregated across those quizzes and the task sends **one consolidated Canvas +message per student** listing every eligible quiz they still have work to do +on (no attempt, imperfect score, page-blur, or perfect-clean), with a single +course-level manifest `course_reminder_sent[_dryrun]_YYYYMMDD.csv` instead of +the per-quiz manifest. Quizzes without a `*_questions_w_tags_*.csv` are +skipped with a warning. Combine with `--dry-run` to preview the consolidated +messages without sending. + --- #### Media-recording check-in workflow diff --git a/canvigator.py b/canvigator.py index fb2f6ea..5497576 100755 --- a/canvigator.py +++ b/canvigator.py @@ -33,7 +33,7 @@ ('delete-old-conversations', 'Delete Canvas conversations older than N months (account-wide; default 6; override with --months/-m)'), ('get-gradebook', 'Export course gradebook'), ('get-roster', 'Export the full course roster (name, id, sis_id, enrollment_type)'), - ('send-quiz-reminder', 'Send quiz reminder messages to students'), + ('send-quiz-reminder', 'Send quiz reminder messages to students (use --all for one consolidated message across every published, future-due quiz)'), ('prep-class-digest', 'Synthesize a 1-page Markdown brief on cohort gaps from the last N days (default 7; override with --days/-n)'), ]), ] From f417cda16446f099afa0ae2c6a567a76653b7300 Mon Sep 17 00:00:00 2001 From: Steve Geinitz Date: Fri, 8 May 2026 16:47:21 -0600 Subject: [PATCH 3/3] Add per-task --help to canvigator.py CLI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `canvigator.py --help` (and `-h`) now renders a focused per-task cheat-sheet: description, prerequisites, file inputs/outputs, applicable flags, example invocations, and a Workflow line listing related tasks to run before/after. Addresses the growing pain of keeping 22 tasks, their dependencies, and their I/O straight. Per-task data lives in a new `canvigator_help.py` module so the CLI file doesn't grow with every task. `FLAG_DESCRIPTIONS` is the single source for flag wording — both the global and per-task renderers share it. The existing global `--help` continues to work; a new footer points users at the per-task form. Tests in `TestPerTaskHelp` cover integrity (every task has an entry, required fields non-empty, flag references resolve, run_before/after targets exist), rendering, and end-to-end CLI dispatch via subprocess. Co-Authored-By: Claude Opus 4.7 --- canvigator.py | 44 +-- canvigator_help.py | 890 +++++++++++++++++++++++++++++++++++++++++++++ test_canvigator.py | 151 ++++++++ 3 files changed, 1061 insertions(+), 24 deletions(-) create mode 100644 canvigator_help.py diff --git a/canvigator.py b/canvigator.py index 5497576..927eab0 100755 --- a/canvigator.py +++ b/canvigator.py @@ -43,28 +43,8 @@ def print_help(): """Print usage information with grouped task descriptions.""" - print("Usage: canvigator.py [OPTIONS] \n") - print("Options (short and long forms are interchangeable):") - print(" -d, --dry-run Preview changes without modifying Canvas (applies to bonus, reminder, follow-up,") - print(" feedback, delete-old-conversations, get-media-recordings — CSV is still written)") - print(" -t, --tag Use a cloud LLM via Ollama to tag questions (get-quiz-questions only)") - print(" -a, --all Run across every quiz in the course (get-quiz-questions, get-quiz-submission-events, send-quiz-reminder)") - print(" -c, --crn Select course by CRN (last 5 digits of course code)") - print(" -m, --months Age threshold in months for delete-old-conversations (default: 6)") - print(" -w, --reply-window-days Days to accept replies after follow-up sent (default: 5, assess-replies only)") - print(" -g, --auto-grade Skip per-student review prompt and auto-award points_possible (get-media-recordings only)") - print(" -n, --days Lookback window in days for prep-class-digest (default: 7)") - print(" -q, --cloud-questions Route the discussion-question step to cloud Gemini 3 with a redacted prompt (prep-class-digest only)") - max_name = max(len(t) for t in tasks) - for header, items in task_groups: - print(f"\n{header}") - for name, desc in items: - print(f" {name:<{max_name}} {desc}") - print("\nNotes:") - print(" generate-follow-up-questions uses a cloud LLM (via Ollama) in two steps:") - print(" 1. Classifies each question as 'explain' (oral) or 'draw' (visual)") - print(" 2. Generates a mode-appropriate open-ended question for instructor review") - print(" Output CSV includes a question_mode column so the instructor can override choices.") + import canvigator_help as ch + ch.print_global_help(task_groups) def _run_assignment_task(task, course, canvas, canv_config, dry_run, auto_grade_flag): @@ -152,9 +132,18 @@ def _run_quiz_task(task, quiz, dry_run, tag, reply_window_days): '-g': '--auto-grade', '-n': '--days', '-q': '--cloud-questions', + '-h': '--help', } args = [_SHORT_TO_LONG.get(a, a) for a in sys.argv[1:]] +# --help is intentionally stripped before the rest of the parser so that +# `canvigator.py --help` and `canvigator.py --help ` behave the +# same. The bare `canvigator.py --help` case falls through to the empty-args +# branch below, which prints global help. +help_requested = '--help' in args +while '--help' in args: + args.remove('--help') + dry_run = '--dry-run' in args if dry_run: args.remove('--dry-run') @@ -238,11 +227,13 @@ def _run_quiz_task(task, quiz, dry_run, tag, reply_window_days): if len(args) < 1: print_help() - sys.exit(1) + # Distinguish bare `canvigator.py --help` (success) from `canvigator.py` + # with no args at all (usage error). + sys.exit(0 if help_requested else 1) task = args[0] -if task in ("help", "--help"): +if task == "help": print_help() sys.exit(0) @@ -250,6 +241,11 @@ def _run_quiz_task(task, quiz, dry_run, tag, reply_window_days): print(f"Invalid task: '{task}'. Run with --help to see available tasks.") sys.exit(1) +if help_requested: + import canvigator_help as ch + ch.print_task_help(task) + sys.exit(0) + # export-anon-data works with local files only — no Canvas API needed if task == 'export-anon-data': import logging diff --git a/canvigator_help.py b/canvigator_help.py new file mode 100644 index 0000000..58db45c --- /dev/null +++ b/canvigator_help.py @@ -0,0 +1,890 @@ +"""Per-task help rendering for canvigator.py. + +Two public functions: + +* ``print_task_help(task_name)`` — used by ``canvigator.py --help``. +* ``print_global_help(task_groups)`` — used by ``canvigator.py --help``. + +``FLAG_DESCRIPTIONS`` is the single source of truth for the long-form text of +each ``--flag`` so wording can't drift between the global and per-task help. +``TASK_HELP`` holds the structured per-task data; see the schema comment above +the dict. +""" +import textwrap + + +# Flags that apply to every task. Rendered automatically by ``print_task_help`` +# even when not listed in a per-task entry's ``flags`` list. +UNIVERSAL_FLAGS = ('--crn',) + + +# (short, value_label, description). ``value_label`` is None for boolean flags. +FLAG_DESCRIPTIONS = { + '--dry-run': ( + '-d', None, + "Preview changes without modifying Canvas (a *_dryrun_* manifest is still " + "written so you can inspect what would have been sent).", + ), + '--tag': ( + '-t', None, + "Use a cloud LLM via Ollama to add a 'keywords' column with 1-3 short " + "topical tags per question.", + ), + '--all': ( + '-a', None, + "Skip the interactive picker and run across every applicable quiz in the " + "course.", + ), + '--crn': ( + '-c', '', + "Select course by CRN (last 5 digits of the course code); skips the " + "course picker. Useful for cron / automated runs.", + ), + '--months': ( + '-m', '', + "Age threshold in months (default: 6).", + ), + '--reply-window-days': ( + '-w', '', + "Days to accept replies after the follow-up was sent (default: 5).", + ), + '--auto-grade': ( + '-g', None, + "Skip the per-student review prompt and auto-award points_possible to " + "every submitter.", + ), + '--days': ( + '-n', '', + "Lookback window in days (default: 7).", + ), + '--cloud-questions': ( + '-q', None, + "Route the discussion-question step to cloud Gemini 3 with a redacted " + "prompt (tag names + integer counts only — no transcripts or themes).", + ), +} + + +# Per-task help. Schema for each value: +# +# description: str — 2-4 sentences of plain prose. +# prerequisites: list[str] — commands or files that must exist first. +# inputs: list[str] — what the task reads (Canvas + files). +# outputs: list[str] — files written + Canvas side-effects. +# flags: list[str] — non-universal flags that apply (e.g. ['--all']). +# Universal flags (--crn) are rendered automatically. +# examples: list[str] — 1-3 representative invocations. +# run_before: list[str] — tasks typically run earlier in the workflow. +# run_after: list[str] — tasks typically run after this one. +# +TASK_HELP = { + 'get-activity': { + 'description': ( + "Export per-student page-view, participation, and weekly-activity " + "data to a CSV. Course-level, no quiz selection. Useful as a " + "baseline engagement signal for a class." + ), + 'prerequisites': [], + 'inputs': ["Canvas course (selected interactively or via --crn)."], + 'outputs': ["data//course_activity_YYYYMMDD.csv"], + 'flags': [], + 'examples': [ + "python canvigator.py --crn 12345 get-activity", + ], + 'run_before': [], + 'run_after': [], + }, + + 'create-pairs': { + 'description': ( + "Create student pairings from quiz scores using a distance matrix " + "to maximize score diversity within each pair. Reads a present_*.csv " + "to know who is in class today; the algorithm prefers the median " + "distance method by default." + ), + 'prerequisites': [ + "A present_*.csv in data// with columns name, id, present " + "(1 = present, 0 = absent).", + "Quiz must have student attempts on Canvas.", + ], + 'inputs': [ + "Canvas quiz (selected interactively or via --crn).", + "data//present_*.csv", + ], + 'outputs': [ + "data//pairings_based_on_quiz_YYYYMMDD.csv", + ], + 'flags': [], + 'examples': [ + "python canvigator.py --crn 12345 create-pairs", + ], + 'run_before': [], + 'run_after': ['award-bonus', 'award-bonus-partner-only'], + }, + + 'award-bonus': { + 'description': ( + "Award both partner and retake bonus points for a quiz. Sets fudge " + "points on each student's BEST attempt (highest score) and leaves a " + "Canvas submission comment with the breakdown. Partners are detected " + "by score + answer-timing match (union-find handles triples); " + "retakers are detected by repeated qualifying attempts on different " + "days. Default bonus is 15% of quiz points each (30% combined)." + ), + 'prerequisites': [ + "get-quiz-submission-events must have been run for the quiz so the " + "*_all_submissions_*.csv and *_all_subs_and_events_*.csv exist.", + ], + 'inputs': [ + "Canvas quiz (selected interactively or via --crn).", + "data//quiz_all_submissions_*.csv", + "data//quiz_all_subs_and_events_*.csv", + ], + 'outputs': [ + "Canvas: fudge points + submission comments on each bonus-eligible " + "student's best attempt.", + "data//quiz_detected_partners_YYYYMMDD.csv", + "data//quiz_retake_qualified_YYYYMMDD.csv", + "data//quiz_scores_w_bonus[_dryrun]_YYYYMMDD.csv", + ], + 'flags': ['--dry-run'], + 'examples': [ + "python canvigator.py --crn 12345 --dry-run award-bonus", + "python canvigator.py --crn 12345 award-bonus", + ], + 'run_before': ['get-quiz-submission-events'], + 'run_after': [], + }, + + 'award-bonus-partner-only': { + 'description': ( + "Award only the partner bonus (no retake). Same partner detection " + "as award-bonus; sets fudge points on each partnered student's best " + "attempt." + ), + 'prerequisites': [ + "get-quiz-submission-events must have been run for the quiz first.", + ], + 'inputs': [ + "Canvas quiz (selected interactively or via --crn).", + "data//quiz_all_subs_and_events_*.csv", + ], + 'outputs': [ + "Canvas: fudge points + submission comments on partnered students.", + "data//quiz_detected_partners_YYYYMMDD.csv", + "data//quiz_scores_w_bonus[_dryrun]_YYYYMMDD.csv", + ], + 'flags': ['--dry-run'], + 'examples': [ + "python canvigator.py --crn 12345 --dry-run award-bonus-partner-only", + ], + 'run_before': ['get-quiz-submission-events'], + 'run_after': [], + }, + + 'award-bonus-retake-only': { + 'description': ( + "Award only the retake bonus (no partner). Same retake detection as " + "award-bonus; sets fudge points on each qualifying retaker's best " + "attempt." + ), + 'prerequisites': [ + "get-quiz-submission-events must have been run for the quiz first.", + ], + 'inputs': [ + "Canvas quiz (selected interactively or via --crn).", + "data//quiz_all_subs_and_events_*.csv", + ], + 'outputs': [ + "Canvas: fudge points + submission comments on retake-qualified " + "students.", + "data//quiz_retake_qualified_YYYYMMDD.csv", + "data//quiz_scores_w_bonus[_dryrun]_YYYYMMDD.csv", + ], + 'flags': ['--dry-run'], + 'examples': [ + "python canvigator.py --crn 12345 --dry-run award-bonus-retake-only", + ], + 'run_before': ['get-quiz-submission-events'], + 'run_after': [], + }, + + 'get-quiz-submission-events': { + 'description': ( + "Export per-attempt submission and event data for a selected quiz " + "(or every published quiz with --all), and generate per-question " + "score histograms plus first-attempt timing and page-blur " + "histograms. Stores raw scores (no fudge points from prior bonus " + "runs) so re-runs are idempotent." + ), + 'prerequisites': [], + 'inputs': [ + "Canvas quiz (selected interactively, or via --crn, or --all for " + "every published quiz in the course).", + ], + 'outputs': [ + "data//quiz_student_analysis_YYYYMMDD.csv", + "data//quiz_all_submissions_YYYYMMDD.csv", + "data//quiz_all_subs_by_question_YYYYMMDD.csv", + "data//quiz_all_subs_and_events_YYYYMMDD.csv", + "figures//quiz_*_YYYYMMDD.png (question, timing, blur " + "histograms)", + ], + 'flags': ['--all'], + 'examples': [ + "python canvigator.py --crn 12345 get-quiz-submission-events", + "python canvigator.py --crn 12345 --all get-quiz-submission-events", + ], + 'run_before': [], + 'run_after': ['award-bonus', 'send-quiz-reminder', 'prep-class-digest'], + }, + + 'get-quiz-questions': { + 'description': ( + "Export quiz metadata and question content (id, type, text, points, " + "answers) to a CSV. Skips student data download. With --tag, adds a " + "'keywords' column via cloud LLM and writes to a separate " + "*_questions_w_tags_*.csv so untagged and tagged exports do not " + "overwrite each other. The 'position' column is a 1..N enumeration " + "in the order Canvas returns the questions, which is what students " + "see." + ), + 'prerequisites': [ + "With --tag: OLLAMA_API_KEY must be set in set_env.sh (cloud " + "endpoint at https://ollama.com is used).", + ], + 'inputs': [ + "Canvas quiz (selected interactively, or via --crn, or --all for " + "every published quiz in the course).", + ], + 'outputs': [ + "data//quiz_questions_YYYYMMDD.csv (no --tag)", + "data//quiz_questions_w_tags_YYYYMMDD.csv (with --tag)", + ], + 'flags': ['--tag', '--all'], + 'examples': [ + "python canvigator.py --crn 12345 --tag get-quiz-questions", + "python canvigator.py --crn 12345 --tag --all get-quiz-questions", + ], + 'run_before': [], + 'run_after': [ + 'generate-follow-up-questions', 'send-quiz-reminder', + 'send-follow-up-question', 'analyze-media-recordings', + ], + }, + + 'create-quiz': { + 'description': ( + "Interactively create an unpublished quiz on Canvas. Per question, " + "prompts [p]laceholder, [g]enerate w/ LLM, or [e]nd. The LLM mode " + "turns a natural-language seed into one of seven auto-gradable " + "Canvas question types (multiple choice, true/false, matching, " + "fill-in-the-blank, calculated, multiple answers, multiple " + "dropdowns) and lets you accept / regenerate / skip each draft. " + "Regenerated drafts diverge — they don't recycle the same angle." + ), + 'prerequisites': [ + "For [g]enerate mode only: OLLAMA_API_KEY must be set. Pure " + "placeholder runs work without it.", + ], + 'inputs': [ + "Canvas course (selected interactively or via --crn).", + "Instructor input at the prompt.", + ], + 'outputs': [ + "Canvas: a new unpublished quiz with the questions you accepted.", + ], + 'flags': [], + 'examples': [ + "python canvigator.py --crn 12345 create-quiz", + ], + 'run_before': [], + 'run_after': [], + }, + + 'export-anon-data': { + 'description': ( + "Anonymize all CSVs in data// by replacing student IDs with " + "hashed anon_ids and removing name / sis_id columns. No Canvas API " + "needed. Output is a zip archive plus a mapping CSV. WARNING: " + "anonymization alone does not make this data shareable — IRB / " + "FERPA still apply." + ), + 'prerequisites': [ + "data// must already contain the CSVs you want to " + "anonymize.", + ], + 'inputs': [ + "data//*.csv (all existing files in the course directory).", + ], + 'outputs': [ + "data//anonymized_YYYYMMDD/ (anonymized copies of every CSV)", + "data//anonymized_YYYYMMDD.zip", + "data//anon_mapping_YYYYMMDD.csv (real_id <-> anon_id " + "mapping; keep this private).", + ], + 'flags': [], + 'examples': [ + "python canvigator.py --crn 12345 export-anon-data", + "python canvigator.py export-anon-data", + ], + 'run_before': [], + 'run_after': [], + }, + + 'get-gradebook': { + 'description': ( + "Export the full course gradebook for all published assignments to " + "a CSV (one row per student-assignment pair). Course-level, no quiz " + "selection." + ), + 'prerequisites': [], + 'inputs': ["Canvas course (selected interactively or via --crn)."], + 'outputs': [ + "data//gradebook_YYYYMMDD.csv (columns: name, " + "sortable_name, user_id, assignment_name, assignment_id, " + "points_possible, grade, score).", + ], + 'flags': [], + 'examples': [ + "python canvigator.py --crn 12345 get-gradebook", + ], + 'run_before': [], + 'run_after': [], + }, + + 'get-roster': { + 'description': ( + "Export the full course roster (students, teachers, TAs, observers) " + "to a CSV. Course-level, no quiz selection. Useful as a stable " + "identity reference for scripts outside Canvigator." + ), + 'prerequisites': [], + 'inputs': ["Canvas course (selected interactively or via --crn)."], + 'outputs': [ + "data//roster_YYYYMMDD.csv (columns: name, id, sis_id, " + "enrollment_type, state).", + ], + 'flags': [], + 'examples': [ + "python canvigator.py --crn 12345 get-roster", + ], + 'run_before': [], + 'run_after': [], + }, + + 'get-conversations': { + 'description': ( + "Export Canvas conversations involving any active student in this " + "course to a CSV (sorted newest first). Pulls the instructor's full " + "inbox + sent folders, since Canvas conversations are not " + "course-scoped, then filters to course participants. Useful for " + "back-filling the conversation_id of follow-up sends made before " + "the per-send manifest existed." + ), + 'prerequisites': [], + 'inputs': ["Canvas course (selected interactively or via --crn)."], + 'outputs': [ + "data//conversations_YYYYMMDD.csv (columns: " + "conversation_id, subject, last_message_at, first_message_at, " + "message_count, workflow_state, student_ids, student_names, " + "n_student_participants, last_message).", + ], + 'flags': [], + 'examples': [ + "python canvigator.py --crn 12345 get-conversations", + ], + 'run_before': [], + 'run_after': [], + }, + + 'delete-old-conversations': { + 'description': ( + "Delete Canvas conversations from the instructor's inbox / sent / " + "archived scopes whose last_message_at is older than N months " + "(default 6). Account-wide — does NOT require course selection. " + "Live mode requires an explicit 'yes' confirmation at the prompt." + ), + 'prerequisites': [], + 'inputs': [ + "All Canvas conversations across the instructor's inbox + sent + " + "archived scopes.", + ], + 'outputs': [ + "Canvas: deleted conversations (irreversible).", + ], + 'flags': ['--dry-run', '--months'], + 'examples': [ + "python canvigator.py --dry-run delete-old-conversations", + "python canvigator.py --months 12 delete-old-conversations", + ], + 'run_before': [], + 'run_after': [], + }, + + 'generate-follow-up-questions': { + 'description': ( + "Generate three candidate open-ended follow-up questions per " + "original quiz question using a cloud LLM. For each candidate, " + "also drafts a structured rubric, an assessment guide, and " + "pass / fail exemplars. Output goes to *_open_ended_*.csv with " + "selected_question=0 — the instructor reviews offline and sets " + "exactly one row per group to selected_question=1." + ), + 'prerequisites': [ + "get-quiz-questions --tag must have produced " + "*_questions_w_tags_*.csv for the quiz.", + "OLLAMA_API_KEY must be set (cloud endpoint at https://ollama.com).", + ], + 'inputs': [ + "data//quiz_questions_w_tags_*.csv (selected from a " + "menu — driven by the local CSV, not by Canvas quiz selection).", + ], + 'outputs': [ + "data//quiz_open_ended_YYYYMMDD.csv (3 rows per " + "original question; columns include question_mode, " + "open_ended_question, assessment_guide, rubric_json, " + "exemplar_pass / exemplar_fail).", + ], + 'flags': [], + 'examples': [ + "python canvigator.py --crn 12345 generate-follow-up-questions", + ], + 'run_before': ['get-quiz-questions'], + 'run_after': ['send-follow-up-question'], + }, + + 'send-quiz-reminder': { + 'description': ( + "Send Canvas reminder messages to students about an open quiz. " + "Each student is classified into one of four states (no attempt / " + "imperfect / page blur / perfect clean) and gets a tailored " + "message; imperfect-score students get a bulleted list of the " + "questions they missed on their most recent attempt. With --all, " + "sends ONE consolidated message per student covering every " + "published, future-due quiz." + ), + 'prerequisites': [ + "get-quiz-questions --tag must have produced " + "*_questions_w_tags_*.csv for the quiz (the task fails fast if " + "missing).", + "Quiz must be published with a future due_at.", + ], + 'inputs': [ + "Canvas quiz (selected interactively, or via --crn, or every " + "published future-due quiz with --all).", + "data//quiz_questions_w_tags_*.csv", + ], + 'outputs': [ + "Canvas messages (Conversations) sent to enrolled students.", + "data//quiz_reminder_sent[_dryrun]_YYYYMMDD.csv " + "(per-quiz manifest).", + "With --all: data//course_reminder_sent[_dryrun]_YYYYMMDD" + ".csv (one row per student listing every quiz reminded about).", + ], + 'flags': ['--dry-run', '--all'], + 'examples': [ + "python canvigator.py --crn 12345 send-quiz-reminder", + "python canvigator.py --crn 12345 --all --dry-run send-quiz-reminder", + ], + 'run_before': ['get-quiz-questions', 'get-quiz-submission-events'], + 'run_after': ['generate-follow-up-questions', 'send-follow-up-question'], + }, + + 'send-follow-up-question': { + 'description': ( + "Send the instructor-selected open-ended follow-up question (the " + "row marked selected_question=1 in *_open_ended_*.csv) via Canvas " + "message to each student who missed the corresponding original " + "quiz question on their latest attempt. Mode-aware instructions: " + "'explain' asks for a voice recording, 'draw' asks for an attached " + "photo. Uses force_new=True so the thread is dedicated." + ), + 'prerequisites': [ + "get-quiz-questions --tag must have produced " + "*_questions_w_tags_*.csv.", + "generate-follow-up-questions must have produced *_open_ended_*" + ".csv with exactly one row per question marked " + "selected_question=1.", + ], + 'inputs': [ + "Canvas quiz (selected interactively or via --crn).", + "data//quiz_questions_w_tags_*.csv", + "data//quiz_open_ended_*.csv", + ], + 'outputs': [ + "Canvas messages sent to students who missed the question.", + "data//quiz_followup_sent[_dryrun]_YYYYMMDD.csv " + "(manifest with conversation_id, question_id, sent_at, " + "question_mode columns).", + ], + 'flags': ['--dry-run'], + 'examples': [ + "python canvigator.py --crn 12345 --dry-run send-follow-up-question", + "python canvigator.py --crn 12345 send-follow-up-question", + ], + 'run_before': ['generate-follow-up-questions'], + 'run_after': ['assess-replies', 'send-follow-up-assessments'], + }, + + 'assess-replies': { + 'description': ( + "Refresh student follow-up replies from Canvas (downloading audio " + "and image attachments) and assess each one with a local LLM. For " + "'explain' mode the audio model transcribes and the main model " + "grades the transcript; for 'draw' mode the main model grades the " + "image directly. Each reply is run N=3 times and majority-voted " + "(confidence='high' on agreement, 'borderline' on split). Merges " + "into a persistent (non-dated) *_followup_assessments.csv — rows " + "with sent_assessment=1 are preserved verbatim across re-runs." + ), + 'prerequisites': [ + "send-follow-up-question must have produced " + "*_followup_sent_*.csv.", + "Local Ollama running with OLLAMA_MODEL (default gemma4:31b) and " + "OLLAMA_AUDIO_MODEL (default gemma4:e4b) available.", + ], + 'inputs': [ + "Canvas quiz (selected interactively or via --crn) — used for " + "context lookup.", + "data//quiz_followup_sent_*.csv (iterated row by row " + "to fetch each conversation).", + "data//quiz_open_ended_*.csv (provides the rubric, " + "assessment guide, and exemplars).", + ], + 'outputs': [ + "data//replies/ (downloaded audio + image attachments).", + "data//quiz_followup_replies_YYYYMMDD.csv (refreshed " + "every run).", + "data//quiz_followup_assessments.csv (persistent; " + "instructor edits 'feedback' before send-follow-up-assessments).", + ], + 'flags': ['--reply-window-days'], + 'examples': [ + "python canvigator.py --crn 12345 assess-replies", + "python canvigator.py --crn 12345 -w 10 assess-replies", + ], + 'run_before': ['send-follow-up-question'], + 'run_after': ['send-follow-up-assessments', 'prep-class-digest'], + }, + + 'send-follow-up-assessments': { + 'description': ( + "For every row in the persistent *_followup_assessments.csv with " + "sent_assessment=0 and a non-empty 'feedback' value, post the " + "feedback as a reply on the existing follow-up conversation using " + "the row's conversation_id. On success, sets sent_assessment=1 and " + "stamps sent_at. The instructor edits the 'feedback' column " + "between assess-replies and this task — feedback already sent is " + "never overwritten." + ), + 'prerequisites': [ + "assess-replies must have produced " + "*_followup_assessments.csv.", + "Instructor has reviewed the 'feedback' column and edited any " + "borderline / fail feedback as needed.", + ], + 'inputs': [ + "Canvas quiz (selected interactively or via --crn).", + "data//quiz_followup_assessments.csv", + ], + 'outputs': [ + "Canvas reply messages on each follow-up conversation thread.", + "data//quiz_followup_assessments.csv (rewritten in " + "place with sent_assessment=1 and updated sent_at on success).", + ], + 'flags': ['--dry-run'], + 'examples': [ + "python canvigator.py --crn 12345 --dry-run " + "send-follow-up-assessments", + "python canvigator.py --crn 12345 send-follow-up-assessments", + ], + 'run_before': ['assess-replies'], + 'run_after': [], + }, + + 'create-media-recording-assignment': { + 'description': ( + "Interactively create a Canvas assignment that accepts only " + "media_recording submissions. Prompts for title, prompt body (HTML " + "description), points_possible (default 1), optional ISO due_at, " + "and a publish-now y/N (default unpublished so the instructor can " + "review before exposure)." + ), + 'prerequisites': [], + 'inputs': [ + "Canvas course (selected interactively or via --crn).", + "Instructor input at the prompt.", + ], + 'outputs': [ + "Canvas: a new media-recording assignment (unpublished by " + "default).", + ], + 'flags': [], + 'examples': [ + "python canvigator.py --crn 12345 " + "create-media-recording-assignment", + ], + 'run_before': [], + 'run_after': ['get-media-recordings'], + }, + + 'get-media-recordings': { + 'description': ( + "Fetch each student's audio submission, transcode to 16 kHz mono " + "WAV via ffmpeg (one pass, handles Canvas's DASH playback URLs), " + "transcribe locally with the audio model, and either prompt for a " + "per-student grade or auto-award points_possible. Default: shows " + "each transcript and prompts for points (Enter = full credit, a " + "number = that value, s/skip = no grade). With --auto-grade: " + "skips the prompt and awards points_possible. Combine " + "--auto-grade with --dry-run to preview without mutating Canvas." + ), + 'prerequisites': [ + "ffmpeg installed and on PATH.", + "Local Ollama running with OLLAMA_AUDIO_MODEL (default " + "gemma4:e4b) available.", + "A media-recording assignment must already exist on Canvas (use " + "create-media-recording-assignment).", + ], + 'inputs': [ + "Canvas assignment (selected interactively from the filtered list " + "of media-recording assignments).", + ], + 'outputs': [ + "data//media_recordings/assignment/_.wav " + "(transcoded audio).", + "data//assignment_recordings_YYYYMMDD.csv (columns: " + "student_id, student_name, submission_id, submitted_at, " + "audio_path, transcript, transcribed_at, grade, graded_at).", + "Canvas: posted_grade on each submission (skipped under --dry-run).", + ], + 'flags': ['--auto-grade', '--dry-run'], + 'examples': [ + "python canvigator.py --crn 12345 get-media-recordings", + "python canvigator.py --crn 12345 --auto-grade --dry-run " + "get-media-recordings", + ], + 'run_before': ['create-media-recording-assignment'], + 'run_after': ['analyze-media-recordings', 'prep-class-digest'], + }, + + 'analyze-media-recordings': { + 'description': ( + "Run two local-LLM passes against the saved transcripts: " + "(1) classify each transcript against the unique tags from a " + "*_questions_w_tags_*.csv (paraphrase-tolerant), and (2) extract " + "3-5 cohort-level recurring themes. Output is a Markdown report " + "with a tag-grounded specialized word cloud table, the " + "LLM-extracted themes, and a roster mapping student indices to " + "names." + ), + 'prerequisites': [ + "get-media-recordings must have produced " + "assignment_recordings_*.csv.", + "A *_questions_w_tags_*.csv must exist in data// (run " + "get-quiz-questions --tag for the relevant quiz first).", + "Local Ollama running with OLLAMA_MODEL (default gemma4:31b) — " + "transcripts stay local.", + ], + 'inputs': [ + "Canvas assignment (selected interactively).", + "data//assignment_recordings_*.csv (most recent).", + "data//quiz_questions_w_tags_*.csv (selected from a " + "menu).", + ], + 'outputs': [ + "data//assignment_analysis_YYYYMMDD.md (tag table + " + "themes + roster).", + ], + 'flags': [], + 'examples': [ + "python canvigator.py --crn 12345 analyze-media-recordings", + ], + 'run_before': ['get-media-recordings', 'get-quiz-questions'], + 'run_after': ['prep-class-digest'], + }, + + 'prep-class-digest': { + 'description': ( + "Synthesize a one-page Markdown brief on cohort gaps from the last " + "N days. Pulls three signal sources (per-tag quiz misses, " + "follow-up reply themes, media-recording themes) and turns the top " + "priorities into 2-3 in-class discussion questions. Default uses " + "local Gemma 4 with the full-fidelity prompt; with " + "--cloud-questions, the discussion-question step routes to cloud " + "Gemini 3 with a redacted prompt (tag names + integer counts only " + "— no transcripts or themes leave the machine)." + ), + 'prerequisites': [ + "data// already populated by upstream tasks " + "(get-quiz-questions --tag + get-quiz-submission-events for " + "misses; assess-replies for follow-up themes; get-media-recordings " + "for transcripts).", + "Local Ollama running with OLLAMA_MODEL (default gemma4:31b) for " + "the default discussion-question step.", + "OLLAMA_API_KEY set if --cloud-questions is used.", + ], + 'inputs': [ + "data//quiz_questions_w_tags_*.csv + " + "quiz_all_subs_by_question_*.csv (joined for per-tag miss " + "counts; both must fall within the lookback window).", + "data//quiz_followup_assessments.csv (filtered to " + "result=fail or confidence=borderline within the window).", + "data//assignment_recordings_*.csv (transcripts " + "within the window).", + ], + 'outputs': [ + "data//class_digest_YYYYMMDD.md (5 sections: header, quiz " + "performance, follow-up themes, media-recording themes, " + "discussion questions). Empty windows short-circuit and write " + "nothing.", + ], + 'flags': ['--days', '--cloud-questions'], + 'examples': [ + "python canvigator.py --crn 12345 prep-class-digest", + "python canvigator.py --crn 12345 --days 14 --cloud-questions " + "prep-class-digest", + ], + 'run_before': [ + 'assess-replies', 'get-quiz-submission-events', + 'analyze-media-recordings', + ], + 'run_after': [], + }, +} + + +# ---------------------------------------------------------------------------- +# Rendering +# ---------------------------------------------------------------------------- + +_BODY_WIDTH = 80 +_BULLET_INDENT = ' - ' +_BULLET_CONT = ' ' +# Wide enough for the longest label, '-w, --reply-window-days ' (27 chars). +_FLAG_LABEL_WIDTH = 28 + + +def _flag_label(flag): + """Return the rendered '-x, --flag ' label for one flag.""" + short, value_label, _ = FLAG_DESCRIPTIONS[flag] + label = f"{short}, {flag}" + if value_label: + label = f"{label} {value_label}" + return label + + +def _print_bullets(items): + """Print a bulleted list with consistent wrapping.""" + for item in items: + print(textwrap.fill( + item, + width=_BODY_WIDTH, + initial_indent=_BULLET_INDENT, + subsequent_indent=_BULLET_CONT, + )) + + +def _print_flags(flags): + """Print the Flags section. Universal flags are appended automatically.""" + rendered = list(flags) + for uf in UNIVERSAL_FLAGS: + if uf not in rendered: + rendered.append(uf) + if not rendered: + return + print("Flags:") + for flag in rendered: + label = _flag_label(flag) + _, _, text = FLAG_DESCRIPTIONS[flag] + cont_pad = ' ' * (2 + _FLAG_LABEL_WIDTH) + wrapped = textwrap.wrap(text, width=_BODY_WIDTH - 2 - _FLAG_LABEL_WIDTH) + first = wrapped[0] if wrapped else '' + print(f" {label:<{_FLAG_LABEL_WIDTH}}{first}") + for cont in wrapped[1:]: + print(f"{cont_pad}{cont}") + print() + + +def print_task_help(task_name): + """Print the per-task help block for ``task_name`` to stdout.""" + if task_name not in TASK_HELP: + print(f"No help entry for task: {task_name}") + return + entry = TASK_HELP[task_name] + + print(f"canvigator.py {task_name} [OPTIONS]") + print() + for line in textwrap.wrap(entry['description'], width=_BODY_WIDTH, + initial_indent=' ', subsequent_indent=' '): + print(line) + print() + + for header, key in (('Prerequisites', 'prerequisites'), + ('Inputs', 'inputs'), + ('Outputs', 'outputs')): + items = entry.get(key) or [] + if not items: + continue + print(f"{header}:") + _print_bullets(items) + print() + + _print_flags(entry.get('flags') or []) + + examples = entry.get('examples') or [] + if examples: + print("Examples:") + for ex in examples: + print(f" {ex}") + print() + + run_before = entry.get('run_before') or [] + run_after = entry.get('run_after') or [] + if run_before or run_after: + print("Workflow:") + if run_before: + print(f" Run before this: {', '.join(run_before)}") + if run_after: + print(f" Run after this: {', '.join(run_after)}") + print() + + print("Run 'canvigator.py --help' for the full task list.") + + +def print_global_help(task_groups): + """Print the global help: usage, options, grouped task list, footer. + + ``task_groups`` is the list of ``(header, [(name, one_line_desc), ...])`` + tuples maintained in canvigator.py. + """ + print("Usage: canvigator.py [OPTIONS] ") + print() + print("Options (short and long forms are interchangeable):") + for flag in ( + '--dry-run', '--tag', '--all', '--crn', '--months', + '--reply-window-days', '--auto-grade', '--days', '--cloud-questions', + ): + label = _flag_label(flag) + _, _, text = FLAG_DESCRIPTIONS[flag] + cont_pad = ' ' * (2 + _FLAG_LABEL_WIDTH) + wrapped = textwrap.wrap(text, width=_BODY_WIDTH - 2 - _FLAG_LABEL_WIDTH) + first = wrapped[0] if wrapped else '' + print(f" {label:<{_FLAG_LABEL_WIDTH}}{first}") + for cont in wrapped[1:]: + print(f"{cont_pad}{cont}") + + all_names = [name for _, items in task_groups for name, _ in items] + if all_names: + max_name = max(len(n) for n in all_names) + else: + max_name = 0 + for header, items in task_groups: + print(f"\n{header}") + for name, desc in items: + print(f" {name:<{max_name}} {desc}") + + print() + print("Run 'canvigator.py --help' for detailed help on a specific " + "task (e.g. 'canvigator.py send-quiz-reminder --help').") diff --git a/test_canvigator.py b/test_canvigator.py index 5584fee..465b277 100644 --- a/test_canvigator.py +++ b/test_canvigator.py @@ -3436,3 +3436,154 @@ def test_empty_window_writes_no_file(self, tmp_path, capsys): # User-facing message confirms the no-op. captured = capsys.readouterr() assert 'nothing to digest' in captured.out + + +# --------------------------------------------------------------------------- +# canvigator_help tests (per-task --help) +# --------------------------------------------------------------------------- + +class TestPerTaskHelp: + """Integrity + rendering + CLI-dispatch tests for the per-task help system.""" + + def _all_task_names(self): + """Pull the list of tasks from canvigator.py without executing it.""" + # canvigator.py is a top-level script; importing it would attempt to + # parse sys.argv. We instead read task_groups via runpy in a guarded + # mode that stops before the parse loop. + import ast + src = Path(__file__).parent.joinpath('canvigator.py').read_text() + tree = ast.parse(src) + names = [] + for node in ast.walk(tree): + if isinstance(node, ast.Assign): + for tgt in node.targets: + if isinstance(tgt, ast.Name) and tgt.id == 'task_groups': + # task_groups = [(header, [(name, desc), ...]), ...] + for group_tuple in node.value.elts: + items_list = group_tuple.elts[1] + for entry in items_list.elts: + names.append(entry.elts[0].value) + return names + + def test_every_task_has_help_entry(self): + """Every task name in canvigator.py has a TASK_HELP entry.""" + from canvigator_help import TASK_HELP + task_names = self._all_task_names() + assert task_names, "expected to find tasks in canvigator.py" + missing = sorted(set(task_names) - set(TASK_HELP)) + assert not missing, f"missing TASK_HELP entries: {missing}" + extra = sorted(set(TASK_HELP) - set(task_names)) + assert not extra, f"TASK_HELP has stale entries: {extra}" + + def test_required_fields_non_empty(self): + """Every entry has non-empty description, inputs, outputs, examples.""" + from canvigator_help import TASK_HELP + for name, entry in TASK_HELP.items(): + assert entry.get('description', '').strip(), f"{name}: empty description" + assert entry.get('inputs'), f"{name}: empty inputs" + assert entry.get('outputs'), f"{name}: empty outputs" + assert entry.get('examples'), f"{name}: empty examples" + # These keys must exist (may be empty lists) so the renderer can + # iterate without KeyError. + for key in ('prerequisites', 'flags', 'run_before', 'run_after'): + assert key in entry, f"{name}: missing key '{key}'" + assert isinstance(entry[key], list), f"{name}: '{key}' must be a list" + + def test_flag_references_resolve(self): + """Every flag listed in any entry is defined in FLAG_DESCRIPTIONS.""" + from canvigator_help import TASK_HELP, FLAG_DESCRIPTIONS + for name, entry in TASK_HELP.items(): + for flag in entry.get('flags', []): + assert flag in FLAG_DESCRIPTIONS, ( + f"{name}: references unknown flag '{flag}'" + ) + + def test_run_before_after_targets_exist(self): + """run_before / run_after entries are themselves valid task names.""" + from canvigator_help import TASK_HELP + all_tasks = set(TASK_HELP) + for name, entry in TASK_HELP.items(): + for ref in entry.get('run_before', []) + entry.get('run_after', []): + assert ref in all_tasks, ( + f"{name}: run_before/run_after references unknown task '{ref}'" + ) + + def test_print_task_help_render(self, capsys): + """The rendered output contains the description, prereq, flags, and example.""" + from canvigator_help import print_task_help + print_task_help('send-quiz-reminder') + out = capsys.readouterr().out + assert 'send-quiz-reminder' in out + assert 'Send Canvas reminder messages' in out + assert 'Prerequisites:' in out + assert 'get-quiz-questions' in out # cited as a prerequisite + assert '--all' in out + assert '--dry-run' in out + assert 'Examples:' in out + assert 'python canvigator.py' in out + assert 'Workflow:' in out + assert 'Run before this:' in out + # Universal --crn flag should be auto-appended to the Flags section. + assert '--crn' in out + + def test_print_task_help_unknown_task(self, capsys): + """Unknown task names render a friendly message instead of crashing.""" + from canvigator_help import print_task_help + print_task_help('not-a-real-task') + out = capsys.readouterr().out + assert 'not-a-real-task' in out + + def _run_cli(self, *args): + """Invoke canvigator.py as a subprocess; return (rc, stdout, stderr). + + Sets CANVAS_URL/CANVAS_TOKEN to empty so the subprocess can prove the + --help path exits before constructing the Canvas client. + """ + env = {'PATH': '/usr/bin:/bin', 'PYTHONPATH': ''} + # Inherit some basics so Python can find its stdlib in the test runner. + import os + for k in ('PATH', 'HOME', 'PYTHONPATH', 'PYTHONHOME'): + if k in os.environ: + env[k] = os.environ[k] + # Explicitly NOT setting CANVAS_URL / CANVAS_TOKEN — the help path + # must short-circuit before the env-var check. + env['CANVAS_URL'] = '' + env['CANVAS_TOKEN'] = '' + import sys + cwd = Path(__file__).parent + result = subprocess.run( + [sys.executable, 'canvigator.py', *args], + capture_output=True, text=True, cwd=cwd, env=env, timeout=15, + ) + return result.returncode, result.stdout, result.stderr + + def test_cli_dispatch_global_help(self): + """`canvigator.py --help` exits 0 and prints the task list.""" + rc, stdout, stderr = self._run_cli('--help') + assert rc == 0, f"stderr={stderr!r}" + assert 'Usage: canvigator.py' in stdout + # The new footer line should be present. + assert " --help" in stdout + # A representative task name should appear. + assert 'send-quiz-reminder' in stdout + + def test_cli_dispatch_task_help(self): + """`canvigator.py --help` exits 0 and prints task-specific help.""" + rc, stdout, stderr = self._run_cli('create-pairs', '--help') + assert rc == 0, f"stderr={stderr!r}" + assert 'create-pairs' in stdout + assert 'Prerequisites:' in stdout + assert 'present_*.csv' in stdout + + def test_cli_dispatch_task_help_short_form(self): + """`canvigator.py -h` works the same as --help.""" + rc, stdout, stderr = self._run_cli('send-quiz-reminder', '-h') + assert rc == 0, f"stderr={stderr!r}" + assert 'send-quiz-reminder' in stdout + assert 'Workflow:' in stdout + + def test_cli_invalid_task_with_help(self): + """`canvigator.py bogus --help` still rejects the unknown task.""" + rc, stdout, _ = self._run_cli('bogus-task', '--help') + assert rc == 1 + assert 'Invalid task' in stdout