From 82a3cb94a8e01afbc746c941bb36cdee7ba7e903 Mon Sep 17 00:00:00 2001 From: erphenheimer Date: Tue, 31 Mar 2026 19:00:06 +0800 Subject: [PATCH 1/3] feat: restore input validation and path traversal protection --- src/cli/research.ts | 5 ++++ src/commands.ts | 10 ++++--- src/utils/security.ts | 63 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 4 deletions(-) create mode 100644 src/utils/security.ts diff --git a/src/cli/research.ts b/src/cli/research.ts index be3cb90..57f5443 100644 --- a/src/cli/research.ts +++ b/src/cli/research.ts @@ -2,6 +2,7 @@ import fs from "node:fs"; import path from "node:path"; import os from "node:os"; import { renderBootstrapMd, renderSoulMd, renderAgentsMd } from "../templates/bootstrap.js"; +import { validateProjectId, ensureWithinDirectory } from "../utils/security.js"; const OPENCLAW_HOME = path.join(os.homedir(), ".openclaw"); const OPENCLAW_CONFIG = path.join(OPENCLAW_HOME, "openclaw.json"); @@ -141,8 +142,10 @@ function listResearchProjects(): ResearchProject[] { } function initProject(id: string, pluginSkillsDir: string): void { + validateProjectId(id); const agentId = `research-${id}`; const workspace = path.join(OPENCLAW_HOME, `workspace-${agentId}`); + ensureWithinDirectory(workspace, OPENCLAW_HOME); // Check if workspace already exists if (fs.existsSync(workspace)) { @@ -244,8 +247,10 @@ function showStatus(id: string): void { } function deleteProject(id: string): void { + validateProjectId(id); const agentId = `research-${id}`; const workspace = path.join(OPENCLAW_HOME, `workspace-${agentId}`); + ensureWithinDirectory(workspace, OPENCLAW_HOME); // Remove workspace if (fs.existsSync(workspace)) { diff --git a/src/commands.ts b/src/commands.ts index 8c34ef8..042e5e5 100644 --- a/src/commands.ts +++ b/src/commands.ts @@ -2,6 +2,7 @@ import * as fs from "node:fs"; import * as path from "node:path"; import * as os from "node:os"; import type { PluginCommandContext, PluginCommandResult } from "./types.js"; +import { ensureWithinDirectory } from "./utils/security.js"; const OPENCLAW_HOME = path.join(os.homedir(), ".openclaw"); @@ -20,10 +21,11 @@ function listResearchAgents(): ResearchAgent[] { const agents = (config.agents as { list?: Array<{ id: string; workspace?: string }> })?.list ?? []; return agents .filter((a) => a.id.startsWith("research-")) - .map((a) => ({ - id: a.id, - workspace: (a.workspace ?? `~/.openclaw/workspace-${a.id}`).replace("~", os.homedir()), - })); + .map((a) => { + const workspace = (a.workspace ?? `~/.openclaw/workspace-${a.id}`).replace("~", os.homedir()); + ensureWithinDirectory(workspace, OPENCLAW_HOME); + return { id: a.id, workspace }; + }); } catch { return []; } diff --git a/src/utils/security.ts b/src/utils/security.ts new file mode 100644 index 0000000..de40756 --- /dev/null +++ b/src/utils/security.ts @@ -0,0 +1,63 @@ +import path from "node:path"; + +/** + * Minimal security utilities for input validation. + * + * These guards protect against path traversal and injection attacks + * on user-supplied identifiers that end up in file paths or shell commands. + */ + +/** Allowed characters for project / agent IDs: lowercase alphanumeric, hyphens, underscores. */ +const PROJECT_ID_RE = /^[a-z0-9][a-z0-9_-]{0,63}$/; + +/** arXiv ID format: YYMM.NNNNN (optionally vN). */ +const ARXIV_ID_RE = /^\d{4}\.\d{4,5}(v\d+)?$/; + +/** DOI format: 10.NNNN/... (conservative). */ +const DOI_RE = /^10\.\d{4,9}\/[^\s]+$/; + +/** + * Validate a project ID used in workspace path construction. + * Rejects path traversal characters (/, \, ..) and non-ASCII. + */ +export function validateProjectId(id: string): string { + if (!PROJECT_ID_RE.test(id)) { + throw new Error( + `Invalid project ID "${id}". Must match ${PROJECT_ID_RE} (lowercase alphanumeric, hyphens, underscores, 1-64 chars).`, + ); + } + return id; +} + +/** + * Validate an arXiv ID before using it in URLs or file paths. + */ +export function validateArxivId(id: string): string { + if (!ARXIV_ID_RE.test(id)) { + throw new Error(`Invalid arXiv ID "${id}". Expected format: YYMM.NNNNN (e.g. 2401.12345).`); + } + return id; +} + +/** + * Validate a DOI string. + */ +export function validateDoi(doi: string): string { + if (!DOI_RE.test(doi)) { + throw new Error(`Invalid DOI "${doi}". Expected format: 10.NNNN/... (e.g. 10.1038/s41586-021-03819-2).`); + } + return doi; +} + +/** + * Ensure a resolved path stays within the expected base directory. + * Prevents path traversal via `../../` or symlink tricks. + */ +export function ensureWithinDirectory(filePath: string, baseDir: string): string { + const resolved = path.resolve(filePath); + const resolvedBase = path.resolve(baseDir); + if (!resolved.startsWith(resolvedBase + path.sep) && resolved !== resolvedBase) { + throw new Error(`Path "${filePath}" escapes base directory "${baseDir}".`); + } + return resolved; +} From 45a0cfb1f05757f178e838a1772f828e542cda58 Mon Sep 17 00:00:00 2001 From: erphenheimer Date: Tue, 31 Mar 2026 19:00:29 +0800 Subject: [PATCH 2/3] feat: add unified paper_download and paper_browser tools --- index.ts | 4 + openclaw.plugin.json | 1 - src/tools/paper-browser.ts | 140 +++++++++++++++++++++ src/tools/paper-download.ts | 243 ++++++++++++++++++++++++++++++++++++ 4 files changed, 387 insertions(+), 1 deletion(-) create mode 100644 src/tools/paper-browser.ts create mode 100644 src/tools/paper-download.ts diff --git a/index.ts b/index.ts index 5ace4bf..a43b396 100644 --- a/index.ts +++ b/index.ts @@ -10,6 +10,8 @@ import { } from "./src/commands.js"; import { createArxivSearchTool } from "./src/tools/arxiv-search.js"; import { createOpenAlexSearchTool } from "./src/tools/openalex-search.js"; +import { createPaperDownloadTool } from "./src/tools/paper-download.js"; +import { createPaperBrowserTool } from "./src/tools/paper-browser.js"; import { createSkillInjectionHook } from "./src/hooks/inject-skill.js"; import { createCronSkillInjectionHook } from "./src/hooks/cron-skill-inject.js"; import { registerResearchCli } from "./src/cli/research.js"; @@ -23,6 +25,8 @@ export default definePluginEntry({ // Register tools api.registerTool(createArxivSearchTool()); api.registerTool(createOpenAlexSearchTool()); + api.registerTool(createPaperDownloadTool()); + api.registerTool(createPaperBrowserTool()); // Register chat commands (bypass LLM) api.registerCommand({ diff --git a/openclaw.plugin.json b/openclaw.plugin.json index b6555ce..79b8450 100644 --- a/openclaw.plugin.json +++ b/openclaw.plugin.json @@ -16,7 +16,6 @@ "skills/research-review", "skills/research-experiment", "skills/research-collect", - "skills/paper-download", "skills/write-review-paper", "skills/metabolism" ] diff --git a/src/tools/paper-browser.ts b/src/tools/paper-browser.ts new file mode 100644 index 0000000..990b441 --- /dev/null +++ b/src/tools/paper-browser.ts @@ -0,0 +1,140 @@ +import { Type } from "@sinclair/typebox"; +import { readFileSync, existsSync, statSync } from "node:fs"; +import { Result } from "./result.js"; + +const DEFAULT_VIEWPORT_SIZE = 100; +const MAX_VIEWPORT_SIZE = 500; + +export const PaperBrowserToolSchema = Type.Object({ + file_path: Type.String({ + description: "Path to the paper file (.tex, .md, or any text file).", + }), + start_line: Type.Optional( + Type.Number({ + description: "Starting line number (1-indexed). Default: 1.", + minimum: 1, + }), + ), + num_lines: Type.Optional( + Type.Number({ + description: `Number of lines to display (default: ${DEFAULT_VIEWPORT_SIZE}, max: ${MAX_VIEWPORT_SIZE}).`, + minimum: 1, + maximum: MAX_VIEWPORT_SIZE, + }), + ), +}); + +function readStringParam(params: Record, key: string, opts?: { required?: boolean }): string | undefined { + const value = params[key]; + if (value === undefined || value === null) { + if (opts?.required) { + throw new Error(`Missing required parameter: ${key}`); + } + return undefined; + } + return String(value); +} + +function readNumberParam(params: Record, key: string, opts?: { integer?: boolean }): number | undefined { + const value = params[key]; + if (value === undefined || value === null) return undefined; + const num = Number(value); + if (isNaN(num)) return undefined; + return opts?.integer ? Math.floor(num) : num; +} + +export function createPaperBrowserTool() { + return { + label: "Paper Browser", + name: "paper_browser", + description: + "Read large paper files (.tex, .md) in paginated chunks. Use this to avoid loading entire multi-thousand-line files into context at once. Returns a viewport of lines with navigation information.", + parameters: PaperBrowserToolSchema, + execute: async (_toolCallId: string, rawArgs: unknown) => { + const params = rawArgs as Record; + const filePath = readStringParam(params, "file_path", { required: true })!; + const startLine = Math.max(1, readNumberParam(params, "start_line", { integer: true }) ?? 1); + const numLines = Math.min( + readNumberParam(params, "num_lines", { integer: true }) ?? DEFAULT_VIEWPORT_SIZE, + MAX_VIEWPORT_SIZE, + ); + + // Validate file exists + if (!existsSync(filePath)) { + return Result.err("file_not_found", `File does not exist: ${filePath}`); + } + + // Check if it's a file (not directory) + let stats; + try { + stats = statSync(filePath); + } catch (error) { + return Result.err("file_error", `Cannot access file: ${error instanceof Error ? error.message : String(error)}`); + } + + if (!stats.isFile()) { + return Result.err("not_a_file", `Path is not a file: ${filePath}`); + } + + // Read file content + let content: string; + try { + content = readFileSync(filePath, "utf-8"); + } catch (error) { + return Result.err("read_error", `Failed to read file: ${error instanceof Error ? error.message : String(error)}`); + } + + // Split into lines + const lines = content.split("\n"); + const totalLines = lines.length; + + // Validate start line + if (startLine > totalLines) { + return Result.err( + "invalid_range", + `Start line ${startLine} exceeds total lines ${totalLines}`, + ); + } + + // Extract viewport + const endLine = Math.min(startLine + numLines - 1, totalLines); + const viewportLines = lines.slice(startLine - 1, endLine); + + // Add line numbers (matching cat -n format for consistency with Read tool) + const numberedLines = viewportLines + .map((line, idx) => { + const lineNum = startLine + idx; + return `${lineNum.toString().padStart(6, " ")}\t${line}`; + }) + .join("\n"); + + // Navigation hints + const hasMore = endLine < totalLines; + const hasPrev = startLine > 1; + + let navigationHint = ""; + if (hasMore && hasPrev) { + navigationHint = `\n\nNavigate: paper_browser({ file_path: "${filePath}", start_line: ${endLine + 1} }) for next page, or start_line: ${Math.max(1, startLine - numLines)} for previous page.`; + } else if (hasMore) { + navigationHint = `\n\nMore content below. Use: paper_browser({ file_path: "${filePath}", start_line: ${endLine + 1} })`; + } else if (hasPrev) { + navigationHint = `\n\nEnd of file. Use: paper_browser({ file_path: "${filePath}", start_line: ${Math.max(1, startLine - numLines)} }) for previous page.`; + } else { + navigationHint = "\n\n[End of file]"; + } + + return Result.ok({ + file_path: filePath, + total_lines: totalLines, + viewport: { + start_line: startLine, + end_line: endLine, + num_lines: viewportLines.length, + }, + content: numberedLines + navigationHint, + has_more: hasMore, + has_prev: hasPrev, + }); + }, + }; +} diff --git a/src/tools/paper-download.ts b/src/tools/paper-download.ts new file mode 100644 index 0000000..9f252aa --- /dev/null +++ b/src/tools/paper-download.ts @@ -0,0 +1,243 @@ +import { Type } from "@sinclair/typebox"; +import * as fs from "node:fs"; +import * as path from "node:path"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { Result } from "./result.js"; +import { validateArxivId, validateDoi } from "../utils/security.js"; + +const execFileAsync = promisify(execFile); + +const ARXIV_RATE_LIMIT_MS = 3000; +const UNPAYWALL_API = "https://api.unpaywall.org/v2"; +const USER_EMAIL = "research@openclaw.ai"; + +const delay = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +export const PaperDownloadSchema = Type.Object({ + ids: Type.Array(Type.String(), { + description: + "List of paper identifiers. arXiv IDs (e.g. '2401.12345') or DOIs (e.g. '10.1038/s41586-021-03819-2'). Max 20.", + minItems: 1, + maxItems: 20, + }), + output_dir: Type.Optional( + Type.String({ description: "Output directory. Defaults to 'papers/' relative to cwd." }), + ), +}); + +type SingleResult = { + id: string; + type: "arxiv" | "doi"; + status: "success" | "not_oa" | "failed"; + format?: "tex" | "pdf"; + path?: string; + files?: string[]; + message: string; + title?: string; +}; + +function isArxivId(id: string): boolean { + return /^\d{4}\.\d{4,5}(v\d+)?$/.test(id); +} + +function readArrayParam(params: Record, key: string): string[] { + const value = params[key]; + if (Array.isArray(value)) return value.map(String); + if (typeof value === "string") { + try { + const parsed = JSON.parse(value); + if (Array.isArray(parsed)) return parsed.map(String); + } catch { /* single value */ } + return [value]; + } + return []; +} + +function readStringParam(params: Record, key: string): string | undefined { + const v = params[key]; + return v == null ? undefined : String(v); +} + +// ── arXiv download helpers ────────────────────────────────────────── + +async function findTexFiles(dir: string): Promise { + const files: string[] = []; + for (const entry of await fs.promises.readdir(dir, { withFileTypes: true })) { + if (entry.isDirectory()) { + const sub = await findTexFiles(path.join(dir, entry.name)); + files.push(...sub.map((f) => path.join(entry.name, f))); + } else if (entry.name.endsWith(".tex")) { + files.push(entry.name); + } + } + return files; +} + +async function downloadArxivTex( + arxivId: string, + outputDir: string, +): Promise { + const paperDir = path.join(outputDir, arxivId); + await fs.promises.mkdir(paperDir, { recursive: true }); + + const srcUrl = `https://arxiv.org/src/${arxivId}`; + const tarPath = path.join(paperDir, "source.tar.gz"); + + try { + const res = await fetch(srcUrl); + if (!res.ok) { + return downloadArxivPdf(arxivId, outputDir, `Source HTTP ${res.status}`); + } + + const buffer = Buffer.from(await res.arrayBuffer()); + await fs.promises.writeFile(tarPath, buffer); + + const isTarGz = buffer[0] === 0x1f && buffer[1] === 0x8b; + if (isTarGz) { + try { + await execFileAsync("tar", ["-xzf", tarPath, "-C", paperDir]); + } catch { + return downloadArxivPdf(arxivId, outputDir, "tar extraction failed"); + } + await fs.promises.unlink(tarPath).catch(() => {}); + + const texFiles = await findTexFiles(paperDir); + if (texFiles.length === 0) { + return downloadArxivPdf(arxivId, outputDir, "No .tex files in archive"); + } + return { id: arxivId, type: "arxiv", status: "success", format: "tex", path: paperDir, files: texFiles, message: `${texFiles.length} .tex files` }; + } else { + const texPath = path.join(paperDir, "main.tex"); + await fs.promises.rename(tarPath, texPath); + return { id: arxivId, type: "arxiv", status: "success", format: "tex", path: paperDir, files: ["main.tex"], message: "Single .tex file" }; + } + } catch (err) { + return downloadArxivPdf(arxivId, outputDir, String(err)); + } +} + +async function downloadArxivPdf( + arxivId: string, + outputDir: string, + fallbackReason: string, +): Promise { + try { + const res = await fetch(`https://arxiv.org/pdf/${arxivId}.pdf`); + if (!res.ok) { + return { id: arxivId, type: "arxiv", status: "failed", message: `PDF HTTP ${res.status} (tex: ${fallbackReason})` }; + } + const pdfPath = path.join(outputDir, `${arxivId}.pdf`); + await fs.promises.writeFile(pdfPath, Buffer.from(await res.arrayBuffer())); + return { id: arxivId, type: "arxiv", status: "success", format: "pdf", path: pdfPath, files: [`${arxivId}.pdf`], message: `PDF fallback (${fallbackReason})` }; + } catch (err) { + return { id: arxivId, type: "arxiv", status: "failed", message: String(err) }; + } +} + +// ── DOI / Unpaywall download helpers ──────────────────────────────── + +async function downloadDoi(doi: string, outputDir: string): Promise { + try { + const apiUrl = `${UNPAYWALL_API}/${encodeURIComponent(doi)}?email=${USER_EMAIL}`; + const res = await fetch(apiUrl, { + headers: { "User-Agent": "scientify-research-agent/1.0 (mailto:research@openclaw.ai)" }, + }); + if (!res.ok) { + return { id: doi, type: "doi", status: "failed", message: `Unpaywall API ${res.status}` }; + } + + const data = (await res.json()) as { + is_oa: boolean; + best_oa_location: { url_for_pdf?: string | null; url?: string } | null; + title?: string; + }; + + if (!data.is_oa || !data.best_oa_location) { + return { id: doi, type: "doi", status: "not_oa", message: "Not open access", title: data.title }; + } + + const pdfUrl = data.best_oa_location.url_for_pdf || data.best_oa_location.url; + if (!pdfUrl) { + return { id: doi, type: "doi", status: "failed", message: "No PDF URL", title: data.title }; + } + + const pdfRes = await fetch(pdfUrl, { + headers: { "User-Agent": "scientify-research-agent/1.0 (mailto:research@openclaw.ai)" }, + redirect: "follow", + }); + if (!pdfRes.ok) { + return { id: doi, type: "doi", status: "failed", message: `PDF download HTTP ${pdfRes.status}`, title: data.title }; + } + + const ct = pdfRes.headers.get("content-type") || ""; + if (!ct.includes("pdf") && !ct.includes("octet-stream")) { + return { id: doi, type: "doi", status: "failed", message: "Response is not a PDF", title: data.title }; + } + + const slug = doi.replace(/[/\\:]/g, "_"); + const pdfPath = path.join(outputDir, `${slug}.pdf`); + await fs.promises.writeFile(pdfPath, Buffer.from(await pdfRes.arrayBuffer())); + return { id: doi, type: "doi", status: "success", format: "pdf", path: pdfPath, files: [`${slug}.pdf`], message: "OK", title: data.title }; + } catch (err) { + return { id: doi, type: "doi", status: "failed", message: String(err) }; + } +} + +// ── Tool factory ──────────────────────────────────────────────────── + +export function createPaperDownloadTool() { + return { + label: "Paper Download", + name: "paper_download", + description: + "Download academic papers by arXiv ID or DOI. arXiv papers: downloads .tex source with PDF fallback. DOI papers: downloads open-access PDF via Unpaywall. Non-OA DOI papers are skipped gracefully.", + parameters: PaperDownloadSchema, + execute: async (_toolCallId: string, rawArgs: unknown) => { + const params = rawArgs as Record; + const rawIds = readArrayParam(params, "ids"); + const outputDir = path.resolve(readStringParam(params, "output_dir") ?? "papers"); + + if (rawIds.length === 0) { + return Result.err("invalid_params", "ids must be a non-empty array"); + } + + await fs.promises.mkdir(outputDir, { recursive: true }); + + // Validate & classify + const tasks: Array<{ id: string; type: "arxiv" | "doi" }> = []; + for (const raw of rawIds) { + const trimmed = raw.trim(); + if (isArxivId(trimmed)) { + validateArxivId(trimmed); + tasks.push({ id: trimmed, type: "arxiv" }); + } else { + validateDoi(trimmed); + tasks.push({ id: trimmed, type: "doi" }); + } + } + + const results: SingleResult[] = []; + for (let i = 0; i < tasks.length; i++) { + const t = tasks[i]; + if (t.type === "arxiv") { + if (i > 0 && tasks[i - 1].type === "arxiv") await delay(ARXIV_RATE_LIMIT_MS); + results.push(await downloadArxivTex(t.id, outputDir)); + } else { + results.push(await downloadDoi(t.id, outputDir)); + await delay(100); // Unpaywall rate limit + } + } + + const success = results.filter((r) => r.status === "success").length; + return Result.ok({ + output_dir: outputDir, + total: tasks.length, + success, + not_oa: results.filter((r) => r.status === "not_oa").length, + failed: tasks.length - success - results.filter((r) => r.status === "not_oa").length, + results, + }); + }, + }; +} From 564e527ab87c671fc445972eb5ac5d5f47e6b0b4 Mon Sep 17 00:00:00 2001 From: erphenheimer Date: Tue, 31 Mar 2026 19:00:55 +0800 Subject: [PATCH 3/3] docs: update skills to reference paper_download tool, remove unsafe bash template --- CLAUDE.md | 26 ++++++++++---- README.en.md | 15 ++------ README.md | 4 +-- skills/idea-generation/SKILL.md | 2 +- skills/metabolism/SKILL.md | 2 +- skills/paper-download/SKILL.md | 61 -------------------------------- skills/research-collect/SKILL.md | 4 +-- 7 files changed, 28 insertions(+), 86 deletions(-) delete mode 100644 skills/paper-download/SKILL.md diff --git a/CLAUDE.md b/CLAUDE.md index 88a8cf6..7935750 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -7,7 +7,8 @@ Scientify 是一个 OpenClaw 插件，提供 AI 驱动的科研工作流自动化功能。 **核心组件：** -- `src/tools/` - 工具实现（arxiv_search, arxiv_download, openalex_search, unpaywall_download, github_search, paper_browser） +- `src/tools/` - 工具实现（arxiv_search, openalex_search, paper_download, paper_browser） +- `src/utils/` - 通用工具（security.ts 输入校验） - `src/commands.ts` - 聊天命令处理 - `skills/` - 技能定义（随 npm 包发布） - `index.ts` - 插件入口 @@ -119,19 +120,30 @@ scientify/ ├── README.zh.md # 中文文档 ├── src/ │ ├── commands.ts # 聊天命令 -│ ├── openclaw.d.ts # 类型声明 +│ ├── types.ts # 类型定义 +│ ├── cli/ +│ │ └── research.ts # CLI 入口（init/delete/list 项目） +│ ├── commands/ +│ │ └── metabolism-status.ts # 新陈代谢状态命令 +│ ├── hooks/ +│ │ ├── inject-skill.ts # Skill 注入 hook +│ │ └── cron-skill-inject.ts # Cron skill 注入 hook +│ ├── templates/ +│ │ └── bootstrap.ts # 项目初始化模板 +│ ├── utils/ +│ │ └── security.ts # 输入校验（路径遍历防护、ID 格式校验） │ └── tools/ │ ├── arxiv-search.ts # ArXiv 搜索工具 -│ ├── arxiv-download.ts # ArXiv 下载工具（含速率限制） │ ├── openalex-search.ts # OpenAlex 跨学科搜索 -│ ├── unpaywall-download.ts # Unpaywall OA PDF 下载 -│ ├── github-search-tool.ts # GitHub 搜索工具 -│ └── paper-browser.ts # 论文分页浏览工具 +│ ├── paper-download.ts # 论文下载（arXiv .tex/PDF + DOI via Unpaywall） +│ ├── paper-browser.ts # 论文分页浏览工具 +│ └── result.ts # 工具结果辅助函数 ├── skills/ │ ├── idea-generation/ │ │ ├── SKILL.md │ │ └── references/idea-template.md -│ ├── research-collect/SKILL.md # 文献搜索 → 筛选 → 下载 → 聚类 +│ ├── metabolism/SKILL.md # 新陈代谢循环（Day 0 + Day 1+） +│ ├── research-collect/SKILL.md # 文献搜索 → 筛选 → 下载 → 聚类 │ ├── research-pipeline/SKILL.md # 编排器，通过 sessions_spawn 调度以下 5 个 skill │ ├── research-survey/SKILL.md # 深度论文分析 + 方法对比 │ ├── research-plan/SKILL.md # 四部分实现计划 diff --git a/README.en.md b/README.en.md index cf163fd..f86c1d1 100644 --- a/README.en.md +++ b/README.en.md @@ -90,7 +90,7 @@ Driven by multi-agent iteration: the orchestrator holds hypotheses and all accum │ │──→│ edit them too │ │ arxiv_search │ └──────────────────────────────┘ │ openalex_search │ -│ github_search │ +│ paper_download │ │ paper_browser │ │ code_executor │ └──────────────────────────┘ @@ -121,7 +121,7 @@ The agents' hands and eyes: | Tool | Capability | |------|-----------| | `arxiv_search` / `openalex_search` | Search academic papers (arXiv + cross-disciplinary) | -| `github_search` | Search open-source code implementations | +| `paper_download` | Download papers by arXiv ID (.tex source with PDF fallback) or DOI (open-access PDF via Unpaywall). Input validation prevents shell injection. | | `paper_browser` | Paginated paper reading, avoids context overflow | | `code_executor` | Execute experiment code in `uv`-isolated environment | @@ -276,7 +276,6 @@ Check status anytime: | Skill | Description | |-------|-------------| | **write-review-paper** | Draft a review/survey paper from project research outputs. | -| **research-subscription** | Create/list/remove scheduled Scientify jobs via `scientify_cron_job` (research digests or plain reminders). | @@ -286,14 +285,9 @@ Check status anytime: | Tool | Description | |------|-------------| | `arxiv_search` | Search arXiv papers. Returns metadata (title, authors, abstract, ID). Supports sorting by relevance/date and date filtering. | -| `arxiv_download` | Batch download papers by arXiv ID. Prefers .tex source files (PDF fallback). | | `openalex_search` | Search cross-disciplinary academic papers via OpenAlex API. Returns DOI, authors, citation count, OA status. | -| `openreview_lookup` | Lookup OpenReview evidence by title/ID/forum. Returns decision, review rating/confidence aggregates, and review summaries. | -| `unpaywall_download` | Download open access PDFs by DOI via Unpaywall API. Non-OA papers are silently skipped. | -| `github_search` | Search GitHub repositories. Returns repo name, description, stars, URL. Supports language filtering and sorting. | +| `paper_download` | Download papers by arXiv ID (prefers .tex source, PDF fallback) or DOI (open-access PDF via Unpaywall). Validates all identifiers to prevent injection. Max 20 papers per call with rate limiting. | | `paper_browser` | Paginated browsing of large paper files (.tex/.md) to avoid context overflow. | -| `scientify_cron_job` | Manage scheduled Scientify jobs (`upsert`/`list`/`remove`). | -| `scientify_literature_state` | Persistent incremental state for subscriptions: dedupe, record, feedback, and status inspection. | @@ -308,9 +302,6 @@ Check status anytime: | `/projects` | List all projects | | `/project-switch ` | Switch active project | | `/project-delete ` | Delete a project | -| `/research-subscribe ...` | Create/update scheduled Scientify jobs | -| `/research-subscriptions` | Show your scheduled Scientify jobs | -| `/research-unsubscribe [job-id]` | Remove your scheduled Scientify jobs | diff --git a/README.md b/README.md index 0c848fe..7b061e6 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ Scientify 采用**新陈代谢模式**——持续地摄入、消化、沉淀、 │ │──→│ │ │ arxiv_search │ └──────────────────────────────┘ │ openalex_search │ -│ github_search │ +│ paper_download │ │ paper_browser │ │ code_executor │ └──────────────────────────┘ @@ -121,7 +121,7 @@ Agent 的手和眼： | 工具 | 能力 | |------|------| | `arxiv_search` / `openalex_search` | 搜索学术论文（arXiv + 跨学科） | -| `github_search` | 搜索开源代码实现 | +| `paper_download` | 下载论文：arXiv ID（优先 .tex 源文件，PDF 兜底）或 DOI（通过 Unpaywall 获取 OA PDF）。内置输入校验防止注入攻击。 | | `paper_browser` | 分页精读论文，避免上下文溢出 | | `code_executor` | 在 `uv` 隔离环境中执行实验代码 | diff --git a/skills/idea-generation/SKILL.md b/skills/idea-generation/SKILL.md index 89ff55a..b1ae4f2 100644 --- a/skills/idea-generation/SKILL.md +++ b/skills/idea-generation/SKILL.md @@ -90,7 +90,7 @@ arxiv_search({ query: "{user_topic}", max_results: 10 }) openalex_search({ query: "{user_topic}", max_results: 10 }) ``` -2. **Download papers:** 按 /paper-download 的方式下载到 `papers/` +2. **Download papers:** 使用 `paper_download` 工具下载到 `papers/` 3. **Clone reference repos (optional):** ```bash diff --git a/skills/metabolism/SKILL.md b/skills/metabolism/SKILL.md index 3b9c96f..f5a5a03 100644 --- a/skills/metabolism/SKILL.md +++ b/skills/metabolism/SKILL.md @@ -96,7 +96,7 @@ openalex_search({ 合并结果，按 arXiv ID / DOI 去重，**跳过 `processed_ids` 中已有的论文**。 -按 /paper-download 的方式下载新论文到 `papers/`（arXiv 优先 .tex 源文件，DOI 通过 Unpaywall 获取 OA PDF）。 +使用 `paper_download` 工具下载新论文到 `papers/`（arXiv 优先 .tex 源文件，DOI 通过 Unpaywall 获取 OA PDF）。 ### Step 2: Read（阅读） diff --git a/skills/paper-download/SKILL.md b/skills/paper-download/SKILL.md deleted file mode 100644 index 41b309c..0000000 --- a/skills/paper-download/SKILL.md +++ /dev/null @@ -1,61 +0,0 @@ ---- -name: paper-download -description: "Download academic papers: arXiv source/PDF by ID, DOI papers via Unpaywall open access. Supports batch download." ---- - -# Paper Download - -将论文下载到当前工作目录的 `papers/` 下。 - -## arXiv 论文 - -**优先下载 .tex 源文件**（可读性远优于 PDF）： - -```bash -mkdir -p papers/{arxiv_id} -curl -L "https://arxiv.org/src/{arxiv_id}" | tar -xz -C papers/{arxiv_id} -``` - -如果 tar 解压失败（部分论文只提供 PDF），回退到 PDF： - -```bash -curl -L -o papers/{arxiv_id}.pdf "https://arxiv.org/pdf/{arxiv_id}" -``` - -> arXiv 限速：连续下载时每篇间隔 3 秒（`sleep 3`）。 - -## DOI 论文（通过 Unpaywall） - -查询开放获取链接，有则下载，无则跳过： - -```bash -curl -s "https://api.unpaywall.org/v2/{doi}?email=research@openclaw.ai" | \ - python3 -c " -import sys, json -d = json.load(sys.stdin) -oa = d.get('best_oa_location') or {} -url = oa.get('url_for_pdf') or oa.get('url') -if url: print(url) -else: print('NO_OA', file=sys.stderr) -" | xargs -I{} curl -L -o papers/{doi_slug}.pdf "{}" -``` - -> `{doi_slug}` = DOI 中的 `/` 替换为 `_`，例如 `10.1000/xyz123` → `10.1000_xyz123`。 -> 非开放获取论文静默跳过，不报错。 - -## 批量下载 - -```bash -# 批量 arXiv -for id in 2401.12345 2403.00001 2405.67890; do - mkdir -p papers/$id - curl -L "https://arxiv.org/src/$id" | tar -xz -C papers/$id || \ - curl -L -o papers/$id.pdf "https://arxiv.org/pdf/$id" - sleep 3 -done -``` - -## 下载后 - -- 下载的论文 ID 应追加到 `config.json` 的 `processed_ids`（如果存在） -- 优先读 `.tex` 源码而非 PDF（信息更完整，公式可直接提取） diff --git a/skills/research-collect/SKILL.md b/skills/research-collect/SKILL.md index 10ec361..a09f24d 100644 --- a/skills/research-collect/SKILL.md +++ b/skills/research-collect/SKILL.md @@ -61,7 +61,7 @@ openalex_search({ query: "", max_results: 20 }) #### 2.3 下载论文 -按 /paper-download 的方式下载论文到 `papers/`。 +使用 `paper_download` 工具下载论文到 `papers/`。 **完成一个检索词后，再进行下一个。** 这样避免上下文被大量搜索结果污染。 @@ -140,5 +140,5 @@ mv "papers/2401.12345" "papers/data-driven/" |----------------|---------| | `arxiv_search` | 搜索 arXiv 论文 | | `openalex_search` | 搜索跨学科论文（覆盖更广） | -| /paper-download | 下载论文（arXiv .tex/PDF、DOI via Unpaywall） | +| paper_download | 下载论文（arXiv .tex/PDF、DOI via Unpaywall） | | `gh search repos "query"` | 搜索 GitHub 仓库 |