diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 00000000..4bef45a1 Binary files /dev/null and b/.DS_Store differ diff --git a/.github/workflows/build_docker_dev.yml b/.github/workflows/build_docker_dev.yml index 4b6564db..966d558b 100644 --- a/.github/workflows/build_docker_dev.yml +++ b/.github/workflows/build_docker_dev.yml @@ -9,62 +9,110 @@ on: - 'docker-compose.yml' - 'docker-compose-warp.yml' - 'docs/**' - - '.github/workflows/build_docker_main.yml' - - '.github/workflows/build_docker_dev.yml' workflow_dispatch: +permissions: + contents: write + packages: write + jobs: main: runs-on: ubuntu-latest steps: - name: Check out the repository - uses: actions/checkout@v2 + uses: actions/checkout@v5 + with: + fetch-depth: 0 - name: Read the version from version.txt id: get_version run: | version=$(cat version.txt) echo "Current version: v$version-dev" - echo "::set-output name=version::v$version-dev" + echo "version=v$version-dev" >> "$GITHUB_OUTPUT" - name: Commit and push version tag - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | version=${{ steps.get_version.outputs.version }} git config --local user.email "action@github.com" git config --local user.name "GitHub Action" - git tag "$version" - git push https://x-access-token:${GHCR_PAT}@github.com/lanqian528/chat2api.git "$version" + if git rev-parse "$version" >/dev/null 2>&1; then + echo "Tag $version already exists" + else + git tag "$version" + git push origin "$version" + fi - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 + + - name: Log in to GHCR + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker meta for GHCR + id: meta_ghcr + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/chat2api + tags: | + type=raw,value=latest-dev + type=raw,value=${{ steps.get_version.outputs.version }} + + - name: Build and push GHCR image + uses: docker/build-push-action@v6 + with: + context: . + platforms: linux/amd64,linux/arm64 + file: Dockerfile + push: true + tags: ${{ steps.meta_ghcr.outputs.tags }} + labels: ${{ steps.meta_ghcr.outputs.labels }} + + - name: Detect Docker Hub credentials + id: dockerhub_creds + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + run: | + if [ -n "$DOCKER_USERNAME" ] && [ -n "$DOCKER_PASSWORD" ]; then + echo "present=true" >> "$GITHUB_OUTPUT" + else + echo "present=false" >> "$GITHUB_OUTPUT" + echo "Docker Hub credentials not configured; skipping Docker Hub publish." + fi - name: Log in to Docker Hub - uses: docker/login-action@v3 + if: ${{ steps.dockerhub_creds.outputs.present == 'true' }} + uses: docker/login-action@v4 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Docker meta - id: meta + - name: Docker meta for Docker Hub + if: ${{ steps.dockerhub_creds.outputs.present == 'true' }} + id: meta_dockerhub uses: docker/metadata-action@v5 with: - images: lanqian528/chat2api + images: nanashiwang/chat2api tags: | type=raw,value=latest-dev type=raw,value=${{ steps.get_version.outputs.version }} - - name: Build and push - uses: docker/build-push-action@v5 + - name: Build and push Docker Hub image + if: ${{ steps.dockerhub_creds.outputs.present == 'true' }} + uses: docker/build-push-action@v6 with: context: . platforms: linux/amd64,linux/arm64 file: Dockerfile push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + tags: ${{ steps.meta_dockerhub.outputs.tags }} + labels: ${{ steps.meta_dockerhub.outputs.labels }} diff --git a/.github/workflows/build_docker_main.yml b/.github/workflows/build_docker_main.yml index 4c7c6386..b833be79 100644 --- a/.github/workflows/build_docker_main.yml +++ b/.github/workflows/build_docker_main.yml @@ -9,62 +9,110 @@ on: - 'docker-compose.yml' - 'docker-compose-warp.yml' - 'docs/**' - - '.github/workflows/build_docker_main.yml' - - '.github/workflows/build_docker_dev.yml' workflow_dispatch: +permissions: + contents: write + packages: write + jobs: main: runs-on: ubuntu-latest steps: - name: Check out the repository - uses: actions/checkout@v2 + uses: actions/checkout@v5 + with: + fetch-depth: 0 - name: Read the version from version.txt id: get_version run: | version=$(cat version.txt) echo "Current version: v$version" - echo "::set-output name=version::v$version" + echo "version=v$version" >> "$GITHUB_OUTPUT" - name: Commit and push version tag - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | version=${{ steps.get_version.outputs.version }} git config --local user.email "action@github.com" git config --local user.name "GitHub Action" - git tag "$version" - git push https://x-access-token:${GHCR_PAT}@github.com/lanqian528/chat2api.git "$version" + if git rev-parse "$version" >/dev/null 2>&1; then + echo "Tag $version already exists" + else + git tag "$version" + git push origin "$version" + fi - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 + + - name: Log in to GHCR + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker meta for GHCR + id: meta_ghcr + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/chat2api + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=raw,value=${{ steps.get_version.outputs.version }} + + - name: Build and push GHCR image + uses: docker/build-push-action@v6 + with: + context: . + platforms: linux/amd64,linux/arm64 + file: Dockerfile + push: true + tags: ${{ steps.meta_ghcr.outputs.tags }} + labels: ${{ steps.meta_ghcr.outputs.labels }} + + - name: Detect Docker Hub credentials + id: dockerhub_creds + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + run: | + if [ -n "$DOCKER_USERNAME" ] && [ -n "$DOCKER_PASSWORD" ]; then + echo "present=true" >> "$GITHUB_OUTPUT" + else + echo "present=false" >> "$GITHUB_OUTPUT" + echo "Docker Hub credentials not configured; skipping Docker Hub publish." + fi - name: Log in to Docker Hub - uses: docker/login-action@v3 + if: ${{ steps.dockerhub_creds.outputs.present == 'true' }} + uses: docker/login-action@v4 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Docker meta - id: meta + - name: Docker meta for Docker Hub + if: ${{ steps.dockerhub_creds.outputs.present == 'true' }} + id: meta_dockerhub uses: docker/metadata-action@v5 with: - images: lanqian528/chat2api + images: nanashiwang/chat2api tags: | type=raw,value=latest,enable={{is_default_branch}} type=raw,value=${{ steps.get_version.outputs.version }} - - name: Build and push - uses: docker/build-push-action@v5 + - name: Build and push Docker Hub image + if: ${{ steps.dockerhub_creds.outputs.present == 'true' }} + uses: docker/build-push-action@v6 with: context: . platforms: linux/amd64,linux/arm64 file: Dockerfile push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file + tags: ${{ steps.meta_dockerhub.outputs.tags }} + labels: ${{ steps.meta_dockerhub.outputs.labels }} diff --git a/README.md b/README.md index e3d090c0..0360c4bd 100644 --- a/README.md +++ b/README.md @@ -10,205 +10,497 @@ 👮 配套用户管理端[Chat-Share](https://github.com/h88782481/Chat-Share)使用前需提前配置好环境变量（ENABLE_GATEWAY设置为True，AUTO_SEED设置为False） +--- -## 交流群 +## ✨ nanashiwang 分支新特性 -[https://t.me/chat2api](https://t.me/chat2api) +> 本分支在上游基础上做了大量工程化增强，**适合生产部署**。完整说明见 [`docs/FEATURES.md`](docs/FEATURES.md)。 -要提问请先阅读完仓库文档，尤其是常见问题部分。 +### 一键部署（零交互） -提问时请提供： +```bash +curl -fsSL https://raw.githubusercontent.com/nanashiwang/chat2api/main/deploy/install.sh | bash +``` -1. 启动日志截图（敏感信息打码，包括环境变量和版本号） -2. 报错的日志信息（敏感信息打码） -3. 接口返回的状态码和响应体 +脚本默认部署多实例编排面板：装 Docker → 下载编排脚本 → 启动 orchestrator → 安装 `chat2api` 管理命令 → 打印访问地址。 + +### 新增能力一览 + +| 能力 | 说明 | 文档 | +|---|---|---| +| 🛡️ **Antiban 风控层** | IP-账号粘性桶 / 账号冷却 / 地域一致性 / 熔断自愈 | [FEATURES#1](docs/FEATURES.md#1-antiban-风控规避层) | +| 🍪 **Harvester 采集** | UI 上粘贴 chatgpt.com session cookie 自动验证+导入 | [COOKIE_HARVEST](docs/COOKIE_HARVEST.md) | +| 📂 **文件上传导入** | .txt / .json 自动解析，预览后确认导入 | [FEATURES#3](docs/FEATURES.md#3-管理后台增强) | +| 📝 **系统日志 UI** | 实时轮询 / 级别筛选 / 关键字搜索 / 一键下载 | [FEATURES#4](docs/FEATURES.md#4-系统日志-ui) | +| 🔐 **安全加固** | IP 白名单 / HttpOnly / CSRF / 密码隔离 / CF 指引 | [SECURITY](docs/SECURITY.md) | +| 🔄 **UI 代理热加载** | 添加/删除代理即时生效，不需重启 | [FEATURES#3](docs/FEATURES.md#3-管理后台增强) | +| 🎯 **新版 Token 识别** | 支持 `rt_*` 新格式 + `sess-*` SessionToken + chat_refresh 现代化 | [FEATURES#6](docs/FEATURES.md#6-新版-token-支持) | +| 🧩 **一容器一账号编排** | `deploy/multi/` 提供生成器 + nginx 路径分发 + orchestrator 面板，N 个账号 = N 个隔离容器 | [部署：多实例](#多实例一容器一账号) | +| 🔗 **LibreChat 会话续接** | request body 携带 `librechat_conversation_id` 即可让同窗口在 ChatGPT 端续会话（节省 token + 用上账号原生记忆） | [LibreChat 集成](#librechat--new-api-集成) | + +### 核心运维流程 + +``` +1. 一键部署 (install.sh) + ↓ +2. 登录管理后台 (URL 从部署脚本最后输出) + ↓ +3. 配置 IP 白名单 (SECURITY.md) ← 强烈建议 + ↓ +4. (可选) 代理与路由 → 添加住宅代理 + ↓ +5. 账号采集 Harvester → 🍪 粘贴 Cookie ← 主流程 + ↓ +6. 开始使用 /v1/chat/completions + ↓ +7. 日常升级 → `chat2api update` + ↓ +8. Cookie 过期 (数月后) → 重抓并替换 +``` + +### Cookie 抓取快速链接 + +需要在**和 chat2api 出口 IP 相似的地区**登录 chatgpt.com 抓 cookie。完整指南（含 SSH 隧道技巧）见 [`docs/COOKIE_HARVEST.md`](docs/COOKIE_HARVEST.md)。 + +速抓命令： +```javascript +// 浏览器登录 chatgpt.com 后，F12 Console 执行 +document.cookie.split(';').filter(x=>x.includes('session-token')).join('; ') +``` + +--- + +## LibreChat / New-API 集成 + +> 适用场景：`LibreChat → New-API → chat2api → ChatGPT` 链路下，希望**同一对话窗口在 ChatGPT 服务端续会话**（账号原生 Memory 自动累积，每次只发最新一条 user message 节省 token）。 + +### 工作原理 + +``` +[LibreChat 窗口 X] [chat2api] + messages = [system, u1, a1, u2, a2, u3] ① 看到 librechat_conversation_id + body 含 librechat_conversation_id ② 查 sqlite 映射 → ChatGPT conv_id + ↓ ③ 命中：注入 conversation_id + +[New-API Channel Affinity] parent_message_id，messages 截短 + 按 lc_conv_id 路由到固定渠道 ④ 转发 ChatGPT + ↓ ⑤ 嗅探响应里的 conv_id 回写映射 +[chat2api 实例 K] + ↓ +[ChatGPT 服务端] 续接 conv，自动用上账号原生记忆 +``` + +### 三段配置（chat2api 已默认开启） + +#### 1. LibreChat（`librechat.yaml`，零代码） + +```yaml +endpoints: + custom: + - name: "chat2api" + apiKey: "${NEWAPI_KEY}" + baseURL: "https://your-newapi/v1" + addParams: + librechat_conversation_id: "{{LIBRECHAT_BODY_CONVERSATIONID}}" + librechat_user_id: "{{LIBRECHAT_USER_ID}}" + models: + default: ["gpt-5-5", "gpt-5", "gpt-4o", "gpt-4o-mini", "o1-preview", "o3-mini"] +``` + +#### 2. New-API Channel Affinity（后台 UI 配置） + +```json +{ + "enabled": true, + "rules": [{ + "name": "librechat_conv_sticky", + "model_regex": ["gpt.*", "o1.*", "o3.*"], + "key_sources": [{"type": "gjson", "path": "librechat_conversation_id"}], + "ttl_seconds": 86400, + "switch_on_success": true, + "skip_retry_on_failure": false + }] +} +``` + +#### 3. chat2api（默认开启，对裸 OpenAI 客户端无影响） + +| 环境变量 | 默认 | 说明 | +|---|---|---| +| `ENABLE_SESSION_STICKY` | `true` | 总开关 | +| `SESSION_TTL_DAYS` | `30` | 多少天未活跃自动清理映射 | +| `SESSION_LC_FIELD` | `librechat_conversation_id` | request body 中携带 LibreChat conversationId 的字段名 | +| `SESSION_TRIM_TO_LAST_USER` | `true` | 命中映射时是否把 messages[] 截到只含最后一条 user（依赖 ChatGPT 服务端续历史） | + +数据存储：`/app/data/sessions.db`（SQLite，跟随实例数据卷）。 +映射失效（ChatGPT 端 conv 被删）→ 自动清理 + `async_retry` 重新建对话。 + +### 验证 + +```bash +# 在某 chat2api 实例容器内 +docker exec -it c2a- sqlite3 /app/data/sessions.db \ + "SELECT * FROM lc_session_map LIMIT 5;" + +# 查看命中日志 +docker logs c2a- | grep session_sticky +# 应有: [session_sticky] hit lc=lc-uuid... → cv=cv-XXX... +``` + +--- + +## 多实例（一容器一账号） + +> 适用场景：N 个 ChatGPT 账号 + 多用户并发；通过 `deploy/multi/` 把每个账号编排到独立容器（独立代理 / 独立指纹 / 独立 cookie 卷），单账号被风控时其他账号不连坐。 + +### 一句话部署 + +```bash +curl -fsSL https://raw.githubusercontent.com/nanashiwang/chat2api/main/deploy/install.sh | bash +``` + +初始化完成后，脚本会直接输出编排面板入口和密码，然后打开： + +```text +http://:60403/orchestrator/ +``` + +如果你已经用一键脚本装了单实例，请走迁移流程，避免 60403 端口冲突： + +```bash +chat2api migrate prep +chat2api migrate apply +``` + +进入面板后点「新增账号」，填写 `slug`、代理和备注即可；不需要手动编辑 `accounts.csv`。 + +`accounts.csv` 仍然保留给批量导入/脚本化部署使用。 + +### 统一调用入口 + +多实例会额外暴露一个统一 OpenAI 兼容入口，由 Orchestrator 自动均衡到各个账号容器： + +```text +Base URL: http://:60403/v1 +API Key: 编排面板右上角「统一 API」查看 +``` + +路由策略： + +- 无会话键：轮询分配到不同容器 +- 有 `librechat_conversation_id` / `conversation_id` / `X-Chat2API-Affinity`：固定到同一容器 +- Orchestrator 会把统一 Key 转成目标容器自己的 `AUTHORIZATION` -## 功能 - -### 最新版本号存于 `version.txt` - -### 逆向API 功能 -> - [x] 流式、非流式传输 -> - [x] 免登录 GPT-3.5 对话 -> - [x] GPT-3.5 模型对话（传入模型名不包含 gpt-4，则默认使用 gpt-3.5，也就是 text-davinci-002-render-sha） -> - [x] GPT-4 系列模型对话（传入模型名包含: gpt-4，gpt-4o，gpt-4o-mini，gpt-4-moblie 即可使用对应模型，需传入 AccessToken） -> - [x] O1 系列模型对话（传入模型名包含 o1-preview，o1-mini 即可使用对应模型，需传入 AccessToken） -> - [x] GPT-4 模型画图、代码、联网 -> - [x] 支持 GPTs（传入模型名：gpt-4-gizmo-g-*） -> - [x] 支持 Team Plus 账号（需传入 team account id） -> - [x] 上传图片、文件（格式为 API 对应格式，支持 URL 和 base64） -> - [x] 可作为网关使用，可多机分布部署 -> - [x] 多账号轮询，同时支持 `AccessToken` 和 `RefreshToken` -> - [x] 请求失败重试，自动轮询下一个 Token -> - [x] Tokens 管理，支持上传、清除 -> - [x] 定时使用 `RefreshToken` 刷新 `AccessToken` / 每次启动将会全部非强制刷新一次，每4天晚上3点全部强制刷新一次。 -> - [x] 支持文件下载，需要开启历史记录 -> - [x] 支持 `O3-mini/high`、`O1/mini/Pro` 等模型推理过程输出 - -### 官网镜像 功能 -> - [x] 支持官网原生镜像 -> - [x] 后台账号池随机抽取，`Seed` 设置随机账号 -> - [x] 输入 `RefreshToken` 或 `AccessToken` 直接登录使用 -> - [x] 支持 `O3-mini/high`、`O1/mini/Pro`、`GPT-4/4o/mini` -> - [x] 敏感信息接口禁用、部分设置接口禁用 -> - [x] /login 登录页面，注销后自动跳转到登录页面 -> - [x] /?token=xxx 直接登录, xxx 为 `RefreshToken` 或 `AccessToken` 或 `SeedToken` (随机种子) -> - [x] 支持不同 SeedToken 会话隔离 -> - [x] 支持 `GPTs` 商店 -> - [x] 支持 `DeepReaserch`、`Canvas` 等官网独有功能 -> - [x] 支持切换各国语言 - - -> TODO -> - [ ] 暂无，欢迎提 `issue` - -## 逆向API - -完全 `OpenAI` 格式的 API ，支持传入 `AccessToken` 或 `RefreshToken`，可用 GPT-4, GPT-4o, GPT-4o-Mini, GPTs, O1-Pro, O1, O1-Mini, O3-Mini, O3-Mini-High： +### 已默认应用的工程加固（`deploy/multi/generate.py`） + +| 类别 | 项 | 默认 | +|---|---|---| +| 风控 | `ENABLE_ANTIBAN` | `true` | +| 风控 | `STRICT_IP_BINDING` | `true` | +| 风控 | `BUCKET_MAX_ACCOUNTS_PER_IP` | `1` | +| 风控 | 账号级冷却 (`ACCOUNT_MIN_INTERVAL_SECONDS`) | `0`（一容器一账号无需自限速） | +| 风控 | 429/403 熔断退避 | 1800/3600s | +| 安全 | `cap_drop: [ALL]` + `no-new-privileges` | ✅ | +| 资源 | `mem_limit / cpus / pids_limit` | 512m / 0.5 / 200 | +| 续会话 | `ENABLE_SESSION_STICKY` | `true` | + +### 从单实例迁移到多实例 + +```bash +chat2api migrate prep # 备份 + 生成 accounts.csv 模板（安全） +chat2api migrate apply # 停单 + 启多（需输 yes 确认） +# 不满意可回滚： +chat2api migrate rollback ~/chat2api.backup-YYYYMMDD-HHMMSS +``` + +迁移完成后打开 orchestrator 面板管理账号；如需批量预置账号，再编辑 `~/chat2api/deploy/multi/accounts.csv`。 + +--- + +## 功能总览 + +### OpenAI 兼容接口 + +- 流式 / 非流式响应 +- 模型支持：免登录 `GPT-3.5`、`GPT-4 / 4o / 4o-mini`、`GPT-5 / 5-mini / 5-thinking / 5-pro / 5-5`、`o1 / o1-mini / o1-preview / o1-pro`、`o3-mini / o3-mini-high` +- GPTs（`gpt-4-gizmo-g-*`）/ Team / Plus 账号 / 文件 / 图片 / 联网 / 画图 +- AccessToken / RefreshToken / SessionToken（`rt_*` 新格式）多 Tokens 轮询 + 失败自动重试 +- O3 / O1 系列推理过程输出 +- conversation_id / parent_message_id 续接（用于 [LibreChat 集成](#librechat--new-api-集成)） + +### 官网镜像（Gateway 模式） + +`ENABLE_GATEWAY=true` 后启用： + +- `/login` 登录页 + 后台账号池随机抽取（`Seed` 设置随机账号） +- `/?token=xxx` 直接登录（值为 RefreshToken / AccessToken / SeedToken） +- 不同 SeedToken 会话隔离 +- 支持 GPTs 商店、DeepResearch、Canvas +- 多语言切换、敏感接口禁用 + +### 工程化能力（nanashiwang 分支） + +完整能力清单见上文 [✨ nanashiwang 分支新特性](#-nanashiwang-分支新特性) 表格。 + +--- + +## API 使用 + +### 调用示例 + +```bash +curl --location 'http://127.0.0.1:5005/${API_PREFIX}/v1/chat/completions' \ + --header 'Content-Type: application/json' \ + --header 'Authorization: Bearer {{Token}}' \ + --data '{ + "model": "gpt-5-5", + "messages": [{"role": "user", "content": "Say this is a test!"}], + "stream": true + }' +``` + +`{{Token}}` 可以是： + +- 你账号的 `AccessToken` / `RefreshToken` —— 单账号直连 +- 你设置的 `AUTHORIZATION` 环境变量值 —— 后台 Tokens 轮询（推荐） +- LibreChat → New-API 流转下来的 New-API key —— 见 [LibreChat 集成](#librechat--new-api-集成) + +### Team 账号 + +传 `ChatGPT-Account-ID` header，或将其拼接在 Authorization： + +``` +Authorization: Bearer , +``` + +### 深度研究（Deep Research） + +API 端兼容 ChatGPT 的深度研究功能，支持**两种触发方式**（任选其一）： + +**方式 A：模型名后缀**（OpenAI 兼容客户端首选） ```bash -curl --location 'http://127.0.0.1:5005/v1/chat/completions' \ ---header 'Content-Type: application/json' \ ---header 'Authorization: Bearer {{Token}}' \ ---data '{ - "model": "gpt-3.5-turbo", - "messages": [{"role": "user", "content": "Say this is a test!"}], - "stream": true - }' +curl -N 'http://127.0.0.1:5005/${API_PREFIX}/v1/chat/completions' \ + -H 'Content-Type: application/json' \ + -H 'Authorization: Bearer {{Token}}' \ + -d '{ + "model": "o4-mini-deep-research", + "stream": true, + "messages": [{"role":"user","content":"2026 年量子计算的主要突破有哪些？请给出引用"}] + }' ``` -将你账号的 `AccessToken` 或 `RefreshToken` 作为 `{{ Token }}` 传入。 -也可填写你设置的环境变量 `Authorization` 的值, 将会随机选择后台账号 +可用别名：`o3-deep-research` / `o4-mini-deep-research` / `gpt-4o-deep-research` / `deep-research` -如果有team账号，可以传入 `ChatGPT-Account-ID`，使用 Team 工作区： +**方式 B：`system_hints` 透传**（高级用法） + +```bash +curl -N 'http://127.0.0.1:5005/${API_PREFIX}/v1/chat/completions' \ + -H 'Content-Type: application/json' \ + -H 'Authorization: Bearer {{Token}}' \ + -d '{ + "model": "gpt-5-5", + "system_hints": ["research"], + "stream": true, + "messages": [{"role":"user","content":"对比主流 LLM 推理引擎的吞吐量与成本"}] + }' +``` -- 传入方式一： -`headers` 中传入 `ChatGPT-Account-ID`值 +也可使用快捷开关 `"deep_research": true`。 -- 传入方式二： -`Authorization: Bearer ,` +**注意事项：** -如果设置了 `AUTHORIZATION` 环境变量，可以将设置的值作为 `{{ Token }}` 传入进行多 Tokens 轮询。 +- 账号需具备深度研究权限（Plus / Pro / Team），且本月额度未耗尽 +- 单次任务耗时通常 5~15 分钟，反向代理需调高读取超时：`nginx proxy_read_timeout ≥ 1800s`、`cloudflare` 建议改用 WebSocket / 直连 +- 流式响应包含：搜索查询提示 → 引用清单 → 中间思考 → 最终研究报告 +- 不支持与 Gizmo（GPTs）同时使用，触发时会自动降级为 `primary_assistant` 模式 -> - `AccessToken` 获取: chatgpt官网登录后，再打开 [https://chatgpt.com/api/auth/session](https://chatgpt.com/api/auth/session) 获取 `accessToken` 这个值。 -> - `RefreshToken` 获取: 此处不提供获取方法。 -> - 免登录 gpt-3.5 无需传入 Token。 +### Tokens 来源 -## Tokens 管理 +- **AccessToken**：登录 chatgpt.com 后访问 [`https://chatgpt.com/api/auth/session`](https://chatgpt.com/api/auth/session) 取 `accessToken` +- **RefreshToken / SessionToken**：浏览器登录后 F12 抓 cookie，**强烈建议走 Harvester UI**（[`docs/COOKIE_HARVEST.md`](docs/COOKIE_HARVEST.md)） +- **免登录 GPT-3.5**：无需 Token -1. 配置环境变量 `AUTHORIZATION` 作为 `授权码` ，然后运行程序。 +### Tokens 管理 -2. 访问 `/tokens` 或者 `/{api_prefix}/tokens` 可以查看现有 Tokens 数量，也可以上传新的 Tokens ，或者清空 Tokens。 +1. 设置 `AUTHORIZATION` 环境变量作为授权码 +2. 访问管理后台（`/{api_prefix}/admin/login`）→ Tokens 管理 / Harvester +3. 请求时把 `AUTHORIZATION` 当成 `APIKEY` 传入即可使用轮询 -3. 请求时传入 `AUTHORIZATION` 中配置的 `授权码` 即可使用轮询的Tokens进行对话 +--- -![tokens.png](docs/tokens.png) +## 环境变量参考 -## 官网原生镜像 +> 表格仅列**最常用**变量；未列出的请勿设置或使用默认值。完整参考可在容器启动日志的 `Environment variables:` 区域查看。 -1. 配置环境变量 `ENABLE_GATEWAY` 为 `true`，然后运行程序, 注意开启后别人也可以直接通过域名访问你的网关。 +| 分类 | 变量 | 默认值 | 说明 | +|---|---|---|---| +| 安全 | `API_PREFIX` | `None` | API 路径前缀（推荐设置，避免被扫描） | +| 安全 | `AUTHORIZATION` | `[]` | 授权码（多个用英文逗号分隔），用于 Tokens 轮询 | +| 安全 | `ADMIN_PASSWORD` | `None` | 管理后台登录密码 | +| 安全 | `ADMIN_IP_WHITELIST` | (空) | 管理后台 IP 白名单（CIDR 支持），强烈建议配置 | +| 请求 | `CHATGPT_BASE_URL` | `https://chatgpt.com` | 上游网关，多个用逗号分隔 | +| 请求 | `PROXY_URL` | `[]` | 全局代理 URL，多个用逗号分隔（也可 UI 配置） | +| 功能 | `HISTORY_DISABLED` | `true` | 不保存聊天记录到 ChatGPT 服务端 | +| 功能 | `ENABLE_LIMIT` | `true` | 不突破官方次数限制（防封号） | +| 功能 | `SCHEDULED_REFRESH` | `false` | 定时刷新 AccessToken | +| 功能 | `RANDOM_TOKEN` | `true` | 随机选取后台 Token（关闭则顺序轮询） | +| 网关 | `ENABLE_GATEWAY` | `false` | 启用官网镜像；开启后默认无认证，需配 `AUTH_KEY` 或 IP 白名单 | +| 网关 | `AUTO_SEED` | `true` | 启用随机账号模式（`seed` 参数自动匹配账号） | +| Antiban | `ENABLE_ANTIBAN` | `false`（multi 默认 `true`） | 风控规避层：IP 粘性桶 / 地域一致性 / 熔断自愈 | +| Antiban | `STRICT_IP_BINDING` | `true` | 无匹配代理时拒绝（不退化到母机直连） | +| Antiban | `BUCKET_MAX_ACCOUNTS_PER_IP` | `5`（multi 默认 `1`） | 每 IP 桶容纳的账号数 | +| Antiban | `CIRCUIT_429_COOLDOWN` | `1800` | 429 触发后账号冷却秒数 | +| Antiban | `CIRCUIT_403_COOLDOWN` | `3600` | 403/cf_chl_opt 触发后 IP 桶冷冻秒数 | +| Session | `ENABLE_SESSION_STICKY` | `false`（一键部署默认 `true`） | LibreChat 窗口级会话续接总开关 | +| Session | `SESSION_LC_FIELD` | `librechat_conversation_id` | LibreChat conversationId 在 request body 的字段名 | +| Session | `SESSION_TTL_DAYS` | `30` | 多少天未活跃自动清理映射 | +| Session | `SESSION_TRIM_TO_LAST_USER` | `true` | 命中映射时把 messages 截到最后一条 user | -2. 在 Tokens 管理页面上传 `RefreshToken` 或 `AccessToken` +详细文档： -3. 访问 `/login` 到登录页面 +- Antiban 工作原理：[`docs/FEATURES.md#1-antiban-风控规避层`](docs/FEATURES.md#1-antiban-风控规避层) +- 安全加固：[`docs/SECURITY.md`](docs/SECURITY.md) +- Cookie 采集：[`docs/COOKIE_HARVEST.md`](docs/COOKIE_HARVEST.md) +- 会话续接（本仓库新增）：[LibreChat 集成](#librechat--new-api-集成) -![login.png](docs/login.png) +--- -4. 进入官网原生镜像页面使用 +## 部署方式 -![chatgpt.png](docs/chatgpt.png) +### 一键部署（推荐） -## 环境变量 +零交互，默认安装多实例编排面板： -每个环境变量都有默认值，如果不懂环境变量的含义，请不要设置，更不要传空值，字符串无需引号。 +```bash +curl -fsSL https://raw.githubusercontent.com/nanashiwang/chat2api/main/deploy/install.sh | bash +``` -| 分类 | 变量名 | 示例值 | 默认值 | 描述 | -|------|-------------------|-------------------------------------------------------------|-----------------------|--------------------------------------------------------------| -| 安全相关 | API_PREFIX | `your_prefix` | `None` | API 前缀密码，不设置容易被人访问，设置后需请求 `/your_prefix/v1/chat/completions` | -| | AUTHORIZATION | `your_first_authorization`,
`your_second_authorization` | `[]` | 你自己为使用多账号轮询 Tokens 设置的授权码，英文逗号分隔 | -| | AUTH_KEY | `your_auth_key` | `None` | 私人网关需要加`auth_key`请求头才设置该项 | -| 请求相关 | CHATGPT_BASE_URL | `https://chatgpt.com` | `https://chatgpt.com` | ChatGPT 网关地址，设置后会改变请求的网站，多个网关用逗号分隔 | -| | PROXY_URL | `http://ip:port`,
`http://username:password@ip:port` | `[]` | 全局代理 URL，出 403 时启用，多个代理用逗号分隔 | -| | EXPORT_PROXY_URL | `http://ip:port`或
`http://username:password@ip:port` | `None` | 出口代理 URL，防止请求图片和文件时泄漏源站 ip | -| 功能相关 | HISTORY_DISABLED | `true` | `true` | 是否不保存聊天记录并返回 conversation_id | -| | POW_DIFFICULTY | `00003a` | `00003a` | 要解决的工作量证明难度，不懂别设置 | -| | RETRY_TIMES | `3` | `3` | 出错重试次数，使用 `AUTHORIZATION` 会自动随机/轮询下一个账号 | -| | CONVERSATION_ONLY | `false` | `false` | 是否直接使用对话接口，如果你用的网关支持自动解决 `POW` 才启用 | -| | ENABLE_LIMIT | `true` | `true` | 开启后不尝试突破官方次数限制，尽可能防止封号 | -| | UPLOAD_BY_URL | `false` | `false` | 开启后按照 `URL+空格+正文` 进行对话，自动解析 URL 内容并上传，多个 URL 用空格分隔 | -| | SCHEDULED_REFRESH | `false` | `false` | 是否定时刷新 `AccessToken` ，开启后每次启动程序将会全部非强制刷新一次，每4天晚上3点全部强制刷新一次。 | -| | RANDOM_TOKEN | `true` | `true` | 是否随机选取后台 `Token` ，开启后随机后台账号，关闭后为顺序轮询 | -| 网关功能 | ENABLE_GATEWAY | `false` | `false` | 是否启用网关模式，开启后可以使用镜像站，但也将会不设防 | -| | AUTO_SEED | `false` | `true` | 是否启用随机账号模式，默认启用，输入`seed`后随机匹配后台`Token`。关闭之后需要手动对接接口，来进行`Token`管控。 | +如需旧的单实例模式： -## 部署 +```bash +CHAT2API_MODE=single bash <(curl -fsSL https://raw.githubusercontent.com/nanashiwang/chat2api/main/deploy/install.sh) +``` -### Zeabur 部署 +可选环境变量： -[![Deploy on Zeabur](https://zeabur.com/button.svg)](https://zeabur.com/templates/6HEGIZ?referralCode=LanQian528) +| 变量 | 用途 | +|---|---| +| `INSTALL_DIR` | 自定义安装目录（默认 `~/chat2api`） | +| `CHAT2API_PORT` | 监听端口（默认 `60403`） | +| `CHAT2API_MODE` | `multi`（默认）或 `single` | +| `INTERACTIVE=1` | 交互式询问密码 / API 前缀 | -### 直接部署 +### 直接源码部署 ```bash -git clone https://github.com/LanQian528/chat2api +git clone https://github.com/nanashiwang/chat2api cd chat2api pip install -r requirements.txt python app.py ``` -### Docker 部署 - -您需要安装 Docker 和 Docker Compose。 +### Docker ```bash -docker run -d \ - --name chat2api \ +docker run -d --name chat2api \ -p 5005:5005 \ - lanqian528/chat2api:latest + -v $(pwd)/data:/app/data \ + -e AUTHORIZATION=sk-your-key \ + ghcr.io/nanashiwang/chat2api:latest ``` -### (推荐，可用 PLUS 账号) Docker Compose 部署 - -创建一个新的目录，例如 chat2api，并进入该目录： +### Docker Compose（自定义部署） ```bash -mkdir chat2api -cd chat2api +mkdir chat2api && cd chat2api +curl -fsSL https://raw.githubusercontent.com/nanashiwang/chat2api/main/deploy/docker-compose.template.yml -o docker-compose.yml +# 创建 .env 写入 ADMIN_PASSWORD / AUTHORIZATION / API_PREFIX +docker compose up -d ``` -在此目录中下载库中的 docker-compose.yml 文件： +### chat2api CLI 命令 + +部署完成后全局命令自动可用，单/多实例自动适配： ```bash -wget https://raw.githubusercontent.com/LanQian528/chat2api/main/docker-compose-warp.yml +# 通用 +chat2api status # 容器状态（多实例下含出口 IP 抽样） +chat2api update # 拉镜像并重建 +chat2api logs [slug] # 实时日志 +chat2api restart / stop / start +chat2api uninstall # 卸载：停止服务并删除安装目录 +chat2api uninstall --keep-data # 只停止服务，保留安装目录和数据 +chat2api path # 打印安装目录 + +# 单实例专属 +chat2api sync-template # 检测并合并上游 docker-compose 模板的新 ENV +chat2api migrate prep # 准备从单实例迁到多实例（仅备份+生成 csv） +chat2api migrate apply # 切换到多实例（destructive，需输 yes 确认） +chat2api migrate rollback # 从备份回滚到单实例 + +# 多实例专属 +chat2api verify # 校验每实例的 admin / tokens 路由 +chat2api secrets # 打印每实例 AUTH/ADMIN 凭据 + orchestrator 入口 +chat2api shell # 进入指定实例容器 shell +chat2api admin # 打印管理后台访问 URL ``` -修改 docker-compose-warp.yml 文件中的环境变量，保存后： +老机器升级最简方式： ```bash -docker-compose up -d +# 拉新版 chat2api.sh 脚本 +sudo curl -fsSL https://raw.githubusercontent.com/nanashiwang/chat2api/main/deploy/chat2api.sh \ + -o /usr/local/bin/chat2api && sudo chmod +x /usr/local/bin/chat2api +chat2api update # 拉镜像 +chat2api sync-template # 单实例：拉新 ENV ``` +--- ## 常见问题 -> - 错误代码： -> - `401`：当前 IP 不支持免登录，请尝试更换 IP 地址，或者在环境变量 `PROXY_URL` 中设置代理，或者你的身份验证失败。 -> - `403`：请在日志中查看具体报错信息。 -> - `429`：当前 IP 请求1小时内请求超过限制，请稍后再试，或更换 IP。 -> - `500`：服务器内部错误，请求失败。 -> - `502`：服务器网关错误，或网络不可用，请尝试更换网络环境。 +### 错误码 + +| 状态码 | 原因 | 处理 | +|---|---|---| +| `401` | 当前 IP 不支持免登录 / 鉴权失败 | 换 IP / 配代理 / 检查 `AUTHORIZATION` | +| `403` | 风控触发（cf_chl_opt / 区域限制） | 看日志，配住宅代理；antiban 会自动冷却 IP 桶 | +| `429` | 1 小时内请求超限 | 等待或换 IP；antiban 会自动账号冷却 | +| `500` | 服务器内部错误 | 看 chat2api 日志 | +| `502` | 上游网关错误 | 换网络环境 / 检查代理 | + +### 已知情况 + +- **日本 IP** 很多不支持免登录 GPT-3.5，建议美国 IP +- **GPT-4o 免费** 99% 账号支持，但按 IP 区域开启（日本/新加坡 IP 概率较高） +- **机房 IP**（DigitalOcean / WARP / Vultr 等）几乎已被风控；优先用住宅或移动蜂窝代理 + +### 提问前请准备 + +1. 启动日志截图（敏感信息打码，环境变量 + 版本号必备） +2. 报错日志（含 chat2api / nginx / 上游） +3. 接口返回的状态码和响应体 -> - 已知情况： -> - 日本 IP 很多不支持免登，免登 GPT-3.5 建议使用美国 IP。 -> - 99%的账号都支持免费 `GPT-4o` ，但根据 IP 地区开启，目前日本和新加坡 IP 已知开启概率较大。 +--- -> - 环境变量 `AUTHORIZATION` 是什么？ -> - 是一个自己给 chat2api 设置的一个身份验证，设置后才可使用已保存的 Tokens 轮询，请求时当作 `APIKEY` 传入。 -> - AccessToken 如何获取？ -> - chatgpt官网登录后，再打开 [https://chatgpt.com/api/auth/session](https://chatgpt.com/api/auth/session) 获取 `accessToken` 这个值。 +## 学习交流声明 +> ⚠️ **本项目仅供学习与技术交流，请勿用于任何商业或违反 [OpenAI 服务条款](https://openai.com/policies/terms-of-use) 的用途。** + +使用本项目即代表你已阅读并同意以下条款： + +- 仅出于个人学习、技术研究、交流目的使用 +- 不得用于任何形式的商业牟利 +- 不得用于任何违反 OpenAI 服务条款或所在地法律法规的活动 +- 一切因使用本项目产生的风险（账号被封、数据丢失、服务中断、法律责任等）由使用者**自行承担** +- 作者及贡献者不对使用本项目造成的任何直接或间接后果负责 + +如果你不接受上述任何一条，请立即停止使用并删除本项目。 + +本分支基于 [LanQian528/chat2api](https://github.com/LanQian528/chat2api) 二次开发，所有上游代码版权归原作者所有。 + +--- ## License MIT License - diff --git a/api/chat2api.py b/api/chat2api.py index 6892199f..29a4b167 100644 --- a/api/chat2api.py +++ b/api/chat2api.py @@ -1,5 +1,7 @@ import asyncio +import time import types +import uuid from apscheduler.schedulers.asyncio import AsyncIOScheduler from fastapi import Request, HTTPException, Form, Security @@ -11,20 +13,206 @@ from app import app, templates, security_scheme from chatgpt.ChatService import ChatService from chatgpt.authorization import refresh_all_tokens +from chatgpt import session_sticky +from utils.bootstrap import initialize_from_env from utils.Logger import logger -from utils.configs import api_prefix, scheduled_refresh +from utils.configs import api_prefix, scheduled_refresh, history_disabled, enable_session_sticky from utils.retry import async_retry +from utils import antiban +from utils.antiban import circuit as antiban_circuit scheduler = AsyncIOScheduler() +def _responses_input_to_text(value): + if value is None: + return "" + if isinstance(value, str): + return value + if isinstance(value, list): + chunks = [] + for item in value: + if isinstance(item, str): + chunks.append(item) + continue + if not isinstance(item, dict): + continue + item_type = item.get("type") + if item_type in {"input_text", "output_text", "text"}: + text = item.get("text") + if isinstance(text, str): + chunks.append(text) + elif item_type == "message": + chunks.append(_responses_input_to_messages(item)) + return "\n".join(part for part in chunks if part) + if isinstance(value, dict): + if isinstance(value.get("text"), str): + return value["text"] + if isinstance(value.get("content"), list): + return _responses_input_to_text(value["content"]) + return str(value) + + +def _responses_input_to_messages(value): + if isinstance(value, str): + return value + if isinstance(value, dict): + role = value.get("role") + if role in {"system", "developer", "user", "assistant"}: + content = _responses_input_to_text(value.get("content")) + return {"role": role, "content": content or ""} + return None + + +def _convert_responses_request_to_chat(payload): + messages = [] + instructions = payload.get("instructions") + if isinstance(instructions, str) and instructions.strip(): + messages.append({"role": "system", "content": instructions.strip()}) + + raw_input = payload.get("input") + if isinstance(raw_input, str): + messages.append({"role": "user", "content": raw_input}) + elif isinstance(raw_input, list): + for item in raw_input: + message = _responses_input_to_messages(item) + if message: + messages.append(message) + else: + text = _responses_input_to_text(item) + if text: + messages.append({"role": "user", "content": text}) + elif raw_input is not None: + text = _responses_input_to_text(raw_input) + if text: + messages.append({"role": "user", "content": text}) + + if not messages: + raise HTTPException(status_code=400, detail={"error": "input is required"}) + + chat_payload = { + "model": payload.get("model"), + "messages": messages, + "stream": bool(payload.get("stream", False)), + } + for key in ( + "temperature", + "top_p", + "max_output_tokens", + "presence_penalty", + "frequency_penalty", + "user", + ): + if key in payload: + value = payload[key] + if key == "max_output_tokens": + chat_payload["max_tokens"] = value + else: + chat_payload[key] = value + return chat_payload + + +def _convert_chat_response_to_responses(chat_response, request_payload): + choice = ((chat_response or {}).get("choices") or [{}])[0] + message = choice.get("message") or {} + output_text = message.get("content", "") or "" + usage = chat_response.get("usage") or {} + created = int(time.time()) + response_id = f"resp_{uuid.uuid4().hex}" + model = chat_response.get("model") or request_payload.get("model") + return { + "id": response_id, + "object": "response", + "created_at": created, + "status": "completed", + "model": model, + "output": [ + { + "id": f"msg_{uuid.uuid4().hex}", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": output_text, + "annotations": [], + } + ], + } + ], + "output_text": output_text, + "usage": { + "input_tokens": usage.get("prompt_tokens", 0), + "output_tokens": usage.get("completion_tokens", 0), + "total_tokens": usage.get("total_tokens", 0), + }, + "finish_reason": choice.get("finish_reason"), + } + + +def _compact_responses_payload(data): + usage = data.get("usage") or {} + return { + "id": data.get("id"), + "object": "response.compact", + "model": data.get("model"), + "output_text": data.get("output_text", ""), + "finish_reason": data.get("finish_reason"), + "usage": usage, + } + + +async def _process_responses_request(request_data, req_token): + chat_request_data = _convert_responses_request_to_chat(request_data) + if chat_request_data.get("stream"): + raise HTTPException(status_code=400, detail={"error": "stream responses is not supported yet"}) + + chat_service, res = await async_retry(process, chat_request_data, req_token) + try: + if isinstance(res, types.AsyncGeneratorType): + raise HTTPException(status_code=400, detail={"error": "stream responses is not supported yet"}) + return _convert_chat_response_to_responses(res, request_data) + finally: + await chat_service.close_client() + + @app.on_event("startup") async def app_start(): + initialize_from_env() + await antiban.init() + + # Session sticky: 启动时初始化 SQLite + 启动 TTL 清理定时任务 + if enable_session_sticky: + session_sticky.init_db() + scheduler.add_job( + id='session_sticky_cleanup', + func=session_sticky.cleanup_expired, + trigger='interval', + hours=24, + ) + + # Antiban 自愈定时任务 + from utils.configs import enable_antiban, circuit_bucket_heal_minutes + if enable_antiban: + scheduler.add_job( + id='antiban_heal', + func=antiban_circuit.scheduled_heal, + trigger='interval', + minutes=max(int(circuit_bucket_heal_minutes), 5), + ) + if scheduled_refresh: scheduler.add_job(id='refresh', func=refresh_all_tokens, trigger='cron', hour=3, minute=0, day='*/2', kwargs={'force_refresh': True}) scheduler.start() asyncio.get_event_loop().call_later(0, lambda: asyncio.create_task(refresh_all_tokens(force_refresh=False))) + elif enable_antiban: + # 只有 antiban 启用、没启用 refresh 时，也需要把 scheduler 跑起来 + scheduler.start() + elif enable_session_sticky: + # 仅 session_sticky 启用时，scheduler 也要启动以执行 cleanup + scheduler.start() async def to_send_conversation(request_data, req_token): @@ -44,9 +232,38 @@ async def to_send_conversation(request_data, req_token): async def process(request_data, req_token): chat_service = await to_send_conversation(request_data, req_token) - await chat_service.prepare_send_conversation() - res = await chat_service.send_conversation() - return chat_service, res + try: + await chat_service.prepare_send_conversation() + res = await chat_service.send_conversation() + return chat_service, res + except HTTPException as e: + await chat_service.close_client() + raise HTTPException(status_code=e.status_code, detail=e.detail) + except Exception as e: + await chat_service.close_client() + logger.error(f"Server error, {str(e)}") + raise HTTPException(status_code=500, detail="Server error") + + +def parse_bool_query(value, default): + if value is None: + return default + return str(value).lower() in ['true', '1', 't', 'y', 'yes'] + + +def format_models_response(model_slugs): + data = [] + for model_slug in sorted(model_slugs): + data.append({ + "id": model_slug, + "object": "model", + "created": 0, + "owned_by": "openai", + }) + return { + "object": "list", + "data": data, + } @app.post(f"/{api_prefix}/v1/chat/completions" if api_prefix else "/v1/chat/completions") @@ -56,7 +273,14 @@ async def send_conversation(request: Request, credentials: HTTPAuthorizationCred request_data = await request.json() except Exception: raise HTTPException(status_code=400, detail={"error": "Invalid JSON body"}) + # Session sticky: LibreChat conv_id → ChatGPT conv_id 翻译注入 + # 副作用: 命中映射时改写 request_data['conversation_id'/'parent_message_id'/'messages'] + # 返回 lc_conv_id 用于流式响应嗅探回写；未启用或无 lc 字段时返回 None + lc_conv_id = session_sticky.inject_session(request_data) if enable_session_sticky else None chat_service, res = await async_retry(process, request_data, req_token) + # 把 lc_conv_id 挂到 chat_service 上，供 stream_response 嗅探时回写 DB + if lc_conv_id: + chat_service.librechat_conv_id = lc_conv_id try: if isinstance(res, types.AsyncGeneratorType): background = BackgroundTask(chat_service.close_client) @@ -76,6 +300,63 @@ async def send_conversation(request: Request, credentials: HTTPAuthorizationCred raise HTTPException(status_code=500, detail="Server error") +@app.post(f"/{api_prefix}/v1/responses" if api_prefix else "/v1/responses") +async def send_responses(request: Request, credentials: HTTPAuthorizationCredentials = Security(security_scheme)): + req_token = credentials.credentials + try: + request_data = await request.json() + except Exception: + raise HTTPException(status_code=400, detail={"error": "Invalid JSON body"}) + try: + response_payload = await _process_responses_request(request_data, req_token) + return JSONResponse(response_payload, media_type="application/json") + except HTTPException as e: + raise HTTPException(status_code=e.status_code, detail=e.detail) + except Exception as e: + logger.error(f"Server error, {str(e)}") + raise HTTPException(status_code=500, detail="Server error") + + +@app.post(f"/{api_prefix}/v1/responses/compact" if api_prefix else "/v1/responses/compact") +async def send_responses_compact(request: Request, credentials: HTTPAuthorizationCredentials = Security(security_scheme)): + req_token = credentials.credentials + try: + request_data = await request.json() + except Exception: + raise HTTPException(status_code=400, detail={"error": "Invalid JSON body"}) + try: + data = await _process_responses_request(request_data, req_token) + return JSONResponse(_compact_responses_payload(data), media_type="application/json") + except HTTPException as e: + raise HTTPException(status_code=e.status_code, detail=e.detail) + except Exception as e: + logger.error(f"Server error, {str(e)}") + raise HTTPException(status_code=500, detail="Server error") + +@app.get(f"/{api_prefix}/v1/models" if api_prefix else "/v1/models") +async def list_models(request: Request, credentials: HTTPAuthorizationCredentials = Security(security_scheme)): + chat_service = ChatService(credentials.credentials) + try: + await chat_service.resolve_auth_context() + chat_service.history_disabled = parse_bool_query( + request.query_params.get("history_disabled", request.query_params.get("history_and_training_disabled")), + history_disabled, + ) + request_account_id = request.headers.get("ChatGPT-Account-ID") or request.headers.get("Chatgpt-Account-Id") + if request_account_id: + chat_service.account_id = request_account_id + await chat_service.initialize_request_context() + model_slugs = await chat_service.fetch_available_models() + return JSONResponse(format_models_response(model_slugs), media_type="application/json") + except HTTPException as e: + raise HTTPException(status_code=e.status_code, detail=e.detail) + except Exception as e: + logger.error(f"Server error, {str(e)}") + raise HTTPException(status_code=500, detail="Server error") + finally: + await chat_service.close_client() + + @app.get(f"/{api_prefix}/tokens" if api_prefix else "/tokens", response_class=HTMLResponse) async def upload_html(request: Request): tokens_count = len(set(globals.token_list) - set(globals.error_token_list)) @@ -133,4 +414,4 @@ async def clear_seed_tokens(): with open(globals.CONVERSATION_MAP_FILE, "w", encoding="utf-8") as f: f.write("{}") logger.info(f"Seed token count: {len(globals.seed_map)}") - return {"status": "success", "seed_tokens_count": len(globals.seed_map)} \ No newline at end of file + return {"status": "success", "seed_tokens_count": len(globals.seed_map)} diff --git a/api/image_generations.py b/api/image_generations.py new file mode 100644 index 00000000..5147c3cc --- /dev/null +++ b/api/image_generations.py @@ -0,0 +1,248 @@ +"""OpenAI 兼容的图片生成端点。 + +POST /v1/images/generations + +内部实现：把图片生成请求转换为强制触发 ChatGPT 网页 dalle.text2im 工具 +的 chat 对话，解析响应里的 markdown 图片 URL，包装为 OpenAI 格式返回。 + +支持 OpenAI 标准参数：prompt / model / n / size / quality / style / +response_format（url | b64_json）。 +""" + +import asyncio +import base64 +import re +import time + +from fastapi import HTTPException, Request, Security +from fastapi.responses import JSONResponse +from fastapi.security import HTTPAuthorizationCredentials + +from app import app, security_scheme +from api.chat2api import process +from chatgpt.ChatService import ChatService +from utils.configs import api_prefix +from utils.Logger import logger +from utils.retry import async_retry + + +IMAGE_MARKDOWN_RE = re.compile(r"!\[(?:image|File\s*\d+)\]\(([^)]+)\)", re.IGNORECASE) +DEFAULT_CHAT_MODEL = "gpt-5-5" +MAX_CONCURRENT_IMAGES = 4 +MAX_IMAGES_PER_REQUEST = 10 +IMAGE_DOWNLOAD_TIMEOUT = 30 +ALLOWED_RESPONSE_FORMATS = ("url", "b64_json") + + +def _build_image_generation_prompt(prompt, size, quality, style): + """构造强制触发 dalle.text2im 的 prompt 文本。""" + lines = [ + "Use the image generation tool dalle.text2im to create an image for the following description:", + f'"{prompt}"', + ] + if size: + lines.append(f"Size: {size}") + if quality: + lines.append(f"Quality: {quality}") + if style: + lines.append(f"Style: {style}") + lines.append("Generate the image directly. Do not ask for clarification.") + return "\n".join(lines) + + +def _extract_image_urls(content): + """从 chat 响应内容里提取所有图片 URL。""" + if not content: + return [] + return [m.group(1).strip() for m in IMAGE_MARKDOWN_RE.finditer(content)] + + +async def _resolve_target_model(req_token, requested_model): + """动态选 model：探测上游可用列表，命中即用，否则 fallback gpt-5-5。 + + 使用临时 ChatService 实例只做模型探测，用完即关闭，与后续图片生成调用不共享。 + """ + probe = ChatService(req_token) + try: + await probe.resolve_auth_context() + await probe.initialize_request_context() + slugs = await probe.fetch_available_models() + if requested_model and requested_model in slugs: + return requested_model + except Exception as e: + logger.warning(f"[image_gen] resolve_target_model failed, fallback gpt-5-5: {e}") + finally: + try: + await probe.close_client() + except Exception: + pass + return DEFAULT_CHAT_MODEL + + +async def _download_image_to_b64(chat_service, url): + """复用 chat_service 的 curl_cffi 客户端下载图片并 base64 编码。""" + headers = (getattr(chat_service, "base_headers", None) or {}).copy() + headers.pop("content-type", None) + try: + r = await chat_service.s.get(url, headers=headers, timeout=IMAGE_DOWNLOAD_TIMEOUT) + except Exception as e: + raise HTTPException(status_code=502, detail=f"Failed to download image: {str(e)}") + if r.status_code != 200: + raise HTTPException(status_code=502, detail=f"Failed to download image: status {r.status_code}") + return base64.b64encode(r.content).decode("ascii") + + +async def _generate_single_image(req_token, target_model, prompt, size, quality, style): + """执行一次图片生成 chat 流程，返回 (chat_service, content, urls)。 + + 成功时 chat_service 不在此处关闭，由上层 finally 统一释放（便于复用其客户端下载图片）。 + """ + chat_request = { + "model": target_model, + "messages": [ + { + "role": "user", + "content": _build_image_generation_prompt(prompt, size, quality, style), + } + ], + "stream": False, + } + chat_service, res = await async_retry(process, chat_request, req_token) + try: + content = ((res or {}).get("choices") or [{}])[0].get("message", {}).get("content", "") + urls = _extract_image_urls(content) + return chat_service, content, urls + except Exception: + await chat_service.close_client() + raise + + +def _validate_payload(payload): + """校验请求参数，返回标准化的字段元组。""" + prompt = (payload.get("prompt") or "").strip() + if not prompt: + raise HTTPException( + status_code=400, + detail={ + "error": { + "message": "prompt is required", + "type": "invalid_request_error", + "param": "prompt", + } + }, + ) + + n = payload.get("n", 1) + if not isinstance(n, int) or n < 1 or n > MAX_IMAGES_PER_REQUEST: + raise HTTPException( + status_code=400, + detail={ + "error": { + "message": f"n must be int in [1, {MAX_IMAGES_PER_REQUEST}]", + "type": "invalid_request_error", + "param": "n", + } + }, + ) + + response_format = (payload.get("response_format") or "url").lower() + if response_format not in ALLOWED_RESPONSE_FORMATS: + raise HTTPException( + status_code=400, + detail={ + "error": { + "message": f"response_format must be one of {ALLOWED_RESPONSE_FORMATS}", + "type": "invalid_request_error", + "param": "response_format", + } + }, + ) + + return { + "prompt": prompt, + "n": n, + "size": payload.get("size") or "1024x1024", + "quality": payload.get("quality"), + "style": payload.get("style"), + "requested_model": payload.get("model"), + "response_format": response_format, + } + + +@app.post(f"/{api_prefix}/v1/images/generations" if api_prefix else "/v1/images/generations") +async def generate_images( + request: Request, + credentials: HTTPAuthorizationCredentials = Security(security_scheme), +): + req_token = credentials.credentials + try: + payload = await request.json() + except Exception: + raise HTTPException(status_code=400, detail={"error": "Invalid JSON body"}) + + params = _validate_payload(payload) + target_model = await _resolve_target_model(req_token, params["requested_model"]) + logger.info(f"[image_gen] target_model={target_model} n={params['n']} format={params['response_format']}") + + # 并发 n 次生成，并发上限 MAX_CONCURRENT_IMAGES + semaphore = asyncio.Semaphore(min(params["n"], MAX_CONCURRENT_IMAGES)) + + async def _one(): + async with semaphore: + return await _generate_single_image( + req_token, + target_model, + params["prompt"], + params["size"], + params["quality"], + params["style"], + ) + + results = await asyncio.gather(*[_one() for _ in range(params["n"])], return_exceptions=True) + + services_to_close = [] + image_items = [] + failure_excerpts = [] + try: + for r in results: + if isinstance(r, Exception): + failure_excerpts.append(str(r)[:200]) + continue + svc, content, urls = r + services_to_close.append(svc) + if not urls: + failure_excerpts.append((content or "")[:200]) + continue + url = urls[0] + if params["response_format"] == "url": + image_items.append({"url": url}) + else: + try: + b64 = await _download_image_to_b64(svc, url) + image_items.append({"b64_json": b64}) + except HTTPException as e: + failure_excerpts.append(f"download failed: {e.detail}") + + if not image_items: + raise HTTPException( + status_code=400, + detail={ + "error": { + "message": "Failed to generate image", + "type": "image_generation_failed", + "code": "no_image_generated", + "upstream_excerpt": failure_excerpts[0] if failure_excerpts else "", + } + }, + ) + + return JSONResponse( + {"created": int(time.time()), "data": image_items}, + media_type="application/json", + ) + finally: + for svc in services_to_close: + try: + await svc.close_client() + except Exception: + pass diff --git a/api/models.py b/api/models.py index 56665f03..de6140aa 100644 --- a/api/models.py +++ b/api/models.py @@ -8,11 +8,20 @@ "gpt-4-turbo": "gpt-4-turbo-2024-04-09", "gpt-4o": "gpt-4o-2024-08-06", "gpt-4o-mini": "gpt-4o-mini-2024-07-18", + "gpt-5": "gpt-5", + "gpt-5-mini": "gpt-5-mini", + "gpt-5-thinking": "gpt-5-thinking", + "gpt-5-pro": "gpt-5-pro", + "gpt-5-5": "gpt-5-5", "o1-preview": "o1-preview-2024-09-12", "o1-mini": "o1-mini-2024-09-12", "o1": "o1-2024-12-18", "o3-mini": "o3-mini-2025-01-31", "o3-mini-high": "o3-mini-high-2025-01-31", + "o3-deep-research": "o3-deep-research-2025-06-26", + "o4-mini-deep-research": "o4-mini-deep-research-2025-06-26", + "gpt-4o-deep-research": "gpt-4o-deep-research", + "deep-research": "deep-research", "claude-3-opus": "claude-3-opus-20240229", "claude-3-sonnet": "claude-3-sonnet-20240229", "claude-3-haiku": "claude-3-haiku-20240307", @@ -29,3 +38,113 @@ "gpt-4o-2024-05-13": ["fp_3aa7262c27"], "gpt-4o-mini-2024-07-18": ["fp_c9aa9c0491"] } + +MODEL_REQUEST_RULES = ( + ("o3-deep-research", "o3-deep-research"), + ("o4-mini-deep-research", "o4-mini-deep-research"), + ("gpt-4o-deep-research", "gpt-4o-deep-research"), + ("deep-research", "deep-research"), + ("o3-mini-high", "o3-mini-high"), + ("o3-mini-medium", "o3-mini-medium"), + ("o3-mini-low", "o3-mini-low"), + ("o3-mini", "o3-mini"), + ("o3", "o3"), + ("o1-preview", "o1-preview"), + ("o1-pro", "o1-pro"), + ("o1-mini", "o1-mini"), + ("o1", "o1"), + ("gpt-5-5", "gpt-5-5"), + ("gpt-5-pro", "gpt-5-pro"), + ("gpt-5-thinking", "gpt-5-thinking"), + ("gpt-5-mini", "gpt-5-mini"), + ("gpt-5", "gpt-5"), + ("gpt-4.5o", "gpt-4.5o"), + ("gpt-4o-canmore", "gpt-4o-canmore"), + ("gpt-4o-mini", "gpt-4o-mini"), + ("gpt-4o", "gpt-4o"), + ("gpt-4-mobile", "gpt-4-mobile"), + ("gpt-4", "gpt-4"), + ("gpt-3.5", "text-davinci-002-render-sha"), + ("auto", "auto"), +) + +DEEP_RESEARCH_MODEL_ALIASES = ( + "o3-deep-research", + "o4-mini-deep-research", + "gpt-4o-deep-research", + "deep-research", +) + + +def should_expose_deep_research_aliases(model_slugs): + paid_markers = ( + "gpt-4", + "gpt-4o", + "gpt-5", + "o1", + "o3", + "o4", + ) + return any( + isinstance(slug, str) and slug.startswith(paid_markers) + for slug in (model_slugs or []) + ) + + +def augment_model_slugs(model_slugs): + slugs = set(model_slugs or []) + if should_expose_deep_research_aliases(slugs): + slugs.update(DEEP_RESEARCH_MODEL_ALIASES) + return slugs + + +def get_response_model(origin_model): + return model_proxy.get(origin_model, origin_model) + + +def match_model_family(origin_model, alias): + return origin_model == alias or origin_model.startswith(f"{alias}-") + + +def resolve_request_model(origin_model): + origin_model = (origin_model or "gpt-5-5").strip() + base_model = origin_model + gizmo_id = None + + if "-gizmo-g-" in origin_model: + base_model, _, gizmo_suffix = origin_model.partition("-gizmo-") + gizmo_id = gizmo_suffix + elif origin_model.startswith("g-"): + gizmo_id = origin_model + base_model = "gpt-5-5" + + for alias, target in MODEL_REQUEST_RULES: + if match_model_family(base_model, alias): + return target, gizmo_id, False + + return base_model, gizmo_id, True + + +def extract_model_slugs(models_payload): + slugs = set() + model_items = models_payload.get("models", []) + if isinstance(model_items, dict): + model_items = model_items.values() + + for model in model_items: + if not isinstance(model, dict): + continue + + for key in ("slug", "id", "model_slug"): + value = model.get(key) + if isinstance(value, str) and value: + slugs.add(value) + + nested_model = model.get("model") + if isinstance(nested_model, dict): + for key in ("slug", "id", "model_slug"): + value = nested_model.get(key) + if isinstance(value, str) and value: + slugs.add(value) + + return slugs diff --git a/app.py b/app.py index 52f3f063..2c1bf445 100644 --- a/app.py +++ b/app.py @@ -37,13 +37,14 @@ from app import app import api.chat2api +import api.image_generations +import gateway.admin if enable_gateway: - import gateway.share import gateway.login import gateway.chatgpt import gateway.gpts - import gateway.admin + import gateway.share import gateway.v1 import gateway.backend else: diff --git a/chatgpt/ChatService.py b/chatgpt/ChatService.py index 7260de01..e661e402 100644 --- a/chatgpt/ChatService.py +++ b/chatgpt/ChatService.py @@ -1,4 +1,3 @@ -import asyncio import hashlib import json import random @@ -7,16 +6,17 @@ from fastapi import HTTPException from starlette.concurrency import run_in_threadpool -from api.files import get_image_size, get_file_extension, determine_file_use_case -from api.models import model_proxy -from chatgpt.authorization import get_req_token, verify_token +from chatgpt.authorization import get_req_token from chatgpt.chatFormat import api_messages_to_chat, stream_response, format_not_stream_response, head_process_response from chatgpt.chatLimit import check_is_limit, handle_request_limit from chatgpt.fp import get_fp from chatgpt.proofofWork import get_config, get_dpl, get_answer_token, get_requirements_token +from chatgpt.services import AuthMixin, FileMixin, ModelMixin +from chatgpt.services._helpers import _sanitize_headers, _stringify_header_value from utils.Client import Client from utils.Logger import logger +from utils import antiban from utils.configs import ( chatgpt_base_url_list, ark0se_token_url_list, @@ -29,10 +29,18 @@ auth_key, turnstile_solver_url, oai_language, + accept_language, + chat_requirements_timeout, + chat_request_timeout, + client_timezone, + client_timezone_offset_min, + enable_antiban, + oai_client_version, + oai_client_build_number, ) -class ChatService: +class ChatService(AuthMixin, ModelMixin, FileMixin): def __init__(self, origin_token=None): # self.user_agent = random.choice(user_agents_list) if user_agents_list else "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36" self.req_token = get_req_token(origin_token) @@ -40,54 +48,41 @@ def __init__(self, origin_token=None): self.s = None self.ss = None self.ws = None + self.dynamic_model = False + self.antiban_ctx = None + # 深度研究相关：system_hints 与请求体透传 / 模型名后缀双模式触发 + self.system_hints = [] + # Session sticky: 由 api 层 inject 后挂载，stream_response 嗅探时用于回写映射 + self.librechat_conv_id = None - async def set_dynamic_data(self, data): - if self.req_token: - req_len = len(self.req_token.split(",")) - if req_len == 1: - self.access_token = await verify_token(self.req_token) - self.account_id = None - else: - self.access_token = await verify_token(self.req_token.split(",")[0]) - self.account_id = self.req_token.split(",")[1] - else: - logger.info("Request token is empty, use no-auth 3.5") - self.access_token = None - self.account_id = None + async def initialize_request_context(self): + # Antiban: 在读取 fp 之前获取上下文（bucket/geo/冷却/熔断） + self.antiban_ctx = await antiban.acquire_context(self.req_token) self.fp = get_fp(self.req_token).copy() self.proxy_url = self.fp.pop("proxy_url", None) self.impersonate = self.fp.pop("impersonate", "safari15_3") self.user_agent = self.fp.get("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36 Edg/130.0.0.0") + + # Antiban 强制粘性 IP：以桶内 proxy 覆盖 fp 中的 proxy_url（若已分配） + if self.antiban_ctx and self.antiban_ctx.enabled and self.antiban_ctx.proxy_url: + if self.proxy_url != self.antiban_ctx.proxy_url: + logger.info( + f"[antiban] proxy overridden by bucket: " + f"{self.proxy_url} -> {self.antiban_ctx.proxy_url}" + ) + self.proxy_url = self.antiban_ctx.proxy_url + logger.info(f"Request token: {self.req_token}") logger.info(f"Request proxy: {self.proxy_url}") logger.info(f"Request UA: {self.user_agent}") logger.info(f"Request impersonate: {self.impersonate}") - self.data = data - await self.set_model() - if enable_limit and self.req_token: - limit_response = await handle_request_limit(self.req_token, self.req_model) - if limit_response: - raise HTTPException(status_code=429, detail=limit_response) - - self.account_id = self.data.get('Chatgpt-Account-Id', self.account_id) - self.parent_message_id = self.data.get('parent_message_id') - self.conversation_id = self.data.get('conversation_id') - self.history_disabled = self.data.get('history_disabled', history_disabled) - - self.api_messages = self.data.get("messages", []) - self.prompt_tokens = 0 - self.max_tokens = self.data.get("max_tokens", 2147483647) - if not isinstance(self.max_tokens, int): - self.max_tokens = 2147483647 - - # self.proxy_url = random.choice(proxy_url_list) if proxy_url_list else None - self.host_url = random.choice(chatgpt_base_url_list) if chatgpt_base_url_list else "https://chatgpt.com" self.ark0se_token_url = random.choice(ark0se_token_url_list) if ark0se_token_url_list else None - session_id = hashlib.md5(self.req_token.encode()).hexdigest() + session_source = self.req_token or "no-auth" + session_id = hashlib.md5(session_source.encode()).hexdigest() proxy_url = self.proxy_url.replace("{}", session_id) if self.proxy_url else None self.s = Client(proxy=proxy_url, impersonate=self.impersonate) if sentinel_proxy_url_list: @@ -107,7 +102,7 @@ async def set_dynamic_data(self, data): self.base_headers = { 'accept': '*/*', 'accept-encoding': 'gzip, deflate, br, zstd', - 'accept-language': 'en-US,en;q=0.9', + 'accept-language': accept_language, 'content-type': 'application/json', 'oai-language': oai_language, 'origin': self.host_url, @@ -117,7 +112,37 @@ async def set_dynamic_data(self, data): 'sec-fetch-mode': 'cors', 'sec-fetch-site': 'same-origin' } - self.base_headers.update(self.fp) + # 反降智关键头：让请求看起来像真实 ChatGPT 前端发出 + if oai_client_version: + self.base_headers['oai-client-version'] = oai_client_version + if oai_client_build_number: + self.base_headers['oai-client-build-number'] = str(oai_client_build_number) + # oai-session-id：与 oai-device-id 类似，token 级稳定（fp.py 在生成 fp 时填入） + session_id_header = self.fp.get("oai-session-id") + if session_id_header: + self.base_headers['oai-session-id'] = session_id_header + # T1: 注入用户偏好相关 CH 头（Chromium 真实浏览器在 same-origin 请求中默认携带） + _pref_color = self.fp.get("color_scheme") + if _pref_color in ("light", "dark"): + self.base_headers['sec-ch-prefers-color-scheme'] = _pref_color + _pref_motion = self.fp.get("prefers_reduced_motion") + if _pref_motion in ("no-preference", "reduce"): + self.base_headers['sec-ch-prefers-reduced-motion'] = _pref_motion + # 过滤掉 fp 中的非 HTTP-header 内部指纹字段（screen/viewport 等仅供 PoW 与 contextual_info 使用） + for _internal_key in ( + "screen", "hardware_concurrency", "device_memory", "pixel_ratio", "viewport", + # 扩展指纹字段：仅供 client_contextual_info / 未来 sentinel 字段使用，绝不能进 HTTP 头 + "nav_platform", "languages", "max_touch_points", "webgl", + "color_scheme", "prefers_reduced_motion", "color_gamut", + "connection", "audio", + # T2/T3/T5/M1/M2 等纯指纹字段 + "canvas_hash", "font_list_hash", "font_list_count", "audio_fp_hash", + "timezone", "intl_locale", "user_pace", "virtual_page_load_ms", + # D2/D3 深耕字段 + "webgpu", "webrtc", + ): + self.fp.pop(_internal_key, None) + self.base_headers.update(_sanitize_headers(self.fp)) if self.access_token: self.base_url = self.host_url + "/backend-api" @@ -130,52 +155,56 @@ async def set_dynamic_data(self, data): if auth_key: self.base_headers['authkey'] = auth_key - await get_dpl(self) + # Antiban: 用 geo 结果覆盖 accept-language / oai-language + if self.antiban_ctx and self.antiban_ctx.enabled and self.antiban_ctx.header_overrides: + for k, v in self.antiban_ctx.header_overrides.items(): + if k.startswith("_") or not v: + continue + normalized = _stringify_header_value(v) + if normalized is not None: + self.base_headers[k] = normalized + logger.info( + f"[antiban] headers overridden by geo: " + f"accept-language={self.base_headers.get('accept-language')} " + f"oai-language={self.base_headers.get('oai-language')}" + ) - async def set_model(self): - self.origin_model = self.data.get("model", "gpt-3.5-turbo-0125") - self.resp_model = model_proxy.get(self.origin_model, self.origin_model) - if "gizmo" in self.origin_model or "g-" in self.origin_model: - self.gizmo_id = "g-" + self.origin_model.split("g-")[-1] - else: - self.gizmo_id = None + async def set_dynamic_data(self, data): + await self.resolve_auth_context() - if "o3-mini-high" in self.origin_model: - self.req_model = "o3-mini-high" - elif "o3-mini-medium" in self.origin_model: - self.req_model = "o3-mini-medium" - elif "o3-mini-low" in self.origin_model: - self.req_model = "o3-mini-low" - elif "o3-mini" in self.origin_model: - self.req_model = "o3-mini" - elif "o3" in self.origin_model: - self.req_model = "o3" - elif "o1-preview" in self.origin_model: - self.req_model = "o1-preview" - elif "o1-pro" in self.origin_model: - self.req_model = "o1-pro" - elif "o1-mini" in self.origin_model: - self.req_model = "o1-mini" - elif "o1" in self.origin_model: - self.req_model = "o1" - elif "gpt-4.5o" in self.origin_model: - self.req_model = "gpt-4.5o" - elif "gpt-4o-canmore" in self.origin_model: - self.req_model = "gpt-4o-canmore" - elif "gpt-4o-mini" in self.origin_model: - self.req_model = "gpt-4o-mini" - elif "gpt-4o" in self.origin_model: - self.req_model = "gpt-4o" - elif "gpt-4-mobile" in self.origin_model: - self.req_model = "gpt-4-mobile" - elif "gpt-4" in self.origin_model: - self.req_model = "gpt-4" - elif "gpt-3.5" in self.origin_model: - self.req_model = "text-davinci-002-render-sha" - elif "auto" in self.origin_model: - self.req_model = "auto" - else: - self.req_model = "gpt-4o" + self.data = data + # 深度研究：双模式触发字段提取（必须在 set_model 之前，便于模型名识别合并） + # 1) 显式透传 system_hints；2) 支持别名 hints；3) 支持 deep_research:bool 快捷开关 + raw_hints = self.data.get("system_hints") + if raw_hints is None: + raw_hints = self.data.get("hints", []) + if not isinstance(raw_hints, list): + raw_hints = [] + if self.data.get("deep_research") is True and "research" not in raw_hints: + raw_hints = raw_hints + ["research"] + self.system_hints = raw_hints + + await self.set_model() + + self.account_id = self.data.get('Chatgpt-Account-Id', self.account_id) + self.parent_message_id = self.data.get('parent_message_id') + self.conversation_id = self.data.get('conversation_id') + self.history_disabled = self.data.get('history_disabled', history_disabled) + + self.api_messages = self.data.get("messages", []) + self.prompt_tokens = 0 + self.max_tokens = self.data.get("max_tokens", 2147483647) + if not isinstance(self.max_tokens, int): + self.max_tokens = 2147483647 + + await self.initialize_request_context() + await get_dpl(self) + await self.validate_model_access() + + if enable_limit and self.req_token: + limit_response = await handle_request_limit(self.req_token, self.req_model) + if limit_response: + raise HTTPException(status_code=429, detail=limit_response) async def get_chat_requirements(self): if conversation_only: @@ -183,10 +212,11 @@ async def get_chat_requirements(self): url = f'{self.base_url}/sentinel/chat-requirements' headers = self.base_headers.copy() try: - config = get_config(self.user_agent, self.req_token) + tz_offset = self.antiban_ctx.tz_offset_min if (self.antiban_ctx and self.antiban_ctx.enabled) else None + config = get_config(self.user_agent, self.req_token, tz_offset) p = get_requirements_token(config) data = {'p': p} - r = await self.ss.post(url, headers=headers, json=data, timeout=5) + r = await self.ss.post(url, headers=headers, json=data, timeout=chat_requirements_timeout) if r.status_code == 200: resp = r.json() @@ -194,15 +224,7 @@ async def get_chat_requirements(self): if self.persona != "chatgpt-paid": if self.req_model == "gpt-4" or self.req_model == "o1-preview": logger.error(f"Model {self.resp_model} not support for {self.persona}") - raise HTTPException( - status_code=404, - detail={ - "message": f"The model `{self.origin_model}` does not exist or you do not have access to it.", - "type": "invalid_request_error", - "param": None, - "code": "model_not_found", - }, - ) + raise self.model_not_found() turnstile = resp.get('turnstile', {}) turnstile_required = turnstile.get('required') @@ -266,6 +288,8 @@ async def get_chat_requirements(self): detail = r.json().get("detail", r.json()) else: detail = r.text + # Antiban: 分级上报错误（IP 降级 / 账号冷却延长 / 黑名单） + await antiban.report_error(self.antiban_ctx, r.status_code, detail) if "cf_chl_opt" in detail: raise HTTPException(status_code=r.status_code, detail="cf_chl_opt") if r.status_code == 429: @@ -274,6 +298,11 @@ async def get_chat_requirements(self): except HTTPException as e: raise HTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: + # M3: transport 层失败（连接被拒/超时/DNS）→ 告知 antiban 桶降级 + try: + await antiban.report_network_error(self.antiban_ctx, type(e).__name__) + except Exception: + pass raise HTTPException(status_code=500, detail=str(e)) async def prepare_send_conversation(self): @@ -308,18 +337,62 @@ async def prepare_send_conversation(self): else: conversation_mode = {"kind": "primary_assistant"} + # 深度研究强制 primary_assistant 模式（原生协议约束） + if "research" in self.system_hints and self.gizmo_id: + logger.warning("Deep research forces primary_assistant mode, ignoring gizmo_id") + conversation_mode = {"kind": "primary_assistant"} + self.gizmo_id = None + logger.info(f"Model mapping: {self.origin_model} -> {self.req_model}") + if self.system_hints: + logger.info(f"System hints: {self.system_hints}") + + # client_contextual_info：token 级稳定（首选）；同账号多次请求保持一致，避免抖动暴露自动化 + ctx_info = None + pace_range = None + try: + if enable_antiban: + from utils.antiban import fingerprint as _fp_mod + # H3 双保险：即使上游 acquire_context 未触发，也保证扩展字段齐全 + _fp_mod.ensure_extended(self.req_token) + ctx_info = _fp_mod.get_contextual_info(self.req_token) + pace_range = _fp_mod.get_user_pace_range(self.req_token) + except Exception: + ctx_info = None + pace_range = None + + # M1: time_since_loaded 按 user_pace 抽样（fast/normal/slow），同账号节奏一致 + if pace_range: + time_since_loaded = random.randint(pace_range[0], pace_range[1]) + else: + time_since_loaded = random.randint(3000, 30000) + + if ctx_info: + # is_dark_mode 不再硬编码 False，改为 token 级稳定的 color_scheme 派生（约 30% 用户偏好暗色） + client_contextual_info = { + "is_dark_mode": ctx_info.get("color_scheme") == "dark", + "time_since_loaded": time_since_loaded, + "page_height": ctx_info["page_height"], + "page_width": ctx_info["page_width"], + "pixel_ratio": ctx_info["pixel_ratio"], + "screen_height": ctx_info["screen_height"], + "screen_width": ctx_info["screen_width"], + } + else: + # antiban 未启用：保持原行为但修正 pixel_ratio 取真实值（1.0/2.0 而非 1.5） + client_contextual_info = { + "is_dark_mode": False, + "time_since_loaded": time_since_loaded, + "page_height": random.randint(700, 1200), + "page_width": random.randint(1200, 2000), + "pixel_ratio": random.choice([1.0, 2.0]), + "screen_height": random.randint(900, 1440), + "screen_width": random.randint(1440, 2560), + } + self.chat_request = { "action": "next", - "client_contextual_info": { - "is_dark_mode": False, - "time_since_loaded": random.randint(50, 500), - "page_height": random.randint(500, 1000), - "page_width": random.randint(1000, 2000), - "pixel_ratio": 1.5, - "screen_height": random.randint(800, 1200), - "screen_width": random.randint(1200, 2200), - }, + "client_contextual_info": client_contextual_info, "conversation_mode": conversation_mode, "conversation_origin": None, "force_paragen": False, @@ -335,23 +408,44 @@ async def prepare_send_conversation(self): "reset_rate_limits": False, "suggestions": [], "supported_encodings": [], - "system_hints": [], - "timezone": "America/Los_Angeles", - "timezone_offset_min": -480, + "system_hints": self.system_hints, + "timezone": client_timezone, + "timezone_offset_min": client_timezone_offset_min, "variant_purpose": "comparison_implicit", "websocket_request_id": f"{uuid.uuid4()}", } + # Antiban: 按 IP 地域覆盖时区（与 UA / accept-language 一致） + if self.antiban_ctx and self.antiban_ctx.enabled and self.antiban_ctx.tz_offset_min is not None: + self.chat_request["timezone_offset_min"] = self.antiban_ctx.tz_offset_min + if self.antiban_ctx.header_overrides.get("_timezone_name"): + self.chat_request["timezone"] = self.antiban_ctx.header_overrides["_timezone_name"] if self.conversation_id: self.chat_request['conversation_id'] = self.conversation_id + # 真实浏览器的 referer 是具体会话 URL（如 /c/），不是首页 + self.chat_headers['referer'] = f"{self.host_url}/c/{self.conversation_id}" return self.chat_request async def send_conversation(self): try: url = f'{self.base_url}/conversation' stream = self.data.get("stream", False) - r = await self.s.post_stream(url, headers=self.chat_headers, json=self.chat_request, timeout=10, stream=True) + r = await self.s.post_stream( + url, + headers=self.chat_headers, + json=self.chat_request, + timeout=chat_request_timeout, + stream=True, + ) if r.status_code != 200: rtext = await r.atext() + # Session sticky: 注入的 conv_id 触发 4xx → 清理映射，让重试新建对话 + if 400 <= r.status_code < 500 and self.data.get("librechat_conversation_id") \ + and self.conversation_id: + try: + from chatgpt import session_sticky as _ss + _ss.drop_mapping(self.data.get("librechat_conversation_id")) + except Exception: + pass if "application/json" == r.headers.get("Content-Type", ""): detail = json.loads(rtext).get("detail", json.loads(rtext)) if r.status_code == 429: @@ -359,14 +453,21 @@ async def send_conversation(self): else: if "cf_chl_opt" in rtext: # logger.error(f"Failed to send conversation: cf_chl_opt") + await antiban.report_error(self.antiban_ctx, r.status_code, "cf_chl_opt") raise HTTPException(status_code=r.status_code, detail="cf_chl_opt") if r.status_code == 429: # logger.error(f"Failed to send conversation: rate-limit") + await antiban.report_error(self.antiban_ctx, r.status_code, "rate-limit") raise HTTPException(status_code=r.status_code, detail="rate-limit") detail = r.text[:100] # logger.error(f"Failed to send conversation: {detail}") + await antiban.report_error(self.antiban_ctx, r.status_code, detail) raise HTTPException(status_code=r.status_code, detail=detail) + # 200 OK: 立即标记账号使用（真正 success 在响应流完成后由上游调度更稳， + # 但 200 即可代表风控校验通过，此处记录冷却足够） + await antiban.report_success(self.antiban_ctx) + content_type = r.headers.get("Content-Type", "") if "text/event-stream" in content_type: res, start = await head_process_response(r.aiter_lines()) @@ -394,157 +495,12 @@ async def send_conversation(self): except HTTPException as e: raise HTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - async def get_download_url(self, file_id): - url = f"{self.base_url}/files/{file_id}/download" - headers = self.base_headers.copy() - try: - r = await self.s.get(url, headers=headers, timeout=10) - if r.status_code == 200: - download_url = r.json().get('download_url') - return download_url - else: - raise HTTPException(status_code=r.status_code, detail=r.text) - except Exception as e: - logger.error(f"Failed to get download url: {e}") - return "" - - async def get_attachment_url(self, file_id, conversation_id): - url = f"{self.base_url}/conversation/{conversation_id}/attachment/{file_id}/download" - headers = self.base_headers.copy() - try: - r = await self.s.get(url, headers=headers, timeout=10) - if r.status_code == 200: - download_url = r.json().get('download_url') - return download_url - else: - raise HTTPException(status_code=r.status_code, detail=r.text) - except Exception as e: - logger.error(f"Failed to get download url: {e}") - return "" - - async def get_download_url_from_upload(self, file_id): - url = f"{self.base_url}/files/{file_id}/uploaded" - headers = self.base_headers.copy() - try: - r = await self.s.post(url, headers=headers, json={}, timeout=10) - if r.status_code == 200: - download_url = r.json().get('download_url') - return download_url - else: - raise HTTPException(status_code=r.status_code, detail=r.text) - except Exception as e: - logger.error(f"Failed to get download url from upload: {e}") - return "" - - async def get_upload_url(self, file_name, file_size, use_case="multimodal"): - url = f'{self.base_url}/files' - headers = self.base_headers.copy() - try: - r = await self.s.post( - url, - headers=headers, - json={"file_name": file_name, "file_size": file_size, "reset_rate_limits": False, "timezone_offset_min": -480, "use_case": use_case}, - timeout=5, - ) - if r.status_code == 200: - res = r.json() - file_id = res.get('file_id') - upload_url = res.get('upload_url') - logger.info(f"file_id: {file_id}, upload_url: {upload_url}") - return file_id, upload_url - else: - raise HTTPException(status_code=r.status_code, detail=r.text) - except Exception as e: - logger.error(f"Failed to get upload url: {e}") - return "", "" - - async def upload(self, upload_url, file_content, mime_type): - headers = self.base_headers.copy() - headers.update( - { - 'accept': 'application/json, text/plain, */*', - 'content-type': mime_type, - 'x-ms-blob-type': 'BlockBlob', - 'x-ms-version': '2020-04-08', - } - ) - headers.pop('authorization', None) - headers.pop('oai-device-id', None) - headers.pop('oai-language', None) - try: - r = await self.s.put(upload_url, headers=headers, data=file_content, timeout=60) - if r.status_code == 201: - return True - else: - raise HTTPException(status_code=r.status_code, detail=r.text) - except Exception as e: - logger.error(f"Failed to upload file: {e}") - return False - - async def upload_file(self, file_content, mime_type): - if not file_content or not mime_type: - return None - - width, height = None, None - if mime_type.startswith("image/"): + # M3: send_conversation transport 层失败也上报，三次连续将触发桶降级 try: - width, height = await get_image_size(file_content) - except Exception as e: - logger.error(f"Error image mime_type, change to text/plain: {e}") - mime_type = 'text/plain' - file_size = len(file_content) - file_extension = await get_file_extension(mime_type) - file_name = f"{uuid.uuid4()}{file_extension}" - use_case = await determine_file_use_case(mime_type) - - file_id, upload_url = await self.get_upload_url(file_name, file_size, use_case) - if file_id and upload_url: - if await self.upload(upload_url, file_content, mime_type): - download_url = await self.get_download_url_from_upload(file_id) - if download_url: - file_meta = { - "file_id": file_id, - "file_name": file_name, - "size_bytes": file_size, - "mime_type": mime_type, - "width": width, - "height": height, - "use_case": use_case, - } - logger.info(f"File_meta: {file_meta}") - return file_meta - - async def check_upload(self, file_id): - url = f'{self.base_url}/files/{file_id}' - headers = self.base_headers.copy() - try: - for i in range(30): - r = await self.s.get(url, headers=headers, timeout=5) - if r.status_code == 200: - res = r.json() - retrieval_index_status = res.get('retrieval_index_status', '') - if retrieval_index_status == "success": - break - await asyncio.sleep(1) - return True - except HTTPException: - return False - - async def get_response_file_url(self, conversation_id, message_id, sandbox_path): - try: - url = f"{self.base_url}/conversation/{conversation_id}/interpreter/download" - params = {"message_id": message_id, "sandbox_path": sandbox_path} - headers = self.base_headers.copy() - r = await self.s.get(url, headers=headers, params=params, timeout=10) - if r.status_code == 200: - return r.json().get("download_url") - else: - return None - except Exception: - logger.info("Failed to get response file url") - return None + await antiban.report_network_error(self.antiban_ctx, type(e).__name__) + except Exception: + pass + raise HTTPException(status_code=500, detail=str(e)) async def close_client(self): if self.s: diff --git a/chatgpt/authorization.py b/chatgpt/authorization.py index 5a3c7219..8436725f 100644 --- a/chatgpt/authorization.py +++ b/chatgpt/authorization.py @@ -6,7 +6,7 @@ import utils.configs as configs import utils.globals as globals -from chatgpt.refreshToken import rt2ac +from chatgpt.refreshToken import rt2ac, sess2ac from utils.Logger import logger @@ -54,7 +54,17 @@ async def verify_token(req_token): if req_token.startswith("eyJhbGciOi") or req_token.startswith("fk-"): access_token = req_token return access_token - elif len(req_token) == 45: + # SessionToken：带 sess- 前缀的 chatgpt.com __Secure-next-auth.session-token + elif req_token.startswith("sess-"): + try: + if req_token in globals.error_token_list: + raise HTTPException(status_code=401, detail="Error SessionToken") + access_token = await sess2ac(req_token, force_refresh=False) + return access_token + except HTTPException as e: + raise HTTPException(status_code=e.status_code, detail=e.detail) + # 识别 RefreshToken：老版 45 字符 或 新版 Auth0 'rt_' 前缀（长度 ≥ 60） + elif (req_token.startswith("rt_") and len(req_token) >= 60) or len(req_token) == 45: try: if req_token in globals.error_token_list: raise HTTPException(status_code=401, detail="Error RefreshToken") @@ -69,10 +79,13 @@ async def verify_token(req_token): async def refresh_all_tokens(force_refresh=False): for token in list(set(globals.token_list) - set(globals.error_token_list)): - if len(token) == 45: - try: + try: + if token.startswith("sess-"): + await asyncio.sleep(0.5) + await sess2ac(token, force_refresh=force_refresh) + elif (token.startswith("rt_") and len(token) >= 60) or len(token) == 45: await asyncio.sleep(0.5) await rt2ac(token, force_refresh=force_refresh) - except HTTPException: - pass + except HTTPException: + pass logger.info("All tokens refreshed.") diff --git a/chatgpt/chatFormat.py b/chatgpt/chatFormat.py index 7459615a..f38f47cd 100644 --- a/chatgpt/chatFormat.py +++ b/chatgpt/chatFormat.py @@ -13,6 +13,8 @@ from api.files import get_file_content from api.models import model_system_fingerprint from api.tokens import split_tokens_from_content, calculate_image_tokens, num_tokens_from_messages +from chatgpt import session_sticky +from utils import antiban from utils.Logger import logger moderation_message = "I'm sorry, I cannot provide or engage in any content related to pornography, violence, or any unethical material. If you have any other questions or need assistance, please feel free to let me know. I'll do my best to provide support and assistance." @@ -173,6 +175,12 @@ async def stream_response(service, response, model, max_tokens): conversation_id = chunk_old_data.get("conversation_id") role = message.get('author', {}).get('role') if role == 'user' or role == 'system': + # 账号风险嗅探：system 消息常携带"账号异常"软警告 banner（Step A：仅记录） + if role == 'system': + try: + antiban.sniff_account_warning(service.antiban_ctx, message, chunk_old_data) + except Exception as _e: + logger.error(f"[account_risk] sniff failed (suppressed): {_e}") continue status = message.get("status") @@ -184,6 +192,11 @@ async def stream_response(service, response, model, max_tokens): model_slug = meta_data.get("model_slug", model_slug) if not message and chunk_old_data.get("type") == "moderation": + # 账号风险嗅探：moderation chunk 偶尔也会携带风险信号 + try: + antiban.sniff_account_warning(service.antiban_ctx, message or {}, chunk_old_data) + except Exception as _e: + logger.error(f"[account_risk] sniff failed (suppressed): {_e}") delta = {"role": "assistant", "content": moderation_message} finish_reason = "stop" end = True @@ -206,10 +219,18 @@ async def stream_response(service, response, model, max_tokens): continue citation = message.get("metadata", {}).get("citations", []) if len(citation) > len_last_citation: - inside_metadata = citation[-1].get("metadata", {}) - citation_title = inside_metadata.get("title", "") - citation_url = inside_metadata.get("url", "") - new_text = f' **[[""]]({citation_url} "{citation_title}")** ' + # 深度研究场景下单次会下发多个引用，聚合输出避免遗漏 + new_citations = citation[len_last_citation:] + citation_parts = [] + for c in new_citations: + inside_metadata = c.get("metadata", {}) or {} + citation_title = inside_metadata.get("title", "ref") + citation_url = inside_metadata.get("url", "") + if citation_url: + citation_parts.append( + f'**[[""]]({citation_url} "{citation_title}")**' + ) + new_text = " " + " ".join(citation_parts) + " " if citation_parts else "" len_last_citation = len(citation) else: if role == 'assistant' and last_role != 'assistant': @@ -321,6 +342,16 @@ async def stream_response(service, response, model, max_tokens): last_message_id = message_id last_role = role last_status = status + # Session sticky: 嗅探到 ChatGPT 端 conv_id + message_id 即回写映射 + # （多次 chunk 触发时最后一次会覆盖，最终 parent_msg_id = 最新一条 assistant message） + if getattr(service, "librechat_conv_id", None) and conversation_id and message_id \ + and role == 'assistant': + try: + session_sticky.sniff_and_save( + service.librechat_conv_id, conversation_id, message_id, + ) + except Exception as _e: + logger.error(f"[session_sticky] sniff error: {_e}") if not end and not delta.get("content"): delta = {"role": "assistant", "content": ""} chunk_new_data["choices"][0]["delta"] = delta diff --git a/chatgpt/fp.py b/chatgpt/fp.py index ae1003aa..e14805e1 100644 --- a/chatgpt/fp.py +++ b/chatgpt/fp.py @@ -8,23 +8,204 @@ import utils.globals as globals from utils import configs +from utils.routing import get_bound_proxy + +MAX_SUPPORTED_CHROME_MAJOR = 124 +MIN_SUPPORTED_CHROME_MAJOR = 119 + + +def _stringify_ch_value(value): + if value is None or isinstance(value, str): + return value + if isinstance(value, bool): + return "?1" if value else "?0" + if isinstance(value, (int, float)): + return str(value) + return json.dumps(value, ensure_ascii=False, separators=(",", ":"), default=str) + + +def _infer_arch(platform_str): + """根据 ch.platform 推断 sec-ch-ua-arch（Chromium 不暴露 arch 库字段，需手动推断）。 + + 主流分布：macOS Apple Silicon (arm) 占 70%、Intel x86 30%； + Windows 几乎全是 x86_64；Linux x86_64 居多。 + """ + p = (platform_str or "").strip('"').lower() + if p == "macos": + # 与真实分布对齐：70% arm + return '"arm"' if random.random() < 0.7 else '"x86"' + if p in ("windows", "linux", "chromeos"): + return '"x86"' + return '"x86"' + + +def _infer_form_factors(device_str): + """根据 ua_generator 返回的 device 类型推断 sec-ch-ua-form-factors 值。 + + Chrome 124+ 在 same-origin 请求中默认携带，未携带会被识别为非主流客户端。 + 返回 CH 字符串形式（双引号包裹的列表字面量）。 + """ + d = (device_str or "").lower() + if d == "mobile": + return '"Mobile"' + if d == "tablet": + return '"Tablet"' + # 桌面：极少数 2-in-1 设备会上报 "Desktop", "Tablet"，但绝大多数仅 "Desktop" + return '"Desktop"' + + +def _extract_full_version(ua_text, brands_full): + """从 UA 或 brands_full_version_list 中提取 Chrome 完整版本号，如 "146.0.7680.121"。""" + try: + if "Chrome/" in ua_text: + ver = ua_text.split("Chrome/")[1].split(" ")[0] + if ver and ver != "0.0.0.0": + return f'"{ver}"' + except Exception: + pass + try: + if brands_full: + # 形如 '"Chromium";v="146.0.7680.121", "Google Chrome";v="146.0.7680.121", ...' + import re as _re + m = _re.search(r'"Google Chrome";v="([0-9.]+)"', brands_full) + if m: + return f'"{m.group(1)}"' + m = _re.search(r'"Chromium";v="([0-9.]+)"', brands_full) + if m: + return f'"{m.group(1)}"' + except Exception: + pass + return None + + +def _extract_chrome_major(ua_text): + """从 UA 中解析 Chrome 主版本号；解析失败返回 None。""" + ua = (ua_text or "").lower() + try: + if "edg/" in ua: + return int(ua.split("edg/")[1].split(".")[0]) + if "chrome/" in ua: + return int(ua.split("chrome/")[1].split(".")[0]) + except (IndexError, ValueError): + return None + return None + + +def _clamp_ua_to_supported(ua_text): + """若 UA Chrome 主版本超过 MAX_SUPPORTED_CHROME_MAJOR，按上限重写 UA。 + + 避免 UA=Chrome 130 但 curl_cffi TLS=chrome124 的 6 版本错配。 + 返回 (clamped_ua, was_clamped)。 + """ + if not ua_text: + return ua_text, False + major = _extract_chrome_major(ua_text) + if major is None or major <= MAX_SUPPORTED_CHROME_MAJOR: + return ua_text, False + import re as _re + # 替换 Chrome/X.Y.Z.W 与可能存在的 Edg/X.Y.Z.W 主版本号 + replaced = _re.sub(r"(Chrome/)(\d+)", lambda m: m.group(1) + str(MAX_SUPPORTED_CHROME_MAJOR), ua_text) + replaced = _re.sub(r"(Edg/)(\d+)", lambda m: m.group(1) + str(MAX_SUPPORTED_CHROME_MAJOR), replaced) + return replaced, True + + +def select_impersonate(user_agent): + """根据 UA 选择最接近的 curl_cffi 浏览器指纹（TLS/JA3 + HTTP2 帧）。 + + 与真实 Chrome 主版本对齐能显著降低风控评分；偏离过远（如 UA=Chrome147 但 TLS=chrome119） + 会被识别为自动化客户端。当前镜像固定 curl_cffi==0.7.3，最高安全使用 chrome124。 + """ + ua = (user_agent or "").lower() + def by_major(ver): + if ver >= 124: + return "chrome124" + if ver >= 123: + return "chrome123" + if ver >= 120: + return "chrome120" + return "chrome119" + + # Edge 与 Chrome 共用 Chromium 内核，按主版本选择 curl_cffi 支持的最近指纹。 + if "edg/" in ua: + try: + ver = int(ua.split("edg/")[1].split(".")[0]) + except (IndexError, ValueError): + return "chrome124" + return by_major(ver) + if "chrome/" in ua or "chromium/" in ua: + # 解析 Chrome 主版本号 + try: + ver = int(ua.split("chrome/")[1].split(".")[0]) + except (IndexError, ValueError): + return "chrome124" + return by_major(ver) + return "chrome124" def get_fp(req_token): fp = globals.fp_map.get(req_token, {}) + bound_proxy = get_bound_proxy(req_token) if fp and fp.get("user-agent") and fp.get("impersonate"): - if "proxy_url" in fp.keys() and (fp["proxy_url"] is None or fp["proxy_url"] not in configs.proxy_url_list): + if bound_proxy: + fp["proxy_url"] = bound_proxy + globals.fp_map[req_token] = fp + with open(globals.FP_FILE, "w", encoding="utf-8") as f: + json.dump(globals.fp_map, f, indent=4, ensure_ascii=False) + elif "proxy_url" in fp.keys() and (fp["proxy_url"] is None or fp["proxy_url"] not in configs.proxy_url_list): fp["proxy_url"] = random.choice(configs.proxy_url_list) if configs.proxy_url_list else None globals.fp_map[req_token] = fp with open(globals.FP_FILE, "w", encoding="utf-8") as f: - json.dump(globals.fp_map, f, indent=4) - if globals.impersonate_list and "impersonate" in fp.keys() and fp["impersonate"] not in globals.impersonate_list: - fp["impersonate"] = random.choice(globals.impersonate_list) + json.dump(globals.fp_map, f, indent=4, ensure_ascii=False) + if "user-agent" in fp.keys(): + fp["impersonate"] = select_impersonate(fp["user-agent"]) + globals.fp_map[req_token] = fp + with open(globals.FP_FILE, "w", encoding="utf-8") as f: + json.dump(globals.fp_map, f, indent=4, ensure_ascii=False) + elif globals.impersonate_list and "impersonate" in fp.keys() and fp["impersonate"] not in globals.impersonate_list: + fp["impersonate"] = globals.impersonate_list[-1] globals.fp_map[req_token] = fp with open(globals.FP_FILE, "w", encoding="utf-8") as f: json.dump(globals.fp_map, f, indent=4) - if configs.user_agents_list and "user-agent" in fp.keys() and fp["user-agent"] not in configs.user_agents_list: - fp["user-agent"] = random.choice(configs.user_agents_list) + # 老 fp 迁移：补齐 oai-session-id 与高熵 sec-ch-ua-* 头（旧账号也能享受加固） + _migrated = False + if "oai-session-id" not in fp: + fp["oai-session-id"] = str(uuid.uuid4()) + _migrated = True + if "sec-ch-ua-platform" in fp and "sec-ch-ua-arch" not in fp: + fp["sec-ch-ua-arch"] = _infer_arch(fp.get("sec-ch-ua-platform")) + _migrated = True + # 老 fp 缺 form-factors → 桌面默认 "Desktop" + if "sec-ch-ua-platform" in fp and "sec-ch-ua-form-factors" not in fp: + fp["sec-ch-ua-form-factors"] = '"Desktop"' + _migrated = True + # 老 fp 缺少完整高熵 CH 头时，仅打提示日志（不强制重生，避免破坏 strict_ip_binding 画像） + # 用户可手动删除 data/fp_map.json 让新账号走完整生成流程 + if "sec-ch-ua-platform" in fp and "sec-ch-ua-full-version-list" not in fp: + # 至少补 full-version（基于 UA 解析出 Chrome 版本号） + full_ver = _extract_full_version(fp.get("user-agent", ""), None) + if full_ver and "sec-ch-ua-full-version" not in fp: + fp["sec-ch-ua-full-version"] = full_ver + _migrated = True + if _migrated: + globals.fp_map[req_token] = fp + with open(globals.FP_FILE, "w", encoding="utf-8") as f: + json.dump(globals.fp_map, f, indent=4, ensure_ascii=False) + # 严格指纹绑定：开启后绝不因 user_agents_list 变化而漂移 UA，保留历史画像 + if (not (configs.enable_antiban and configs.strict_ip_binding) + and configs.user_agents_list + and "user-agent" in fp.keys() + and fp["user-agent"] not in configs.user_agents_list): + picked_ua = random.choice(configs.user_agents_list) + # H4: 用户配置的 UA 若主版本号超过 curl_cffi 支持上限，自动降级避免 TLS/UA 错配 + picked_ua, clamped = _clamp_ua_to_supported(picked_ua) + if clamped: + from utils.Logger import logger as _logger + _logger.warning( + f"[fp] UA Chrome major > {MAX_SUPPORTED_CHROME_MAJOR}, " + f"clamped to avoid TLS mismatch" + ) + fp["user-agent"] = picked_ua + fp["impersonate"] = select_impersonate(picked_ua) globals.fp_map[req_token] = fp with open(globals.FP_FILE, "w", encoding="utf-8") as f: json.dump(globals.fp_map, f, indent=4) @@ -32,30 +213,55 @@ def get_fp(req_token): return fp else: options = Options(version_ranges={ - 'chrome': VersionRange(min_version=124), - 'edge': VersionRange(min_version=124), + 'chrome': VersionRange(min_version=MIN_SUPPORTED_CHROME_MAJOR, max_version=MAX_SUPPORTED_CHROME_MAJOR), + 'edge': VersionRange(min_version=MIN_SUPPORTED_CHROME_MAJOR, max_version=MAX_SUPPORTED_CHROME_MAJOR), }) ua = ua_generator.generate( device=configs.device_tuple if configs.device_tuple else ('desktop'), - browser=configs.browser_tuple if configs.browser_tuple else ('chrome', 'edge', 'firefox', 'safari'), + browser=configs.browser_tuple if configs.browser_tuple else ('chrome', 'edge'), platform=configs.platform_tuple if configs.platform_tuple else ('windows', 'macos'), options=options ) + user_agent = ua.text if not configs.user_agents_list else random.choice(configs.user_agents_list) fp = { - "user-agent": ua.text if not configs.user_agents_list else random.choice(configs.user_agents_list), - "impersonate": random.choice(globals.impersonate_list), - "proxy_url": random.choice(configs.proxy_url_list) if configs.proxy_url_list else None, - "oai-device-id": str(uuid.uuid4()) + "user-agent": user_agent, + "impersonate": select_impersonate(user_agent), + "proxy_url": bound_proxy or (random.choice(configs.proxy_url_list) if configs.proxy_url_list else None), + "oai-device-id": str(uuid.uuid4()), + # 浏览器 tab/session 级稳定标识（真实浏览器同一 tab 内不变） + "oai-session-id": str(uuid.uuid4()), } if ua.device == "desktop" and ua.browser in ("chrome", "edge"): - fp["sec-ch-ua-platform"] = ua.ch.platform - fp["sec-ch-ua"] = ua.ch.brands - fp["sec-ch-ua-mobile"] = ua.ch.mobile + # 标准 3 个低熵 CH 头 + fp["sec-ch-ua-platform"] = _stringify_ch_value(ua.ch.platform) + fp["sec-ch-ua"] = _stringify_ch_value(ua.ch.brands) + fp["sec-ch-ua-mobile"] = _stringify_ch_value(ua.ch.mobile) + # 高熵 CH 头（Chromium 124+ 默认在 same-origin 请求中携带） + brands_full = getattr(ua.ch, "brands_full_version_list", None) + if brands_full: + fp["sec-ch-ua-full-version-list"] = _stringify_ch_value(brands_full) + full_ver = _extract_full_version(user_agent, brands_full) + if full_ver: + fp["sec-ch-ua-full-version"] = full_ver + bitness = getattr(ua.ch, "bitness", None) + if bitness: + fp["sec-ch-ua-bitness"] = _stringify_ch_value(bitness) + model = getattr(ua.ch, "model", None) + if model is not None: + fp["sec-ch-ua-model"] = _stringify_ch_value(model) + platform_version = getattr(ua.ch, "platform_version", None) + if platform_version: + fp["sec-ch-ua-platform-version"] = _stringify_ch_value(platform_version) + # arch 需手动推断（ua_generator 不直接提供） + fp["sec-ch-ua-arch"] = _infer_arch(fp.get("sec-ch-ua-platform")) + # D1: sec-ch-ua-form-factors（Chrome 124+ 默认携带；桌面恒为 "Desktop"） + # 真实浏览器返回数组形式："Desktop"，移动端 "Mobile"，二合一 "Desktop", "Tablet" + fp["sec-ch-ua-form-factors"] = _infer_form_factors(ua.device) if not req_token: return fp else: globals.fp_map[req_token] = fp with open(globals.FP_FILE, "w", encoding="utf-8") as f: - json.dump(globals.fp_map, f, indent=4) + json.dump(globals.fp_map, f, indent=4, ensure_ascii=False) return fp diff --git a/chatgpt/proofofWork.py b/chatgpt/proofofWork.py index d6aa8667..00ff84c0 100644 --- a/chatgpt/proofofWork.py +++ b/chatgpt/proofofWork.py @@ -11,7 +11,7 @@ import diskcache as dc from utils.Logger import logger -from utils.configs import conversation_only +from utils.configs import conversation_only, client_timezone, client_timezone_offset_min, accept_language, oai_language cores = [8, 16, 24, 32] timeLayout = "%a %b %d %Y %H:%M:%S" @@ -92,7 +92,7 @@ "requestMediaKeySystemAccess−function requestMediaKeySystemAccess() { [native code] }", "vendor−Google Inc.", "pdfViewerEnabled−true", - "language−zh-CN", + "language−en-US", "setAppBadge−function setAppBadge() { [native code] }", "geolocation−[object Geolocation]", "userAgentData−[object NavigatorUAData]", @@ -408,6 +408,12 @@ async def get_dpl(service): if int(time.time()) - cached_time < 15 * 60: return True headers = service.base_headers.copy() + # T4: 首页 GET 用 HTML Accept（真实浏览器导航请求） + headers["accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8" + headers["sec-fetch-dest"] = "document" + headers["sec-fetch-mode"] = "navigate" + headers["sec-fetch-site"] = "none" + headers["sec-fetch-user"] = "?1" cached_scripts = [] cached_dpl = "" try: @@ -427,32 +433,80 @@ async def get_dpl(service): return False -def get_parse_time(): - now = datetime.now(timezone(timedelta(hours=-5))) - return now.strftime(timeLayout) + " GMT-0500 (Eastern Standard Time)" +def get_parse_time(tz_offset_min=None, tz_name=None): + """支持 antiban 动态覆盖时区，默认沿用全局配置。""" + offset = tz_offset_min if tz_offset_min is not None else client_timezone_offset_min + name = tz_name if tz_name else client_timezone + now = datetime.now(timezone(timedelta(minutes=offset))) + offset_hours = int(offset / 60) + offset_label = f"GMT{offset_hours:+03d}00" + timezone_name = name.split("/")[-1].replace("_", " ") + return now.strftime(timeLayout) + f" {offset_label} ({timezone_name})" @cache.memoize(expire=3600 * 24 * 7) -def get_config(user_agent, req_token=None): +def _get_static_config_meta(req_token): + """Token 级缓存：仅缓存稳定的静态 metadata，避免每次重读 fp_map。 + + 动态字段（time / perf_counter / uuid / 随机 navigator key）NOT cached，每次重算。 + 旧实现把整个 config 缓存了 7 天，导致同一 token 多次 PoW 输入完全一致 → 重放特征。 + """ + screen_sum = None + cores_val = None + page_load_ms = None + try: + from utils import configs as _configs + if _configs.enable_antiban and req_token: + from utils.antiban import fingerprint as _fp + screen_sum = _fp.get_screen_resolution_sum(req_token) + cores_val = _fp.get_hardware_concurrency(req_token) + page_load_ms = _fp.get_virtual_page_load_ms(req_token) + except Exception: + pass + return {"screen_sum": screen_sum, "cores": cores_val, "page_load_ms": page_load_ms} + + +def get_config(user_agent, req_token=None, tz_offset_min=None, tz_name=None): + """生成 PoW config。静态字段 token 级缓存；动态字段（时间/UUID/随机 key）每次重算。""" + meta = _get_static_config_meta(req_token) + screen_sum = meta.get("screen_sum") + cores_val = meta.get("cores") + page_load_ms = meta.get("page_load_ms") + + # perf_counter：真实浏览器从 page load 起算（秒级到分钟级），不是进程级累加 + now_perf_ms = time.perf_counter() * 1000 + if page_load_ms is not None: + # 用 token 级稳定的"虚拟页面加载偏移"：模拟用户已在页面停留若干秒 + perf_relative = now_perf_ms - page_load_ms + else: + perf_relative = now_perf_ms + + # T6: navigator_key 池含 "hardwareConcurrency−32" 等硬编码键； + # 若随机选中这类与 fp.hardware_concurrency 不一致的字符串，会暴露指纹矛盾。 + # 优先选不含数值的键（vendor/cookieEnabled 等），仅当抽中 hardwareConcurrency-* 时改写为真实值。 + chosen_nav_key = random.choice(navigator_key) + if cores_val is not None and chosen_nav_key.startswith("hardwareConcurrency−"): + chosen_nav_key = f"hardwareConcurrency−{cores_val}" + config = [ - random.choice([1920 + 1080, 2560 + 1440, 1920 + 1200, 2560 + 1600]), - get_parse_time(), + screen_sum if screen_sum is not None else random.choice([1920 + 1080, 2560 + 1440, 1920 + 1200, 2560 + 1600]), + get_parse_time(tz_offset_min, tz_name), 4294705152, 0, user_agent, random.choice(cached_scripts) if cached_scripts else "", cached_dpl, - "en-US", - "en-US,es-US,en,es", + oai_language, + accept_language, 0, - random.choice(navigator_key), + chosen_nav_key, random.choice(document_key), random.choice(window_key), - time.perf_counter() * 1000, + perf_relative, str(uuid.uuid4()), "", - random.choice(cores), - time.time() * 1000 - (time.perf_counter() * 1000), + cores_val if cores_val is not None else random.choice(cores), + time.time() * 1000 - now_perf_ms, ] return config diff --git a/chatgpt/refreshToken.py b/chatgpt/refreshToken.py index a580a75f..40f981be 100644 --- a/chatgpt/refreshToken.py +++ b/chatgpt/refreshToken.py @@ -1,15 +1,170 @@ +import asyncio +import base64 import hashlib import json import random import time +from urllib.parse import urlencode from fastapi import HTTPException from utils.Client import Client from utils.Logger import logger -from utils.configs import proxy_url_list +from utils.configs import ( + openai_auth_client_id, + openai_auth_redirect_uri, + openai_auth_scope, + openai_auth_token_url, + proxy_url_list, +) +from utils.routing import get_bound_proxy, save_routing_config import utils.globals as globals +# 跨结构 key 迁移锁，防止多个并发刷新同时改 token_list/refresh_map/routing_config +_session_key_lock = asyncio.Lock() + + +def persist_refresh_map(): + with open(globals.REFRESH_MAP_FILE, "w", encoding="utf-8") as f: + json.dump(globals.refresh_map, f, indent=4, ensure_ascii=False) + + +def persist_error_tokens(): + with open(globals.ERROR_TOKENS_FILE, "w", encoding="utf-8") as f: + for token in globals.error_token_list: + f.write(token + "\n") + + +def _decode_jwt_exp(jwt_token): + """解析 JWT payload 的 'exp' 字段（秒级时间戳）；任何失败返回 0。""" + if not jwt_token or "." not in jwt_token: + return 0 + try: + payload_b64 = jwt_token.split(".")[1] + payload_b64 += "=" * (-len(payload_b64) % 4) + payload = json.loads(base64.urlsafe_b64decode(payload_b64.encode("ascii"))) + return int(payload.get("exp") or 0) + except Exception: + return 0 + + +def _extract_rotated_cookie(response, original_cookie): + """从响应里取 NextAuth 滚动续期回写的新 session-token。 + + 支持： + - 单片：`__Secure-next-auth.session-token=` + - 多片：`__Secure-next-auth.session-token.0=...; .1=...; .2=...` + 多片按下标排序后用 SESS_CHUNK_SEPARATOR（'|||'）拼接，与存储格式一致 + + 返回： + - 新 cookie 值（已剥除 cookie 名）；与 original_cookie 相同则返回 None + - 解析失败或服务端未续期返回 None + """ + base_name = NEXTAUTH_COOKIE_NAME # __Secure-next-auth.session-token + chunks = {} # idx -> value；-1 代表单片 + try: + # curl_cffi 响应的 cookies 实际是 RequestsCookieJar（dict-like + .items()） + jar = getattr(response, "cookies", None) + if jar is not None: + try: + items = list(jar.items()) + except Exception: + items = [(c.name, c.value) for c in jar] + for name, value in items: + if not name or not value: + continue + if name == base_name: + chunks[-1] = value + elif name.startswith(base_name + "."): + suffix = name[len(base_name) + 1:] + if suffix.isdigit(): + chunks[int(suffix)] = value + except Exception as e: + logger.warning(f"[rotated_cookie] parse cookies failed: {e!r}") + return None + + if not chunks: + return None + + if -1 in chunks and len(chunks) == 1: + new_value = chunks[-1] + else: + ordered = [chunks[i] for i in sorted(k for k in chunks if k >= 0)] + if not ordered: + return None + new_value = SESS_CHUNK_SEPARATOR.join(ordered) + + if not new_value or new_value == original_cookie: + return None + return new_value + + +async def _migrate_session_key(old_key, new_key, new_access_token, jwt_exp, proxy_url=""): + """cookie 滚动后，把所有以 old_key 为索引的数据迁移到 new_key 并持久化。 + + 覆盖：refresh_map / token_list(+ token.txt) / fp_map(+ fp_map.json) + / routing_config.bindings & account_meta(+ routing_config.json) + / error_token_list(+ error_token.txt) + """ + if old_key == new_key: + return + async with _session_key_lock: + now = int(time.time()) + + # 1) refresh_map：复制旧条目并合并新字段，再删旧 key + meta = dict(globals.refresh_map.get(old_key, {})) + meta.update({ + "token": new_access_token, + "timestamp": now, + "last_success_at": now, + "last_error": "", + "last_error_at": 0, + "fail_count": 0, + "jwt_exp": jwt_exp, + "last_proxy": proxy_url or meta.get("last_proxy", ""), + "rotated_from": old_key[:24] + "...", + "rotated_at": now, + }) + globals.refresh_map[new_key] = meta + globals.refresh_map.pop(old_key, None) + persist_refresh_map() + + # 2) token_list / token.txt + if old_key in globals.token_list: + idx = globals.token_list.index(old_key) + globals.token_list[idx] = new_key + globals.persist_token_list() + + # 3) fp_map / fp_map.json + if old_key in globals.fp_map: + globals.fp_map[new_key] = globals.fp_map.pop(old_key) + globals.persist_fp_map() + + # 4) routing_config bindings & account_meta + routing_changed = False + bindings = globals.routing_config.get("bindings", {}) if isinstance(globals.routing_config, dict) else {} + if old_key in bindings: + bindings[new_key] = bindings.pop(old_key) + routing_changed = True + account_meta = globals.routing_config.get("account_meta", {}) if isinstance(globals.routing_config, dict) else {} + if old_key in account_meta: + account_meta[new_key] = account_meta.pop(old_key) + routing_changed = True + if routing_changed: + save_routing_config(globals.routing_config) + + # 5) error_token_list（理论上 rotation 发生时 old_key 不在 error 里，但兜底） + if old_key in globals.error_token_list: + globals.error_token_list[:] = [ + (new_key if t == old_key else t) for t in globals.error_token_list + ] + persist_error_tokens() + + logger.info( + f"[rotation] session-token rotated: {old_key[:16]}... -> {new_key[:16]}... " + f"(jwt_exp={jwt_exp}, +{jwt_exp - now}s)" + ) + async def rt2ac(refresh_token, force_refresh=False): if not force_refresh and (refresh_token in globals.refresh_map and int(time.time()) - globals.refresh_map.get(refresh_token, {}).get("timestamp", 0) < 5 * 24 * 60 * 60): @@ -19,42 +174,301 @@ async def rt2ac(refresh_token, force_refresh=False): else: try: access_token = await chat_refresh(refresh_token) - globals.refresh_map[refresh_token] = {"token": access_token, "timestamp": int(time.time())} - with open(globals.REFRESH_MAP_FILE, "w") as f: - json.dump(globals.refresh_map, f, indent=4) + refresh_meta = globals.refresh_map.get(refresh_token, {}) + now = int(time.time()) + refresh_meta.update({ + "token": access_token, + "timestamp": now, + "last_success_at": now, + "last_error": "", + "last_error_at": 0, + "fail_count": 0, + }) + globals.refresh_map[refresh_token] = refresh_meta + if refresh_token in globals.error_token_list: + globals.error_token_list[:] = [item for item in globals.error_token_list if item != refresh_token] + persist_error_tokens() + persist_refresh_map() logger.info(f"refresh_token -> access_token with openai: {access_token}") return access_token except HTTPException as e: raise HTTPException(status_code=e.status_code, detail=e.detail) +async def sess2ac(session_token, force_refresh=False): + """Session cookie → access_token（带 NextAuth 滚动续期）。 + + `session_token` 是带 'sess-' 前缀的存储形态（外部传入时已剥除 or 保留都支持）。 + 缓存条件： + 1) timestamp 距今 < 8 分钟（绝对节流，避免高频请求打爆 NextAuth） + 2) 解码后的 JWT exp 距今 > 5 分钟（确保 token 真的还能用，避免拿死 token） + 任一条件不满足都强制刷新。 + """ + # 统一 key：带前缀的是存储形态，剥除后的是 cookie 真实值 + if session_token.startswith("sess-"): + storage_key = session_token + cookie_value = session_token[5:] + else: + storage_key = "sess-" + session_token + cookie_value = session_token + + # 缓存命中：timestamp 节流 + JWT exp 真实有效性双重校验 + now = int(time.time()) + cached_meta = globals.refresh_map.get(storage_key, {}) + cached_token = cached_meta.get("token", "") + cached_ts = int(cached_meta.get("timestamp", 0)) + cached_exp = int(cached_meta.get("jwt_exp", 0)) or _decode_jwt_exp(cached_token) + if (not force_refresh + and cached_token + and now - cached_ts < 8 * 60 + and (cached_exp == 0 or cached_exp - now > 300)): + return cached_token + + try: + access_token, effective_key, jwt_exp = await fetch_session_access_token(cookie_value) + # 若 fetch 内部完成了 cookie rotation，effective_key != storage_key， + # 此时 refresh_map[effective_key] 已被 _migrate_session_key 完整填充，无需重复写 + if effective_key == storage_key: + now = int(time.time()) + refresh_meta = globals.refresh_map.get(storage_key, {}) + refresh_meta.update({ + "token": access_token, + "timestamp": now, + "last_success_at": now, + "last_error": "", + "last_error_at": 0, + "fail_count": 0, + "jwt_exp": jwt_exp, + }) + globals.refresh_map[storage_key] = refresh_meta + if storage_key in globals.error_token_list: + globals.error_token_list[:] = [item for item in globals.error_token_list if item != storage_key] + persist_error_tokens() + persist_refresh_map() + logger.info( + f"session_cookie -> access_token OK (key={effective_key[:12]}..., " + f"jwt_exp_in={(jwt_exp - int(time.time())) if jwt_exp else 'n/a'}s, " + f"rotated={'yes' if effective_key != storage_key else 'no'})" + ) + return access_token + except HTTPException as e: + raise HTTPException(status_code=e.status_code, detail=e.detail) + + +async def fetch_session_access_token(session_cookie): + """带 __Secure-next-auth.session-token cookie 访问 chatgpt.com/api/auth/session。 + + 支持两种 storage_key 格式： + - 单片：sess- + - 多片（JWE 超 4KB 被 NextAuth 分片）：sess-|||||| + + 返回响应 JSON 中的 accessToken 字段（JWT，调 chatgpt.com/backend-api 的 Bearer）。 + """ + session_id = hashlib.md5(session_cookie.encode()).hexdigest() + storage_key = "sess-" + session_cookie + bound_proxy = get_bound_proxy(storage_key) + proxy_url = bound_proxy or (random.choice(proxy_url_list).replace("{}", session_id) if proxy_url_list else None) + if proxy_url: + proxy_url = proxy_url.replace("{}", session_id) + refresh_meta = globals.refresh_map.get(storage_key, {}) + refresh_meta["last_proxy"] = proxy_url or "" + globals.refresh_map[storage_key] = refresh_meta + + # 按 NextAuth 协议组装 Cookie header + # 单片 → 一条 __Secure-next-auth.session-token= + # 多片 → 多条 __Secure-next-auth.session-token.0=xxx; .1=yyy; ... + cookie_header = _build_nextauth_cookie_header(session_cookie) + + client = Client(proxy=proxy_url, impersonate="chrome124") + try: + r = await client.get( + "https://chatgpt.com/api/auth/session", + headers={ + "Accept": "application/json", + "Accept-Language": "en-US,en;q=0.9", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", + "Cookie": cookie_header, + }, + timeout=15, + ) + raw_text = (r.text or "").strip() + content_type = r.headers.get("content-type", "") + logger.info( + f"[sess2ac] key={storage_key[:12]}... status={r.status_code} " + f"ctype={content_type} body_len={len(raw_text)} " + f"proxy={'yes' if proxy_url else 'no'} chunks={cookie_header.count('session-token')}" + ) + + if r.status_code != 200: + if storage_key not in globals.error_token_list and r.status_code in (401, 403): + globals.error_token_list.append(storage_key) + persist_error_tokens() + raise Exception( + f"chatgpt.com/api/auth/session status={r.status_code}: {raw_text[:200]}" + ) + if not raw_text: + raise Exception("chatgpt.com/api/auth/session 返回空响应；cookie 可能已失效") + + try: + payload = json.loads(raw_text) + except json.JSONDecodeError: + raise Exception(f"非 JSON 响应 ctype={content_type}: {raw_text[:200]}") + + # 未登录时 NextAuth 返回 {} 或 {"user": null} + access_token = payload.get("accessToken") or payload.get("access_token") + if not access_token: + if storage_key not in globals.error_token_list: + globals.error_token_list.append(storage_key) + persist_error_tokens() + raise Exception( + f"session cookie 无效或过期（response keys={list(payload.keys())}）。" + f"提示：NextAuth session token 可能分片，请确保同时提供 .0 和 .1（若存在）" + ) + + # NextAuth 滚动续期：尝试从响应 Set-Cookie 中提取新 session-token； + # 若拿到，立刻把所有数据结构里的旧 key 替换为新 key 并持久化，达成"永不过期" + jwt_exp = _decode_jwt_exp(access_token) + effective_storage_key = storage_key + try: + rotated = _extract_rotated_cookie(r, session_cookie) + except Exception as e: + rotated = None + logger.warning(f"[sess2ac] extract rotated cookie failed (non-fatal): {e!r}") + if rotated: + new_storage_key = "sess-" + rotated + await _migrate_session_key( + old_key=storage_key, + new_key=new_storage_key, + new_access_token=access_token, + jwt_exp=jwt_exp, + proxy_url=proxy_url or "", + ) + effective_storage_key = new_storage_key + return access_token, effective_storage_key, jwt_exp + except Exception as e: + now = int(time.time()) + refresh_meta = globals.refresh_map.get(storage_key, {}) + refresh_meta.update({ + "last_error": str(e)[:300], + "last_error_at": now, + "fail_count": int(refresh_meta.get("fail_count", 0)) + 1, + "last_proxy": proxy_url or "", + }) + globals.refresh_map[storage_key] = refresh_meta + persist_refresh_map() + logger.error(f"[sess2ac] key={storage_key[:12]}... failed: {str(e)[:400]}") + raise HTTPException(status_code=500, detail=str(e)[:300]) + finally: + await client.close() + del client + + +# 分片分隔符（内部用，不可与 base64url 字符冲突） +SESS_CHUNK_SEPARATOR = "|||" +NEXTAUTH_COOKIE_NAME = "__Secure-next-auth.session-token" + + +def _build_nextauth_cookie_header(session_cookie: str) -> str: + """根据存储的 session_cookie 字符串，构造发给 chatgpt.com 的 Cookie header。 + + Args: + session_cookie: 已去除 'sess-' 前缀的原始值。 + - 单片：直接是 cookie value + - 多片：||||||... + + Returns: + Cookie header 字符串（NextAuth 规范分片格式） + """ + if SESS_CHUNK_SEPARATOR in session_cookie: + chunks = session_cookie.split(SESS_CHUNK_SEPARATOR) + return "; ".join( + f"{NEXTAUTH_COOKIE_NAME}.{i}={chunk}" + for i, chunk in enumerate(chunks) + if chunk.strip() + ) + return f"{NEXTAUTH_COOKIE_NAME}={session_cookie}" + + async def chat_refresh(refresh_token): - data = { - "client_id": "pdlLIX2Y72MIl2rhLhTE9VV9bN905kBh", + # 使用 Codex CLI 风格：application/x-www-form-urlencoded + auth.openai.com + # 老版 auth0.openai.com + iOS client_id 已返回 404 + form_body = urlencode({ "grant_type": "refresh_token", - "redirect_uri": "com.openai.chat://auth0.openai.com/ios/com.openai.chat/callback", - "refresh_token": refresh_token + "client_id": openai_auth_client_id, + "refresh_token": refresh_token, + "scope": openai_auth_scope, + }) + headers = { + "Accept": "application/json", + "Content-Type": "application/x-www-form-urlencoded", + "User-Agent": "Codex_CLI/0.1.0", } session_id = hashlib.md5(refresh_token.encode()).hexdigest() - proxy_url = random.choice(proxy_url_list).replace("{}", session_id) if proxy_url_list else None - client = Client(proxy=proxy_url) + bound_proxy = get_bound_proxy(refresh_token) + proxy_url = bound_proxy or (random.choice(proxy_url_list).replace("{}", session_id) if proxy_url_list else None) + if proxy_url: + proxy_url = proxy_url.replace("{}", session_id) + refresh_meta = globals.refresh_map.get(refresh_token, {}) + refresh_meta["last_proxy"] = proxy_url or "" + globals.refresh_map[refresh_token] = refresh_meta + client = Client(proxy=proxy_url, impersonate=None) + token_prefix = refresh_token[:8] try: - r = await client.post("https://auth0.openai.com/oauth/token", json=data, timeout=15) + r = await client.post(openai_auth_token_url, data=form_body, headers=headers, timeout=15) + raw_text = (r.text or "").strip() + content_type = r.headers.get("content-type", "") + + # 诊断日志：每次刷新都记录上游返回的关键元数据 + logger.info( + f"[chat_refresh] token={token_prefix}... status={r.status_code} " + f"ctype={content_type} body_len={len(raw_text)} proxy={'yes' if proxy_url else 'no'} " + f"endpoint={openai_auth_token_url}" + ) + + # 200 路径：仍需防御解析 if r.status_code == 200: - access_token = r.json()['access_token'] - return access_token - else: - if "invalid_grant" in r.text or "access_denied" in r.text: - if refresh_token not in globals.error_token_list: - globals.error_token_list.append(refresh_token) - with open(globals.ERROR_TOKENS_FILE, "a", encoding="utf-8") as f: - f.write(refresh_token + "\n") - raise Exception(r.text) - else: - raise Exception(r.text[:300]) + if not raw_text: + raise Exception("OpenAI returned empty body with status 200") + try: + payload = json.loads(raw_text) + except json.JSONDecodeError: + raise Exception( + f"OpenAI non-JSON response (status 200, ctype={content_type}): " + f"{raw_text[:200]}" + ) + if "access_token" not in payload: + raise Exception( + f"OpenAI JSON missing access_token: keys={list(payload.keys())} " + f"body={raw_text[:200]}" + ) + return payload["access_token"] + + # 非 200 路径：详细分流并记录 + error_body_hint = raw_text[:300] if raw_text else "(empty body)" + if "invalid_grant" in raw_text or "access_denied" in raw_text or "refresh_token_expired" in raw_text: + if refresh_token not in globals.error_token_list: + globals.error_token_list.append(refresh_token) + persist_error_tokens() + raise Exception( + f"OpenAI rejected refresh_token (status {r.status_code}): {error_body_hint}" + ) + raise Exception( + f"OpenAI refresh failed (status {r.status_code}, ctype={content_type}): {error_body_hint}" + ) except Exception as e: - logger.error(f"Failed to refresh access_token `{refresh_token}`: {str(e)}") - raise HTTPException(status_code=500, detail=f"Failed to refresh access_token.") + now = int(time.time()) + refresh_meta = globals.refresh_map.get(refresh_token, {}) + refresh_meta.update({ + "last_error": str(e)[:300], + "last_error_at": now, + "fail_count": int(refresh_meta.get("fail_count", 0)) + 1, + "last_proxy": proxy_url or "", + }) + globals.refresh_map[refresh_token] = refresh_meta + persist_refresh_map() + logger.error(f"[chat_refresh] token={token_prefix}... failed: {str(e)[:400]}") + raise HTTPException(status_code=500, detail=str(e)[:300]) finally: await client.close() del client diff --git a/chatgpt/services/__init__.py b/chatgpt/services/__init__.py new file mode 100644 index 00000000..177761fc --- /dev/null +++ b/chatgpt/services/__init__.py @@ -0,0 +1,13 @@ +"""ChatService 子职责 Mixin 包。 + +各 Mixin 通过 self.* 共享 ChatService 实例状态： +- AuthMixin：rese_auth_context（req_token -> access_token + account_id） +- ModelMixin：模型解析、上游模型列表缓存、可用性校验 +- FileMixin：文件上传/下载相关的 8 个上游端点封装 +""" + +from chatgpt.services.auth_mixin import AuthMixin +from chatgpt.services.file_mixin import FileMixin +from chatgpt.services.model_mixin import ModelMixin + +__all__ = ["AuthMixin", "FileMixin", "ModelMixin"] diff --git a/chatgpt/services/_helpers.py b/chatgpt/services/_helpers.py new file mode 100644 index 00000000..1e8e6073 --- /dev/null +++ b/chatgpt/services/_helpers.py @@ -0,0 +1,30 @@ +"""无状态头部工具函数。 + +供 ChatService 及其 Mixin 类型化和清洗 HTTP 头部使用。 +原始位置：chatgpt/ChatService.py:46-66 +""" + +import json + + +def _stringify_header_value(value): + if value is None: + return None + if isinstance(value, str): + return value + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, (int, float)): + return str(value) + return json.dumps(value, ensure_ascii=False, separators=(",", ":"), default=str) + + +def _sanitize_headers(headers): + clean = {} + for key, value in (headers or {}).items(): + if not key: + continue + value = _stringify_header_value(value) + if value is not None: + clean[str(key)] = value + return clean diff --git a/chatgpt/services/auth_mixin.py b/chatgpt/services/auth_mixin.py new file mode 100644 index 00000000..e20d7eb5 --- /dev/null +++ b/chatgpt/services/auth_mixin.py @@ -0,0 +1,24 @@ +"""认证上下文解析 Mixin。 + +封装从 req_token 解析 access_token 与 account_id 的逻辑。 +原始位置：chatgpt/ChatService.py:146-158 +""" + +from chatgpt.authorization import verify_token +from utils.Logger import logger + + +class AuthMixin: + async def resolve_auth_context(self): + if self.req_token: + req_len = len(self.req_token.split(",")) + if req_len == 1: + self.access_token = await verify_token(self.req_token) + self.account_id = None + else: + self.access_token = await verify_token(self.req_token.split(",")[0]) + self.account_id = self.req_token.split(",")[1] + else: + logger.info("Request token is empty, use no-auth 3.5") + self.access_token = None + self.account_id = None diff --git a/chatgpt/services/file_mixin.py b/chatgpt/services/file_mixin.py new file mode 100644 index 00000000..9cba77c9 --- /dev/null +++ b/chatgpt/services/file_mixin.py @@ -0,0 +1,170 @@ +"""文件上传/下载操作 Mixin。 + +封装与 ChatGPT /files 与 /conversation 文件相关的 8 个端点： +- 上传：get_upload_url → upload → check_upload → get_download_url_from_upload +- 下载：get_download_url / get_attachment_url / get_response_file_url +- 高层入口：upload_file（图片尺寸探测、扩展名/用途推断） + +原始位置：chatgpt/ChatService.py:580-728 +""" + +import asyncio +import uuid + +from fastapi import HTTPException + +from api.files import get_image_size, get_file_extension, determine_file_use_case +from utils.configs import client_timezone_offset_min +from utils.Logger import logger + + +class FileMixin: + async def get_download_url(self, file_id): + url = f"{self.base_url}/files/{file_id}/download" + headers = self.base_headers.copy() + try: + r = await self.s.get(url, headers=headers, timeout=10) + if r.status_code == 200: + download_url = r.json().get('download_url') + return download_url + else: + raise HTTPException(status_code=r.status_code, detail=r.text) + except Exception as e: + logger.error(f"Failed to get download url: {e}") + return "" + + async def get_attachment_url(self, file_id, conversation_id): + url = f"{self.base_url}/conversation/{conversation_id}/attachment/{file_id}/download" + headers = self.base_headers.copy() + try: + r = await self.s.get(url, headers=headers, timeout=10) + if r.status_code == 200: + download_url = r.json().get('download_url') + return download_url + else: + raise HTTPException(status_code=r.status_code, detail=r.text) + except Exception as e: + logger.error(f"Failed to get download url: {e}") + return "" + + async def get_download_url_from_upload(self, file_id): + url = f"{self.base_url}/files/{file_id}/uploaded" + headers = self.base_headers.copy() + try: + r = await self.s.post(url, headers=headers, json={}, timeout=10) + if r.status_code == 200: + download_url = r.json().get('download_url') + return download_url + else: + raise HTTPException(status_code=r.status_code, detail=r.text) + except Exception as e: + logger.error(f"Failed to get download url from upload: {e}") + return "" + + async def get_upload_url(self, file_name, file_size, use_case="multimodal"): + url = f'{self.base_url}/files' + headers = self.base_headers.copy() + try: + r = await self.s.post( + url, + headers=headers, + json={"file_name": file_name, "file_size": file_size, "reset_rate_limits": False, "timezone_offset_min": client_timezone_offset_min, "use_case": use_case}, + timeout=5, + ) + if r.status_code == 200: + res = r.json() + file_id = res.get('file_id') + upload_url = res.get('upload_url') + logger.info(f"file_id: {file_id}, upload_url: {upload_url}") + return file_id, upload_url + else: + raise HTTPException(status_code=r.status_code, detail=r.text) + except Exception as e: + logger.error(f"Failed to get upload url: {e}") + return "", "" + + async def upload(self, upload_url, file_content, mime_type): + headers = self.base_headers.copy() + headers.update( + { + 'accept': 'application/json, text/plain, */*', + 'content-type': mime_type, + 'x-ms-blob-type': 'BlockBlob', + 'x-ms-version': '2020-04-08', + } + ) + headers.pop('authorization', None) + headers.pop('oai-device-id', None) + headers.pop('oai-language', None) + try: + r = await self.s.put(upload_url, headers=headers, data=file_content, timeout=60) + if r.status_code == 201: + return True + else: + raise HTTPException(status_code=r.status_code, detail=r.text) + except Exception as e: + logger.error(f"Failed to upload file: {e}") + return False + + async def upload_file(self, file_content, mime_type): + if not file_content or not mime_type: + return None + + width, height = None, None + if mime_type.startswith("image/"): + try: + width, height = await get_image_size(file_content) + except Exception as e: + logger.error(f"Error image mime_type, change to text/plain: {e}") + mime_type = 'text/plain' + file_size = len(file_content) + file_extension = await get_file_extension(mime_type) + file_name = f"{uuid.uuid4()}{file_extension}" + use_case = await determine_file_use_case(mime_type) + + file_id, upload_url = await self.get_upload_url(file_name, file_size, use_case) + if file_id and upload_url: + if await self.upload(upload_url, file_content, mime_type): + download_url = await self.get_download_url_from_upload(file_id) + if download_url: + file_meta = { + "file_id": file_id, + "file_name": file_name, + "size_bytes": file_size, + "mime_type": mime_type, + "width": width, + "height": height, + "use_case": use_case, + } + logger.info(f"File_meta: {file_meta}") + return file_meta + + async def check_upload(self, file_id): + url = f'{self.base_url}/files/{file_id}' + headers = self.base_headers.copy() + try: + for i in range(30): + r = await self.s.get(url, headers=headers, timeout=5) + if r.status_code == 200: + res = r.json() + retrieval_index_status = res.get('retrieval_index_status', '') + if retrieval_index_status == "success": + break + await asyncio.sleep(1) + return True + except HTTPException: + return False + + async def get_response_file_url(self, conversation_id, message_id, sandbox_path): + try: + url = f"{self.base_url}/conversation/{conversation_id}/interpreter/download" + params = {"message_id": message_id, "sandbox_path": sandbox_path} + headers = self.base_headers.copy() + r = await self.s.get(url, headers=headers, params=params, timeout=10) + if r.status_code == 200: + return r.json().get("download_url") + else: + return None + except Exception: + logger.info("Failed to get response file url") + return None diff --git a/chatgpt/services/model_mixin.py b/chatgpt/services/model_mixin.py new file mode 100644 index 00000000..32d82b39 --- /dev/null +++ b/chatgpt/services/model_mixin.py @@ -0,0 +1,94 @@ +"""模型解析与上游模型校验 Mixin。 + +负责： +- 上游可用模型列表的拉取与缓存（host + account + token 维度，TTL 300s） +- 请求模型解析（origin -> req/resp/gizmo/dynamic） +- 模型可用性校验，统一 404 model_not_found 语义 + +原始位置：chatgpt/ChatService.py:87-144 / 293-303 +""" + +import hashlib +import time + +from fastapi import HTTPException + +from api.models import augment_model_slugs, extract_model_slugs, get_response_model, resolve_request_model +from utils.configs import check_model +from utils.Logger import logger + + +class ModelMixin: + available_model_cache = {} + available_model_cache_ttl = 300 + + def model_not_found(self): + return HTTPException( + status_code=404, + detail={ + "message": f"The model `{self.origin_model}` does not exist or you do not have access to it.", + "type": "invalid_request_error", + "param": None, + "code": "model_not_found", + }, + ) + + def get_model_cache_key(self): + token = self.req_token.split(",")[0] if self.req_token else "anon" + token_hash = hashlib.sha256(token.encode()).hexdigest() + account_id = self.account_id or "default" + return f"{self.host_url}:{account_id}:{token_hash}" + + async def fetch_available_models(self): + cache_key = self.get_model_cache_key() + now = time.time() + cached = self.available_model_cache.get(cache_key) + if cached and now - cached["time"] < self.available_model_cache_ttl: + return cached["slugs"] + + url = f"{self.host_url}/backend-api/models?history_and_training_disabled={str(self.history_disabled).lower()}" + headers = self.base_headers.copy() + r = await self.s.get(url, headers=headers, timeout=10) + if r.status_code != 200: + detail = r.text + if "application/json" in r.headers.get("Content-Type", ""): + detail = r.json().get("detail", r.json()) + raise HTTPException(status_code=r.status_code, detail=detail) + + models_payload = r.json() + model_slugs = augment_model_slugs(extract_model_slugs(models_payload)) + self.available_model_cache[cache_key] = { + "time": now, + "slugs": model_slugs, + } + logger.info(f"Available models exposed: {len(model_slugs)}") + return model_slugs + + async def validate_model_access(self): + if self.gizmo_id: + return + + if not self.access_token: + if self.req_model != "text-davinci-002-render-sha": + raise self.model_not_found() + return + + if not (self.dynamic_model or check_model): + return + + available_models = await self.fetch_available_models() + if self.req_model not in available_models: + logger.error(f"Model {self.req_model} not found in upstream models") + raise self.model_not_found() + + async def set_model(self): + self.origin_model = self.data.get("model", "gpt-5-5") + self.resp_model = get_response_model(self.origin_model) + self.req_model, self.gizmo_id, self.dynamic_model = resolve_request_model(self.origin_model) + + # 深度研究：模型名后缀识别（双模式触发之二） + # 当模型名包含 deep-research / deepresearch 时，自动注入 system_hints=["research"] + lower_origin = (self.origin_model or "").lower() + if "deep-research" in lower_origin or "deepresearch" in lower_origin: + if "research" not in self.system_hints: + self.system_hints = list(self.system_hints) + ["research"] diff --git a/chatgpt/session_sticky.py b/chatgpt/session_sticky.py new file mode 100644 index 00000000..c98829dc --- /dev/null +++ b/chatgpt/session_sticky.py @@ -0,0 +1,212 @@ +"""LibreChat 会话粘性：LibreChat conv_id ↔ ChatGPT conv_id 翻译层。 + +设计 (KISS / YAGNI / DRY)： + - SQLite 单表持久化映射；WAL 模式 + 短连接，async 友好 + - 入口 inject_session：命中映射 → 在 request_data 注入 conversation_id / parent_message_id； + 未命中 → 不动 body，让 ChatService 走"新建对话"流程 + - 出口 sniff_and_save：流式响应每个 chunk 含 conv_id / message_id 时回写 DB + - 不影响现有逻辑：开关 enable_session_sticky 关闭时所有 API 直接 return + +外部 API： + init_db() + inject_session(request_data) -> Optional[str] # 返回 lc_conv_id 用于后续嗅探 + sniff_and_save(lc_conv_id, chatgpt_conv_id, parent_msg_id) + drop_mapping(lc_conv_id) # 命中后 ChatGPT 返回 404 时清理 + cleanup_expired() # 超 TTL 自动清理 +""" +from __future__ import annotations + +import os +import sqlite3 +import threading +import time +from typing import Optional, Tuple + +from utils.Logger import logger +from utils import configs + +_DB_INITIALIZED = False +_INIT_LOCK = threading.Lock() +_WRITE_LOCK = threading.Lock() # 避免并发写竞争 (uvicorn 单 worker async 通常不需要，但 worker>1 时保险) + + +def _enabled() -> bool: + return bool(getattr(configs, "enable_session_sticky", False)) + + +def _db_path() -> str: + return getattr(configs, "session_db_path", "data/sessions.db") + + +def _lc_field() -> str: + return getattr(configs, "session_lc_field", "librechat_conversation_id") + + +def _connect() -> sqlite3.Connection: + path = _db_path() + parent = os.path.dirname(path) + if parent and not os.path.exists(parent): + os.makedirs(parent, exist_ok=True) + conn = sqlite3.connect(path, timeout=5.0, isolation_level=None) + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA synchronous=NORMAL") + conn.execute("PRAGMA busy_timeout=3000") + return conn + + +def init_db() -> None: + """容器启动时调用一次。多次调用安全。""" + global _DB_INITIALIZED + if not _enabled(): + return + with _INIT_LOCK: + if _DB_INITIALIZED: + return + try: + with _connect() as conn: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS lc_session_map ( + librechat_conv_id TEXT PRIMARY KEY, + chatgpt_conv_id TEXT NOT NULL, + parent_msg_id TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ) + """ + ) + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_lc_updated ON lc_session_map(updated_at)" + ) + _DB_INITIALIZED = True + logger.info(f"[session_sticky] db ready at {_db_path()}") + except Exception as e: + logger.error(f"[session_sticky] init failed: {e}") + + +def _get_mapping(lc_conv_id: str) -> Optional[Tuple[str, Optional[str]]]: + try: + with _connect() as conn: + row = conn.execute( + "SELECT chatgpt_conv_id, parent_msg_id FROM lc_session_map WHERE librechat_conv_id=?", + (lc_conv_id,), + ).fetchone() + return (row[0], row[1]) if row else None + except Exception as e: + logger.error(f"[session_sticky] get_mapping error: {e}") + return None + + +def _upsert_mapping(lc_conv_id: str, chatgpt_conv_id: str, parent_msg_id: Optional[str]) -> None: + now = int(time.time()) + try: + with _WRITE_LOCK, _connect() as conn: + conn.execute( + """ + INSERT INTO lc_session_map (librechat_conv_id, chatgpt_conv_id, parent_msg_id, created_at, updated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(librechat_conv_id) DO UPDATE SET + chatgpt_conv_id = excluded.chatgpt_conv_id, + parent_msg_id = excluded.parent_msg_id, + updated_at = excluded.updated_at + """, + (lc_conv_id, chatgpt_conv_id, parent_msg_id, now, now), + ) + except Exception as e: + logger.error(f"[session_sticky] upsert_mapping error: {e}") + + +def drop_mapping(lc_conv_id: str) -> None: + """ChatGPT 返回 404 / conv_id 失效时清理。下次同 lc_conv_id 会新建对话。""" + if not _enabled() or not lc_conv_id: + return + try: + with _WRITE_LOCK, _connect() as conn: + conn.execute("DELETE FROM lc_session_map WHERE librechat_conv_id=?", (lc_conv_id,)) + logger.info(f"[session_sticky] dropped mapping for {lc_conv_id[:12]}...") + except Exception as e: + logger.error(f"[session_sticky] drop_mapping error: {e}") + + +def cleanup_expired() -> int: + """删除超过 TTL 未更新的映射；返回删除条数。""" + if not _enabled(): + return 0 + ttl_days = int(getattr(configs, "session_ttl_days", 30)) + cutoff = int(time.time()) - ttl_days * 86400 + try: + with _WRITE_LOCK, _connect() as conn: + cur = conn.execute("DELETE FROM lc_session_map WHERE updated_at < ?", (cutoff,)) + deleted = cur.rowcount or 0 + if deleted: + logger.info(f"[session_sticky] cleanup_expired removed {deleted} rows") + return deleted + except Exception as e: + logger.error(f"[session_sticky] cleanup_expired error: {e}") + return 0 + + +def inject_session(request_data: dict) -> Optional[str]: + """请求入口注入。 + + 返回： + lc_conv_id（用于后续 sniff_and_save 回写）；功能未启用或请求体无 lc_conv_id 时返回 None。 + + 副作用： + 命中映射 → 写入 request_data['conversation_id'] / ['parent_message_id'] + → 默认把 messages 截到只保留最后一条 user message（节省 token， + 依赖 ChatGPT 服务端续接历史） + """ + if not _enabled() or not isinstance(request_data, dict): + return None + lc_conv_id = request_data.get(_lc_field()) + if not lc_conv_id or not isinstance(lc_conv_id, str): + return None + + init_db() # 懒初始化 + mapping = _get_mapping(lc_conv_id) + if not mapping: + # 首次见到该 lc_conv_id；让 ChatGPT 创建新 conv，嗅探阶段再回写 + logger.info(f"[session_sticky] miss lc={lc_conv_id[:12]}... → new conv") + return lc_conv_id + + chatgpt_conv_id, parent_msg_id = mapping + # 仅当用户没有显式传入 conversation_id 时才注入（用户显式传入优先级最高） + if not request_data.get("conversation_id"): + request_data["conversation_id"] = chatgpt_conv_id + if parent_msg_id and not request_data.get("parent_message_id"): + request_data["parent_message_id"] = parent_msg_id + # 续接 ChatGPT 端历史 → 强制 history_disabled=False + request_data["history_disabled"] = False + + # 截短 messages：仅保留最后一条 user message + 可选 system + if getattr(configs, "session_trim_to_last_user", True): + msgs = request_data.get("messages") or [] + if isinstance(msgs, list) and len(msgs) > 1: + trimmed = [] + # 保留首条 system（若有） + if msgs and isinstance(msgs[0], dict) and msgs[0].get("role") == "system": + trimmed.append(msgs[0]) + # 取最后一条 user + last_user = next( + (m for m in reversed(msgs) if isinstance(m, dict) and m.get("role") == "user"), + None, + ) + if last_user: + trimmed.append(last_user) + request_data["messages"] = trimmed + + logger.info( + f"[session_sticky] hit lc={lc_conv_id[:12]}... → cv={chatgpt_conv_id[:8]}... " + f"parent={(parent_msg_id or '')[:8]}..." + ) + return lc_conv_id + + +def sniff_and_save(lc_conv_id: Optional[str], chatgpt_conv_id: Optional[str], + parent_msg_id: Optional[str]) -> None: + """流式响应嗅探回写。同一对话多次 chunk 触发时，最后一次 message_id 会覆盖。""" + if not _enabled() or not lc_conv_id or not chatgpt_conv_id: + return + init_db() + _upsert_mapping(lc_conv_id, chatgpt_conv_id, parent_msg_id) diff --git a/deploy/.env.template b/deploy/.env.template new file mode 100644 index 00000000..cbd25eb7 --- /dev/null +++ b/deploy/.env.template @@ -0,0 +1,29 @@ +# ============ 敏感凭据（勿提交到 git）============ +# 管理后台密码（请修改为强密码） +ADMIN_PASSWORD=CHANGE_ME_STRONG_ADMIN_PASSWORD + +# API 调用方凭据（你分发给客户端用） +AUTHORIZATION=CHANGE_ME_AUTHORIZATION_KEY + +# 访问路径前缀（避免被扫描器发现） +API_PREFIX=CHANGE_ME_API_PREFIX + +# ============ 运行参数 ============ +TZ=Asia/Shanghai +CHAT2API_PORT=60403 +CHAT2API_IMAGE=ghcr.io/nanashiwang/chat2api:latest + +# ============ 可选安全加固 ============ +# 管理后台 IP 白名单（推荐配置，多条用逗号，支持 CIDR） +# ADMIN_IP_WHITELIST=1.2.3.4,10.0.0.0/8 +# 是否信任反代的 X-Forwarded-For（在 Cloudflare/Nginx 后开启） +# ADMIN_TRUST_PROXY=true + +# ============ 可选：代理 ============ +# 如果服务器直连 OpenAI 有问题，配置代理 +# PROXY_URL=socks5://user:pass@gate.residential-proxy.com:10000 +# EXPORT_PROXY_URL=socks5://user:pass@gate.residential-proxy.com:10000 + +# ============ 可选：OAuth 覆盖（一般不用动） ============ +# OPENAI_AUTH_CLIENT_ID=app_EMoamEEZ73f0CkXaXp7hrann +# OPENAI_AUTH_TOKEN_URL=https://auth.openai.com/oauth/token diff --git a/deploy/chat2api.sh b/deploy/chat2api.sh new file mode 100644 index 00000000..ac38d386 --- /dev/null +++ b/deploy/chat2api.sh @@ -0,0 +1,814 @@ +#!/usr/bin/env bash +set -euo pipefail + +CONFIG_FILE="/etc/chat2api.env" +GITHUB_RAW_DEFAULT="https://raw.githubusercontent.com/nanashiwang/chat2api/main" +GITHUB_REPO_DEFAULT="https://github.com/nanashiwang/chat2api" + +if [[ -f "$CONFIG_FILE" ]]; then + # shellcheck disable=SC1091 + source "$CONFIG_FILE" +fi + +GITHUB_RAW="${GITHUB_RAW:-$GITHUB_RAW_DEFAULT}" +GITHUB_REPO="${GITHUB_REPO:-$GITHUB_REPO_DEFAULT}" + +detect_install_dir() { + if [[ -n "${INSTALL_DIR:-}" && -d "${INSTALL_DIR:-}" ]]; then + printf "%s\n" "$INSTALL_DIR" + return + fi + + if [[ -f "./docker-compose.yml" || -f "./compose.yml" || -f "./compose.yaml" ]]; then + pwd + return + fi + + local candidates=( + "/opt/chat2api" + "/opt/chat2api/data" + "/srv/chat2api" + "/root/chat2api" + "$HOME/chat2api" + ) + + local dir + for dir in "${candidates[@]}"; do + if [[ -f "$dir/docker-compose.yml" || -f "$dir/compose.yml" || -f "$dir/compose.yaml" || -x "$dir/deploy/multi/manage.sh" ]]; then + printf "%s\n" "$dir" + return + fi + done + + return 1 +} + +INSTALL_DIR="$(detect_install_dir || true)" +if [[ -z "$INSTALL_DIR" ]]; then + echo "Cannot find chat2api compose directory." + echo "Run this command inside your deployment directory," + echo "or create /etc/chat2api.env," + echo "or run deploy/install.sh / deploy/install-command.sh first." + exit 1 +fi + +cd "$INSTALL_DIR" + +is_multi_install() { + [[ -x "$INSTALL_DIR/deploy/multi/manage.sh" ]] +} + +is_multi_generated() { + [[ -f "$INSTALL_DIR/deploy/multi/generated/docker-compose.yml" ]] +} + +run_multi_manage() { + (cd "$INSTALL_DIR/deploy/multi" && ./manage.sh "$@") +} + +install_latest_cli_from_repo() { + local src="$INSTALL_DIR/deploy/chat2api.sh" + [[ -f "$src" ]] || return 0 + if [[ "$(id -u)" -eq 0 ]]; then + install -m 0755 "$src" /usr/local/bin/chat2api 2>/dev/null && echo "[✓] 管理命令已更新: /usr/local/bin/chat2api" || true + elif command -v sudo >/dev/null 2>&1; then + sudo install -m 0755 "$src" /usr/local/bin/chat2api 2>/dev/null && echo "[✓] 管理命令已更新: /usr/local/bin/chat2api" || true + fi +} + +install_latest_cli_from_remote() { + local tmp_script + tmp_script="$(mktemp)" || return 0 + if curl -fsSL --max-time 15 "$GITHUB_RAW/deploy/chat2api.sh" -o "$tmp_script"; then + if [[ "$(id -u)" -eq 0 ]]; then + install -m 0755 "$tmp_script" /usr/local/bin/chat2api 2>/dev/null && echo "[✓] 管理命令已更新: /usr/local/bin/chat2api" || true + elif command -v sudo >/dev/null 2>&1; then + sudo install -m 0755 "$tmp_script" /usr/local/bin/chat2api 2>/dev/null && echo "[✓] 管理命令已更新: /usr/local/bin/chat2api" || true + fi + else + echo "[!] 管理命令更新失败，继续执行当前操作" + fi + rm -f "$tmp_script" +} + +sync_deploy_assets_from_remote() { + if [[ "${NO_PULL:-0}" == "1" ]]; then + return 0 + fi + if ! command -v tar >/dev/null 2>&1; then + echo "[!] tar 不可用，跳过部署脚本同步" + return 0 + fi + + local tmp_dir archive_dir + tmp_dir="$(mktemp -d)" || return 0 + echo "[*] 同步部署脚本与编排面板..." + if curl -fsSL --max-time 30 "${GITHUB_REPO}/archive/refs/heads/main.tar.gz" | tar -xz -C "$tmp_dir"; then + archive_dir="$(find "$tmp_dir" -maxdepth 1 -type d -name 'chat2api-*' | head -1)" + if [[ -n "$archive_dir" && -d "$archive_dir/deploy" ]]; then + mkdir -p "$INSTALL_DIR/deploy" + cp -R "$archive_dir/deploy"/. "$INSTALL_DIR/deploy/" + chmod +x "$INSTALL_DIR/deploy/chat2api.sh" "$INSTALL_DIR/deploy/multi/manage.sh" 2>/dev/null || true + install_latest_cli_from_repo + echo "[✓] 部署脚本与编排面板已同步" + else + echo "[!] 仓库包缺少 deploy/，跳过同步" + fi + else + echo "[!] 部署脚本同步失败，继续使用当前版本" + fi + rm -rf "$tmp_dir" +} + +update_repo_for_multi() { + # 让 `chat2api update` 自己先更新部署脚本，再调用 deploy/multi/manage.sh。 + # 本地跟踪文件改动会自动放入 git stash；不 reset、不删除用户改动。 + if [[ "${NO_PULL:-0}" == "1" ]]; then + echo "[*] NO_PULL=1，跳过 git pull" + return 0 + fi + + local repo_root branch stash_msg="" + if ! repo_root="$(git -C "$INSTALL_DIR" rev-parse --show-toplevel 2>/dev/null)"; then + echo "[*] 非 git 仓库，改用 GitHub 包同步" + sync_deploy_assets_from_remote + return 0 + fi + branch="$(git -C "$repo_root" rev-parse --abbrev-ref HEAD 2>/dev/null || echo HEAD)" + if [[ "$branch" == "HEAD" ]]; then + echo "[*] detached HEAD，跳过 git pull" + return 0 + fi + + if [[ -n "$(git -C "$repo_root" status --porcelain 2>/dev/null)" ]]; then + stash_msg="chat2api auto-stash before update $(date -u +%Y%m%dT%H%M%SZ)" + echo "[*] 检测到本地改动，自动暂存到 git stash..." + if git -C "$repo_root" stash push -m "$stash_msg" -- . >/dev/null 2>&1; then + echo "[✓] 本地改动已暂存：$stash_msg" + else + echo "[!] 自动暂存失败，继续使用当前代码" + return 0 + fi + fi + + echo "[*] git pull --ff-only ($branch)..." + if git -C "$repo_root" pull --ff-only --quiet; then + echo "[✓] 代码已同步到 $(git -C "$repo_root" log -1 --pretty='%h %s')" + install_latest_cli_from_repo + if [[ -n "$stash_msg" ]]; then + echo "[i] 被暂存的本地改动可用以下命令查看：git -C \"$repo_root\" stash list" + fi + else + if [[ -n "$stash_msg" ]]; then + git -C "$repo_root" stash pop --quiet >/dev/null 2>&1 || true + fi + echo "[!] git pull 失败，已继续使用当前代码" + fi +} + +as_root() { + if [[ "$(id -u)" -eq 0 ]]; then + "$@" + elif command -v sudo >/dev/null 2>&1; then + sudo "$@" + else + echo "需要 root 权限或安装 sudo" + return 1 + fi +} + +run_compose() { + if ! command -v docker >/dev/null 2>&1 || ! docker compose version >/dev/null 2>&1; then + echo "docker compose is not available" + exit 1 + fi + + if [[ "$(id -u)" -eq 0 ]] || docker info >/dev/null 2>&1; then + docker compose "$@" + elif command -v sudo >/dev/null 2>&1; then + sudo docker compose "$@" + else + echo "docker requires root permission or sudo" + exit 1 + fi +} + +stop_single_compose_if_present() { + if [[ -f "$INSTALL_DIR/docker-compose.yml" || -f "$INSTALL_DIR/compose.yml" || -f "$INSTALL_DIR/compose.yaml" ]]; then + run_compose down --remove-orphans + return + fi + return 1 +} + +remove_known_chat2api_containers() { + if ! command -v docker >/dev/null 2>&1; then + return 0 + fi + docker rm -f chat2api watchtower c2a-nginx c2a-orchestrator c2a-watchtower >/dev/null 2>&1 || true +} + +confirm_uninstall() { + local keep_data="$1" + if [[ "${CHAT2API_UNINSTALL_CONFIRM:-}" == "yes" ]]; then + return 0 + fi + if [[ "${2:-}" == "--yes" || "${2:-}" == "-y" ]]; then + return 0 + fi + + echo "============================================================" + echo " chat2api uninstall" + echo "============================================================" + echo "安装目录: $INSTALL_DIR" + if [[ "$keep_data" == "1" ]]; then + echo "操作: 停止服务，保留安装目录和数据" + else + echo "操作: 停止服务，并删除安装目录、/usr/local/bin/chat2api、/etc/chat2api.env" + fi + echo + read -r -p 'Type "yes" to proceed: ' ans }" >&2 + return 1 + fi + resolved="$(cd "$INSTALL_DIR" && pwd -P)" || return 1 + case "$resolved" in + /|/root|/home|/Users|/opt|/srv|/tmp) + echo "[!] Refusing to remove unsafe install directory: $resolved" >&2 + return 1 + ;; + esac + if [[ ! -f "$resolved/docker-compose.yml" && ! -f "$resolved/compose.yml" && ! -f "$resolved/compose.yaml" && ! -x "$resolved/deploy/multi/manage.sh" ]]; then + echo "[!] Refusing to remove non-chat2api directory: $resolved" >&2 + return 1 + fi + printf "%s\n" "$resolved" +} + +cmd_uninstall() { + local keep_data=0 assume_yes=0 arg install_dir_resolved + for arg in "$@"; do + case "$arg" in + --keep-data) keep_data=1 ;; + --yes|-y) assume_yes=1 ;; + *) + echo "Usage: chat2api uninstall [--keep-data] [--yes]" + return 1 + ;; + esac + done + + install_dir_resolved="$(resolve_uninstall_dir)" || return 1 + + if ! confirm_uninstall "$keep_data" "$([[ "$assume_yes" == "1" ]] && printf -- --yes)"; then + echo "Aborted." + return 0 + fi + + echo "[*] Stopping chat2api services..." + if is_multi_install && is_multi_generated; then + if ! run_multi_manage down; then + echo "[!] Failed to stop multi-instance services; uninstall aborted." + return 1 + fi + elif ! stop_single_compose_if_present; then + if is_multi_install; then + echo "[*] Multi-instance config is not generated; removing known chat2api containers..." + remove_known_chat2api_containers + else + echo "[!] Failed to stop services; uninstall aborted." + return 1 + fi + fi + + if [[ "$keep_data" == "1" ]]; then + echo "[✓] Services stopped. Data kept at: $INSTALL_DIR" + return 0 + fi + + echo "[*] Removing install directory and command..." + cd / + rm -rf -- "$install_dir_resolved" || as_root rm -rf -- "$install_dir_resolved" + as_root rm -f /usr/local/bin/chat2api "$CONFIG_FILE" + echo "[✓] chat2api uninstalled." +} + +# ============================================================ +# B: 模板同步（仅单实例模式） +# ============================================================ +# 提取 environment 块下的 ENV 名（" KEY: 'value'" 格式） +extract_env_keys() { + local file="$1" + awk ' + /^ environment:/ { in_env=1; next } + /^ [a-z]/ { in_env=0 } + in_env && /^ [A-Z][A-Z0-9_]*:/ { + sub(/:.*/, "") + sub(/^[ ]+/, "") + print + } + ' "$file" +} + +cmd_sync_template_check() { + # 仅打印是否有差异，返回 0=无差异 1=有差异 2=错误 + if is_multi_install; then + return 0 + fi + local local_compose="$INSTALL_DIR/docker-compose.yml" + [[ -f "$local_compose" ]] || return 2 + + local tmp_template + tmp_template="$(mktemp)" || return 2 + if ! curl -fsSL --max-time 8 "$GITHUB_RAW/deploy/docker-compose.template.yml" -o "$tmp_template" 2>/dev/null; then + rm -f "$tmp_template" + return 2 + fi + + local new_keys + new_keys="$(comm -23 \ + <(extract_env_keys "$tmp_template" | sort -u) \ + <(extract_env_keys "$local_compose" | sort -u))" + rm -f "$tmp_template" + + if [[ -z "$new_keys" ]]; then + return 0 + fi + echo "[i] Upstream template has new ENV keys:" + echo "$new_keys" | sed 's/^/ + /' + echo "[i] Run: chat2api sync-template # to merge" + return 1 +} + +cmd_sync_template() { + if is_multi_install; then + echo "Multi-instance mode auto-syncs via 'chat2api update' (regenerates from generate.py)." + echo "No template sync needed." + return 0 + fi + + local local_compose="$INSTALL_DIR/docker-compose.yml" + if [[ ! -f "$local_compose" ]]; then + echo "[!] $local_compose not found" + return 1 + fi + + local tmp_template + tmp_template="$(mktemp)" + echo "[*] Fetching latest template from $GITHUB_RAW ..." + if ! curl -fsSL --max-time 15 "$GITHUB_RAW/deploy/docker-compose.template.yml" -o "$tmp_template"; then + echo "[!] Download failed" + rm -f "$tmp_template" + return 1 + fi + + local local_keys remote_keys new_keys removed_keys + local_keys="$(extract_env_keys "$local_compose" | sort -u)" + remote_keys="$(extract_env_keys "$tmp_template" | sort -u)" + new_keys="$(comm -23 <(echo "$remote_keys") <(echo "$local_keys"))" + removed_keys="$(comm -13 <(echo "$remote_keys") <(echo "$local_keys"))" + + if [[ -z "$new_keys" && -z "$removed_keys" ]]; then + echo "[✓] Template already in sync." + rm -f "$tmp_template" + return 0 + fi + + if [[ -n "$new_keys" ]]; then + echo "[+] New ENV keys to merge:" + echo "$new_keys" | sed 's/^/ + /' + fi + if [[ -n "$removed_keys" ]]; then + echo "[-] Local-only ENV keys (will be kept untouched):" + echo "$removed_keys" | sed 's/^/ - /' + fi + + if [[ -z "$new_keys" ]]; then + rm -f "$tmp_template" + return 0 + fi + + echo + read -r -p "Merge new ENV keys into local docker-compose.yml? [y/N]: " ans "${local_compose}.new" + else + awk -v block="$insert_block" ' + /^ environment:/ { in_env=1 } + in_env && /^ [a-z]/ && !/^ environment:/ && !inserted { + printf "%s", block + inserted = 1 + } + { print } + ' "$local_compose" > "${local_compose}.new" + fi + + mv "${local_compose}.new" "$local_compose" + rm -f "$tmp_template" + echo "[✓] Merged. Run 'chat2api restart' to apply." +} + +# ============================================================ +# D: 单实例 → 多实例迁移 +# ============================================================ +cmd_migrate() { + local mode="${1:-prep}" + + if is_multi_install; then + echo "[i] Already in multi-instance mode. Nothing to do." + return 0 + fi + + case "$mode" in + prep) cmd_migrate_prep ;; + apply) cmd_migrate_apply ;; + rollback) cmd_migrate_rollback "${2:-}" ;; + *) + echo "Usage: chat2api migrate [prep|apply|rollback ]" + echo " prep Backup current install + generate accounts.csv (safe)" + echo " apply Stop single-instance + start multi (destructive)" + echo " rollback Restore from a previous backup directory" + return 1 + ;; + esac +} + +cmd_migrate_prep() { + local multi_dir="$INSTALL_DIR/deploy/multi" + local backup_dir="${INSTALL_DIR}.backup-$(date +%Y%m%d-%H%M%S)" + + if [[ ! -f "$INSTALL_DIR/.env" ]]; then + echo "[!] $INSTALL_DIR/.env not found; cannot migrate" + return 1 + fi + if [[ ! -d "$multi_dir" ]]; then + echo "[!] $multi_dir not found." + echo " Update repo first (re-run install.sh or git pull) so deploy/multi/ exists." + return 1 + fi + + echo "============================================================" + echo " chat2api migrate prep (Single → Multi-instance)" + echo "============================================================" + echo " Source: $INSTALL_DIR (single-instance)" + echo " Backup: $backup_dir" + echo " Multi: $multi_dir" + echo + echo "[!] WARNING: This step is non-destructive (only backups + generates csv)." + echo " 'chat2api migrate apply' is the destructive step." + echo + read -r -p "Continue? [y/N]: " ans /dev/null || true + echo "[✓] Backup ready." + + local csv="$multi_dir/accounts.csv" + local example_csv="$multi_dir/accounts.example.csv" + local tokens_file="$INSTALL_DIR/data/token.txt" + + if [[ -f "$csv" ]]; then + echo "[i] $csv already exists; skipping CSV generation" + else + echo "slug,proxy_url,note" > "$csv" + if [[ -f "$tokens_file" ]]; then + local count=0 i=0 + while IFS= read -r line; do + [[ -z "$line" || "$line" =~ ^# ]] && continue + i=$((i+1)) + printf 'acc%d,,migrated token #%d\n' "$i" "$i" >> "$csv" + count=$i + done < "$tokens_file" + if [[ "$count" -eq 0 ]]; then + echo "acc1,,migrated (please replace with real account)" >> "$csv" + echo "[!] $tokens_file empty; created template csv with 1 placeholder" + else + echo "[✓] Generated $csv with $count account slot(s)" + fi + else + echo "acc1,,migrated (please replace with real account)" >> "$csv" + echo "[!] $tokens_file not found; created template csv with 1 placeholder" + fi + fi + + cat <&1 | tail -5) || true + + echo "[*] Bringing up multi-instance..." + if (cd "$multi_dir" && ./manage.sh init); then + echo + echo "[✓] Migration applied." + echo " chat2api status # verify" + echo " chat2api secrets # see new credentials" + else + echo "[!] manage.sh init failed; you can rollback:" + echo " chat2api migrate rollback " + return 1 + fi +} + +cmd_migrate_rollback() { + local backup_dir="${1:-}" + if [[ -z "$backup_dir" ]]; then + echo "Usage: chat2api migrate rollback " + echo + echo "Available backups:" + ls -d "${INSTALL_DIR}.backup-"* 2>/dev/null || echo " (none found)" + return 1 + fi + if [[ ! -d "$backup_dir" ]]; then + echo "[!] $backup_dir not found" + return 1 + fi + + echo "[!] Rollback will:" + echo " 1. Stop multi-instance (if running)" + echo " 2. Restore $INSTALL_DIR from $backup_dir" + echo " 3. Restart single-instance" + echo + read -r -p 'Type "yes" to proceed: ' ans &1 | tail -5) || true + fi + + local trash="${INSTALL_DIR}.discarded-$(date +%Y%m%d-%H%M%S)" + mv "$INSTALL_DIR" "$trash" + cp -a "$backup_dir" "$INSTALL_DIR" + echo "[*] Restored from backup; old install moved to $trash (delete after verifying)" + + (cd "$INSTALL_DIR" && run_compose up -d) + echo "[✓] Rollback done. Run 'chat2api status' to verify." +} + +# ============================================================ +# Help +# ============================================================ +show_help() { + if is_multi_install; then + cat <<'EOF' +Usage: chat2api + +Multi-instance commands: + update Re-generate config and recreate services + start Same as update + restart Same as update + stop Stop all multi-instance services + uninstall Stop and remove chat2api; use --keep-data to keep files + status Show multi-instance status and sampled egress IPs + verify Verify admin/tokens routing for all instances + logs Tail one instance logs + shell Enter one instance shell + secrets Print instance auth/admin secrets + admin Print orchestrator/admin entry hints + path Print install directory + help Show this help + +Any other command is passed through to: deploy/multi/manage.sh +EOF + return + fi + cat <<'EOF' +Usage: chat2api + +Single-instance commands: + update Pull latest image and recreate containers + (also reports if upstream template has new ENV keys) + sync-template Merge new ENV keys from upstream template into + local docker-compose.yml (interactive, makes backup) + migrate prep Prepare migration to multi-instance: + backup install dir + generate deploy/multi/accounts.csv + migrate apply Stop single, start multi (destructive) + migrate rollback + Restore single-instance from a previous backup + restart Restart services + stop Stop services + uninstall Stop and remove chat2api; use --keep-data to keep files + start Start services + status Show compose status + logs Tail chat2api logs + admin Print admin login URL + api Print API base URL + path Print install directory + help Show this help +EOF +} + +# ============================================================ +# Dispatcher +# ============================================================ +command_name="${1:-help}" + +case "$command_name" in + update) + if is_multi_install; then + update_repo_for_multi + run_multi_manage apply + else + sync_deploy_assets_from_remote + run_compose pull + run_compose up -d + # 提示是否有新 ENV 待合并（不强制） + cmd_sync_template_check || true + fi + ;; + sync-template) + cmd_sync_template + ;; + migrate) + cmd_migrate "${@:2}" + ;; + restart) + if is_multi_install; then + run_multi_manage apply + else + run_compose restart + fi + ;; + stop) + if is_multi_install; then + run_multi_manage down + else + run_compose stop + fi + ;; + uninstall|unstaill) + cmd_uninstall "${@:2}" + ;; + start) + if is_multi_install; then + run_multi_manage apply + else + run_compose up -d + fi + ;; + status) + if is_multi_install; then + run_multi_manage status + else + run_compose ps + fi + ;; + logs) + if is_multi_install; then + run_multi_manage logs "${@:2}" + else + run_compose logs -f chat2api + fi + ;; + shell) + if is_multi_install; then + run_multi_manage shell "${@:2}" + else + echo "shell command is only available in multi-instance mode" + exit 1 + fi + ;; + verify) + if is_multi_install; then + run_multi_manage verify + else + echo "verify command is only available in multi-instance mode" + exit 1 + fi + ;; + secrets) + if is_multi_install; then + run_multi_manage secrets + else + echo "secrets command is only available in multi-instance mode" + exit 1 + fi + ;; + admin) + if is_multi_install; then + run_multi_manage secrets + elif [[ -n "${PORT:-}" && -n "${API_PREFIX:-}" ]]; then + echo "http://:${PORT}/${API_PREFIX}/admin/login" + else + echo "PORT/API_PREFIX not found in /etc/chat2api.env" + fi + ;; + api) + if is_multi_install; then + run_multi_manage secrets + elif [[ -n "${PORT:-}" && -n "${API_PREFIX:-}" ]]; then + echo "http://:${PORT}/${API_PREFIX}/v1/chat/completions" + else + echo "PORT/API_PREFIX not found in /etc/chat2api.env" + fi + ;; + path) + echo "$INSTALL_DIR" + ;; + help|--help|-h) + show_help + ;; + *) + if is_multi_install; then + run_multi_manage "$command_name" "${@:2}" + else + echo "Unknown command: $command_name" + show_help + exit 1 + fi + ;; +esac diff --git a/deploy/docker-compose.template.yml b/deploy/docker-compose.template.yml new file mode 100644 index 00000000..5746d333 --- /dev/null +++ b/deploy/docker-compose.template.yml @@ -0,0 +1,61 @@ +services: + chat2api: + image: ${CHAT2API_IMAGE:-ghcr.io/nanashiwang/chat2api:latest} + container_name: chat2api + restart: unless-stopped + pull_policy: always + env_file: + - .env + ports: + - '${CHAT2API_PORT:-60403}:5005' + volumes: + - ./data:/app/data + environment: + # ============ 基础（从 .env 读敏感项） ============ + TZ: '${TZ:-Asia/Shanghai}' + CHATGPT_BASE_URL: 'https://chatgpt.com' + + # ============ 功能开关 ============ + HISTORY_DISABLED: 'true' + RETRY_TIMES: '3' + RANDOM_TOKEN: 'false' + SCHEDULED_REFRESH: 'true' + ENABLE_LIMIT: 'true' + CHECK_MODEL: 'false' + UPLOAD_BY_URL: 'false' + OAI_LANGUAGE: 'zh-CN' + ENABLE_GATEWAY: 'false' + AUTO_SEED: 'true' + + # ============ Antiban 风控规避层 ============ + ENABLE_ANTIBAN: 'true' + STRICT_IP_BINDING: 'false' + BUCKET_MAX_ACCOUNTS_PER_IP: '30' + ACCOUNT_MIN_INTERVAL_SECONDS: '60' + ACCOUNT_COOLDOWN_JITTER: '0.3' + ACCOUNT_MAX_WAIT_SECONDS: '30' + IP_GEO_PROVIDER: 'ip-api' + CIRCUIT_429_COOLDOWN: '1800' + CIRCUIT_403_COOLDOWN: '3600' + CIRCUIT_BUCKET_HEAL_MINUTES: '30' + + # ============ 日志 ============ + LOG_BUFFER_SIZE: '3000' + + # ============ 冷启动初始化 ============ + INIT_APPLY_ON_EMPTY: 'true' + INIT_FORCE: 'false' + + # ============ Session Sticky（LibreChat → New-API → chat2api 链路下的窗口级会话续接） ============ + # 默认开启；仅在 request body 携带 librechat_conversation_id 时生效，对裸 OpenAI 客户端无影响 + ENABLE_SESSION_STICKY: 'true' + SESSION_TTL_DAYS: '30' + SESSION_LC_FIELD: 'librechat_conversation_id' + SESSION_TRIM_TO_LAST_USER: 'true' + + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:5005/${API_PREFIX}/admin/login > /dev/null || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 30s diff --git a/deploy/install-command.sh b/deploy/install-command.sh new file mode 100644 index 00000000..ed164249 --- /dev/null +++ b/deploy/install-command.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +set -euo pipefail + +DEFAULT_INSTALL_DIR="${INSTALL_DIR:-$HOME/chat2api}" +DEFAULT_PORT="60403" +DEFAULT_API_PREFIX="nanapi-2026-a1" + +if [ "$(id -u)" -eq 0 ]; then + SUDO="" +elif command -v sudo >/dev/null 2>&1; then + SUDO="sudo" +else + echo "需要 root 权限或安装 sudo" + exit 1 +fi + +prompt() { + local var_name="$1" + local prompt_text="$2" + local default_value="${3:-}" + local current_value="" + read -r -p "$prompt_text [$default_value]: " current_value + if [[ -z "$current_value" ]]; then + current_value="$default_value" + fi + printf -v "$var_name" '%s' "$current_value" +} + +shell_escape() { + printf "%s" "$1" | sed "s/'/'\\\\''/g" +} + +echo "== Install chat2api manage command ==" +prompt INSTALL_DIR "Compose directory" "$DEFAULT_INSTALL_DIR" +prompt PORT "Host port" "$DEFAULT_PORT" +prompt API_PREFIX "API prefix" "$DEFAULT_API_PREFIX" + +if [[ ! -f "$INSTALL_DIR/docker-compose.yml" && ! -f "$INSTALL_DIR/compose.yml" && ! -f "$INSTALL_DIR/compose.yaml" ]]; then + echo "No docker compose file found in: $INSTALL_DIR" + exit 1 +fi + +SCRIPT_SOURCE="${BASH_SOURCE[0]-}" +SCRIPT_SOURCE_DIR="" +if [ -n "$SCRIPT_SOURCE" ] && [ -f "$SCRIPT_SOURCE" ]; then + SCRIPT_SOURCE_DIR="$(cd "$(dirname "$SCRIPT_SOURCE")" && pwd)" +fi +$SUDO mkdir -p /etc +$SUDO tee /etc/chat2api.env >/dev/null <&2; } + +# ----- 配置默认值 ----- +INSTALL_DIR="${INSTALL_DIR:-$HOME/chat2api}" +GITHUB_RAW="${GITHUB_RAW:-https://raw.githubusercontent.com/nanashiwang/chat2api/main}" +GITHUB_REPO="${GITHUB_REPO:-https://github.com/nanashiwang/chat2api}" +CHAT2API_PORT="${CHAT2API_PORT:-60403}" +CHAT2API_MODE="${CHAT2API_MODE:-multi}" +INTERACTIVE="${INTERACTIVE:-0}" +SCRIPT_SOURCE="${BASH_SOURCE[0]-}" +SCRIPT_SOURCE_DIR="" +if [ -n "$SCRIPT_SOURCE" ] && [ -f "$SCRIPT_SOURCE" ]; then + SCRIPT_SOURCE_DIR="$(cd "$(dirname "$SCRIPT_SOURCE")" 2>/dev/null && pwd || true)" +fi + +# ----- sudo / root 判定 ----- +if [ "$(id -u)" -eq 0 ]; then + SUDO="" +else + if command -v sudo >/dev/null 2>&1; then + SUDO="sudo" + else + err "需要 root 权限或安装 sudo" + exit 1 + fi +fi + +# ----- 操作系统 ----- +if [ -f /etc/os-release ]; then + . /etc/os-release + OS_ID="${ID,,}" + OS_NAME="${PRETTY_NAME:-$ID}" +else + err "无法识别操作系统（缺 /etc/os-release）" + exit 1 +fi +ok "操作系统: $OS_NAME" + +# ----- 架构 ----- +ARCH="$(uname -m)" +case "$ARCH" in + x86_64|amd64) ARCH_DOCKER="amd64" ;; + aarch64|arm64) ARCH_DOCKER="arm64" ;; + *) err "不支持的架构: $ARCH"; exit 1 ;; +esac +ok "架构: $ARCH_DOCKER" + +# ----- 依赖工具 ----- +command -v curl >/dev/null 2>&1 || { + log "安装 curl" + case "$OS_ID" in + ubuntu|debian) $SUDO apt-get update -qq && $SUDO apt-get install -y -qq curl ;; + centos|rhel|rocky|almalinux) $SUDO yum install -y -q curl ;; + *) err "请先安装 curl"; exit 1 ;; + esac +} + +# ----- 安装 Docker ----- +if command -v docker >/dev/null 2>&1; then + ok "Docker 已安装: $(docker --version | head -1)" +else + log "Docker 未安装，使用官方脚本自动安装..." + curl -fsSL https://get.docker.com | $SUDO sh + $SUDO systemctl enable --now docker 2>/dev/null || true + ok "Docker 安装完成" +fi + +# ----- docker compose 插件 ----- +if ! docker compose version >/dev/null 2>&1; then + log "安装 docker compose 插件" + case "$OS_ID" in + ubuntu|debian) $SUDO apt-get install -y -qq docker-compose-plugin ;; + centos|rhel|rocky|almalinux) $SUDO yum install -y -q docker-compose-plugin ;; + *) err "docker compose 插件不可用，请手动装"; exit 1 ;; + esac +fi +ok "docker compose 可用: $(docker compose version --short 2>/dev/null || echo installed)" + +# ----- 目录 ----- +log "安装目录: $INSTALL_DIR" +mkdir -p "$INSTALL_DIR/data" +cd "$INSTALL_DIR" + +# ----- 随机凭据生成 ----- +gen_random() { + local length="${1:-32}" + LC_ALL=C tr -dc 'A-Za-z0-9' /dev/null 2>&1; then + return 0 + fi + if docker ps -a --format '{{.Names}}' | grep -qx 'chat2api'; then + log "检测到旧单实例容器占用端口，先停止并移除..." + docker rm -f chat2api >/dev/null 2>&1 || true + fi +} + +sync_deploy_assets() { + if [ -x "$INSTALL_DIR/deploy/multi/manage.sh" ] && [ -f "$INSTALL_DIR/deploy/chat2api.sh" ]; then + return 0 + fi + + log "准备部署辅助脚本（含多实例编排）..." + mkdir -p "$INSTALL_DIR/deploy" + + if [ -n "$SCRIPT_SOURCE_DIR" ] && [ -d "$SCRIPT_SOURCE_DIR/multi" ]; then + local src_deploy dest_deploy + src_deploy="$(cd "$SCRIPT_SOURCE_DIR" 2>/dev/null && pwd -P || true)" + dest_deploy="$(cd "$INSTALL_DIR/deploy" 2>/dev/null && pwd -P || true)" + if [ "$src_deploy" != "$dest_deploy" ]; then + cp -R "$SCRIPT_SOURCE_DIR"/. "$INSTALL_DIR/deploy/" + fi + chmod +x "$INSTALL_DIR/deploy/chat2api.sh" "$INSTALL_DIR/deploy/multi/manage.sh" 2>/dev/null || true + ok "部署辅助脚本已准备" + return 0 + fi + + if ! command -v tar >/dev/null 2>&1; then + warn "tar 不可用，跳过 deploy/multi 下载；单实例部署不受影响" + return 0 + fi + + local tmp_dir archive_dir + tmp_dir="$(mktemp -d)" || { + warn "临时目录创建失败，跳过 deploy/multi 下载" + return 0 + } + + if curl -fsSL "${GITHUB_REPO}/archive/refs/heads/main.tar.gz" | tar -xz -C "$tmp_dir"; then + archive_dir="$(find "$tmp_dir" -maxdepth 1 -type d -name 'chat2api-*' | head -1)" + if [ -n "$archive_dir" ] && [ -d "$archive_dir/deploy" ]; then + cp -R "$archive_dir/deploy"/. "$INSTALL_DIR/deploy/" + chmod +x "$INSTALL_DIR/deploy/chat2api.sh" "$INSTALL_DIR/deploy/multi/manage.sh" 2>/dev/null || true + ok "部署辅助脚本已准备（含 deploy/multi）" + else + warn "仓库包缺少 deploy/，跳过 deploy/multi 下载" + fi + else + warn "deploy/multi 下载失败；单实例部署不受影响" + fi + rm -rf "$tmp_dir" +} + +install_manage_command() { + local script_src tmp_script + + script_src="" + if [ -n "$SCRIPT_SOURCE_DIR" ] && [ -f "$SCRIPT_SOURCE_DIR/chat2api.sh" ]; then + script_src="${SCRIPT_SOURCE_DIR}/chat2api.sh" + elif [ -f "$INSTALL_DIR/deploy/chat2api.sh" ]; then + script_src="$INSTALL_DIR/deploy/chat2api.sh" + fi + + $SUDO mkdir -p /etc || return 1 + if ! $SUDO tee /etc/chat2api.env >/dev/null < .env <&1 | grep -v "^$" || true +$SUDO docker compose up -d + +# ----- 宿主管理命令 ----- +log "安装管理命令..." +if install_manage_command; then + ok "管理命令已安装: chat2api update" +else + warn "管理命令安装失败；服务已启动，可先用 docker compose pull && docker compose up -d 更新" +fi + +# ----- 健康检查 ----- +log "等待服务就绪..." +for i in $(seq 1 60); do + if curl -fsS "http://127.0.0.1:${CHAT2API_PORT}/${API_PREFIX}/admin/login" > /dev/null 2>&1; then + ok "服务就绪（第 ${i} 秒）" + break + fi + if [ "$i" -eq 60 ]; then + warn "服务 60 秒内未响应，检查日志: cd $INSTALL_DIR && docker compose logs" + fi + sleep 1 +done + +# ----- 公网 IP 获取 ----- +PUBLIC_IP="$(curl -fsSL --max-time 5 https://api.ipify.org 2>/dev/null || \ + curl -fsSL --max-time 5 https://ifconfig.me 2>/dev/null || \ + echo 'your-server-ip')" + +# ----- 结果摘要 ----- +cat <x.includes('session-token')).join('; ') + 复制结果粘到 UI） + 3. "代理与路由"（可选）→ 添加住宅代理 → 给账号绑定 + 4. 测试: curl -H "Authorization: Bearer ${AUTHORIZATION}" \\ + http://localhost:${CHAT2API_PORT}/${API_PREFIX}/v1/models + +🛡️ 安全加固（强烈建议）: + - 配置 IP 白名单: + vim ${INSTALL_DIR}/.env + ADMIN_IP_WHITELIST=你的办公/家庭 IP + chat2api restart + - 或接入 Cloudflare 免费版隐藏真实 IP + - 详见: ${GITHUB_RAW}/docs/SECURITY.md + +📋 常用命令: + chat2api status # 查看状态 + chat2api logs # 实时日志 + chat2api restart # 重启 + chat2api update # 升级 + chat2api stop # 停止 + +============================================================ + +EOF diff --git a/deploy/multi/.gitignore b/deploy/multi/.gitignore new file mode 100644 index 00000000..d2010efd --- /dev/null +++ b/deploy/multi/.gitignore @@ -0,0 +1,12 @@ +# 真实账号清单（含代理凭证），不入库 +accounts.csv + +# 生成产物（含密钥），不入库 +generated/ + +# 每实例数据目录（含 cookie / token），不入库 +data/ + +# 本地临时 +*.tmp +*.bak diff --git a/deploy/multi/accounts.example.csv b/deploy/multi/accounts.example.csv new file mode 100644 index 00000000..e879f727 --- /dev/null +++ b/deploy/multi/accounts.example.csv @@ -0,0 +1,4 @@ +slug,proxy_url,note +acc1,socks5://user1:pass1@residential1.example:8000,plus 主号 +acc2,socks5://user2:pass2@residential2.example:8000,plus 备号 +acc3,,免代理直连示例 diff --git a/deploy/multi/generate.py b/deploy/multi/generate.py new file mode 100755 index 00000000..4351eb4e --- /dev/null +++ b/deploy/multi/generate.py @@ -0,0 +1,580 @@ +#!/usr/bin/env python3 +"""chat2api 多实例部署生成器（KISS / YAGNI / DRY） + +输入：./accounts.csv [slug, proxy_url, note] +输出：./generated/docker-compose.yml + ./generated/nginx.conf + ./generated/env/.env + ./generated/secrets.txt (chmod 600) + ./data// (空目录，作为容器数据卷) + +设计要点： +1. 纯标准库（stdlib），无 jinja2 依赖 +2. 已有 env 文件中的 AUTHORIZATION/ADMIN_PASSWORD/API_PREFIX 复用，避免重生成导致客户端 key 失效 +3. 每实例独立 SOCKS5 代理（可空）；空时 PROXY_URL 不写入 +4. 不映射 chat2api 实例的宿主端口，仅 nginx 暴露 60403 +""" +from __future__ import annotations + +import csv +import os +import re +import secrets as pysecrets +import string +import sys +from pathlib import Path +from typing import Iterator, NamedTuple + +ROOT = Path(__file__).resolve().parent +CSV_FILE = ROOT / "accounts.csv" +GEN_DIR = ROOT / "generated" +ENV_DIR = GEN_DIR / "env" +DATA_DIR = ROOT / "data" +SECRETS_FILE = GEN_DIR / "secrets.txt" +COMPOSE_FILE = GEN_DIR / "docker-compose.yml" +NGINX_FILE = GEN_DIR / "nginx.conf" +ORCH_ENV = GEN_DIR / "orch.env" + +SLUG_RE = re.compile(r"^[a-z0-9-]{1,16}$") +PROXY_RE = re.compile(r"^(socks5|socks5h|http|https)://[^\s]+$") + +NGINX_PORT = int(os.environ.get("CHAT2API_GATEWAY_PORT", "60403")) +CHAT2API_IMAGE = os.environ.get( + "CHAT2API_IMAGE", "ghcr.io/nanashiwang/chat2api:latest" +) +WATCHTOWER_IMAGE = os.environ.get( + "WATCHTOWER_IMAGE", "nickfedor/watchtower:latest" +) +ORCH_ENABLED = os.environ.get("ORCH_ENABLED", "true").lower() != "false" + + +class Account(NamedTuple): + slug: str + proxy_url: str + note: str + auth: str + admin_password: str + api_prefix: str + + +# ---------- helpers ---------- + +def fail(msg: str) -> None: + sys.stderr.write(f"\033[1;31m[!]\033[0m {msg}\n") + sys.exit(1) + + +def info(msg: str) -> None: + sys.stdout.write(f"\033[1;34m[*]\033[0m {msg}\n") + + +def ok(msg: str) -> None: + sys.stdout.write(f"\033[1;32m[\u2713]\033[0m {msg}\n") + + +def gen_auth() -> str: + return "sk-" + pysecrets.token_hex(16) + + +def gen_admin_password() -> str: + alphabet = string.ascii_letters + string.digits + return "".join(pysecrets.choice(alphabet) for _ in range(24)) + + +def gen_api_prefix() -> str: + return "api-" + pysecrets.token_hex(4) + + +def parse_env_file(path: Path) -> dict[str, str]: + """简易 .env 解析：KEY=VALUE，忽略 # 注释与空行；不解析 quote。""" + if not path.exists(): + return {} + out: dict[str, str] = {} + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + if "=" not in line: + continue + k, v = line.split("=", 1) + out[k.strip()] = v.strip() + return out + + +def read_csv() -> list[tuple[str, str, str]]: + if not CSV_FILE.exists(): + fail(f"未找到 {CSV_FILE}；请基于 accounts.example.csv 创建") + rows: list[tuple[str, str, str]] = [] + with CSV_FILE.open("r", encoding="utf-8", newline="") as f: + reader = csv.DictReader(f) + required = {"slug", "proxy_url", "note"} + if not reader.fieldnames or not required.issubset(reader.fieldnames): + fail(f"CSV 表头必须包含: {sorted(required)}") + seen: set[str] = set() + for i, row in enumerate(reader, start=2): + slug = (row.get("slug") or "").strip() + proxy = (row.get("proxy_url") or "").strip() + note = (row.get("note") or "").strip() + if not slug: + continue # 跳过空行 + if not SLUG_RE.match(slug): + fail(f"第 {i} 行 slug='{slug}' 不合法（需 [a-z0-9-]{{1,16}}）") + if slug in seen: + fail(f"第 {i} 行 slug='{slug}' 重复") + seen.add(slug) + if proxy and not PROXY_RE.match(proxy): + fail(f"第 {i} 行 proxy_url='{proxy}' 不合法") + rows.append((slug, proxy, note)) + if not rows: + info("CSV 暂无账号，仅启动 orchestrator + nginx + watchtower（可在面板内增加）") + return rows + + +def resolve_secrets(slug: str) -> tuple[str, str, str]: + """已有 env 优先复用其密钥，否则生成新值。""" + env = parse_env_file(ENV_DIR / f"{slug}.env") + auth = env.get("AUTHORIZATION") or gen_auth() + pwd = env.get("ADMIN_PASSWORD") or gen_admin_password() + prefix = env.get("API_PREFIX") or gen_api_prefix() + return auth, pwd, prefix + + +def load_accounts() -> list[Account]: + out: list[Account] = [] + for slug, proxy, note in read_csv(): + auth, pwd, prefix = resolve_secrets(slug) + out.append(Account(slug, proxy, note, auth, pwd, prefix)) + return out + + +# ---------- renderers ---------- + +ENV_TEMPLATE = """\ +# 自动生成 — 由 generate.py 维护，请勿手工修改 +AUTHORIZATION={auth} +ADMIN_PASSWORD={admin_password} +API_PREFIX={api_prefix} +""" + + +def render_env(acc: Account) -> str: + body = ENV_TEMPLATE.format( + auth=acc.auth, + admin_password=acc.admin_password, + api_prefix=acc.api_prefix, + ) + if acc.proxy_url: + body += f"PROXY_URL={acc.proxy_url}\n" + return body + + +COMPOSE_HEADER = """\ +# 自动生成 — 由 generate.py 维护，请勿手工修改 +# 来源：deploy/multi/accounts.csv + +x-chat2api-common: &c2a-common + image: {image} + restart: unless-stopped + pull_policy: always + networks: [c2a-net] + labels: + com.centurylinklabs.watchtower.enable: 'true' + # 资源/权限边界：单实例故障不污染母机与同行容器 + cap_drop: [ALL] + security_opt: + - no-new-privileges:true + pids_limit: 200 + mem_limit: 512m + cpus: '0.5' + environment: + TZ: 'Asia/Shanghai' + CHATGPT_BASE_URL: 'https://chatgpt.com' + HISTORY_DISABLED: 'true' + SCHEDULED_REFRESH: 'true' + ENABLE_LIMIT: 'true' + OAI_LANGUAGE: 'zh-CN' + ENABLE_GATEWAY: 'true' + AUTO_SEED: 'true' + RANDOM_TOKEN: 'false' + RETRY_TIMES: '3' + # 一容器一账号 + 独立住宅 IP：默认开启风控规避层并强制 IP 绑定 + ENABLE_ANTIBAN: 'true' + STRICT_IP_BINDING: 'true' + BUCKET_MAX_ACCOUNTS_PER_IP: '1' + # 账号级冷却已关闭（一容器一账号，外部客户端节奏决定 QPS）； + # 429/403 熔断退避走 CIRCUIT_* 独立路径，不受影响。 + ACCOUNT_MIN_INTERVAL_SECONDS: '0' + FREE_ACCOUNT_MIN_INTERVAL_SECONDS: '0' + ACCOUNT_COOLDOWN_JITTER: '0.3' + ACCOUNT_MAX_WAIT_SECONDS: '30' + IP_GEO_PROVIDER: 'ip-api' + CIRCUIT_429_COOLDOWN: '1800' + CIRCUIT_403_COOLDOWN: '3600' + CIRCUIT_BUCKET_HEAL_MINUTES: '30' + INIT_APPLY_ON_EMPTY: 'true' + LOG_BUFFER_SIZE: '3000' + # Session Sticky (LibreChat → New-API → chat2api 链路下的窗口级会话续接) + # 默认开启：依赖 LibreChat 在 request body 注入 librechat_conversation_id（librechat.yaml addParams） + # 与 New-API 的 Channel Affinity（同 lc_conv_id 永远命中同一渠道）协作；不会影响不带该字段的请求 + ENABLE_SESSION_STICKY: 'true' + SESSION_TTL_DAYS: '30' + SESSION_LC_FIELD: 'librechat_conversation_id' + SESSION_TRIM_TO_LAST_USER: 'true' + +services: +""" + +COMPOSE_INSTANCE = """\ + chat2api-{slug}: + <<: *c2a-common + container_name: c2a-{slug} + env_file: + - ./generated/env/{slug}.env + volumes: + - ./data/{slug}:/app/data + healthcheck: + # 镜像不含 curl，用 python TCP 探活 5005 端口（最简存活检查） + test: ["CMD", "python", "-c", "import socket; s=socket.socket(); s.settimeout(3); s.connect(('127.0.0.1',5005)); s.close()"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 30s + +""" + +COMPOSE_ORCHESTRATOR = """\ + orchestrator: + image: c2a-orchestrator:local + build: + context: ./orchestrator + container_name: c2a-orchestrator + restart: unless-stopped + networks: [c2a-net] + env_file: + - ./generated/orch.env + environment: + MULTI_HOST_PATH: '{host_path}' + ORCH_PORT: '8080' + TZ: 'Asia/Shanghai' + # 关键：用宿主路径作为容器内挂载点，让 docker compose 客户端 + # （在容器内运行）与 daemon（在宿主运行）能用同一路径找到 env_file 与 volumes + volumes: + - '{host_path}:{host_path}' + - /var/run/docker.sock:/var/run/docker.sock + labels: + com.centurylinklabs.watchtower.enable: 'true' + +""" + +COMPOSE_FOOTER = """\ + nginx: + image: nginx:alpine + container_name: c2a-nginx + restart: unless-stopped + ports: + - '{port}:80' + volumes: + - ./generated/nginx.conf:/etc/nginx/nginx.conf:ro + networks: [c2a-net] + depends_on: +{depends_on} + + watchtower: + image: {watchtower_image} + container_name: c2a-watchtower + restart: unless-stopped + volumes: + - /var/run/docker.sock:/var/run/docker.sock + command: --label-enable --cleanup --interval 300 + +networks: + c2a-net: + driver: bridge +""" + + +def render_compose(accounts: list[Account]) -> str: + services = "".join(COMPOSE_INSTANCE.format(slug=a.slug) for a in accounts) + depends_on_lines = [f" - chat2api-{a.slug}" for a in accounts] + if ORCH_ENABLED: + depends_on_lines.append(" - orchestrator") + depends_on = "\n".join(depends_on_lines) + body = COMPOSE_HEADER.format(image=CHAT2API_IMAGE) + services + if ORCH_ENABLED: + body += COMPOSE_ORCHESTRATOR.format(host_path=str(ROOT).replace("'", "''")) + body += COMPOSE_FOOTER.format(port=NGINX_PORT, depends_on=depends_on, watchtower_image=WATCHTOWER_IMAGE) + return body + + +NGINX_HEADER = """\ +# 自动生成 — 由 generate.py 维护，请勿手工修改 +worker_processes auto; +events { worker_connections 1024; } + +http { + server_tokens off; + proxy_http_version 1.1; + client_max_body_size 50m; + + # SSE / 流式响应基线 + proxy_buffering off; + proxy_cache off; + proxy_set_header Connection ''; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 600s; + proxy_send_timeout 600s; + # Docker 内部 DNS。配合变量 proxy_pass，避免容器重建后 nginx 继续缓存旧 IP。 + resolver 127.0.0.11 ipv6=off valid=30s; + + # 简单访问日志（生产可换 json） + log_format upstream_log '$remote_addr - [$time_local] "$request" ' + '$status $body_bytes_sent upstream=$upstream_addr ' + 'rt=$request_time urt=$upstream_response_time'; + access_log /var/log/nginx/access.log upstream_log; + + server { + listen 80 default_server; + server_name _; + + location = / { + default_type text/plain; + return 200 "chat2api multi-instance gateway\\n"; + } + + location = /healthz { + default_type text/plain; + return 200 "ok\\n"; + } + +""" + +NGINX_ORCH_LOCATION = """\ + # ---- unified OpenAI-compatible API (load-balanced by orchestrator) ---- + location /v1/ { + proxy_pass http://c2a-orchestrator:8080/v1/; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection ''; + } + + # ---- orchestrator (编排面板) ---- + location /orchestrator/ { + proxy_pass http://c2a-orchestrator:8080/; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + } + +""" + +NGINX_LOCATION = """\ + # ---- {slug} ({note}) ---- + location /{slug}/ {{ + set $chat2api_upstream c2a-{slug}:5005; + rewrite ^/{slug}/(.*)$ /{api_prefix}/$1 break; + proxy_pass http://$chat2api_upstream; + # 把外部访问域名/协议传给 chat2api，避免页面里出现容器内网地址 + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Forwarded-Proto $scheme; + # chat2api 内部硬编码 API_PREFIX 到 redirect / cookie / HTML 链接里， + # 经 nginx 反代后必须把 /{api_prefix}/ 改回 /{slug}/，否则用户登录后跳转 404 + # 用 $scheme://$http_host 拼完整 URL（含客户端原始端口如 :60403） + proxy_redirect ~^/{api_prefix}/(.*)$ $scheme://$http_host/{slug}/$1; + proxy_cookie_path /{api_prefix} /{slug}; + # sub_filter 改写响应体内的硬编码链接（CSS/JS/HTML）；强制无 gzip 才能改 + proxy_set_header Accept-Encoding ""; + sub_filter "/{api_prefix}/" "/{slug}/"; + sub_filter_once off; + sub_filter_types text/html text/css application/javascript application/json; + }} + + # 反向兜底：chat2api 前端 JS 在运行时动态拼接 /{api_prefix}/...， + # sub_filter 只改静态字符串改不到。用 307 让浏览器跳到 /{slug}/... + # （而非 internal rewrite），保证浏览器按新 URL 的 cookie path 发送 admin_auth。 + # 307 保留 method/body，POST/PATCH 不会变 GET。 + location ~ ^/{api_prefix}/(.*)$ {{ + return 307 $scheme://$http_host/{slug}/$1$is_args$args; + }} + +""" + +NGINX_FOOTER = """\ + } +} +""" + + +def render_nginx(accounts: list[Account]) -> str: + locations = "" + if ORCH_ENABLED: + locations += NGINX_ORCH_LOCATION + locations += "".join( + NGINX_LOCATION.format( + slug=a.slug, + api_prefix=a.api_prefix, + note=a.note or "-", + ) + for a in accounts + ) + return NGINX_HEADER + locations + NGINX_FOOTER + + +SECRETS_HEADER = """\ +# chat2api 多实例访问凭证（自动生成） +# 字段：slug | path-base | AUTHORIZATION | ADMIN_PASSWORD | API_PREFIX | proxy +# 警告：含敏感数据，请妥善保管（已 chmod 600） + +""" + + +def render_secrets(accounts: list[Account]) -> str: + lines = [SECRETS_HEADER] + for a in accounts: + proxy = a.proxy_url or "-" + lines.append( + f"slug={a.slug}\n" + f" path = /{a.slug}/v1/...\n" + f" AUTH = {a.auth}\n" + f" ADMIN_PWD = {a.admin_password}\n" + f" API_PREFIX = {a.api_prefix}\n" + f" PROXY = {proxy}\n" + f" note = {a.note or '-'}\n\n" + ) + return "".join(lines) + + +# ---------- writer ---------- + +def write_file(path: Path, content: str, mode: int | None = None) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + if mode is not None: + path.chmod(mode) + + +def cleanup_orphan_envs(accounts: list[Account]) -> None: + """CSV 中已删除的 slug，env 文件应该清理。data/ 故意保留以避免误删。""" + if not ENV_DIR.exists(): + return + keep = {a.slug for a in accounts} + for env_file in ENV_DIR.glob("*.env"): + if env_file.stem in {"orch"} or env_file.stem in keep: + continue + info(f"清理孤儿 env: {env_file.name}") + env_file.unlink() + + +def ensure_orch_env() -> tuple[str, bool]: + """orchestrator 凭证：首次自动生成登录密码、会话密钥和统一 API key。 + + 返回 (password, was_generated)。 + """ + if ORCH_ENV.exists(): + env = parse_env_file(ORCH_ENV) + username = env.get("ORCH_USERNAME", "") + pwd = env.get("ORCH_PASSWORD", "") + secret = env.get("ORCH_SESSION_SECRET", "") + api_key = env.get("ORCH_API_KEY", "") + changed = False + if not username: + env["ORCH_USERNAME"] = "admin" + changed = True + if not api_key: + env["ORCH_API_KEY"] = "sk-orch-" + pysecrets.token_hex(24) + changed = True + if pwd and secret: + if changed: + body = "# 自动生成 — orchestrator 凭证（generate.py 维护）\n" + for key in ("ORCH_USERNAME", "ORCH_PASSWORD", "ORCH_SESSION_SECRET", "ORCH_API_KEY"): + if env.get(key): + body += f"{key}={env[key]}\n" + write_file(ORCH_ENV, body, mode=0o600) + return pwd, False + username = "admin" + pwd = gen_admin_password() + secret = pysecrets.token_hex(32) + api_key = "sk-orch-" + pysecrets.token_hex(24) + body = ( + "# 自动生成 — orchestrator 凭证（generate.py 维护）\n" + f"ORCH_USERNAME={username}\n" + f"ORCH_PASSWORD={pwd}\n" + f"ORCH_SESSION_SECRET={secret}\n" + f"ORCH_API_KEY={api_key}\n" + ) + write_file(ORCH_ENV, body, mode=0o600) + return pwd, True + + +def parse_env_file(path: Path) -> dict[str, str]: + """简易 .env 解析，仅本模块用。""" + out: dict[str, str] = {} + if not path.exists(): + return out + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, v = line.split("=", 1) + out[k.strip()] = v.strip() + return out + + +def main() -> int: + accounts = load_accounts() + info(f"读到 {len(accounts)} 个账号") + + GEN_DIR.mkdir(parents=True, exist_ok=True) + ENV_DIR.mkdir(parents=True, exist_ok=True) + DATA_DIR.mkdir(parents=True, exist_ok=True) + + # 1. env files (per-instance) + for a in accounts: + write_file(ENV_DIR / f"{a.slug}.env", render_env(a), mode=0o600) + (DATA_DIR / a.slug).mkdir(exist_ok=True) + cleanup_orphan_envs(accounts) + + # 2. compose + write_file(COMPOSE_FILE, render_compose(accounts)) + + # 3. nginx + write_file(NGINX_FILE, render_nginx(accounts)) + + # 4. secrets + write_file(SECRETS_FILE, render_secrets(accounts), mode=0o600) + + # 5. orchestrator 凭证（首次生成时打印） + orch_first_pwd: str | None = None + if ORCH_ENABLED: + pwd, was_generated = ensure_orch_env() + if was_generated: + orch_first_pwd = pwd + + ok(f"compose: {COMPOSE_FILE}") + ok(f"nginx: {NGINX_FILE}") + ok(f"env dir: {ENV_DIR}") + ok(f"secrets: {SECRETS_FILE} (chmod 600)") + ok(f"网关端口: {NGINX_PORT} (可用 CHAT2API_GATEWAY_PORT 覆盖)") + if orch_first_pwd: + sys.stdout.write( + "\n" + "============================================================\n" + " Orchestrator 首次访问用户名：admin\n" + f" Orchestrator 首次访问密码：{orch_first_pwd}\n" + " 请立即记录！可通过 ./manage.sh orch-password 重置\n" + " 访问入口：http://:{port}/orchestrator/\n" + "============================================================\n" + .format(port=NGINX_PORT) + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/deploy/multi/manage.sh b/deploy/multi/manage.sh new file mode 100755 index 00000000..4c719159 --- /dev/null +++ b/deploy/multi/manage.sh @@ -0,0 +1,473 @@ +#!/usr/bin/env bash +# chat2api 多实例运维包装（KISS） +# 全部子命令幂等：底层都是 generate.py + docker compose +set -euo pipefail + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$DIR" + +C_RESET="\033[0m"; C_INFO="\033[1;34m"; C_OK="\033[1;32m"; C_ERR="\033[1;31m" +log() { echo -e "${C_INFO}[*]${C_RESET} $*"; } +ok() { echo -e "${C_OK}[\u2713]${C_RESET} $*"; } +err() { echo -e "${C_ERR}[\u2717]${C_RESET} $*" >&2; } + +CSV="$DIR/accounts.csv" +EXAMPLE_CSV="$DIR/accounts.example.csv" +GEN_DIR="$DIR/generated" +COMPOSE="$GEN_DIR/docker-compose.yml" +REPO_ROOT="$(cd "$DIR/../.." && pwd)" + +# orchestrator 必需：让容器内的 docker compose --project-directory 指向宿主路径 +export MULTI_HOST_PATH="$DIR" + +ensure_csv() { + if [ ! -f "$CSV" ]; then + printf 'slug,proxy_url,note\n' > "$CSV" + log "已创建空 accounts.csv；可在编排面板里新增账号" + fi +} + +require_compose() { + if [ ! -f "$COMPOSE" ]; then + err "尚未生成，请先 ./manage.sh init" + exit 1 + fi +} + +dc() { + docker compose -f "$COMPOSE" --project-directory "$DIR" "$@" +} + +compose_up() { + if dc up -d --remove-orphans --pull missing "$@"; then + return 0 + fi + log "镜像拉取失败，尝试使用本地已有镜像继续..." + dc up -d --remove-orphans --pull never "$@" +} + +slugs() { + awk -F, 'NR>1 && $1!="" {print $1}' "$CSV" +} + +as_root() { + if [ "$(id -u)" -eq 0 ]; then + "$@" + elif command -v sudo >/dev/null 2>&1; then + sudo "$@" + else + err "需要 root 权限或已安装 sudo" + exit 1 + fi +} + +public_host() { + curl -fsSL --max-time 5 https://api.ipify.org 2>/dev/null \ + || curl -fsSL --max-time 5 https://ifconfig.me 2>/dev/null \ + || printf '' +} + +orch_password() { + awk -F= '$1=="ORCH_PASSWORD"{print $2}' "$GEN_DIR/orch.env" 2>/dev/null | tail -1 +} + +orch_api_key() { + awk -F= '$1=="ORCH_API_KEY"{print $2}' "$GEN_DIR/orch.env" 2>/dev/null | tail -1 +} + +cmd_access_summary() { + [ -f "$GEN_DIR/orch.env" ] || return 0 + local port host orch_pwd + port="${CHAT2API_GATEWAY_PORT:-60403}" + host="$(public_host)" + orch_pwd="$(orch_password)" + cat </dev/null 2>&1 || true + fi +} + +auto_pull() { + # cmd_apply 前自动 git pull --ff-only。 + # 有本地跟踪文件改动时自动 stash，避免旧部署脚本挡住更新。 + # 设计原则：永不 reset；本地改动只暂存到 git stash，方便需要时找回。 + if [ "${NO_PULL:-0}" = "1" ]; then + log "NO_PULL=1，跳过 git pull" + return 0 + fi + local repo_root + if ! repo_root="$(git -C "$DIR" rev-parse --show-toplevel 2>/dev/null)"; then + log "非 git 仓库，由 chat2api update 负责同步部署文件" + return 0 + fi + local branch + branch="$(git -C "$repo_root" rev-parse --abbrev-ref HEAD 2>/dev/null || echo HEAD)" + if [ "$branch" = "HEAD" ]; then + log "detached HEAD，跳过 git pull（手动 checkout 分支后再 update）" + return 0 + fi + local stash_msg="" + if [ -n "$(git -C "$repo_root" status --porcelain 2>/dev/null)" ]; then + stash_msg="chat2api auto-stash before update $(date -u +%Y%m%dT%H%M%SZ)" + log "检测到本地改动，自动暂存到 git stash..." + if git -C "$repo_root" stash push -m "$stash_msg" -- . >/dev/null 2>&1; then + ok "本地改动已暂存：$stash_msg" + else + log "自动暂存失败，跳过 git pull（可手动处理后重试）" + return 0 + fi + fi + log "git pull --ff-only ($branch)..." + if git -C "$repo_root" pull --ff-only --quiet 2>/dev/null; then + ok "代码已同步到 $(git -C "$repo_root" log -1 --pretty='%h %s')" + if [ -n "$stash_msg" ]; then + log "如需查看被暂存的本地改动：git -C \"$repo_root\" stash list" + fi + else + if [ -n "$stash_msg" ]; then + git -C "$repo_root" stash pop --quiet >/dev/null 2>&1 || true + fi + log "git pull 跳过（非 fast-forward 或远端不可达），继续用当前版本" + fi +} + +cmd_apply() { + auto_pull + ensure_csv + log "生成配置..." + python3 "$DIR/generate.py" + cleanup_renamed_containers + local has_orchestrator=0 + if dc config --services 2>/dev/null | grep -qx orchestrator; then + has_orchestrator=1 + log "构建 orchestrator 镜像..." + dc build orchestrator + fi + log "应用 docker compose..." + if [ "$has_orchestrator" -eq 1 ]; then + # orchestrator 是本地 build 镜像，静态文件变更后必须替换容器才能加载新面板。 + compose_up --force-recreate orchestrator + fi + compose_up + # nginx.conf 变化时 compose 不会重启 nginx，主动 reload + if docker ps --format '{{.Names}}' | grep -qx c2a-nginx; then + docker exec c2a-nginx nginx -s reload 2>/dev/null \ + && log "nginx reload OK" \ + || log "nginx reload 失败（首次启动可忽略）" + fi + cmd_verify + if [ "$has_orchestrator" -eq 1 ]; then + cmd_verify_orchestrator + cmd_access_summary + fi + ok "完成。优先使用上面的编排面板管理所有容器。" +} + +cmd_init() { + cmd_apply +} + +cmd_add() { + local slug="${1:-}" proxy="${2:-}" note="${3:-}" + if [ -z "$slug" ]; then + err "用法: ./manage.sh add [proxy_url] [note]" + exit 1 + fi + ensure_csv + if grep -q "^${slug}," "$CSV" 2>/dev/null; then + err "slug='$slug' 已存在于 accounts.csv" + exit 1 + fi + echo "${slug},${proxy},${note}" >> "$CSV" + ok "已追加到 accounts.csv: $slug" + cmd_apply +} + +cmd_remove() { + local slug="${1:-}" + if [ -z "$slug" ]; then + err "用法: ./manage.sh remove " + exit 1 + fi + ensure_csv + if ! grep -q "^${slug}," "$CSV"; then + err "slug='$slug' 不在 accounts.csv 中" + exit 1 + fi + log "停止并清理容器 c2a-${slug}..." + if [ -f "$COMPOSE" ]; then + dc stop "chat2api-${slug}" 2>/dev/null || true + dc rm -f "chat2api-${slug}" 2>/dev/null || true + fi + log "从 accounts.csv 移除..." + grep -v "^${slug}," "$CSV" > "$CSV.tmp" && mv "$CSV.tmp" "$CSV" + log "保留 data/${slug}/ 目录（如确认无用，请手动 rm -rf）" + cmd_apply +} + +cmd_list() { + require_compose + dc ps +} + +cmd_logs() { + local slug="${1:-}" tail="${2:-200}" + if [ -z "$slug" ]; then + err "用法: ./manage.sh logs [行数]" + exit 1 + fi + docker logs --tail "$tail" -f "c2a-${slug}" +} + +cmd_shell() { + local slug="${1:-}" + if [ -z "$slug" ]; then + err "用法: ./manage.sh shell " + exit 1 + fi + docker exec -it "c2a-${slug}" sh +} + +cmd_secrets() { + if [ ! -f "$GEN_DIR/secrets.txt" ]; then + err "secrets.txt 不存在，先 ./manage.sh init" + exit 1 + fi + cat "$GEN_DIR/secrets.txt" + if [ -f "$GEN_DIR/orch.env" ]; then + echo + echo "============ Orchestrator 编排面板（管理所有容器） ============" + grep '^ORCH_USERNAME=' "$GEN_DIR/orch.env" 2>/dev/null \ + | sed 's/^/ /' + grep '^ORCH_PASSWORD=' "$GEN_DIR/orch.env" 2>/dev/null \ + | sed 's/^/ /' + local port="${CHAT2API_GATEWAY_PORT:-60403}" + local host + host="$(public_host)" + echo " URL: http://${host}:${port}/orchestrator/" + echo + echo "============ 统一 API（自动均衡所有实例） ============" + grep '^ORCH_API_KEY=' "$GEN_DIR/orch.env" 2>/dev/null \ + | sed 's/^/ /' + echo " BASE_URL: http://${host}:${port}/v1" + echo " CHAT: http://${host}:${port}/v1/chat/completions" + echo "===============================================" + fi +} + +cmd_orch_password() { + if [ ! -f "$GEN_DIR/orch.env" ]; then + err "orch.env 不存在；先 ./manage.sh init 生成" + exit 1 + fi + local new_pwd + if [ -n "${1:-}" ]; then + new_pwd="$1" + else + new_pwd=$(python3 -c 'import secrets,string; print("".join(secrets.choice(string.ascii_letters+string.digits) for _ in range(24)))') + fi + awk -v new="$new_pwd" '/^ORCH_PASSWORD=/{print "ORCH_PASSWORD="new; next} {print}' \ + "$GEN_DIR/orch.env" > "$GEN_DIR/orch.env.tmp" + mv "$GEN_DIR/orch.env.tmp" "$GEN_DIR/orch.env" + chmod 600 "$GEN_DIR/orch.env" + log "已写入新 ORCH_PASSWORD，强制重建 orchestrator（让 env_file 重新加载）..." + dc up -d --force-recreate orchestrator + ok "新密码：$new_pwd" + log "提示：HMAC SESSION_SECRET 未变，旧 cookie 仍有效到 8h 过期；如需立刻全部失效，编辑 orch.env 改 ORCH_SESSION_SECRET 后再次重建" +} + +cmd_down() { + require_compose + dc down + ok "已停止所有实例（数据保留）" +} + +cmd_status() { + require_compose + log "容器状态:" + dc ps + echo + log "出口 IP 抽样（前 5 个实例）:" + local count=0 + for s in $(awk -F, 'NR>1 && $1!="" {print $1}' "$CSV"); do + [ "$count" -ge 5 ] && break + printf " %-12s -> " "$s" + docker exec "c2a-${s}" curl -s --max-time 8 https://api.ipify.org 2>/dev/null \ + || echo "(unreachable)" + echo + count=$((count+1)) + done +} + +check_contains() { + local url="$1" needle="$2" tries="${3:-8}" body="" + local i + for i in $(seq 1 "$tries"); do + body="$(curl -fsS --max-time 15 "$url" 2>/dev/null || true)" + if printf '%s' "$body" | grep -q "$needle"; then + return 0 + fi + sleep 1 + done + printf '%s' "$body" + return 1 +} + +cmd_verify() { + require_compose + ensure_csv + local port="${CHAT2API_GATEWAY_PORT:-60403}" + local failed=0 slug env_prefix container_prefix nginx_block admin_out tokens_out + log "校验路由与后台页面..." + for slug in $(slugs); do + env_prefix="$(awk -F= '$1=="API_PREFIX"{print $2}' "$GEN_DIR/env/${slug}.env" 2>/dev/null | tail -1)" + container_prefix="$(docker exec "c2a-${slug}" sh -lc 'printf %s "${API_PREFIX:-}"' 2>/dev/null || true)" + if [ -z "$env_prefix" ] || [ "$env_prefix" != "$container_prefix" ]; then + err "${slug}: API_PREFIX 不一致（env=${env_prefix:-} container=${container_prefix:-}）" + failed=1 + continue + fi + + nginx_block="$(docker exec c2a-nginx nginx -T 2>/dev/null | grep -A8 "location /${slug}/" || true)" + if ! printf '%s\n' "$nginx_block" | grep -q "$env_prefix"; then + err "${slug}: nginx 生效配置未指向 ${env_prefix}" + failed=1 + continue + fi + + if ! admin_out="$(check_contains "http://127.0.0.1:${port}/${slug}/admin/login" '管理后台登录\|Admin Login')"; then + err "${slug}: admin/login 校验失败" + printf '%s\n' "$admin_out" | head -3 + failed=1 + continue + fi + + if ! tokens_out="$(check_contains "http://127.0.0.1:${port}/${slug}/tokens" 'Tokens 管理')"; then + err "${slug}: /tokens 校验失败" + printf '%s\n' "$tokens_out" | head -3 + failed=1 + continue + fi + + ok "${slug}: admin/tokens 正常" + done + + if [ "$failed" -ne 0 ]; then + err "校验失败：请先处理上面的实例" + return 1 + fi +} + +cmd_verify_orchestrator() { + require_compose + local port="${CHAT2API_GATEWAY_PORT:-60403}" + local js_out app_out models_out + log "校验 orchestrator 静态资源..." + if ! js_out="$(check_contains "http://127.0.0.1:${port}/orchestrator/static/app.js" 'pg-custom-model' 6)"; then + err "orchestrator: app.js 仍是旧版本（缺少 Playground 自定义模型代码）" + printf '%s\n' "$js_out" | head -3 + return 1 + fi + if ! app_out="$(docker exec c2a-orchestrator grep -q 'X-Chat2API-Stream-Compat' /app/app.py 2>&1)"; then + err "orchestrator: app.py 仍是旧版本（缺少统一入口流式兼容代码）" + printf '%s\n' "$app_out" | head -3 + return 1 + fi + if ! models_out="$(check_contains "http://127.0.0.1:${port}/orchestrator/static/models_by_plan.json" '"plans"' 6)"; then + err "orchestrator: models_by_plan.json 不可访问" + printf '%s\n' "$models_out" | head -3 + return 1 + fi + ok "orchestrator: 模型看板静态资源正常" +} + +cmd_install_cli() { + local config_tmp + config_tmp="$(mktemp)" + cat > "$config_tmp" < [proxy] [note] 追加单个账号 + apply + ./manage.sh remove 移除单个账号 + apply（保留 data/） + ./manage.sh list 所有容器状态（docker compose ps） + ./manage.sh status 状态 + 抽样验证出口 IP + ./manage.sh verify 校验每个实例的后台/路由是否串线 + ./manage.sh verify-orchestrator 校验编排面板模型看板静态资源 + ./manage.sh logs [N] 跟随该实例日志（默认 200 行） + ./manage.sh shell 进入该实例容器 shell + ./manage.sh secrets 打印所有 AUTH / ADMIN_PWD（敏感） + ./manage.sh orch-password [pwd] 重置编排面板密码（不传则随机生成） + ./manage.sh install-cli 安装全局 chat2api 命令 + ./manage.sh down 停止全部（数据保留） + ./manage.sh help 显示本帮助 + +环境变量（可选）: + CHAT2API_GATEWAY_PORT nginx 对外端口（默认 60403） + CHAT2API_IMAGE 覆盖镜像（默认 ghcr.io/nanashiwang/chat2api:latest） + +文件: + accounts.csv 真实账号清单（敏感，git 忽略） + generated/ 生成产物（含密钥，git 忽略） + data// 每实例数据卷（含 cookie/token，git 忽略） +EOF +} + +cmd="${1:-help}" +shift || true +case "$cmd" in + init) cmd_init "$@" ;; + apply) cmd_apply "$@" ;; + add) cmd_add "$@" ;; + remove) cmd_remove "$@" ;; + list) cmd_list "$@" ;; + status) cmd_status "$@" ;; + verify) cmd_verify "$@" ;; + verify-orchestrator) cmd_verify_orchestrator "$@" ;; + logs) cmd_logs "$@" ;; + shell) cmd_shell "$@" ;; + secrets) cmd_secrets "$@" ;; + orch-password) cmd_orch_password "$@" ;; + install-cli) cmd_install_cli "$@" ;; + down) cmd_down "$@" ;; + help|-h|--help) cmd_help ;; + *) err "未知命令: $cmd"; cmd_help; exit 1 ;; +esac diff --git a/deploy/multi/orchestrator/Dockerfile b/deploy/multi/orchestrator/Dockerfile new file mode 100644 index 00000000..42a65de7 --- /dev/null +++ b/deploy/multi/orchestrator/Dockerfile @@ -0,0 +1,26 @@ +FROM python:3.11-alpine + +# docker-cli 用于 subprocess 调 docker compose；curl 用于 healthcheck +RUN apk add --no-cache docker-cli docker-cli-compose curl tzdata \ + && rm -rf /var/cache/apk/* + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY app.py . +COPY templates ./templates +COPY static ./static + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + TZ=Asia/Shanghai \ + ORCH_PORT=8080 + +EXPOSE 8080 + +HEALTHCHECK --interval=30s --timeout=5s --retries=3 --start-period=15s \ + CMD curl -fsS http://127.0.0.1:8080/healthz || exit 1 + +CMD ["python", "-m", "uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8080", "--proxy-headers"] diff --git a/deploy/multi/orchestrator/app.py b/deploy/multi/orchestrator/app.py new file mode 100644 index 00000000..fc44af81 --- /dev/null +++ b/deploy/multi/orchestrator/app.py @@ -0,0 +1,2060 @@ +"""chat2api 多实例编排面板 (Orchestrator) + +职责： +- 单密码登录（HMAC 签名 cookie + CSRF 双 cookie） +- 增删改账号实例（写 accounts.csv → 调 generate.py → docker compose up） +- 启停重启单实例 +- 状态仪表盘（docker inspect + exit IP 抽样 + cookie age） +- 操作审计（jsonl 追加写） + +所有 docker 调用走容器内 docker-cli + 挂入的 /var/run/docker.sock。 +所有 compose 操作必须 --project-directory $MULTI_HOST_PATH 让 daemon 用宿主路径解析 volumes。 +""" +from __future__ import annotations + +import base64 +import csv +import functools +import hashlib +import io +import json +import logging +import os +import re +import secrets as pysecrets +import subprocess +import time +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import httpx +from fastapi import ( + Cookie, + Depends, + FastAPI, + Form, + HTTPException, + Query, + Request, + Response, + status, +) +from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse, StreamingResponse +from fastapi.staticfiles import StaticFiles +from fastapi.templating import Jinja2Templates +from itsdangerous import BadSignature, SignatureExpired, URLSafeTimedSerializer +from pydantic import BaseModel, Field, field_validator +from starlette.background import BackgroundTask + +# ---------- 配置 ---------- + +# WORK 目录策略：让容器内 WORK 路径 = 宿主 deploy/multi 绝对路径， +# 因为 docker compose 客户端（在容器内）发命令前会先解析 env_file 并检查存在性， +# 必须用容器内能访问的路径；而 daemon（在宿主）解析 volumes 用宿主路径。 +# 通过 compose volume `${MULTI_HOST_PATH}:${MULTI_HOST_PATH}` 让两者重合。 +HOST_PATH = (os.environ.get("MULTI_HOST_PATH") or "/work").rstrip("/") +WORK = Path(HOST_PATH) +COMPOSE_FILE_C = WORK / "generated" / "docker-compose.yml" +ACCOUNTS_CSV = WORK / "accounts.csv" +SECRETS_FILE = WORK / "generated" / "secrets.txt" +ORCH_ENV = WORK / "generated" / "orch.env" +DATA_DIR = WORK / "data" +AUDIT_FILE = WORK / "audit.jsonl" +USAGE_FILE = WORK / "usage.jsonl" + +USERNAME = (os.environ.get("ORCH_USERNAME") or "admin").strip() or "admin" +PASSWORD = (os.environ.get("ORCH_PASSWORD") or "").strip() +SESSION_SECRET = (os.environ.get("ORCH_SESSION_SECRET") or "").strip() +SESSION_MAX_AGE = 8 * 3600 # 8h +SESSION_COOKIE = "orch_session" +CSRF_COOKIE = "orch_csrf" +UNIFIED_API_KEY_ENV = "ORCH_API_KEY" + +if not PASSWORD or not SESSION_SECRET: + raise RuntimeError( + "ORCH_PASSWORD 与 ORCH_SESSION_SECRET 必须在 generated/orch.env 中设置" + ) + +SLUG_RE = re.compile(r"^[a-z0-9-]{1,16}$") +PROXY_RE = re.compile(r"^(socks5|socks5h|http|https)://[^\s]+$") +ORCH_USERNAME_RE = re.compile(r"^[A-Za-z0-9_.@-]{3,64}$") + +LOG_LEVEL = os.environ.get("ORCH_LOG_LEVEL", "INFO").upper() +logging.basicConfig( + level=LOG_LEVEL, + format="%(asctime)s | %(levelname)s | %(message)s", +) +logger = logging.getLogger("orchestrator") + +serializer = URLSafeTimedSerializer(SESSION_SECRET, salt="orch-session-v1") + +app = FastAPI(title="chat2api Orchestrator", docs_url=None, redoc_url=None) +templates = Jinja2Templates(directory="templates") +APP_DIR = Path(__file__).parent +STATIC_DIR = APP_DIR / "static" + + +@functools.lru_cache(maxsize=1) +def static_version() -> str: + """根据前端资源内容生成版本号，避免部署后浏览器继续使用旧 JS/CSS。""" + digest = hashlib.sha256() + for name in ("static/app.js", "static/styles.css", "static/models_by_plan.json"): + try: + digest.update((APP_DIR / name).read_bytes()) + except FileNotFoundError: + digest.update(name.encode("utf-8")) + return digest.hexdigest()[:12] + + +@app.middleware("http") +async def no_cache_orchestrator_ui(request: Request, call_next): + response = await call_next(request) + if request.url.path.startswith(("/static/", "/orchestrator/static/")) or request.url.path in {"/", "/orchestrator/"}: + response.headers["Cache-Control"] = "no-store, no-cache, must-revalidate, max-age=0" + response.headers["Pragma"] = "no-cache" + response.headers["Expires"] = "0" + return response + + +app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static") + +# ---------- 工具：subprocess + docker ---------- + +class DockerError(Exception): + pass + + +def run(cmd: list[str], timeout: int = 60) -> tuple[int, str, str]: + """同步执行命令，返回 (rc, stdout, stderr)。绝不抛 stderr 给前端原文（避免泄漏路径）。""" + logger.debug("run: %s", " ".join(cmd)) + try: + proc = subprocess.run( + cmd, capture_output=True, text=True, timeout=timeout + ) + except subprocess.TimeoutExpired: + raise DockerError(f"命令超时（{timeout}s）：{cmd[0]}") + return proc.returncode, proc.stdout or "", proc.stderr or "" + + +def dc(*args: str, timeout: int = 180) -> tuple[int, str, str]: + """docker compose 包装，自动加 -f 与 --project-directory。""" + cmd = [ + "docker", "compose", + "-f", str(COMPOSE_FILE_C), + "--project-directory", HOST_PATH, + *args, + ] + return run(cmd, timeout=timeout) + + +def inspect(container: str) -> dict | None: + rc, out, _ = run(["docker", "inspect", container], timeout=10) + if rc != 0: + return None + try: + data = json.loads(out) + return data[0] if data else None + except json.JSONDecodeError: + return None + + +def regenerate_and_apply() -> None: + """写 csv 后必须调用：先 generate.py，再 docker compose up -d --remove-orphans， + 再 nginx -s reload（compose 不会因 nginx.conf 改动重启 nginx）。""" + rc, out, err = run( + ["python3", str(WORK / "generate.py")], timeout=30 + ) + if rc != 0: + logger.error("generate.py 失败：rc=%s out=%s err=%s", rc, out, err) + raise DockerError(f"配置生成失败：{(err or out)[:300]}") + + rc, out, err = dc("up", "-d", "--remove-orphans", timeout=240) + if rc != 0: + logger.error("compose up 失败：rc=%s err=%s", rc, err) + raise DockerError(f"docker compose 失败：{(err or out)[:300]}") + + # nginx.conf 变化必须 reload，不然新 location 不生效 + rc, out, err = run( + ["docker", "exec", "c2a-nginx", "nginx", "-s", "reload"], timeout=10 + ) + if rc != 0: + logger.warning("nginx reload 失败（非致命）：%s", (err or out)[:200]) + + +# ---------- 工具：CSV / env / secrets ---------- + +def read_accounts() -> list[dict[str, str]]: + if not ACCOUNTS_CSV.exists(): + return [] + out: list[dict[str, str]] = [] + with ACCOUNTS_CSV.open("r", encoding="utf-8", newline="") as f: + reader = csv.DictReader(f) + for row in reader: + slug = (row.get("slug") or "").strip() + if not slug: + continue + out.append({ + "slug": slug, + "proxy_url": (row.get("proxy_url") or "").strip(), + "note": (row.get("note") or "").strip(), + }) + return out + + +def write_accounts(rows: list[dict[str, str]]) -> None: + """原子写：tmp + rename。csv 头固定。""" + tmp = ACCOUNTS_CSV.with_suffix(".csv.tmp") + with tmp.open("w", encoding="utf-8", newline="") as f: + writer = csv.DictWriter(f, fieldnames=["slug", "proxy_url", "note"]) + writer.writeheader() + for r in rows: + writer.writerow({ + "slug": r["slug"], + "proxy_url": r.get("proxy_url", ""), + "note": r.get("note", ""), + }) + tmp.replace(ACCOUNTS_CSV) + + +def read_env_file(path: Path) -> dict[str, str]: + if not path.exists(): + return {} + out: dict[str, str] = {} + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, v = line.split("=", 1) + out[k.strip()] = v.strip() + return out + + +def write_env_file(path: Path, values: dict[str, str]) -> None: + tmp = path.with_suffix(path.suffix + ".tmp") + with tmp.open("w", encoding="utf-8") as f: + for key, value in values.items(): + f.write(f"{key}={value}\n") + tmp.chmod(0o600) + tmp.replace(path) + + +def orch_credentials() -> dict[str, str]: + env = read_env_file(ORCH_ENV) + username = (env.get("ORCH_USERNAME") or USERNAME or "admin").strip() or "admin" + password = (env.get("ORCH_PASSWORD") or PASSWORD).strip() + secret = (env.get("ORCH_SESSION_SECRET") or SESSION_SECRET).strip() + version = hashlib.sha256(f"{username}\0{password}".encode("utf-8")).hexdigest()[:16] + return { + "username": username, + "password": password, + "secret": secret, + "version": version, + } + + +def mask_proxy(url: str) -> str: + """socks5://user:pass@host:port → socks5://****@host:port""" + if not url: + return "" + m = re.match(r"^(\w+)://([^@/]+@)?(.+)$", url) + if not m: + return url + scheme, _, host = m.groups() + return f"{scheme}://****@{host}" if _ else f"{scheme}://{host}" + + +def mask_secret(s: str, head: int = 6, tail: int = 4) -> str: + if not s: + return "" + if len(s) <= head + tail + 3: + return "*" * len(s) + return f"{s[:head]}...{s[-tail:]}" + + +def public_origin(request: Request) -> str: + proto = (request.headers.get("x-forwarded-proto") or request.url.scheme).split(",", 1)[0].strip() + host = ( + request.headers.get("x-forwarded-host") + or request.headers.get("host") + or "" + ).split(",", 1)[0].strip() + bare_host = host + if bare_host.startswith("[") and "]" in bare_host: + bare_host = bare_host[1:bare_host.index("]")] + elif ":" in bare_host: + bare_host = bare_host.rsplit(":", 1)[0] + if not host or bare_host in {"localhost", "127.0.0.1", "0.0.0.0"} or bare_host.startswith("c2a-"): + return "" + return f"{proto}://{host}" + + +def public_url(request: Request, path: str) -> str: + clean_path = "/" + path.lstrip("/") + origin = public_origin(request).rstrip("/") + return f"{origin}{clean_path}" if origin else clean_path + + +def unified_api_key() -> str: + env = read_env_file(ORCH_ENV) + return (env.get(UNIFIED_API_KEY_ENV) or os.environ.get(UNIFIED_API_KEY_ENV) or "").strip() + + +def require_unified_api_key(request: Request) -> None: + expected = unified_api_key() + if not expected: + raise HTTPException(status_code=503, detail="统一 API Key 未生成，请重新 ./manage.sh apply") + auth = (request.headers.get("authorization") or "").strip() + token = auth[7:].strip() if auth.lower().startswith("bearer ") else auth + if not pysecrets.compare_digest(token, expected): + raise HTTPException( + status_code=401, + detail="Invalid API key", + headers={"WWW-Authenticate": "Bearer"}, + ) + + +def _extract_json_body(body: bytes, content_type: str) -> dict[str, Any]: + if not body or "json" not in content_type.lower(): + return {} + try: + data = json.loads(body.decode("utf-8")) + return data if isinstance(data, dict) else {} + except Exception: + return {} + + +def _affinity_key(request: Request, body_json: dict[str, Any]) -> str: + for header in ("x-chat2api-affinity", "x-conversation-id", "x-request-affinity"): + val = (request.headers.get(header) or "").strip() + if val: + return val + for key in ( + "chat2api_affinity_key", + "librechat_conversation_id", + "conversation_id", + "parent_conversation_id", + ): + val = body_json.get(key) + if isinstance(val, str) and val.strip(): + return val.strip() + return "" + + +def _candidate_models(slug: str) -> set[str]: + try: + info = _get_instance_info(slug) + return { + (m.get("id") if isinstance(m, dict) else str(m)) + for m in info.get("models", []) + if m + } + except Exception: + return set() + + +def _backend_candidates(model: str = "") -> list[dict[str, str]]: + all_rows: list[dict[str, str]] = [] + matched: list[dict[str, str]] = [] + for row in read_accounts(): + slug = row["slug"] + env = read_env_file(WORK / "generated" / "env" / f"{slug}.env") + auth = env.get("AUTHORIZATION", "") + prefix = env.get("API_PREFIX", "") + if not auth or not prefix: + continue + item = {"slug": slug, "auth": auth, "api_prefix": prefix} + all_rows.append(item) + models = _candidate_models(slug) if model else set() + if model and models and model in models: + matched.append(item) + return matched or all_rows + + +_proxy_rr_cursor = 0 + + +def _ordered_backends(candidates: list[dict[str, str]], affinity: str) -> list[dict[str, str]]: + global _proxy_rr_cursor + if not candidates: + return [] + if affinity: + digest = hashlib.sha256(affinity.encode("utf-8")).hexdigest() + start = int(digest, 16) % len(candidates) + else: + start = _proxy_rr_cursor % len(candidates) + _proxy_rr_cursor += 1 + return candidates[start:] + candidates[:start] + + +HOP_BY_HOP_HEADERS = { + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailer", + "transfer-encoding", + "upgrade", + "host", + "content-length", + "authorization", + "cookie", +} + + +def _client_ip(request: Request) -> str: + xff = (request.headers.get("x-forwarded-for") or "").split(",", 1)[0].strip() + return xff or (request.client.host if request.client else "?") + + +def _usage_from_body(data: Any) -> dict[str, int | None]: + usage = data.get("usage") if isinstance(data, dict) else {} + if not isinstance(usage, dict): + usage = {} + prompt = usage.get("prompt_tokens") + completion = usage.get("completion_tokens") + total = usage.get("total_tokens") + if prompt is None: + prompt = usage.get("input_tokens") + if completion is None: + completion = usage.get("output_tokens") + if total is None and prompt is not None and completion is not None: + try: + total = int(prompt) + int(completion) + except (TypeError, ValueError): + total = None + + def as_int(value: Any) -> int | None: + try: + return int(value) + except (TypeError, ValueError): + return None + + return { + "prompt_tokens": as_int(prompt), + "completion_tokens": as_int(completion), + "total_tokens": as_int(total), + } + + +def _write_usage(record: dict[str, Any]) -> None: + try: + with USAGE_FILE.open("a", encoding="utf-8") as f: + f.write(json.dumps(record, ensure_ascii=False) + "\n") + except OSError as e: + logger.error("usage write failed: %s", e) + + +def _record_usage( + request: Request, + *, + slug: str, + path: str, + model: str, + stream: bool, + status_code: int, + duration_ms: int, + ok: bool, + request_id: str, + usage: dict[str, int | None] | None = None, + error: str = "", + stream_compat: bool = False, +) -> None: + token_usage = usage or {} + _write_usage({ + "ts": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "request_id": request_id, + "ip": _client_ip(request), + "slug": slug, + "path": path, + "endpoint": path.rsplit("/", 1)[-1] if path else "", + "model": model, + "stream": stream, + "stream_compat": stream_compat, + "status": status_code, + "ok": ok, + "prompt_tokens": token_usage.get("prompt_tokens"), + "completion_tokens": token_usage.get("completion_tokens"), + "total_tokens": token_usage.get("total_tokens"), + "duration_ms": duration_ms, + "error": error[:300], + }) + + +def read_usage(limit: int = 200) -> list[dict[str, Any]]: + if not USAGE_FILE.exists(): + return [] + lines = USAGE_FILE.read_text(encoding="utf-8").splitlines() + out: list[dict[str, Any]] = [] + for line in reversed(lines[-limit * 3:]): + if not line.strip(): + continue + try: + out.append(json.loads(line)) + except json.JSONDecodeError: + continue + if len(out) >= limit: + break + return out + + +def usage_summary(records: list[dict[str, Any]]) -> dict[str, Any]: + total_requests = len(records) + success = sum(1 for r in records if r.get("ok")) + failed = total_requests - success + total_tokens = sum(int(r.get("total_tokens") or 0) for r in records) + prompt_tokens = sum(int(r.get("prompt_tokens") or 0) for r in records) + completion_tokens = sum(int(r.get("completion_tokens") or 0) for r in records) + durations = [int(r.get("duration_ms") or 0) for r in records if r.get("duration_ms") is not None] + avg_duration_ms = round(sum(durations) / len(durations)) if durations else 0 + sorted_durations = sorted(durations) + p95_index = max(0, min(len(sorted_durations) - 1, int((len(sorted_durations) - 1) * 0.95))) if sorted_durations else 0 + p95_duration_ms = sorted_durations[p95_index] if sorted_durations else 0 + by_slug: dict[str, dict[str, Any]] = {} + by_model: dict[str, int] = {} + by_endpoint: dict[str, int] = {} + by_hour: dict[str, dict[str, int]] = {} + for r in records: + slug = str(r.get("slug") or "-") + slug_row = by_slug.setdefault(slug, {"slug": slug, "requests": 0, "tokens": 0, "errors": 0}) + slug_row["requests"] += 1 + slug_row["tokens"] += int(r.get("total_tokens") or 0) + if not r.get("ok"): + slug_row["errors"] += 1 + model = str(r.get("model") or "-") + by_model[model] = by_model.get(model, 0) + 1 + endpoint = str(r.get("endpoint") or "-") + by_endpoint[endpoint] = by_endpoint.get(endpoint, 0) + 1 + hour = str(r.get("ts") or "")[:13] + if hour: + hour_row = by_hour.setdefault(hour, {"requests": 0, "tokens": 0, "duration_ms": 0, "duration_count": 0}) + hour_row["requests"] += 1 + hour_row["tokens"] += int(r.get("total_tokens") or 0) + if r.get("duration_ms") is not None: + hour_row["duration_ms"] += int(r.get("duration_ms") or 0) + hour_row["duration_count"] += 1 + busiest = sorted(by_slug.values(), key=lambda x: (x["requests"], x["tokens"]), reverse=True) + return { + "total_requests": total_requests, + "success": success, + "failed": failed, + "success_rate": round(success * 100 / total_requests, 1) if total_requests else 0, + "total_tokens": total_tokens, + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "avg_duration_ms": avg_duration_ms, + "p95_duration_ms": p95_duration_ms, + "top_slug": busiest[0]["slug"] if busiest else "-", + "by_slug": busiest, + "by_model": sorted( + [{"model": k, "requests": v} for k, v in by_model.items()], + key=lambda x: x["requests"], + reverse=True, + ), + "by_endpoint": sorted( + [{"endpoint": k, "requests": v} for k, v in by_endpoint.items()], + key=lambda x: x["requests"], + reverse=True, + ), + "timeline": [ + { + "hour": hour, + "requests": row["requests"], + "tokens": row["tokens"], + "avg_duration_ms": round(row["duration_ms"] / row["duration_count"]) if row["duration_count"] else 0, + } + for hour, row in sorted(by_hour.items()) + ][-24:], + } + + +async def _close_upstream(resp: httpx.Response, client: httpx.AsyncClient) -> None: + await resp.aclose() + await client.aclose() + + +async def _forward_unified_request( + request: Request, + backend: dict[str, str], + path: str, + body: bytes, + body_json: dict[str, Any], + model: str, + request_id: str, + started: float, +) -> StreamingResponse: + upstream_path = f"/{backend['api_prefix']}/v1/{path.lstrip('/')}" + url = f"http://c2a-{backend['slug']}:5005{upstream_path}" + if request.url.query: + url += f"?{request.url.query}" + + headers = { + k: v + for k, v in request.headers.items() + if k.lower() not in HOP_BY_HOP_HEADERS + } + headers["Authorization"] = f"Bearer {backend['auth']}" + headers["X-Chat2API-Orchestrator"] = "1" + headers["X-Chat2API-Upstream-Slug"] = backend["slug"] + + client = httpx.AsyncClient(timeout=httpx.Timeout(600.0, connect=15.0)) + req = client.build_request( + request.method, + url, + headers=headers, + content=body, + ) + try: + resp = await client.send(req, stream=True) + except Exception: + await client.aclose() + raise + duration_ms = int((time.perf_counter() - started) * 1000) + + response_headers = { + k: v + for k, v in resp.headers.items() + if k.lower() not in HOP_BY_HOP_HEADERS + and k.lower() not in {"content-length", "content-encoding"} + } + response_headers["X-Chat2API-Upstream-Slug"] = backend["slug"] + content_type = resp.headers.get("content-type", "") + is_stream = _is_truthy(body_json.get("stream")) or content_type.startswith("text/event-stream") + if not is_stream and content_type.startswith("application/json"): + raw = await resp.aread() + parsed: Any = {} + try: + parsed = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + parsed = {} + _record_usage( + request, + slug=backend["slug"], + path=f"/v1/{path}", + model=model, + stream=False, + status_code=resp.status_code, + duration_ms=duration_ms, + ok=200 <= resp.status_code < 400, + request_id=request_id, + usage=_usage_from_body(parsed), + error="" if 200 <= resp.status_code < 400 else str(parsed)[:300], + ) + await resp.aclose() + await client.aclose() + return StreamingResponse( + iter([raw]), + status_code=resp.status_code, + headers=response_headers, + media_type=content_type or None, + ) + _record_usage( + request, + slug=backend["slug"], + path=f"/v1/{path}", + model=model, + stream=is_stream, + status_code=resp.status_code, + duration_ms=duration_ms, + ok=200 <= resp.status_code < 400, + request_id=request_id, + error="" if 200 <= resp.status_code < 400 else f"HTTP {resp.status_code}", + ) + return StreamingResponse( + resp.aiter_raw(), + status_code=resp.status_code, + headers=response_headers, + background=BackgroundTask(_close_upstream, resp, client), + ) + + +def _openai_sse_chunk(data: dict[str, Any]) -> str: + return f"data: {json.dumps(data, ensure_ascii=False, separators=(',', ':'))}\n\n" + + +def _is_truthy(value: Any) -> bool: + if isinstance(value, bool): + return value + if isinstance(value, (int, float)): + return value == 1 + if isinstance(value, str): + return value.strip().lower() in {"1", "true", "yes", "y", "on"} + return False + + +def _as_openai_stream_events(data: dict[str, Any], model: str) -> list[str]: + created = int(data.get("created") or time.time()) + chat_id = str(data.get("id") or f"chatcmpl-orch-{pysecrets.token_hex(12)}") + model_id = str(data.get("model") or model or "unknown") + choice = ((data.get("choices") or [{}])[0] or {}) + message = choice.get("message") or {} + content = message.get("content") + if content is None: + content = data.get("output_text", "") + if not isinstance(content, str): + content = json.dumps(content, ensure_ascii=False) + finish_reason = choice.get("finish_reason") or data.get("finish_reason") or "stop" + + base = { + "id": chat_id, + "object": "chat.completion.chunk", + "created": created, + "model": model_id, + } + first = { + **base, + "choices": [{"index": 0, "delta": {"role": "assistant"}, "finish_reason": None}], + } + delta = { + **base, + "choices": [{"index": 0, "delta": {"content": content}, "finish_reason": None}], + } + final = { + **base, + "choices": [{"index": 0, "delta": {}, "finish_reason": finish_reason}], + } + return [_openai_sse_chunk(first), _openai_sse_chunk(delta), _openai_sse_chunk(final), "data: [DONE]\n\n"] + + +async def _forward_unified_stream_compat( + request: Request, + backend: dict[str, str], + path: str, + body_json: dict[str, Any], + model: str, + request_id: str, + started: float, +) -> StreamingResponse: + compat_body = {**body_json, "stream": False} + upstream_path = f"/{backend['api_prefix']}/v1/{path.lstrip('/')}" + url = f"http://c2a-{backend['slug']}:5005{upstream_path}" + if request.url.query: + url += f"?{request.url.query}" + + headers = { + k: v + for k, v in request.headers.items() + if k.lower() not in HOP_BY_HOP_HEADERS + } + headers["Authorization"] = f"Bearer {backend['auth']}" + headers["Content-Type"] = "application/json" + headers["X-Chat2API-Orchestrator"] = "1" + headers["X-Chat2API-Upstream-Slug"] = backend["slug"] + content = json.dumps(compat_body, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + + async with httpx.AsyncClient(timeout=httpx.Timeout(600.0, connect=15.0)) as client: + resp = await client.post(url, headers=headers, content=content) + resp.raise_for_status() + data = resp.json() + duration_ms = int((time.perf_counter() - started) * 1000) + _record_usage( + request, + slug=backend["slug"], + path=f"/v1/{path}", + model=model, + stream=True, + status_code=resp.status_code, + duration_ms=duration_ms, + ok=True, + request_id=request_id, + usage=_usage_from_body(data), + stream_compat=True, + ) + + async def event_generator(): + for event in _as_openai_stream_events(data, model): + yield event + + return StreamingResponse( + event_generator(), + media_type="text/event-stream", + headers={ + "Cache-Control": "no-cache", + "X-Accel-Buffering": "no", + "X-Chat2API-Upstream-Slug": backend["slug"], + "X-Chat2API-Stream-Compat": "1", + }, + ) + + +# ---------- 出口 IP 缓存 ---------- + +_exit_ip_cache: dict[str, tuple[str, float]] = {} +EXIT_IP_TTL = 60.0 + + +def get_exit_ip(slug: str, force: bool = False) -> str | None: + now = time.time() + if not force and slug in _exit_ip_cache: + ip, ts = _exit_ip_cache[slug] + if now - ts < EXIT_IP_TTL: + return ip + rc, out, _ = run( + ["docker", "exec", f"c2a-{slug}", "curl", "-s", "--max-time", "6", + "https://api.ipify.org"], + timeout=10, + ) + ip = out.strip() if rc == 0 and out.strip() else None + if ip: + _exit_ip_cache[slug] = (ip, now) + return ip + + +def get_cookie_last_success(slug: str) -> int | None: + """读 data/{slug}/refresh_map.json，返回最新 last_success_at（unix ts）。""" + p = DATA_DIR / slug / "refresh_map.json" + if not p.exists(): + return None + try: + data = json.loads(p.read_text(encoding="utf-8")) + if not isinstance(data, dict): + return None + ts_list = [ + int(v.get("last_success_at") or v.get("timestamp") or 0) + for v in data.values() + if isinstance(v, dict) + ] + ts_list = [t for t in ts_list if t > 0] + return max(ts_list) if ts_list else None + except Exception: + return None + + +# ---------- 审计 ---------- + +def audit(action: str, request: Request, ok: bool, **fields: Any) -> None: + rec = { + "ts": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "actor": "admin", + "ip": request.client.host if request.client else "?", + "action": action, + "ok": ok, + **fields, + } + try: + with AUDIT_FILE.open("a", encoding="utf-8") as f: + f.write(json.dumps(rec, ensure_ascii=False) + "\n") + except OSError as e: + logger.error("audit write failed: %s", e) + + +def read_audit(limit: int = 200) -> list[dict]: + if not AUDIT_FILE.exists(): + return [] + lines = AUDIT_FILE.read_text(encoding="utf-8").splitlines() + out: list[dict] = [] + for line in reversed(lines[-limit * 2:]): + if not line.strip(): + continue + try: + out.append(json.loads(line)) + except json.JSONDecodeError: + continue + if len(out) >= limit: + break + return out + + +def redact_log_text(text: str) -> str: + """日志对前端展示前做基础脱敏，避免误复制密钥。""" + patterns = [ + (r"(Bearer\s+)[A-Za-z0-9._=-]{20,}", r"\1***"), + (r"(AUTHORIZATION=)[^\s]+", r"\1***"), + (r"(sk-[A-Za-z0-9_-]{8})[A-Za-z0-9_-]+", r"\1***"), + (r"(sess-)[^\s\"']{24,}", r"\1***"), + (r"(rt_[^\s\"']{8})[^\s\"']+", r"\1***"), + (r"(eyJ[A-Za-z0-9_-]{16})[A-Za-z0-9._-]+", r"\1***"), + ] + for pattern, repl in patterns: + text = re.sub(pattern, repl, text) + return text + + +def log_targets() -> list[dict[str, str]]: + targets = [ + {"id": "orchestrator", "label": "orchestrator 面板", "container": "c2a-orchestrator"}, + {"id": "nginx", "label": "nginx 网关", "container": "c2a-nginx"}, + {"id": "watchtower", "label": "watchtower 更新器", "container": "c2a-watchtower"}, + ] + for row in read_accounts(): + slug = row["slug"] + targets.append({"id": slug, "label": f"实例 {slug}", "container": f"c2a-{slug}"}) + return targets + + +def read_container_logs(container: str, tail: int = 80) -> tuple[bool, str]: + rc, out, err = run( + ["docker", "logs", "--tail", str(tail), "--timestamps", container], + timeout=20, + ) + raw = "\n".join(part for part in (out, err) if part.strip()) + if rc != 0 and not raw: + raw = f"docker logs failed: rc={rc}" + return rc == 0, redact_log_text(raw or "(无日志输出)") + + +# ---------- 鉴权 ---------- + +def issue_session_token(username: str, version: str) -> str: + return serializer.dumps({"u": username, "v": version}) + + +def verify_session_token(token: str | None) -> bool: + if not token: + return False + try: + data = serializer.loads(token, max_age=SESSION_MAX_AGE) + creds = orch_credentials() + return ( + isinstance(data, dict) + and data.get("u") == creds["username"] + and data.get("v") == creds["version"] + ) + except (BadSignature, SignatureExpired): + return False + + +def gen_csrf() -> str: + return pysecrets.token_hex(16) + + +def require_session( + request: Request, + orch_session: str | None = Cookie(default=None), +) -> None: + if not verify_session_token(orch_session): + raise HTTPException(status_code=401, detail="未登录") + + +def require_csrf(request: Request) -> None: + """双 cookie 模式：cookie orch_csrf 必须等于 header X-CSRF-Token。""" + cookie_val = request.cookies.get(CSRF_COOKIE) or "" + header_val = request.headers.get("x-csrf-token") or "" + if not cookie_val or not pysecrets.compare_digest(cookie_val, header_val): + raise HTTPException(status_code=403, detail="CSRF 校验失败") + + +# 登录速率限制（同 IP 60s 内最多 5 次失败） +_login_attempts: dict[str, list[float]] = {} + + +def check_login_rate(ip: str) -> bool: + now = time.time() + bucket = _login_attempts.setdefault(ip, []) + bucket[:] = [t for t in bucket if now - t < 60] + return len(bucket) < 5 + + +def record_login_failure(ip: str) -> None: + _login_attempts.setdefault(ip, []).append(time.time()) + + +# ---------- 路由：基础 ---------- + +@app.get("/healthz") +async def healthz() -> JSONResponse: + return JSONResponse({"ok": True}) + + +@app.get("/v1/models") +async def unified_models(request: Request) -> JSONResponse: + require_unified_api_key(request) + model_ids: set[str] = set() + for row in read_accounts(): + model_ids.update(_candidate_models(row["slug"])) + data = [ + { + "id": mid, + "object": "model", + "created": 0, + "owned_by": "chat2api-orchestrator", + } + for mid in sorted(model_ids) + ] + audit("unified_models", request, True, model_count=len(data)) + return JSONResponse({"object": "list", "data": data}) + + +@app.api_route( + "/v1/{path:path}", + methods=["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"], +) +async def unified_proxy(path: str, request: Request) -> Response: + require_unified_api_key(request) + body = await request.body() + body_json = _extract_json_body(body, request.headers.get("content-type", "")) + model = str(body_json.get("model") or "").strip() + affinity = _affinity_key(request, body_json) + candidates = _backend_candidates(model) + if not candidates: + raise HTTPException(status_code=503, detail="暂无可用实例，请先在编排面板新增账号") + + last_error = "" + stream_compat = path.lstrip("/") == "chat/completions" and _is_truthy(body_json.get("stream")) + for backend in _ordered_backends(candidates, affinity): + request_id = request.headers.get("x-request-id") or f"orch-{int(time.time() * 1000)}-{pysecrets.token_hex(4)}" + started = time.perf_counter() + try: + if stream_compat: + resp = await _forward_unified_stream_compat(request, backend, path, body_json, model, request_id, started) + else: + resp = await _forward_unified_request(request, backend, path, body, body_json, model, request_id, started) + audit( + "unified_proxy", + request, + True, + slug=backend["slug"], + path=f"/v1/{path}", + model=model or "", + affinity=bool(affinity), + stream_compat=stream_compat, + ) + return resp + except (httpx.RequestError, httpx.HTTPStatusError, ValueError) as e: + last_error = str(e)[:200] + status_code = e.response.status_code if isinstance(e, httpx.HTTPStatusError) else 502 + _record_usage( + request, + slug=backend["slug"], + path=f"/v1/{path}", + model=model or "", + stream=_is_truthy(body_json.get("stream")), + status_code=status_code, + duration_ms=int((time.perf_counter() - started) * 1000), + ok=False, + request_id=request_id, + error=last_error, + stream_compat=stream_compat, + ) + audit( + "unified_proxy", + request, + False, + slug=backend["slug"], + path=f"/v1/{path}", + model=model or "", + error=last_error, + ) + continue + raise HTTPException(status_code=502, detail=f"所有实例转发失败：{last_error}") + + +@app.get("/login", response_class=HTMLResponse) +async def login_page(request: Request) -> HTMLResponse: + return templates.TemplateResponse( + "login.html", {"request": request, "error": None} + ) + + +@app.post("/login") +async def login( + request: Request, + response: Response, + username: str = Form("admin"), + password: str = Form(...), +) -> Response: + ip = request.client.host if request.client else "?" + if not check_login_rate(ip): + audit("login", request, False, reason="rate_limited") + return templates.TemplateResponse( + "login.html", + {"request": request, "error": "尝试过多，请稍候再试"}, + status_code=429, + ) + creds = orch_credentials() + if ( + not pysecrets.compare_digest(username.strip(), creds["username"]) + or not pysecrets.compare_digest(password, creds["password"]) + ): + record_login_failure(ip) + audit("login", request, False, reason="bad_credentials", username=username[:80]) + return templates.TemplateResponse( + "login.html", + {"request": request, "error": "用户名或密码错误", "username": username}, + status_code=401, + ) + + token = issue_session_token(creds["username"], creds["version"]) + csrf = gen_csrf() + is_https = request.url.scheme == "https" or \ + request.headers.get("x-forwarded-proto") == "https" + resp = RedirectResponse(url="./", status_code=303) + resp.set_cookie( + SESSION_COOKIE, token, + max_age=SESSION_MAX_AGE, + httponly=True, samesite="strict", secure=is_https, path="/", + ) + resp.set_cookie( + CSRF_COOKIE, csrf, + max_age=SESSION_MAX_AGE, + httponly=False, samesite="strict", secure=is_https, path="/", + ) + audit("login", request, True, username=creds["username"]) + return resp + + +@app.post("/logout") +async def logout(request: Request, _: None = Depends(require_session)) -> Response: + audit("logout", request, True) + resp = RedirectResponse(url="./login", status_code=303) + resp.delete_cookie(SESSION_COOKIE, path="/") + resp.delete_cookie(CSRF_COOKIE, path="/") + return resp + + +@app.get("/", response_class=HTMLResponse) +async def dashboard( + request: Request, + orch_session: str | None = Cookie(default=None), +) -> Response: + if not verify_session_token(orch_session): + return RedirectResponse(url="./login", status_code=303) + return templates.TemplateResponse( + "dashboard.html", + {"request": request, "static_version": static_version()}, + ) + + +# ---------- API: accounts CRUD ---------- + +class AccountIn(BaseModel): + slug: str + proxy_url: str = "" + note: str = "" + + @field_validator("slug") + @classmethod + def _slug(cls, v: str) -> str: + v = v.strip() + if not SLUG_RE.match(v): + raise ValueError("slug 不合法（需 [a-z0-9-]{1,16}）") + return v + + @field_validator("proxy_url") + @classmethod + def _proxy(cls, v: str) -> str: + v = v.strip() + if v and not PROXY_RE.match(v): + raise ValueError("proxy_url 必须以 socks5/socks5h/http/https:// 开头") + return v + + +class AccountPatch(BaseModel): + proxy_url: str | None = None + note: str | None = None + + @field_validator("proxy_url") + @classmethod + def _proxy(cls, v: str | None) -> str | None: + if v is None: + return None + v = v.strip() + if v and not PROXY_RE.match(v): + raise ValueError("proxy_url 必须以 socks5/socks5h/http/https:// 开头") + return v + + +class OrchestratorCredentialPatch(BaseModel): + username: str + current_password: str + new_password: str + + @field_validator("username") + @classmethod + def _username(cls, v: str) -> str: + v = v.strip() + if not ORCH_USERNAME_RE.match(v): + raise ValueError("用户名需 3-64 位，仅支持字母、数字、_.@-") + return v + + @field_validator("current_password") + @classmethod + def _current_password(cls, v: str) -> str: + if not v or "\n" in v or "\r" in v: + raise ValueError("当前密码必填") + return v + + @field_validator("new_password") + @classmethod + def _new_password(cls, v: str) -> str: + if len(v) < 8: + raise ValueError("新密码至少 8 位") + if "\n" in v or "\r" in v: + raise ValueError("新密码不能包含换行") + return v + + +@app.get("/api/orchestrator/account", dependencies=[Depends(require_session)]) +async def api_orchestrator_account() -> JSONResponse: + creds = orch_credentials() + return JSONResponse({"username": creds["username"]}) + + +@app.get("/api/unified", dependencies=[Depends(require_session), Depends(require_csrf)]) +async def api_unified_credentials(request: Request) -> JSONResponse: + key = unified_api_key() + if not key: + raise HTTPException(status_code=503, detail="统一 API Key 未生成，请重新 ./manage.sh apply") + base_path = "/v1" + chat_path = "/v1/chat/completions" + responses_path = "/v1/responses" + compact_path = "/v1/responses/compact" + audit("reveal_unified_api", request, True) + return JSONResponse({ + "base_path": base_path, + "chat_completions_path": chat_path, + "responses_path": responses_path, + "responses_compact_path": compact_path, + "base_url": public_url(request, base_path), + "chat_completions_url": public_url(request, chat_path), + "responses_url": public_url(request, responses_path), + "responses_compact_url": public_url(request, compact_path), + "api_key": key, + "strategy": "支持 /v1/chat/completions、/v1/responses、/v1/responses/compact；无会话键时轮询，有会话键时固定到同一容器", + }) + + +@app.patch( + "/api/orchestrator/account", + dependencies=[Depends(require_session), Depends(require_csrf)], +) +async def api_update_orchestrator_account( + payload: OrchestratorCredentialPatch, + request: Request, +) -> JSONResponse: + creds = orch_credentials() + if not pysecrets.compare_digest(payload.current_password, creds["password"]): + audit("update_orchestrator_account", request, False, reason="bad_current_password") + raise HTTPException(status_code=403, detail="当前密码错误") + + env = read_env_file(ORCH_ENV) + env["ORCH_USERNAME"] = payload.username + env["ORCH_PASSWORD"] = payload.new_password + env["ORCH_SESSION_SECRET"] = env.get("ORCH_SESSION_SECRET") or creds["secret"] + write_env_file(ORCH_ENV, env) + audit("update_orchestrator_account", request, True, username=payload.username) + return JSONResponse({"ok": True, "username": payload.username, "relogin": True}) + + +@app.get("/api/accounts", dependencies=[Depends(require_session)]) +async def api_list_accounts() -> JSONResponse: + rows = read_accounts() + out = [] + for r in rows: + slug_env = read_env_file(WORK / "generated" / "env" / f"{r['slug']}.env") + out.append({ + "slug": r["slug"], + "proxy_url_masked": mask_proxy(r["proxy_url"]), + "has_proxy": bool(r["proxy_url"]), + "note": r["note"], + "auth_masked": mask_secret(slug_env.get("AUTHORIZATION", "")), + "admin_pwd_masked": mask_secret(slug_env.get("ADMIN_PASSWORD", "")), + "api_prefix": slug_env.get("API_PREFIX", ""), + }) + return JSONResponse({"accounts": out}) + + +@app.post( + "/api/accounts", + dependencies=[Depends(require_session), Depends(require_csrf)], +) +async def api_add_account(payload: AccountIn, request: Request) -> JSONResponse: + rows = read_accounts() + if any(r["slug"] == payload.slug for r in rows): + raise HTTPException(status_code=409, detail=f"slug={payload.slug} 已存在") + rows.append(payload.model_dump()) + write_accounts(rows) + try: + regenerate_and_apply() + except DockerError as e: + # 回滚 csv + write_accounts([r for r in rows if r["slug"] != payload.slug]) + audit("add_account", request, False, slug=payload.slug, error=str(e)) + raise HTTPException(status_code=500, detail=str(e)) + audit("add_account", request, True, slug=payload.slug) + return JSONResponse({"ok": True, "slug": payload.slug}) + + +@app.patch( + "/api/accounts/{slug}", + dependencies=[Depends(require_session), Depends(require_csrf)], +) +async def api_patch_account( + slug: str, payload: AccountPatch, request: Request +) -> JSONResponse: + if not SLUG_RE.match(slug): + raise HTTPException(status_code=400, detail="slug 不合法") + rows = read_accounts() + target = next((r for r in rows if r["slug"] == slug), None) + if not target: + raise HTTPException(status_code=404, detail=f"slug={slug} 不存在") + if payload.proxy_url is not None: + target["proxy_url"] = payload.proxy_url + if payload.note is not None: + target["note"] = payload.note + write_accounts(rows) + try: + regenerate_and_apply() + except DockerError as e: + audit("patch_account", request, False, slug=slug, error=str(e)) + raise HTTPException(status_code=500, detail=str(e)) + audit("patch_account", request, True, slug=slug, fields=payload.model_dump(exclude_none=True)) + return JSONResponse({"ok": True, "slug": slug}) + + +@app.delete( + "/api/accounts/{slug}", + dependencies=[Depends(require_session), Depends(require_csrf)], +) +async def api_delete_account(slug: str, request: Request) -> JSONResponse: + if not SLUG_RE.match(slug): + raise HTTPException(status_code=400, detail="slug 不合法") + rows = read_accounts() + if not any(r["slug"] == slug for r in rows): + raise HTTPException(status_code=404, detail=f"slug={slug} 不存在") + write_accounts([r for r in rows if r["slug"] != slug]) + try: + regenerate_and_apply() + except DockerError as e: + audit("delete_account", request, False, slug=slug, error=str(e)) + raise HTTPException(status_code=500, detail=str(e)) + audit("delete_account", request, True, slug=slug, note="data/ 目录保留") + return JSONResponse({"ok": True, "slug": slug}) + + +# ---------- API: 实例运维 ---------- + +ALLOWED_OPS = {"start", "stop", "restart"} + + +@app.post( + "/api/instances/{slug}/{op}", + dependencies=[Depends(require_session), Depends(require_csrf)], +) +async def api_instance_op(slug: str, op: str, request: Request) -> JSONResponse: + if op == "probe-models": + # 兼容旧版前端路径：/api/instances/{slug}/probe-models + return await api_probe_models(slug, request) + if op not in ALLOWED_OPS: + raise HTTPException(status_code=400, detail=f"非法操作 {op}") + if not SLUG_RE.match(slug): + raise HTTPException(status_code=400, detail="slug 不合法") + if not any(r["slug"] == slug for r in read_accounts()): + raise HTTPException(status_code=404, detail=f"slug={slug} 不存在") + rc, out, err = dc(op, f"chat2api-{slug}", timeout=120) + success = rc == 0 + audit(f"instance_{op}", request, success, slug=slug, + err=(err or "")[:200] if not success else "") + if not success: + raise HTTPException(status_code=500, detail=(err or out)[:300]) + return JSONResponse({"ok": True, "slug": slug, "op": op}) + + +# ---------- API: 状态 ---------- + +def _cached_instance_info(slug: str) -> dict: + """带 5min 内存缓存的 info 取数。api_status 每 5s 轮询用，避免每次重算 JWT+IO。""" + now = time.time() + cached = _models_cache.get(slug) + if cached and now - cached[1] < INFO_CACHE_TTL: + return cached[0] + info = _get_instance_info(slug) + _models_cache[slug] = (info, now) + return info + + +@app.get("/api/status", dependencies=[Depends(require_session)]) +async def api_status() -> JSONResponse: + rows = read_accounts() + instances = [] + for r in rows: + slug = r["slug"] + info = inspect(f"c2a-{slug}") or {} + state = info.get("State", {}) + health = state.get("Health", {}).get("Status") if state else None + started_at = state.get("StartedAt") if state else None + uptime_seconds: int | None = None + if started_at: + try: + # docker 的 StartedAt 是 RFC3339 + 纳秒；切到 26 位再 fromisoformat + ts_str = started_at.replace("Z", "+00:00")[:26] + "+00:00" \ + if "." in started_at and "Z" in started_at else started_at.replace("Z", "+00:00") + dt = datetime.fromisoformat(ts_str) + uptime_seconds = int( + (datetime.now(timezone.utc) - dt).total_seconds() + ) + except (ValueError, TypeError): + uptime_seconds = None + # 调用层信息（带 5min cache，不影响轮询性能） + try: + call_info = _cached_instance_info(slug) + except Exception as e: + logger.warning("status: get info for %s failed: %s", slug, e) + call_info = {} + instances.append({ + "slug": slug, + "container": f"c2a-{slug}", + "state": state.get("Status") if state else "absent", + "health": health or "n/a", + "started_at": started_at, + "uptime_seconds": uptime_seconds, + "proxy_masked": mask_proxy(r["proxy_url"]), + "has_proxy": bool(r["proxy_url"]), + "exit_ip": _exit_ip_cache.get(slug, (None, 0))[0], + "cookie_last_success_at": get_cookie_last_success(slug), + "note": r["note"], + # 新增：调用层（不含 AUTHORIZATION 原文） + "plan_type": call_info.get("plan_type", "unknown"), + "plan_label": call_info.get("plan_label", "未知"), + "plan_color": call_info.get("plan_color", "rose"), + "models": [m.get("id") for m in (call_info.get("models") or []) if isinstance(m, dict) and m.get("id")], + }) + return JSONResponse({ + "instances": instances, + "server_time": int(time.time()), + }) + + +@app.get( + "/api/instances/{slug}/exit-ip", + dependencies=[Depends(require_session)], +) +async def api_exit_ip(slug: str, force: int = Query(0)) -> JSONResponse: + if not SLUG_RE.match(slug): + raise HTTPException(status_code=400, detail="slug 不合法") + ip = get_exit_ip(slug, force=bool(force)) + return JSONResponse({"slug": slug, "exit_ip": ip or ""}) + + +# ---------- API: 凭证查看（敏感） ---------- + +@app.get( + "/api/secrets/{slug}", + dependencies=[Depends(require_session), Depends(require_csrf)], +) +async def api_reveal_secret(slug: str, request: Request) -> JSONResponse: + if not SLUG_RE.match(slug): + raise HTTPException(status_code=400, detail="slug 不合法") + env = read_env_file(WORK / "generated" / "env" / f"{slug}.env") + if not env: + audit("reveal_secret", request, False, slug=slug, reason="not_found") + raise HTTPException(status_code=404, detail=f"slug={slug} 凭证不存在") + audit("reveal_secret", request, True, slug=slug) + return JSONResponse({ + "slug": slug, + "AUTHORIZATION": env.get("AUTHORIZATION", ""), + "ADMIN_PASSWORD": env.get("ADMIN_PASSWORD", ""), + "API_PREFIX": env.get("API_PREFIX", ""), + "PROXY_URL": env.get("PROXY_URL", ""), + }) + + +# ---------- API: 审计 ---------- + +@app.get("/api/audit", dependencies=[Depends(require_session)]) +async def api_audit(limit: int = Query(200, ge=1, le=2000)) -> JSONResponse: + return JSONResponse({"records": read_audit(limit)}) + + +@app.get( + "/api/usage", + dependencies=[Depends(require_session), Depends(require_csrf)], +) +async def api_usage(limit: int = Query(500, ge=1, le=5000)) -> JSONResponse: + records = read_usage(limit) + return JSONResponse({"summary": usage_summary(records), "records": records}) + + +@app.get( + "/api/log-targets", + dependencies=[Depends(require_session), Depends(require_csrf)], +) +async def api_log_targets() -> JSONResponse: + return JSONResponse({"targets": log_targets()}) + + +@app.get( + "/api/logs", + dependencies=[Depends(require_session), Depends(require_csrf)], +) +async def api_logs( + request: Request, + target: str = Query("orchestrator"), + tail: int = Query(200, ge=20, le=2000), +) -> JSONResponse: + target_map = {item["id"]: item for item in log_targets()} + item = target_map.get(target) + if not item: + audit("view_logs", request, False, target=target, reason="invalid_target") + raise HTTPException(status_code=400, detail="日志目标不存在") + + container = item["container"] + ok, text = read_container_logs(container, tail=tail) + audit("view_logs", request, ok, target=target, tail=tail) + if not ok: + return JSONResponse( + { + "ok": False, + "target": target, + "container": container, + "tail": tail, + "logs": text, + }, + status_code=200, + ) + return JSONResponse({ + "ok": True, + "target": target, + "container": container, + "tail": tail, + "logs": text, + }) + + +# ==================================================================== +# 调用层信息聚合 (info / probe / playground / export) +# ==================================================================== + +MODELS_BY_PLAN_FILE = Path(__file__).parent / "static" / "models_by_plan.json" +DEEP_RESEARCH_MODEL_ALIASES = ( + "o3-deep-research", + "o4-mini-deep-research", + "gpt-4o-deep-research", + "deep-research", +) +DEEP_RESEARCH_CAPABLE_PREFIXES = ( + "gpt-4", + "gpt-4o", + "gpt-5", + "o1", + "o3", + "o4", +) +_models_cache: dict[str, tuple[dict, float]] = {} # slug -> (info_dict, ts) +INFO_CACHE_TTL = 300.0 # 5 分钟 +PROBE_MIN_INTERVAL = 30.0 # 单 slug 探测最小间隔 +_probe_last: dict[str, float] = {} + + +@functools.lru_cache(maxsize=1) +def _load_static_models() -> dict: + """读 static/models_by_plan.json；启动后只读一次（除非进程重启）。""" + try: + return json.loads(MODELS_BY_PLAN_FILE.read_text(encoding="utf-8")) + except (FileNotFoundError, json.JSONDecodeError) as e: + logger.error("加载 models_by_plan.json 失败：%s", e) + return {"plans": {"unknown": {"label": "未知", "color": "rose", "models": []}}} + + +def _is_deep_research_capable(model_id: str) -> bool: + if not model_id or model_id in DEEP_RESEARCH_MODEL_ALIASES: + return False + return model_id.startswith(DEEP_RESEARCH_CAPABLE_PREFIXES) + + +def _probe_model_entries(model_ids: list[str]) -> list[dict[str, str]]: + ids = {str(model_id) for model_id in (model_ids or []) if model_id} + if any(_is_deep_research_capable(model_id) for model_id in ids): + ids.update(DEEP_RESEARCH_MODEL_ALIASES) + return [ + { + "id": model_id, + "source": "alias" if model_id in DEEP_RESEARCH_MODEL_ALIASES else "probe", + } + for model_id in sorted(ids) + ] + + +def _parse_jwt_plan(access_token: str) -> str: + """从 OpenAI access_token JWT 的 payload 取 chatgpt_plan_type。 + + JWT 结构: header.payload.signature。payload 是 base64url 编码的 JSON， + 里面 https://api.openai.com/auth.chatgpt_plan_type = "free"/"plus"/"team"/"pro"。 + 任何失败返回 "unknown"。 + """ + if not access_token or "." not in access_token: + return "unknown" + try: + payload_b64 = access_token.split(".")[1] + payload_b64 += "=" * (-len(payload_b64) % 4) + payload = json.loads(base64.urlsafe_b64decode(payload_b64.encode("ascii"))) + auth_claims = payload.get("https://api.openai.com/auth") or {} + plan = (auth_claims.get("chatgpt_plan_type") or "").strip().lower() + return plan or "unknown" + except Exception: + return "unknown" + + +def _read_latest_access_token(slug: str) -> str: + """读 data/{slug}/refresh_map.json，返回 last_success_at 最新那条的 token 字段。""" + p = DATA_DIR / slug / "refresh_map.json" + if not p.exists(): + return "" + try: + data = json.loads(p.read_text(encoding="utf-8")) + if not isinstance(data, dict): + return "" + latest: tuple[int, str] = (0, "") + for v in data.values(): + if not isinstance(v, dict): + continue + ts = int(v.get("last_success_at") or v.get("timestamp") or 0) + tok = v.get("token") or "" + if ts > latest[0] and tok: + latest = (ts, tok) + return latest[1] + except Exception: + return "" + + +def _get_instance_info(slug: str) -> dict: + """汇总实例的调用层信息（不含 AUTHORIZATION 原文，仅 masked）。""" + env = read_env_file(WORK / "generated" / "env" / f"{slug}.env") + api_prefix = env.get("API_PREFIX", "") + authorization = env.get("AUTHORIZATION", "") + # gateway 暴露在容器外端口 60403，nginx 反代到 c2a-{slug}:5005 + # 这里给"对外可调用"的 URL；调用方自己拼 /v1 + gateway_port = os.environ.get("ORCH_GATEWAY_PUBLIC_PORT", "60403") + gateway_host = os.environ.get("ORCH_GATEWAY_PUBLIC_HOST", "") + if gateway_host: + base_url = f"http://{gateway_host}:{gateway_port}/{api_prefix}/v1" if api_prefix else "" + else: + # 没配公开 host 就给相对路径，浏览器拼当前 origin + base_url = f"/{api_prefix}/v1" if api_prefix else "" + + access_token = _read_latest_access_token(slug) + plan_type = _parse_jwt_plan(access_token) if access_token else "unknown" + static = _load_static_models() + plan_entry = static.get("plans", {}).get(plan_type) or static.get("plans", {}).get("unknown", {}) + models = [{"id": m, "source": "plan"} for m in plan_entry.get("models", [])] + + return { + "slug": slug, + "base_url": base_url, + "api_prefix": api_prefix, + "authorization": authorization, # 内部端点返回原文，前端展示前 mask + "auth_masked": mask_secret(authorization, head=6, tail=4), + "plan_type": plan_type, + "plan_label": plan_entry.get("label", "未知"), + "plan_color": plan_entry.get("color", "rose"), + "plan_source": "jwt" if access_token else "default", + "models": models, + "generated_at": int(time.time()), + } + + +@app.get( + "/api/instances/{slug}/info", + dependencies=[Depends(require_session)], +) +async def api_instance_info(slug: str) -> JSONResponse: + """单实例调用层信息（5min 缓存）。AUTHORIZATION 不下发，仅 masked。""" + if not SLUG_RE.match(slug): + raise HTTPException(status_code=400, detail="slug 不合法") + if not any(r["slug"] == slug for r in read_accounts()): + raise HTTPException(status_code=404, detail=f"slug={slug} 不存在") + + now = time.time() + cached = _models_cache.get(slug) + info = _cached_instance_info(slug) + + # 出口前再次剥离 AUTHORIZATION 原文，浏览器只见 masked + safe = {k: v for k, v in info.items() if k != "authorization"} + safe["cached"] = bool(cached and now - cached[1] < INFO_CACHE_TTL) + return JSONResponse(safe) + + +async def _probe_models(slug: str, api_prefix: str, auth: str) -> list[dict[str, str]]: + """容器内网 GET c2a-{slug}:5005/{api_prefix}/v1/models，返回模型展示条目。 + + chat2api 的 /v1/models 是 OpenAI 兼容协议，返回 {"object":"list","data":[{"id":"...",...}]}。 + 深度研究是 chat2api 支持的调用别名，上游模型列表不一定直接返回，所以这里补充展示。 + 认证用 AUTHORIZATION 作 Bearer token。 + """ + url = f"http://c2a-{slug}:5005/{api_prefix}/v1/models" if api_prefix else f"http://c2a-{slug}:5005/v1/models" + headers = {"Authorization": f"Bearer {auth}"} if auth else {} + async with httpx.AsyncClient(timeout=10.0) as client: + r = await client.get(url, headers=headers) + r.raise_for_status() + data = r.json() + items = data.get("data") if isinstance(data, dict) else None + if not isinstance(items, list): + return [] + model_ids = [str(it.get("id")) for it in items if isinstance(it, dict) and it.get("id")] + return _probe_model_entries(model_ids) + + +@app.post( + "/api/probe-models/{slug}", + dependencies=[Depends(require_session), Depends(require_csrf)], +) +async def api_probe_models(slug: str, request: Request) -> JSONResponse: + """强制调实例 /v1/models 取真实可用模型；单 slug 30s 限频；写审计。""" + if not SLUG_RE.match(slug): + raise HTTPException(status_code=400, detail="slug 不合法") + if not any(r["slug"] == slug for r in read_accounts()): + raise HTTPException(status_code=404, detail=f"slug={slug} 不存在") + + now = time.time() + last = _probe_last.get(slug, 0) + if now - last < PROBE_MIN_INTERVAL: + wait = int(PROBE_MIN_INTERVAL - (now - last)) + audit("probe_models", request, False, slug=slug, reason="rate_limited", wait_s=wait) + raise HTTPException(status_code=429, detail=f"探测过快，请 {wait}s 后再试") + _probe_last[slug] = now + + env = read_env_file(WORK / "generated" / "env" / f"{slug}.env") + if not env: + audit("probe_models", request, False, slug=slug, reason="env_missing") + raise HTTPException(status_code=404, detail=f"slug={slug} env 不存在") + + try: + model_entries = await _probe_models(slug, env.get("API_PREFIX", ""), env.get("AUTHORIZATION", "")) + except httpx.HTTPStatusError as e: + _, logs = read_container_logs(f"c2a-{slug}", tail=80) + audit("probe_models", request, False, slug=slug, http_status=e.response.status_code) + raise HTTPException( + status_code=502, + detail=( + f"实例返回 {e.response.status_code}: {e.response.text[:300]}\n\n" + f"--- c2a-{slug} 最近日志 ---\n{logs[-6000:]}" + ), + ) + except Exception as e: + _, logs = read_container_logs(f"c2a-{slug}", tail=80) + audit("probe_models", request, False, slug=slug, error=str(e)[:200]) + raise HTTPException( + status_code=500, + detail=f"探测失败：{str(e)[:300]}\n\n--- c2a-{slug} 最近日志 ---\n{logs[-6000:]}", + ) + + # 把 probe 结果合入 cache（增量），保留 plan_type 等信息 + cached = _models_cache.get(slug) + base = cached[0] if cached else _get_instance_info(slug) + model_ids = [m["id"] for m in model_entries] + base = {**base, "models": model_entries, "probed_at": int(now)} + _models_cache[slug] = (base, now) + + audit("probe_models", request, True, slug=slug, model_count=len(model_ids)) + return JSONResponse({ + "slug": slug, + "models": model_ids, + "model_entries": model_entries, + "probed_at": int(now), + }) + + +# ---------- 调用汇总 & 导出 ---------- + +def _build_aggregate() -> list[dict]: + """同步聚合所有实例的 info（含 AUTHORIZATION 原文，供导出使用）。""" + rows = [] + for r in read_accounts(): + slug = r["slug"] + # 强制取实时 info（含原文 auth），不走 _models_cache（cache 已剥离 auth） + info = _get_instance_info(slug) + # 附带容器状态供前端着色 + cont = inspect(f"c2a-{slug}") or {} + state = cont.get("State", {}) + info["container_state"] = state.get("Status") if state else "absent" + info["container_health"] = (state.get("Health", {}) or {}).get("Status") or "n/a" + rows.append(info) + return rows + + +def _strip_auth(rows: list[dict]) -> list[dict]: + return [{k: v for k, v in r.items() if k != "authorization"} for r in rows] + + +@app.get( + "/api/instances/aggregate", + dependencies=[Depends(require_session)], +) +async def api_aggregate() -> JSONResponse: + """全实例聚合（不下发 AUTHORIZATION 原文）。""" + rows = _build_aggregate() + return JSONResponse({"instances": _strip_auth(rows), "server_time": int(time.time())}) + + +def _gen_litellm_yaml(rows: list[dict], gateway_origin: str) -> str: + """生成 LiteLLM proxy 兼容的 config.yaml。 + + 每个 (slug × model) 组合一条 model_list 条目；model_name 用 `{slug}-{model}` 命名以便区分。 + """ + lines = [ + "# 由 chat2api Orchestrator 自动生成", + f"# 生成时间: {datetime.now(timezone.utc).isoformat(timespec='seconds')}", + "model_list:", + ] + for r in rows: + slug = r["slug"] + prefix = r.get("api_prefix", "") + base = f"{gateway_origin}/{prefix}/v1" if prefix else r.get("base_url", "") + auth = r.get("authorization", "") + for m in r.get("models", []): + mid = m.get("id") if isinstance(m, dict) else str(m) + if not mid: + continue + lines.extend([ + f" - model_name: {slug}-{mid}", + f" litellm_params:", + f" model: openai/{mid}", + f" api_base: {base}", + f" api_key: {auth}", + ]) + return "\n".join(lines) + "\n" + + +def _gen_oneapi_json(rows: list[dict], gateway_origin: str) -> str: + """生成 OneAPI / new-api 兼容的渠道导入 JSON 数组。""" + channels = [] + for r in rows: + prefix = r.get("api_prefix", "") + base = f"{gateway_origin}/{prefix}" if prefix else gateway_origin + models = [m.get("id") for m in r.get("models", []) if isinstance(m, dict) and m.get("id")] + channels.append({ + "name": f"chat2api-{r['slug']}", + "type": 1, # OpenAI + "base_url": base, + "key": r.get("authorization", ""), + "models": ",".join(models), + "group": r.get("plan_type", "default"), + "status": 1, + }) + return json.dumps(channels, indent=2, ensure_ascii=False) + "\n" + + +def _gen_librechat_yaml(rows: list[dict], gateway_origin: str) -> str: + """生成 LibreChat endpoints.custom 片段。""" + lines = [ + "# 由 chat2api Orchestrator 自动生成 — 复制 endpoints.custom 部分到你的 librechat.yaml", + f"# 生成时间: {datetime.now(timezone.utc).isoformat(timespec='seconds')}", + "endpoints:", + " custom:", + ] + for r in rows: + prefix = r.get("api_prefix", "") + base = f"{gateway_origin}/{prefix}/v1" if prefix else r.get("base_url", "") + auth = r.get("authorization", "") + models = [m.get("id") for m in r.get("models", []) if isinstance(m, dict) and m.get("id")] + models_csv = ", ".join(f'"{m}"' for m in models) or '"gpt-5-5"' + lines.extend([ + f" - name: \"chat2api-{r['slug']}\"", + f" apiKey: \"{auth}\"", + f" baseURL: \"{base}\"", + f" models:", + f" default: [{models_csv}]", + f" fetch: false", + f" titleConvo: true", + f" titleModel: \"current_model\"", + f" modelDisplayLabel: \"chat2api-{r['slug']} ({r.get('plan_label', '?')})\"", + ]) + return "\n".join(lines) + "\n" + + +_EXPORT_FORMATS = { + "litellm": ("litellm-config.yaml", "application/x-yaml", _gen_litellm_yaml), + "oneapi": ("oneapi-channels.json", "application/json", _gen_oneapi_json), + "librechat": ("librechat-endpoints.yaml","application/x-yaml", _gen_librechat_yaml), +} + + +@app.get( + "/api/export/{fmt}", + dependencies=[Depends(require_session)], +) +async def api_export(fmt: str, request: Request) -> Response: + """导出多上游配置到三种主流网关 / 客户端格式。包含 AUTHORIZATION 原文。 + + 安全：仅 session 鉴权（无 CSRF, 因为 GET 触发下载），写审计。 + """ + if fmt not in _EXPORT_FORMATS: + raise HTTPException(status_code=400, detail=f"不支持的格式: {fmt}") + filename, mime, generator = _EXPORT_FORMATS[fmt] + # 拼对外可访问的 origin（导出文件里需要全 URL） + gateway_port = os.environ.get("ORCH_GATEWAY_PUBLIC_PORT", "60403") + gateway_host = os.environ.get("ORCH_GATEWAY_PUBLIC_HOST") or request.url.hostname or "localhost" + gateway_origin = f"http://{gateway_host}:{gateway_port}" + rows = _build_aggregate() + body = generator(rows, gateway_origin) + audit("export_config", request, True, fmt=fmt, instance_count=len(rows)) + return Response( + content=body, + media_type=mime, + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + + +# ---------- Playground ---------- + +PG_PER_SLUG_LIMIT = 6 # 每分钟 +PG_GLOBAL_LIMIT = 30 # 每分钟 +PG_WINDOW = 60.0 # 1 分钟滑动 +PG_MAX_TOKENS_HARD = 4096 +PG_TIMEOUT_CONNECT = 5.0 +PG_TIMEOUT_READ = 30.0 +_pg_attempts: dict[str, list[float]] = {} +_pg_global_attempts: list[float] = [] + + +def _pg_rate_limit_check(slug: str) -> tuple[bool, str]: + """滑动窗口限流。返回 (ok, reason)。""" + now = time.time() + cutoff = now - PG_WINDOW + # 全局 + _pg_global_attempts[:] = [t for t in _pg_global_attempts if t > cutoff] + if len(_pg_global_attempts) >= PG_GLOBAL_LIMIT: + return False, f"全局限流（{PG_GLOBAL_LIMIT}/min 已满）" + # 单 slug + lst = _pg_attempts.setdefault(slug, []) + lst[:] = [t for t in lst if t > cutoff] + if len(lst) >= PG_PER_SLUG_LIMIT: + return False, f"slug={slug} 限流（{PG_PER_SLUG_LIMIT}/min 已满）" + lst.append(now) + _pg_global_attempts.append(now) + return True, "" + + +@app.get( + "/api/playground/options", + dependencies=[Depends(require_session)], +) +async def api_playground_options() -> JSONResponse: + """返回所有实例 + 它们当前可用模型，供前端下拉框联动。""" + instances = [] + for r in read_accounts(): + slug = r["slug"] + cached = _models_cache.get(slug) + info = cached[0] if cached else _get_instance_info(slug) + instances.append({ + "slug": slug, + "plan_type": info.get("plan_type", "unknown"), + "plan_label": info.get("plan_label", "未知"), + "models": [m.get("id") for m in info.get("models", []) if isinstance(m, dict) and m.get("id")], + }) + return JSONResponse({"instances": instances}) + + +@app.post( + "/api/playground/invoke", + dependencies=[Depends(require_session), Depends(require_csrf)], +) +async def api_playground_invoke(request: Request) -> JSONResponse: + """服务端代发 chat completions（不流式 MVP）。""" + try: + body = await request.json() + except Exception: + raise HTTPException(status_code=400, detail="invalid JSON body") + slug = (body.get("slug") or "").strip() + model = (body.get("model") or "").strip() + system_prompt = body.get("system") or "" + user_prompt = body.get("user") or "" + temperature = body.get("temperature") + max_tokens = body.get("max_tokens") + + if not SLUG_RE.match(slug): + raise HTTPException(status_code=400, detail="slug 不合法") + if not any(r["slug"] == slug for r in read_accounts()): + raise HTTPException(status_code=404, detail=f"slug={slug} 不存在") + if not model: + raise HTTPException(status_code=400, detail="model 必填") + if not user_prompt: + raise HTTPException(status_code=400, detail="user prompt 不能为空") + + # 参数夹紧 + try: + temperature = float(temperature) if temperature is not None else 0.7 + except (TypeError, ValueError): + temperature = 0.7 + temperature = max(0.0, min(2.0, temperature)) + try: + max_tokens = int(max_tokens) if max_tokens is not None else 512 + except (TypeError, ValueError): + max_tokens = 512 + max_tokens = max(1, min(PG_MAX_TOKENS_HARD, max_tokens)) + + # 限流 + ok, reason = _pg_rate_limit_check(slug) + if not ok: + audit("playground_invoke", request, False, slug=slug, model=model, reason="rate_limited") + raise HTTPException(status_code=429, detail=reason) + + env = read_env_file(WORK / "generated" / "env" / f"{slug}.env") + if not env: + raise HTTPException(status_code=404, detail=f"slug={slug} env 不存在") + api_prefix = env.get("API_PREFIX", "") + auth = env.get("AUTHORIZATION", "") + + url = f"http://c2a-{slug}:5005/{api_prefix}/v1/chat/completions" if api_prefix else f"http://c2a-{slug}:5005/v1/chat/completions" + messages = [] + if system_prompt: + messages.append({"role": "system", "content": system_prompt}) + messages.append({"role": "user", "content": user_prompt}) + payload = { + "model": model, + "messages": messages, + "temperature": temperature, + "max_tokens": max_tokens, + "stream": False, + } + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {auth}", + } + + t0 = time.time() + try: + async with httpx.AsyncClient( + timeout=httpx.Timeout(PG_TIMEOUT_READ, connect=PG_TIMEOUT_CONNECT), + ) as client: + r = await client.post(url, json=payload, headers=headers) + latency_ms = int((time.time() - t0) * 1000) + data: Any = None + try: + data = r.json() + except Exception: + data = None + if r.status_code != 200: + audit("playground_invoke", request, False, slug=slug, model=model, + http_status=r.status_code, latency_ms=latency_ms, + prompt_chars=len(user_prompt) + len(system_prompt)) + return JSONResponse({ + "ok": False, + "status": r.status_code, + "latency_ms": latency_ms, + "error": (data.get("error") if isinstance(data, dict) else None) or r.text[:300], + }) + content = "" + usage = {} + if isinstance(data, dict): + choices = data.get("choices") or [] + if choices and isinstance(choices[0], dict): + msg = choices[0].get("message") or {} + content = msg.get("content") or "" + usage = data.get("usage") or {} + audit("playground_invoke", request, True, slug=slug, model=model, + latency_ms=latency_ms, + prompt_chars=len(user_prompt) + len(system_prompt), + completion_chars=len(content), + prompt_tokens=usage.get("prompt_tokens"), + completion_tokens=usage.get("completion_tokens")) + return JSONResponse({ + "ok": True, + "status": 200, + "latency_ms": latency_ms, + "content": content, + "usage": usage, + }) + except httpx.TimeoutException: + latency_ms = int((time.time() - t0) * 1000) + audit("playground_invoke", request, False, slug=slug, model=model, + reason="timeout", latency_ms=latency_ms) + return JSONResponse({"ok": False, "latency_ms": latency_ms, + "error": f"超时（>{int(PG_TIMEOUT_READ)}s）"}, status_code=200) + except Exception as e: + latency_ms = int((time.time() - t0) * 1000) + audit("playground_invoke", request, False, slug=slug, model=model, + reason="exception", error=str(e)[:200], latency_ms=latency_ms) + return JSONResponse({"ok": False, "latency_ms": latency_ms, + "error": str(e)[:300]}, status_code=200) diff --git a/deploy/multi/orchestrator/requirements.txt b/deploy/multi/orchestrator/requirements.txt new file mode 100644 index 00000000..d7f4eab5 --- /dev/null +++ b/deploy/multi/orchestrator/requirements.txt @@ -0,0 +1,6 @@ +fastapi==0.115.3 +uvicorn[standard]==0.32.0 +jinja2==3.1.4 +python-multipart==0.0.12 +itsdangerous==2.2.0 +httpx==0.27.2 diff --git a/deploy/multi/orchestrator/static/app.js b/deploy/multi/orchestrator/static/app.js new file mode 100644 index 00000000..0612ac34 --- /dev/null +++ b/deploy/multi/orchestrator/static/app.js @@ -0,0 +1,1360 @@ +// chat2api Orchestrator 前端 +// 原生 fetch + 简单 DOM 操作，无框架 + +const $ = (sel) => document.querySelector(sel); +const $$ = (sel) => Array.from(document.querySelectorAll(sel)); + +function getCookie(name) { + return document.cookie.split('; ') + .find(r => r.startsWith(name + '='))?.split('=')[1] || ''; +} + +function csrf() { + return getCookie('orch_csrf'); +} + +async function api(method, path, body) { + const opts = { method, headers: {} }; + if (body !== undefined) { + opts.headers['Content-Type'] = 'application/json'; + opts.body = JSON.stringify(body); + } + // 所有请求都带 CSRF 头：少数 GET（如 /api/secrets/{slug} reveal）也走 CSRF 校验 + const c = csrf(); + if (c) opts.headers['X-CSRF-Token'] = c; + const r = await fetch('.' + path, opts); + if (r.status === 401) { + location.href = './login'; + return; + } + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(formatApiError(data.detail) || `HTTP ${r.status}`); + return data; +} + +function formatApiError(detail) { + if (!detail) return ''; + if (typeof detail === 'string') return detail; + if (Array.isArray(detail)) { + return detail.map((item) => { + if (typeof item === 'string') return item; + const loc = Array.isArray(item.loc) ? item.loc.join('.') : ''; + return [loc, item.msg].filter(Boolean).join(': '); + }).join('\n'); + } + if (typeof detail === 'object') { + return detail.message || detail.msg || JSON.stringify(detail); + } + return String(detail); +} + +function toast(msg, isErr = false) { + const el = $('#toast'); + el.textContent = msg; + el.classList.remove('hidden', 'bg-gray-900', 'bg-red-600'); + el.classList.add(isErr ? 'bg-red-600' : 'bg-gray-900'); + setTimeout(() => el.classList.add('hidden'), 3000); +} + +function fmtUptime(sec) { + if (sec == null) return '-'; + if (sec < 60) return sec + 's'; + if (sec < 3600) return Math.floor(sec / 60) + 'm'; + if (sec < 86400) return Math.floor(sec / 3600) + 'h'; + return Math.floor(sec / 86400) + 'd'; +} + +function fmtCookieAge(ts) { + if (!ts) return '未刷新'; + const age = Math.floor(Date.now() / 1000 - ts); + let txt, color; + if (age < 600) { txt = age + 's 前'; color = 'text-green-600'; } + else if (age < 3600) { txt = Math.floor(age / 60) + 'm 前'; color = 'text-green-600'; } + else if (age < 86400) { txt = Math.floor(age / 3600) + 'h 前'; color = 'text-yellow-600'; } + else { txt = Math.floor(age / 86400) + 'd 前'; color = 'text-red-600'; } + return `${txt}`; +} + +function healthBadge(state, health) { + if (state !== 'running') { + return `${state}`; + } + if (health === 'healthy') return `healthy`; + if (health === 'unhealthy') return `unhealthy`; + if (health === 'starting') return `starting`; + return `${health}`; +} + +function escapeHtml(s) { + return String(s ?? '').replace(/[&<>"']/g, c => ({ + '&':'&','<':'<','>':'>','"':'"',"'":''' + }[c])); +} + +function fmtNum(n) { + const value = Number(n || 0); + if (value >= 1000000) return (value / 1000000).toFixed(value >= 10000000 ? 0 : 2) + 'M'; + if (value >= 1000) return value.toLocaleString(); + return String(value); +} + +function fmtMs(ms) { + const value = Number(ms || 0); + if (value >= 1000) return (value / 1000).toFixed(value >= 10000 ? 1 : 2) + 's'; + return Math.round(value) + 'ms'; +} + +function parseUsageTs(ts) { + const t = Date.parse(ts || ''); + return Number.isFinite(t) ? t : 0; +} + +/** + * 紧凑型行内模型 chip 渲染：最多展示 maxVisible 个，超出显示 "+N"。 + * 入参：models 可以是 ["gpt-5", ...] 或 [{id, source}, ...] + */ +function renderInlineModels(models, maxVisible = 4) { + if (!models || !models.length) { + return '
无可用模型
'; + } + const ids = models.map(m => (typeof m === 'string') ? m : (m && m.id)); + const visible = ids.slice(0, maxVisible); + const more = ids.length - visible.length; + const chips = visible.map(id => `${escapeHtml(id)}`).join(''); + const moreBadge = more > 0 + ? `+${more}` + : ''; + return `
${chips}${moreBadge}
`; +} + +let latestInstances = []; +let selectedSlug = ''; +let instanceSearchQuery = ''; +let activePanel = 'instances'; +let usageRecords = []; +let usageFilteredRecords = []; +let usageLoaded = false; + +function switchPanel(panel) { + activePanel = panel || 'instances'; + $$('[data-panel-body]').forEach(el => { + el.classList.toggle('hidden', el.dataset.panelBody !== activePanel); + }); + $$('.orch-nav-item[data-panel]').forEach(btn => { + btn.classList.toggle('orch-nav-active', btn.dataset.panel === activePanel); + }); + if (activePanel === 'usage' && !usageLoaded) { + loadUsage(); + } +} + +$$('.orch-nav-item[data-panel]').forEach(btn => { + btn.addEventListener('click', () => switchPanel(btn.dataset.panel)); +}); + +function isHealthy(it) { + return it.state === 'running' && it.health === 'healthy'; +} + +function isCookieFresh(it) { + if (!it.cookie_last_success_at) return false; + return Math.floor(Date.now() / 1000 - it.cookie_last_success_at) < 3600; +} + +function instanceInitial(it) { + return String(it.note || it.slug || 'A').trim().slice(0, 1).toUpperCase(); +} + +function statusPill(it) { + if (isHealthy(it)) { + return 'Healthy'; + } + if (it.state === 'running') { + return `${escapeHtml(it.health || 'running')}`; + } + return `${escapeHtml(it.state || 'absent')}`; +} + +function planPill(it) { + const label = it.plan_label || it.plan_type || '未知'; + return `${escapeHtml(label)}`; +} + +function getFilteredInstances() { + const q = instanceSearchQuery.trim().toLowerCase(); + if (!q) return latestInstances; + return latestInstances.filter(it => [ + it.slug, + it.note, + it.proxy_masked, + it.exit_ip, + it.state, + it.health, + it.plan_label, + it.plan_type, + ...(it.models || []), + ].some(v => String(v || '').toLowerCase().includes(q))); +} + +function renderMetrics(instances) { + const total = instances.length; + const healthy = instances.filter(isHealthy).length; + const degraded = total - healthy; + const proxied = instances.filter(it => it.has_proxy).length; + const models = instances.reduce((sum, it) => sum + ((it.models || []).length), 0); + const fresh = instances.filter(isCookieFresh).length; + const pairs = [ + ['#metric-total', total], + ['#metric-healthy', healthy], + ['#metric-degraded', degraded], + ['#metric-proxy', proxied], + ['#metric-models', models], + ['#metric-cookie', fresh], + ]; + for (const [sel, val] of pairs) { + const el = $(sel); + if (el) el.textContent = val; + } +} + +function renderDetail(it) { + const empty = $('#detail-empty'); + const body = $('#detail-body'); + if (!empty || !body) return; + if (!it) { + empty.classList.remove('hidden'); + body.classList.add('hidden'); + body.innerHTML = ''; + return; + } + empty.classList.add('hidden'); + body.classList.remove('hidden'); + body.innerHTML = ` +
+
${escapeHtml(instanceInitial(it))}
+
+
${escapeHtml(it.note || it.slug)}
+
${escapeHtml(it.slug)}
+
+
+
${statusPill(it)}${planPill(it)}
+
+
+
代理
+
${escapeHtml(it.proxy_masked || '未绑定')}
+
+
+
出口 IP
${escapeHtml(it.exit_ip || '?')}
+
运行时长
${fmtUptime(it.uptime_seconds)}
+
+
+
Cookie 鲜度
+
${fmtCookieAge(it.cookie_last_success_at)}
+
+
+
可用模型
+ ${renderInlineModels(it.models, 12)} +
+
+
+ + + + + ${it.state === 'running' + ? `` + : ``} + +
+ `; + $$('#detail-body button').forEach(btn => { + btn.addEventListener('click', () => onRowAction(btn.dataset)); + }); +} + +function selectInstance(slug) { + selectedSlug = slug || ''; + renderRows(getFilteredInstances(), false); + renderDetail(latestInstances.find(it => it.slug === selectedSlug)); +} + +function renderRows(instances, updateDetail = true) { + const tbody = $('#tbody'); + if (!tbody) return; + const count = $('#table-count'); + if (count) count.textContent = `${instances.length} / ${latestInstances.length} 个实例`; + if (!instances.length) { + tbody.innerHTML = '暂无匹配实例，清空搜索或新增账号。'; + if (updateDetail) renderDetail(null); + return; + } + if (!selectedSlug || !latestInstances.some(it => it.slug === selectedSlug)) { + selectedSlug = instances[0].slug; + } + tbody.innerHTML = instances.map(it => { + const selected = it.slug === selectedSlug; + return ` + + +
+
${escapeHtml(instanceInitial(it))}
+
${escapeHtml(it.note || it.slug)}
${escapeHtml(it.slug)}
+
+ +
${planPill(it)}
${renderInlineModels(it.models, 3)} + ${escapeHtml(it.proxy_masked || '-')} + ${statusPill(it)} + ${escapeHtml(it.exit_ip || '?')} + ${fmtCookieAge(it.cookie_last_success_at)} + ${fmtUptime(it.uptime_seconds)} + + + + + + ${it.state === 'running' + ? `` + : ``} + + + `; + }).join(''); + + $$('#tbody button').forEach(btn => { + btn.addEventListener('click', (e) => { + e.stopPropagation(); + onRowAction(btn.dataset); + }); + }); + $$('#tbody .instance-row').forEach(row => { + row.addEventListener('click', () => selectInstance(row.dataset.slug)); + }); + if (updateDetail) renderDetail(latestInstances.find(it => it.slug === selectedSlug)); +} + +async function loadStatus() { + try { + const data = await api('GET', '/api/status'); + latestInstances = data.instances || []; + renderMetrics(latestInstances); + renderRows(getFilteredInstances()); + const serverTime = new Date(data.server_time * 1000); + const statusText = `共 ${latestInstances.length} 个实例 · 服务器时间 ${serverTime.toLocaleTimeString()}`; + $('#server-status').textContent = statusText; + const serverTimeEl = $('#server-time'); + if (serverTimeEl) serverTimeEl.textContent = serverTime.toLocaleString('zh-CN', { hour12: false }); + } catch (e) { + toast('加载状态失败：' + e.message, true); + } +} + +const instanceSearchInput = $('#instance-search'); +if (instanceSearchInput) { + instanceSearchInput.addEventListener('input', () => { + instanceSearchQuery = instanceSearchInput.value || ''; + renderRows(getFilteredInstances()); + }); +} +const clearDetailButton = $('#btn-clear-detail'); +if (clearDetailButton) { + clearDetailButton.addEventListener('click', () => selectInstance('')); +} + +// ---------- 模态 ---------- + +let modalMode = 'add'; // 'add' | 'edit' +let modalSlug = ''; + +function openModal(mode, prefill = {}) { + modalMode = mode; + modalSlug = prefill.slug || ''; + $('#modal-title').textContent = mode === 'add' ? '新增账号' : `编辑 ${prefill.slug}`; + $('#f-slug').value = prefill.slug || ''; + $('#f-slug').disabled = mode === 'edit'; + $('#slug-help').textContent = mode === 'edit' + ? 'slug 是容器路径/数据目录 ID，创建后不可改；列表会优先显示备注' + : '小写字母 / 数字 / 连字符，最多 16 位'; + $('#f-proxy').value = mode === 'edit' ? '' : (prefill.proxy_url || ''); + $('#f-proxy').placeholder = mode === 'edit' + ? `当前：${prefill.proxy || '(无)'}; 留空则不变` + : 'socks5://user:pass@host:port'; + $('#f-note').value = prefill.note || ''; + $('#modal-error').classList.add('hidden'); + $('#modal').classList.remove('hidden'); + $('#modal').classList.add('flex'); +} + +function closeModal() { + $('#modal').classList.add('hidden'); + $('#modal').classList.remove('flex'); +} + +$('#btn-add').addEventListener('click', () => openModal('add')); +$('#btn-cancel').addEventListener('click', closeModal); +$('#btn-refresh').addEventListener('click', () => loadStatus()); + +$('#modal-form').addEventListener('submit', async (e) => { + e.preventDefault(); + const slug = $('#f-slug').value.trim(); + const proxy_url = $('#f-proxy').value.trim(); + const note = $('#f-note').value.trim(); + $('#btn-submit').disabled = true; + $('#btn-submit').textContent = '处理中...'; + try { + if (modalMode === 'add') { + await api('POST', '/api/accounts', { slug, proxy_url, note }); + toast('新增成功，等待容器启动...'); + } else { + const body = { note }; + if (proxy_url) body.proxy_url = proxy_url; + await api('PATCH', '/api/accounts/' + encodeURIComponent(modalSlug), body); + toast('编辑成功，正在重建...'); + } + closeModal(); + await loadStatus(); + } catch (e) { + $('#modal-error').textContent = e.message; + $('#modal-error').classList.remove('hidden'); + } finally { + $('#btn-submit').disabled = false; + $('#btn-submit').textContent = '保存'; + } +}); + +// ---------- 行操作 ---------- + +async function onRowAction({ action, slug, proxy, note }) { + if (action === 'edit') { + openModal('edit', { slug, proxy, note }); + return; + } + if (action === 'secret') { + await showSecret(slug); + return; + } + if (action === 'invoke') { + await openInvokeModal(slug); + return; + } + if (action === 'delete') { + if (!confirm(`确认删除 ${slug}？\n容器将被销毁，data/${slug}/ 会保留。`)) return; + try { + await api('DELETE', '/api/accounts/' + encodeURIComponent(slug)); + toast(`已删除 ${slug}`); + await loadStatus(); + } catch (e) { + toast('删除失败：' + e.message, true); + } + return; + } + if (['start', 'stop', 'restart'].includes(action)) { + try { + await api('POST', `/api/instances/${encodeURIComponent(slug)}/${action}`); + toast(`${action} ${slug} 已发出`); + setTimeout(loadStatus, 1500); + } catch (e) { + toast(`${action} 失败：` + e.message, true); + } + } +} + +// ---------- 凭证查看 ---------- + +async function showSecret(slug) { + if (!confirm(`查看 ${slug} 的明文凭证？\n该操作会写入审计日志。`)) return; + try { + const d = await api('GET', '/api/secrets/' + encodeURIComponent(slug)); + const origin = location.origin; // e.g. http://107.172.96.31:60403 + const adminUrl = `${origin}/${d.slug}/admin/login`; + const apiUrl = `${origin}/${d.slug}/v1/chat/completions`; + $('#secret-body').innerHTML = ` +
仅展示用户侧需要的凭证；后端 API_PREFIX 由 nginx 自动改写，不应直接访问。
+
slug: ${escapeHtml(d.slug)}
+
AUTHORIZATION: ${escapeHtml(d.AUTHORIZATION)}
+
ADMIN_PASSWORD: ${escapeHtml(d.ADMIN_PASSWORD)}
+
PROXY_URL: ${escapeHtml(d.PROXY_URL || '(无)')}
+
+
+
① Admin 后台（粘 cookie 用）
+ ${escapeHtml(adminUrl)} +
用上面 ADMIN_PASSWORD 登录
+
+
+
② API 调用示例
+ curl ${escapeHtml(apiUrl)} -H "Authorization: Bearer ${escapeHtml(d.AUTHORIZATION)}" -H "Content-Type: application/json" -d '{"model":"gpt-5-5","messages":[{"role":"user","content":"hi"}]}' +
+
+ `; + $('#modal-secret').classList.remove('hidden'); + $('#modal-secret').classList.add('flex'); + } catch (e) { + toast('获取凭证失败：' + e.message, true); + } +} + +$('#btn-close-secret').addEventListener('click', () => { + $('#modal-secret').classList.add('hidden'); + $('#modal-secret').classList.remove('flex'); + $('#secret-body').innerHTML = ''; // 立即清屏 +}); + +// ---------- 统一 API ---------- + +async function showUnifiedApi() { + if (!confirm('查看统一 API Key？\n该 Key 可调用所有实例，请勿泄露。')) return; + try { + const d = await api('GET', '/api/unified'); + const baseUrl = location.origin + (d.base_path || '/v1'); + const chatUrl = location.origin + (d.chat_completions_path || '/v1/chat/completions'); + const responsesUrl = location.origin + (d.responses_path || '/v1/responses'); + const compactUrl = location.origin + (d.responses_compact_path || '/v1/responses/compact'); + $('#unified-base-url').textContent = baseUrl; + $('#unified-api-key').textContent = d.api_key; + $('#unified-chat-url').textContent = chatUrl; + $('#unified-responses-url').textContent = responsesUrl; + $('#unified-compact-url').textContent = compactUrl; + $('#unified-curl').textContent = +`curl ${chatUrl} \\ + -H "Authorization: Bearer ${d.api_key}" \\ + -H "Content-Type: application/json" \\ + -d '{"model":"gpt-5-5","messages":[{"role":"user","content":"你好"}]}' + +curl ${responsesUrl} \\ + -H "Authorization: Bearer ${d.api_key}" \\ + -H "Content-Type: application/json" \\ + -d '{"model":"gpt-5-5","input":"你好"}'`; + $('#unified-strategy').textContent = d.strategy || ''; + $('#modal-unified-api').classList.remove('hidden'); + $('#modal-unified-api').classList.add('flex'); + } catch (e) { + toast('获取统一 API 失败：' + e.message, true); + } +} + +$('#btn-unified-api').addEventListener('click', showUnifiedApi); +$('#btn-close-unified-api').addEventListener('click', () => { + $('#modal-unified-api').classList.add('hidden'); + $('#modal-unified-api').classList.remove('flex'); + $('#unified-api-key').textContent = ''; + $('#unified-curl').textContent = ''; +}); + +// ---------- 管理中心 ---------- + +async function openAdminCenter() { + $('#admin-center-error').classList.add('hidden'); + $('#admin-center-error').textContent = ''; + $('#admin-current-password').value = ''; + $('#admin-new-password').value = ''; + $('#modal-admin-center').classList.remove('hidden'); + $('#modal-admin-center').classList.add('flex'); + try { + const d = await api('GET', '/api/orchestrator/account'); + $('#admin-username').value = d.username || 'admin'; + } catch (e) { + $('#admin-center-error').textContent = e.message; + $('#admin-center-error').classList.remove('hidden'); + } +} + +function closeAdminCenter() { + $('#modal-admin-center').classList.add('hidden'); + $('#modal-admin-center').classList.remove('flex'); +} + +$('#btn-admin-center').addEventListener('click', openAdminCenter); +$('#btn-close-admin-center').addEventListener('click', closeAdminCenter); +$('#btn-cancel-admin-center').addEventListener('click', closeAdminCenter); + +$('#admin-center-form').addEventListener('submit', async (e) => { + e.preventDefault(); + const btn = $('#btn-save-admin-center'); + btn.disabled = true; + btn.textContent = '保存中...'; + $('#admin-center-error').classList.add('hidden'); + try { + await api('PATCH', '/api/orchestrator/account', { + username: $('#admin-username').value.trim(), + current_password: $('#admin-current-password').value, + new_password: $('#admin-new-password').value, + }); + toast('已更新，请重新登录'); + setTimeout(() => { location.href = './login'; }, 600); + } catch (e) { + $('#admin-center-error').textContent = e.message; + $('#admin-center-error').classList.remove('hidden'); + } finally { + btn.disabled = false; + btn.textContent = '保存'; + } +}); + +// ---------- 审计 ---------- + +$('#btn-audit').addEventListener('click', async () => { + try { + const d = await api('GET', '/api/audit?limit=200'); + $('#audit-body').innerHTML = d.records.map(r => ` + + ${escapeHtml(r.ts || '')} + ${escapeHtml(r.ip || '')} + ${escapeHtml(r.action || '')} + ${escapeHtml(r.slug || '-')} + ${r.ok ? '✓' : '✗'} + ${escapeHtml(JSON.stringify({...r, ts:undefined, ip:undefined, action:undefined, slug:undefined, ok:undefined, actor:undefined}).replace(/^\{\}$/, ''))} + + `).join(''); + $('#modal-audit').classList.remove('hidden'); + $('#modal-audit').classList.add('flex'); + } catch (e) { + toast('加载审计失败：' + e.message, true); + } +}); + +$('#btn-close-audit').addEventListener('click', () => { + $('#modal-audit').classList.add('hidden'); + $('#modal-audit').classList.remove('flex'); +}); + +// ---------- 诊断日志 ---------- + +let logTargetsLoaded = false; + +async function loadLogTargets() { + const d = await api('GET', '/api/log-targets'); + const targets = d.targets || []; + $('#log-target').innerHTML = targets.map(t => + `` + ).join(''); + logTargetsLoaded = true; +} + +async function refreshLogs() { + const target = $('#log-target').value || 'orchestrator'; + const tail = $('#log-tail').value || '200'; + $('#logs-body').textContent = '加载中...'; + $('#logs-meta').textContent = ''; + try { + const d = await api('GET', `/api/logs?target=${encodeURIComponent(target)}&tail=${encodeURIComponent(tail)}`); + $('#logs-body').textContent = d.logs || '(无日志输出)'; + $('#logs-meta').textContent = `${d.ok ? 'OK' : '异常'} · ${d.container || target} · 最近 ${d.tail || tail} 行`; + if (!d.ok) toast('日志目标返回异常，内容已显示', true); + } catch (e) { + $('#logs-body').textContent = '加载日志失败：' + e.message; + $('#logs-meta').textContent = '加载失败'; + toast('加载日志失败：' + e.message, true); + } +} + +async function openLogsModal() { + $('#modal-logs').classList.remove('hidden'); + $('#modal-logs').classList.add('flex'); + try { + if (!logTargetsLoaded) await loadLogTargets(); + await refreshLogs(); + } catch (e) { + $('#logs-body').textContent = '加载日志失败：' + e.message; + toast('加载日志失败：' + e.message, true); + } +} + +$('#btn-diag-logs').addEventListener('click', openLogsModal); +$('#btn-close-logs').addEventListener('click', () => { + $('#modal-logs').classList.add('hidden'); + $('#modal-logs').classList.remove('flex'); +}); +$('#btn-refresh-logs').addEventListener('click', refreshLogs); +$('#log-target').addEventListener('change', refreshLogs); +$('#log-tail').addEventListener('change', refreshLogs); + +// ---------- 使用日志 ---------- + +function usageInRange(record, range) { + if (!range || range === 'all') return true; + const seconds = Number(range); + if (!seconds) return true; + const ts = parseUsageTs(record.ts); + if (!ts) return true; + return Date.now() - ts <= seconds * 1000; +} + +function uniqueSorted(values) { + return Array.from(new Set(values.filter(Boolean).map(v => String(v)))).sort(); +} + +function syncUsageFilterOptions() { + const slugValue = $('#usage-slug').value; + const modelValue = $('#usage-model').value; + const endpointValue = $('#usage-endpoint').value; + const slugs = uniqueSorted(usageRecords.map(r => r.slug)); + const models = uniqueSorted(usageRecords.map(r => r.model)); + const endpoints = uniqueSorted(usageRecords.map(r => r.endpoint || String(r.path || '').split('/').pop())); + $('#usage-slug').innerHTML = '' + slugs.map(v => ``).join(''); + $('#usage-model').innerHTML = '' + models.map(v => ``).join(''); + $('#usage-endpoint').innerHTML = '' + endpoints.map(v => ``).join(''); + $('#usage-slug').value = slugs.includes(slugValue) ? slugValue : ''; + $('#usage-model').value = models.includes(modelValue) ? modelValue : ''; + $('#usage-endpoint').value = endpoints.includes(endpointValue) ? endpointValue : ''; +} + +function applyUsageFilters() { + const range = $('#usage-range').value; + const slug = $('#usage-slug').value; + const model = $('#usage-model').value; + const endpoint = $('#usage-endpoint').value; + const ok = $('#usage-ok').value; + const stream = $('#usage-stream').value; + const q = ($('#usage-search').value || '').trim().toLowerCase(); + usageFilteredRecords = usageRecords.filter(r => { + if (!usageInRange(r, range)) return false; + if (slug && r.slug !== slug) return false; + if (model && r.model !== model) return false; + const rowEndpoint = r.endpoint || String(r.path || '').split('/').pop(); + if (endpoint && rowEndpoint !== endpoint) return false; + if (ok && String(Boolean(r.ok)) !== ok) return false; + if (stream && String(Boolean(r.stream)) !== stream) return false; + if (q) { + const haystack = [ + r.request_id, r.slug, r.model, r.path, rowEndpoint, r.error, r.status + ].join(' ').toLowerCase(); + if (!haystack.includes(q)) return false; + } + return true; + }); + renderUsage(); +} + +function summarizeUsage(records) { + const total = records.length; + const success = records.filter(r => r.ok).length; + const failed = total - success; + const prompt = records.reduce((sum, r) => sum + Number(r.prompt_tokens || 0), 0); + const completion = records.reduce((sum, r) => sum + Number(r.completion_tokens || 0), 0); + const tokens = records.reduce((sum, r) => sum + Number(r.total_tokens || 0), 0); + const durations = records.map(r => Number(r.duration_ms || 0)).filter(n => Number.isFinite(n)); + const avg = durations.length ? Math.round(durations.reduce((a, b) => a + b, 0) / durations.length) : 0; + const sorted = [...durations].sort((a, b) => a - b); + const p95 = sorted.length ? sorted[Math.max(0, Math.ceil(sorted.length * 0.95) - 1)] : 0; + const bySlug = {}; + for (const r of records) { + const slug = r.slug || '-'; + if (!bySlug[slug]) bySlug[slug] = { slug, requests: 0, tokens: 0, errors: 0 }; + bySlug[slug].requests += 1; + bySlug[slug].tokens += Number(r.total_tokens || 0); + if (!r.ok) bySlug[slug].errors += 1; + } + const slugs = Object.values(bySlug).sort((a, b) => (b.tokens - a.tokens) || (b.requests - a.requests)); + return { + total, success, failed, prompt, completion, tokens, avg, p95, + successRate: total ? (success * 100 / total).toFixed(1) : '0', + topSlug: slugs[0]?.slug || '-', + slugs, + }; +} + +function renderUsageMetrics(summary) { + $('#usage-total').textContent = fmtNum(summary.total); + $('#usage-success-rate').textContent = `${summary.successRate}%`; + $('#usage-success-note').textContent = `${fmtNum(summary.success)} 成功`; + $('#usage-tokens').textContent = fmtNum(summary.tokens); + $('#usage-tokens-note').textContent = `prompt ${fmtNum(summary.prompt)} · output ${fmtNum(summary.completion)}`; + $('#usage-avg-duration').textContent = fmtMs(summary.avg); + $('#usage-p95-duration').textContent = `p95 ${fmtMs(summary.p95)}`; + $('#usage-top-slug').textContent = summary.topSlug; + $('#usage-failed').textContent = fmtNum(summary.failed); +} + +function buildUsageTimeline(records) { + const buckets = new Map(); + for (const r of records) { + const ts = parseUsageTs(r.ts); + if (!ts) continue; + const d = new Date(ts); + const key = `${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')} ${String(d.getHours()).padStart(2, '0')}:00`; + const row = buckets.get(key) || { key, requests: 0, tokens: 0, duration: 0, count: 0 }; + row.requests += 1; + row.tokens += Number(r.total_tokens || 0); + row.duration += Number(r.duration_ms || 0); + row.count += 1; + buckets.set(key, row); + } + return Array.from(buckets.values()).sort((a, b) => a.key.localeCompare(b.key)).slice(-24).map(row => ({ + ...row, + avg: row.count ? Math.round(row.duration / row.count) : 0, + })); +} + +function pointsFor(values, width, height, maxValue) { + if (!values.length) return ''; + const max = maxValue || Math.max(...values, 1); + return values.map((v, i) => { + const x = values.length === 1 ? width : i * (width / (values.length - 1)); + const y = height - ((v || 0) / max) * (height - 18) - 9; + return `${x.toFixed(1)},${y.toFixed(1)}`; + }).join(' '); +} + +function renderUsageTrend(records) { + const el = $('#usage-trend'); + const data = buildUsageTimeline(records); + if (!data.length) { + el.innerHTML = '
暂无趋势数据
'; + return; + } + const width = 760; + const height = 190; + const req = data.map(d => d.requests); + const tokens = data.map(d => d.tokens); + const avg = data.map(d => d.avg); + el.innerHTML = ` + + + + + ${escapeHtml(data[0].key)} + ${escapeHtml(data[data.length - 1].key)} + + `; +} + +function renderUsageSlugBars(summary) { + const rows = summary.slugs.slice(0, 8); + const max = Math.max(...rows.map(r => r.tokens), 1); + $('#usage-slug-bars').innerHTML = rows.length ? rows.map(row => ` +
+
+ ${escapeHtml(row.slug)} + ${fmtNum(row.tokens)} tokens · ${fmtNum(row.requests)} 次 +
+
+
+ `).join('') : '
暂无容器消耗数据
'; +} + +function renderUsageTable(records) { + $('#usage-table-count').textContent = `${records.length} / ${usageRecords.length} 条记录`; + $('#usage-meta').textContent = usageLoaded ? `最近加载 ${new Date().toLocaleTimeString()}` : ''; + $('#usage-body').innerHTML = records.length ? records.slice(0, 500).map(r => { + const time = parseUsageTs(r.ts) ? new Date(parseUsageTs(r.ts)).toLocaleString('zh-CN', { hour12: false }) : (r.ts || '-'); + const endpoint = r.endpoint || String(r.path || '').split('/').pop() || '-'; + return ` + + ${escapeHtml(time)} + ${escapeHtml(r.slug || '-')} + ${escapeHtml(r.model || '-')} + ${escapeHtml(endpoint)} + ${r.stream ? (r.stream_compat ? 'stream compat' : 'stream') : 'normal'} + ${escapeHtml(r.prompt_tokens ?? '-')} + ${escapeHtml(r.completion_tokens ?? '-')} + ${escapeHtml(r.total_tokens ?? '-')} + ${fmtMs(r.duration_ms)} + ${r.ok ? '成功' : `失败 ${escapeHtml(r.status || '')}`} + ${escapeHtml(r.error || '-')} + + `; + }).join('') : '暂无匹配的使用日志'; + $$('#usage-body .usage-row').forEach(row => { + row.addEventListener('click', () => { + const rec = records.find(r => (r.request_id || '') === row.dataset.requestId); + if (rec) showUsageDetail(rec); + }); + }); +} + +function renderUsage() { + const summary = summarizeUsage(usageFilteredRecords); + renderUsageMetrics(summary); + renderUsageTrend(usageFilteredRecords); + renderUsageSlugBars(summary); + renderUsageTable(usageFilteredRecords); +} + +function showUsageDetail(r) { + const mode = r.stream ? (r.stream_compat ? 'stream compat' : 'stream') : 'normal'; + const item = (label, value, wide = false) => ` +
+
${escapeHtml(label)}
+
${escapeHtml(value ?? '-')}
+
+ `; + $('#usage-detail-body').innerHTML = [ + item('Request ID', r.request_id, true), + item('时间', r.ts), + item('客户端 IP', r.ip), + item('容器', r.slug), + item('模型', r.model), + item('路径', r.path, true), + item('模式', mode), + item('状态', `${r.status || '-'} ${r.ok ? 'OK' : 'ERROR'}`), + item('Tokens', `${r.prompt_tokens ?? '-'} + ${r.completion_tokens ?? '-'} = ${r.total_tokens ?? '-'}`), + item('耗时', fmtMs(r.duration_ms)), + item('错误', r.error || '-', true), + ].join(''); + $('#modal-usage-detail').classList.remove('hidden'); + $('#modal-usage-detail').classList.add('flex'); +} + +async function loadUsage() { + $('#usage-body').innerHTML = '加载中...'; + try { + const d = await api('GET', '/api/usage?limit=5000'); + usageRecords = d.records || []; + usageLoaded = true; + syncUsageFilterOptions(); + applyUsageFilters(); + } catch (e) { + $('#usage-body').innerHTML = `加载失败：${escapeHtml(e.message)}`; + toast('加载使用日志失败：' + e.message, true); + } +} + +function exportUsageCsv() { + const headers = ['ts','request_id','ip','slug','model','path','stream','stream_compat','status','ok','prompt_tokens','completion_tokens','total_tokens','duration_ms','error']; + const lines = [headers.join(',')]; + for (const r of usageFilteredRecords) { + lines.push(headers.map(h => { + const value = String(r[h] ?? ''); + return `"${value.replace(/"/g, '""')}"`; + }).join(',')); + } + const blob = new Blob([lines.join('\n')], { type: 'text/csv;charset=utf-8' }); + const a = document.createElement('a'); + a.href = URL.createObjectURL(blob); + a.download = `chat2api-usage-${Date.now()}.csv`; + a.click(); + URL.revokeObjectURL(a.href); +} + +['usage-range','usage-slug','usage-model','usage-endpoint','usage-ok','usage-stream'].forEach(id => { + const el = document.getElementById(id); + if (el) el.addEventListener('change', applyUsageFilters); +}); +const usageSearch = $('#usage-search'); +if (usageSearch) usageSearch.addEventListener('input', applyUsageFilters); +const usageRefresh = $('#btn-usage-refresh'); +if (usageRefresh) usageRefresh.addEventListener('click', loadUsage); +const usageClear = $('#btn-usage-clear'); +if (usageClear) { + usageClear.addEventListener('click', () => { + $('#usage-range').value = '86400'; + $('#usage-slug').value = ''; + $('#usage-model').value = ''; + $('#usage-endpoint').value = ''; + $('#usage-ok').value = ''; + $('#usage-stream').value = ''; + $('#usage-search').value = ''; + applyUsageFilters(); + }); +} +const usageExport = $('#btn-usage-export'); +if (usageExport) usageExport.addEventListener('click', exportUsageCsv); +const usageDetailClose = $('#btn-close-usage-detail'); +if (usageDetailClose) { + usageDetailClose.addEventListener('click', () => { + $('#modal-usage-detail').classList.add('hidden'); + $('#modal-usage-detail').classList.remove('flex'); + $('#usage-detail-body').innerHTML = ''; + }); +} + +// ---------- 调用信息 (单实例) ---------- + +let invokeCurrent = null; // 当前 modal 展示的 info dict +let invokeSnippetTab = 'curl'; + +const PLAN_COLOR_CLASS = { + free: 'bg-gray-200 text-gray-700', + plus: 'bg-blue-100 text-blue-700', + team: 'bg-emerald-100 text-emerald-700', + pro: 'bg-amber-100 text-amber-700', + enterprise: 'bg-violet-100 text-violet-700', + unknown: 'bg-rose-100 text-rose-700', +}; + +function absoluteBaseUrl(rawBaseUrl) { + if (!rawBaseUrl) return ''; + if (/^https?:\/\//.test(rawBaseUrl)) return rawBaseUrl; + // 相对路径 → 拼当前 origin + return location.origin + (rawBaseUrl.startsWith('/') ? rawBaseUrl : '/' + rawBaseUrl); +} + +function genSnippets(baseUrl, apiKey, model) { + const url = (baseUrl || '').replace(/\/$/, ''); + const k = apiKey || 'YOUR_API_KEY'; + const m = model || 'gpt-5-5'; + return { + curl: `curl ${url}/chat/completions \\ + -H "Content-Type: application/json" \\ + -H "Authorization: Bearer ${k}" \\ + -d '{ + "model": "${m}", + "messages": [{"role":"user","content":"Hello"}] + }'`, + python: `from openai import OpenAI + +client = OpenAI( + base_url="${url}", + api_key="${k}", +) +resp = client.chat.completions.create( + model="${m}", + messages=[{"role": "user", "content": "Hello"}], +) +print(resp.choices[0].message.content)`, + node: `import OpenAI from "openai"; + +const client = new OpenAI({ + baseURL: "${url}", + apiKey: "${k}", +}); + +const resp = await client.chat.completions.create({ + model: "${m}", + messages: [{ role: "user", content: "Hello" }], +}); +console.log(resp.choices[0].message.content);`, + }; +} + +function fallbackCopyToClipboard(text) { + const ta = document.createElement('textarea'); + ta.value = text; + ta.setAttribute('readonly', ''); + ta.style.position = 'fixed'; + ta.style.left = '-9999px'; + ta.style.top = '0'; + document.body.appendChild(ta); + ta.focus(); + ta.select(); + let ok = false; + try { + ok = document.execCommand('copy'); + } finally { + document.body.removeChild(ta); + } + if (!ok) throw new Error('浏览器拒绝复制，请手动选中文本复制'); +} + +async function copyToClipboard(text, btn) { + const orig = btn.textContent; + try { + if (navigator.clipboard && window.isSecureContext) { + await navigator.clipboard.writeText(text); + } else { + fallbackCopyToClipboard(text); + } + btn.textContent = '✓ 已复制'; + setTimeout(() => { btn.textContent = orig; }, 1500); + } catch (e) { + btn.textContent = orig; + toast('复制失败：' + (e.message || '当前浏览器不允许在 HTTP 页面直接复制'), true); + } +} + +function renderModelChips(models, sourceHint) { + const wrap = $('#invoke-models-chips'); + if (!models || !models.length) { + wrap.innerHTML = '无可用模型'; + } else { + wrap.innerHTML = models.map(m => { + const modelId = (typeof m === 'string') ? m : (m && m.id); + const source = (typeof m === 'string') ? '' : (m && m.source); + const sourceTag = source === 'probe' + ? '实测' + : (source === 'alias' + ? '别名' + : ''); + return `${escapeHtml(modelId)}${sourceTag}`; + }).join(''); + } + $('#invoke-models-source-hint').textContent = sourceHint || ''; +} + +function renderInvokeSnippet() { + if (!invokeCurrent) return; + const baseUrl = absoluteBaseUrl(invokeCurrent.base_url); + const firstModel = (invokeCurrent.models && invokeCurrent.models[0] && invokeCurrent.models[0].id) || 'gpt-5-5'; + // 注意：因为后端没下发原文 auth，前端代码示例里只能填 masked key。提示用户从「凭证」按钮取原文。 + const apiKey = invokeCurrent.auth_masked || 'YOUR_API_KEY'; + const snippets = genSnippets(baseUrl, apiKey, firstModel); + $('#invoke-snippet').textContent = snippets[invokeSnippetTab] || snippets.curl; +} + +async function openInvokeModal(slug) { + $('#invoke-slug-label').textContent = slug; + $('#invoke-base-url').textContent = '加载中...'; + $('#invoke-auth-key').textContent = '加载中...'; + $('#invoke-plan').textContent = '...'; + $('#invoke-plan-source').textContent = ''; + $('#invoke-cached-state').textContent = ''; + $('#invoke-models-chips').innerHTML = ''; + $('#invoke-error').classList.add('hidden'); + $('#invoke-error').textContent = ''; + $('#invoke-snippet').textContent = ''; + $('#modal-invoke').classList.remove('hidden'); + $('#modal-invoke').classList.add('flex'); + + try { + const info = await api('GET', '/api/instances/' + encodeURIComponent(slug) + '/info'); + invokeCurrent = info; + $('#invoke-base-url').textContent = absoluteBaseUrl(info.base_url) || '(未配置)'; + $('#invoke-auth-key').textContent = info.auth_masked || '(空)'; + const planEl = $('#invoke-plan'); + planEl.textContent = info.plan_label || info.plan_type || 'unknown'; + planEl.className = 'inline-block px-2 py-1 rounded text-xs ' + (PLAN_COLOR_CLASS[info.plan_type] || PLAN_COLOR_CLASS.unknown); + $('#invoke-plan-source').textContent = info.plan_source === 'jwt' ? '(从 JWT 解析)' : '(无 token, 默认 unknown)'; + $('#invoke-cached-state').textContent = info.cached ? '✓ 使用 5min 缓存' : '✓ 新生成'; + renderModelChips(info.models, '(套餐默认表)'); + $('#invoke-error').classList.add('hidden'); + $('#invoke-error').textContent = ''; + invokeSnippetTab = 'curl'; + $$('.snippet-tab').forEach(b => { + const active = b.dataset.snippet === 'curl'; + b.classList.toggle('border-blue-600', active); + b.classList.toggle('text-blue-600', active); + b.classList.toggle('border-transparent', !active); + b.classList.toggle('text-gray-500', !active); + }); + renderInvokeSnippet(); + } catch (e) { + $('#invoke-base-url').textContent = '加载失败'; + toast('加载调用信息失败：' + e.message, true); + } +} + +function closeInvokeModal() { + $('#modal-invoke').classList.add('hidden'); + $('#modal-invoke').classList.remove('flex'); + invokeCurrent = null; +} + +$('#btn-close-invoke').addEventListener('click', closeInvokeModal); + +// 实时探测按钮 +$('#btn-probe-models').addEventListener('click', async () => { + if (!invokeCurrent) return; + const slug = invokeCurrent.slug; + const btn = $('#btn-probe-models'); + const originalText = btn.textContent; + btn.disabled = true; + btn.textContent = '探测中...'; + try { + const d = await api('POST', '/api/probe-models/' + encodeURIComponent(slug), {}); + const models = d.model_entries || (d.models || []).map(id => ({ id, source: 'probe' })); + if (invokeCurrent) { + invokeCurrent.models = models; + const hasAlias = models.some(m => m.source === 'alias'); + renderModelChips( + models, + (hasAlias ? '(实测 + 深度研究别名 @ ' : '(实测 @ ') + new Date(d.probed_at * 1000).toLocaleTimeString() + ')' + ); + renderInvokeSnippet(); // 用新的第一个 model 刷新代码示例 + } + $('#invoke-error').classList.add('hidden'); + $('#invoke-error').textContent = ''; + toast('探测成功：' + models.length + ' 个模型'); + // 30s 内置灰 + btn.textContent = '⏳ 30s 冷却中'; + setTimeout(() => { + btn.disabled = false; + btn.textContent = originalText; + }, 30000); + } catch (e) { + $('#invoke-error').textContent = e.message; + $('#invoke-error').classList.remove('hidden'); + toast('探测失败：' + e.message, true); + btn.disabled = false; + btn.textContent = originalText; + } +}); + +// snippet tab 切换 +$$('.snippet-tab').forEach(btn => { + btn.addEventListener('click', () => { + invokeSnippetTab = btn.dataset.snippet; + $$('.snippet-tab').forEach(b => { + const active = b === btn; + b.classList.toggle('border-blue-600', active); + b.classList.toggle('text-blue-600', active); + b.classList.toggle('border-transparent', !active); + b.classList.toggle('text-gray-500', !active); + }); + renderInvokeSnippet(); + }); +}); + +// 通用 copy 按钮事件委托 +document.addEventListener('click', (e) => { + const btn = e.target.closest('.copy-btn'); + if (!btn) return; + const targetId = btn.dataset.copyTarget; + if (!targetId) return; + const el = document.getElementById(targetId); + if (!el) return; + const text = el.tagName === 'PRE' ? el.textContent : el.textContent.trim(); + copyToClipboard(text, btn); +}); + +// ---------- 调用汇总（跨实例）+ 导出 ---------- + +async function openSummaryModal() { + $('#summary-body').innerHTML = '加载中...'; + $('#summary-empty').classList.add('hidden'); + $('#modal-summary').classList.remove('hidden'); + $('#modal-summary').classList.add('flex'); + try { + const d = await api('GET', '/api/instances/aggregate'); + const rows = d.instances || []; + if (!rows.length) { + $('#summary-body').innerHTML = ''; + $('#summary-empty').classList.remove('hidden'); + return; + } + $('#summary-body').innerHTML = rows.map(r => { + const endpoint = absoluteBaseUrl(r.base_url) || '(未配置)'; + const planClass = PLAN_COLOR_CLASS[r.plan_type] || PLAN_COLOR_CLASS.unknown; + const health = r.container_state === 'running' + ? (r.container_health === 'healthy' + ? '● healthy' + : '● ' + escapeHtml(r.container_health || '?') + '') + : '○ ' + escapeHtml(r.container_state || 'absent') + ''; + const modelIds = (r.models || []).map(m => (typeof m === 'string') ? m : (m && m.id)).filter(Boolean); + const modelsHtml = modelIds.length + ? `
${modelIds.map(id => `${escapeHtml(id)}`).join('')}
` + : '无'; + return ` + + ${escapeHtml(r.slug)} + ${escapeHtml(r.plan_label || r.plan_type)} + ${escapeHtml(endpoint)} + ${escapeHtml(r.auth_masked || '-')} + ${modelsHtml} + ${health} + + `; + }).join(''); + } catch (e) { + $('#summary-body').innerHTML = `加载失败：${escapeHtml(e.message)}`; + } +} + +function exportConfig(fmt) { + // 触发文件下载：浏览器会保留 session cookie，FastAPI Response 带 Content-Disposition + window.location.href = './api/export/' + encodeURIComponent(fmt); + toast('开始下载 ' + fmt + ' 配置'); +} + +$('#btn-summary').addEventListener('click', openSummaryModal); +$('#btn-close-summary').addEventListener('click', () => { + $('#modal-summary').classList.add('hidden'); + $('#modal-summary').classList.remove('flex'); +}); + +document.addEventListener('click', (e) => { + const btn = e.target.closest('.btn-export'); + if (!btn) return; + const fmt = btn.dataset.fmt; + if (fmt) exportConfig(fmt); +}); + +// ---------- Playground 试调用 ---------- + +let pgInstances = []; // 缓存 options 返回值 + +async function openPlaygroundModal() { + $('#modal-playground').classList.remove('hidden'); + $('#modal-playground').classList.add('flex'); + $('#pg-result').classList.add('hidden'); + $('#pg-result-error').classList.add('hidden'); + try { + const d = await api('GET', '/api/playground/options'); + pgInstances = d.instances || []; + const slugSel = $('#pg-slug'); + if (!pgInstances.length) { + slugSel.innerHTML = ''; + $('#pg-model').innerHTML = ''; + return; + } + slugSel.innerHTML = pgInstances.map(it => + `` + ).join(''); + renderPgModels(); + } catch (e) { + toast('加载实例列表失败：' + e.message, true); + } +} + +function renderPgModels() { + const slug = $('#pg-slug').value; + const inst = pgInstances.find(x => x.slug === slug); + const models = inst ? (inst.models || []) : []; + $('#pg-model').innerHTML = models.length + ? models.map(m => ``).join('') + : ''; + $('#pg-custom-model').value = ''; +} + +async function runPlayground() { + const slug = $('#pg-slug').value; + const model = ($('#pg-custom-model').value.trim() || $('#pg-model').value).trim(); + const system = $('#pg-system').value; + const user = $('#pg-user').value.trim(); + const temperature = parseFloat($('#pg-temp').value); + const max_tokens = parseInt($('#pg-max-tokens').value, 10); + + if (!slug || !model) { toast('请先选择实例，或填写自定义模型', true); return; } + if (!user) { toast('user prompt 不能为空', true); return; } + + const btn = $('#btn-pg-run'); + const origText = btn.textContent; + btn.disabled = true; + btn.innerHTML = '运行中...'; + $('#pg-result').classList.remove('hidden'); + $('#pg-result-status').textContent = '...'; + $('#pg-result-latency').textContent = '...'; + $('#pg-result-usage').textContent = '...'; + $('#pg-result-content').textContent = ''; + $('#pg-result-error').classList.add('hidden'); + + try { + const d = await api('POST', '/api/playground/invoke', { + slug, model, system, user, temperature, max_tokens + }); + $('#pg-result-latency').textContent = (d.latency_ms || 0) + 'ms'; + if (d.ok) { + $('#pg-result-status').innerHTML = '✓ 200'; + const u = d.usage || {}; + $('#pg-result-usage').textContent = `${u.prompt_tokens || '-'} + ${u.completion_tokens || '-'} = ${u.total_tokens || '-'}`; + $('#pg-result-content').textContent = d.content || '(空响应)'; + } else { + $('#pg-result-status').innerHTML = `✗ ${escapeHtml(String(d.status || '?'))}`; + $('#pg-result-usage').textContent = '-'; + $('#pg-result-content').textContent = ''; + $('#pg-result-error').textContent = typeof d.error === 'string' ? d.error : JSON.stringify(d.error); + $('#pg-result-error').classList.remove('hidden'); + } + } catch (e) { + $('#pg-result-status').innerHTML = '✗ 异常'; + $('#pg-result-error').textContent = e.message; + $('#pg-result-error').classList.remove('hidden'); + } finally { + btn.disabled = false; + btn.textContent = origText; + } +} + +$('#btn-playground').addEventListener('click', openPlaygroundModal); +$('#btn-close-playground').addEventListener('click', () => { + $('#modal-playground').classList.add('hidden'); + $('#modal-playground').classList.remove('flex'); +}); +$('#pg-slug').addEventListener('change', renderPgModels); +$('#pg-temp').addEventListener('input', () => { + $('#pg-temp-label').textContent = $('#pg-temp').value; +}); +$('#btn-pg-run').addEventListener('click', runPlayground); + +// ---------- 启动 ---------- + +loadStatus(); +setInterval(loadStatus, 5000); diff --git a/deploy/multi/orchestrator/static/models_by_plan.json b/deploy/multi/orchestrator/static/models_by_plan.json new file mode 100644 index 00000000..85b97a80 --- /dev/null +++ b/deploy/multi/orchestrator/static/models_by_plan.json @@ -0,0 +1,37 @@ +{ + "version": 3, + "updated_at": "2026-05-16", + "comment": "每个套餐能用的模型清单。OpenAI 改版后只需更新此文件，无需改 Python。plan_type 从 access_token JWT 的 chatgpt_plan_type 字段解码得到；deep-research 项是 chat2api 支持的调用别名。", + "plans": { + "free": { + "label": "Free", + "color": "slate", + "models": ["gpt-5-5", "gpt-5-mini", "gpt-4o-mini", "o4-mini"] + }, + "plus": { + "label": "Plus", + "color": "blue", + "models": ["gpt-5-5", "gpt-5", "gpt-5-thinking", "gpt-4o", "o4-mini", "o3", "o3-deep-research", "o4-mini-deep-research", "gpt-4o-deep-research", "deep-research", "gpt-4o-mini"] + }, + "team": { + "label": "Team", + "color": "emerald", + "models": ["gpt-5-5", "gpt-5", "gpt-5-thinking", "gpt-5-pro", "o3", "o4-mini", "o3-deep-research", "o4-mini-deep-research", "gpt-4o-deep-research", "deep-research", "gpt-4o"] + }, + "pro": { + "label": "Pro", + "color": "amber", + "models": ["gpt-5-5", "gpt-5-pro", "gpt-5-thinking", "o3-pro", "gpt-5", "o3", "o3-deep-research", "o4-mini-deep-research", "gpt-4o-deep-research", "deep-research", "gpt-4o"] + }, + "enterprise": { + "label": "Enterprise", + "color": "violet", + "models": ["gpt-5-5", "gpt-5", "gpt-5-thinking", "gpt-5-pro", "o3", "o4-mini", "o3-deep-research", "o4-mini-deep-research", "gpt-4o-deep-research", "deep-research", "gpt-4o"] + }, + "unknown": { + "label": "未知", + "color": "rose", + "models": ["gpt-5-5", "gpt-4o-mini"] + } + } +} diff --git a/deploy/multi/orchestrator/static/styles.css b/deploy/multi/orchestrator/static/styles.css new file mode 100644 index 00000000..a8dfb1ce --- /dev/null +++ b/deploy/multi/orchestrator/static/styles.css @@ -0,0 +1,238 @@ +:root { + --panel: rgba(255, 255, 255, 0.88); + --line: #e6edf7; + --brand: #2563eb; + --text-strong: #0f172a; + --text-body: #334155; + --text-soft: #64748b; +} +body { + background: + radial-gradient(circle at top left, rgba(37, 99, 235, 0.12), transparent 34rem), + linear-gradient(135deg, #f8fbff 0%, #eef4fb 48%, #f8fafc 100%); + font-family: "SF Pro Display", "SF Pro Text", "PingFang SC", "Hiragino Sans GB", "Noto Sans CJK SC", "Microsoft YaHei", sans-serif; + color: var(--text-body); + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; + text-rendering: optimizeLegibility; +} +.glass-panel { + background: var(--panel); + border: 1px solid rgba(226, 232, 240, 0.88); + box-shadow: 0 14px 40px rgba(15, 23, 42, 0.055); + backdrop-filter: blur(18px); +} +.metric-card { + min-height: 112px; + border: 1px solid var(--line); + border-radius: 1.25rem; + background: linear-gradient(145deg, #fff, #f8fbff); + box-shadow: 0 10px 28px rgba(15, 23, 42, 0.045); +} +.metric-card > div:first-child, +.metric-card > div:last-child { + letter-spacing: 0.01em; +} +.orch-nav-item { + display: flex; + align-items: center; + gap: 0.75rem; + width: 100%; + border-radius: 1rem; + padding: 0.78rem 0.9rem; + color: #526179; + transition: 160ms ease; + text-align: left; +} +.orch-nav-item:hover { + background: #f1f5f9; + color: var(--brand); +} +.orch-nav-active { + background: linear-gradient(90deg, #eaf2ff, #f8fbff); + color: var(--brand); + box-shadow: inset 3px 0 0 var(--brand); + font-weight: 700; +} +.admin-table th { + color: var(--text-soft); + font-size: 0.72rem; + font-weight: 700; + letter-spacing: 0.04em; + text-transform: uppercase; +} +.admin-table tbody tr { transition: 140ms ease; } +.admin-table tbody tr:hover { background: #f8fbff; } +.status-dot, .dot { + display: inline-block; + width: 8px; + height: 8px; + border-radius: 999px; + margin-right: 6px; +} +.dot-healthy { background: #10b981; } +.dot-unhealthy { background: #ef4444; } +.dot-starting { background: #f59e0b; } +.dot-na { background: #9ca3af; } +.kbd-row { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; } +.row-action-btn { + padding: 0.45rem 0.65rem; + font-size: 0.75rem; + border-radius: 0.75rem; + margin-right: 0.2rem; + font-weight: 600; +} +.row-action-btn:hover { background: #f1f5f9; } +.model-chip { + display: inline-flex; + align-items: center; + padding: 0.2rem 0.55rem; + font-size: 0.72rem; + font-family: ui-monospace, SFMono-Regular, monospace; + background: #eef2ff; + color: #4338ca; + border: 1px solid #c7d2fe; + border-radius: 9999px; + line-height: 1.2; +} +.usage-control { + width: 100%; + height: 2.5rem; + border-radius: 1rem; + border: 1px solid #dbe4ef; + background: rgba(255, 255, 255, 0.92); + padding: 0 0.85rem; + font-size: 0.875rem; + color: var(--text-body); + outline: none; +} +.usage-control:focus { + border-color: #93c5fd; + box-shadow: 0 0 0 4px rgba(147, 197, 253, 0.28); +} +.usage-dot { + display: inline-block; + width: 0.5rem; + height: 0.5rem; + border-radius: 999px; + margin-right: 0.35rem; +} +.usage-chart { + min-height: 220px; + border-radius: 1.5rem; + background: + linear-gradient(to bottom, transparent 0 24%, rgba(203, 213, 225, 0.55) 24% 25%, transparent 25% 49%, rgba(203, 213, 225, 0.55) 49% 50%, transparent 50% 74%, rgba(203, 213, 225, 0.55) 74% 75%, transparent 75% 100%), + linear-gradient(180deg, rgba(248, 250, 252, 0.92), rgba(255,255,255,0.92)); + border: 1px solid #e2e8f0; + overflow: hidden; +} +.usage-bar-track { + height: 0.55rem; + border-radius: 999px; + background: #edf2f7; + overflow: hidden; +} +.usage-bar-fill { + height: 100%; + border-radius: 999px; + background: linear-gradient(90deg, #2563eb, #14b8a6); +} +.usage-row { + cursor: pointer; +} +.usage-row:hover { + background: #f8fbff; +} +.usage-status-ok { + display: inline-flex; + align-items: center; + border-radius: 999px; + background: #dcfce7; + color: #047857; + padding: 0.2rem 0.55rem; + font-size: 0.75rem; + font-weight: 700; +} +.usage-status-error { + display: inline-flex; + align-items: center; + border-radius: 999px; + background: #fee2e2; + color: #b91c1c; + padding: 0.2rem 0.55rem; + font-size: 0.75rem; + font-weight: 700; +} +.usage-mode-pill { + display: inline-flex; + align-items: center; + border-radius: 999px; + background: #e0f2fe; + color: #0369a1; + padding: 0.2rem 0.55rem; + font-size: 0.72rem; + font-weight: 700; +} +.usage-mode-pill-normal { + background: #f1f5f9; + color: #475569; +} +.orch-title, +.orch-section-title, +.orch-metric-value, +.orch-detail-title { + color: var(--text-strong); + letter-spacing: -0.015em; +} +.orch-title { + font-weight: 700; +} +.orch-section-title { + font-weight: 650; +} +.orch-metric-value { + font-weight: 700; +} +.orch-label-strong { + font-weight: 600; +} +.orch-subtle { + color: var(--text-soft); +} +.code-block { + font-family: ui-monospace, SFMono-Regular, monospace; + line-height: 1.45; + white-space: pre; +} +.pg-loading-spinner { + display: inline-block; + width: 12px; + height: 12px; + border: 2px solid rgba(255,255,255,0.35); + border-top-color: white; + border-radius: 50%; + animation: pg-spin 0.7s linear infinite; + vertical-align: middle; + margin-right: 4px; +} +@keyframes pg-spin { to { transform: rotate(360deg); } } +@media (max-width: 1023px) { + .orch-shell { display: block; } + .orch-sidebar { + position: sticky; + top: 0; + z-index: 40; + width: 100%; + min-height: 0; + border-right: 0; + border-bottom: 1px solid var(--line); + } + .orch-nav { + display: grid; + grid-auto-flow: column; + grid-auto-columns: max-content; + overflow-x: auto; + padding-bottom: 0.25rem; + } + .orch-nav-item { white-space: nowrap; } +} diff --git a/deploy/multi/orchestrator/templates/dashboard.html b/deploy/multi/orchestrator/templates/dashboard.html new file mode 100644 index 00000000..f04f9775 --- /dev/null +++ b/deploy/multi/orchestrator/templates/dashboard.html @@ -0,0 +1,586 @@ + + + + + + chat2api Orchestrator + + + + +
+ + +
+
+
+
+ 生产环境 + 服务器时间 -- +
+
+ + +
+ +
+
+
+
+ +
+
+
+

Account Instances

+

账号实例

+

集中查看实例健康、代理、Cookie 鲜度和模型可用性。

+
+ +
+ +
+
总实例数
0
全部账号
+
健康实例
0
Healthy
+
异常实例
0
需关注
+
代理绑定
0
已配置出口
+
模型可用
0
模型总数
+
Cookie 新鲜
0
1 小时内
+
+ +
+
+
+
+

实例列表

+

--

+
+
+ 全部 + Healthy + Degraded +
+
+
+ + + + + + + + + + + + + + + + +
账号套餐/模型代理状态出口 IPCookie运行时长操作
加载中...
+
+
+ + +
+
+ + +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/deploy/multi/orchestrator/templates/login.html b/deploy/multi/orchestrator/templates/login.html new file mode 100644 index 00000000..f1f38235 --- /dev/null +++ b/deploy/multi/orchestrator/templates/login.html @@ -0,0 +1,51 @@ + + + + + + Orchestrator 登录 + + + +
+

chat2api Orchestrator

+

多账号实例编排管理面板

+ +
+
+ + +
+
+ + +
+ {% if error %} +

{{ error }}

+ {% endif %} + +
+ +

+ 忘记密码？SSH 服务器执行：./manage.sh orch-password +

+
+ + diff --git a/docs/COOKIE_HARVEST.md b/docs/COOKIE_HARVEST.md new file mode 100644 index 00000000..4d8fd318 --- /dev/null +++ b/docs/COOKIE_HARVEST.md @@ -0,0 +1,231 @@ +# Cookie 抓取完整指南 + +> 本文教你从 ChatGPT 网页版抓取 **session-token cookie**，喂给 chat2api 使用。 +> 这是目前**唯一可持续**的获取 ChatGPT 网页 API 访问凭证的方式（auth0 OAuth 已废弃）。 + +--- + +## 核心原理 + +### 你抓的是什么？ + +chatgpt.com 用 **NextAuth** 管理登录会话，用户登录后浏览器里会有一个加密 cookie： + +``` +__Secure-next-auth.session-token.0 = +__Secure-next-auth.session-token.1 = +[可能还有 .2 .3 ...] +``` + +为什么会分成 `.0 .1`？因为单个浏览器 cookie 上限约 4KB，而 NextAuth 的加密 session token 长度超过这个限制，就被**自动切片**存成多个 cookie。 + +chat2api 拿到 cookie 后，用它访问 `https://chatgpt.com/api/auth/session`，返回 JSON 中的 `accessToken`（JWT）就是调用 `/backend-api/conversation` 等接口的 Bearer token。 + +### 寿命对比 + +| 凭据 | 寿命 | 用途 | +|---|---|---| +| **session-token cookie** | **数月**（只要你不登出） | 长期凭证 → 存进 chat2api | +| accessToken (JWT) | **10-15 分钟** | 短命，chat2api 用 cookie 每次自动刷新 | + +**所以必须抓 cookie，不能只抓 accessToken**。 + +--- + +## 🎯 设备无关，IP 地区强相关 + +Cookie 本身不绑设备——JWE 里没有"浏览器指纹"。但 **OpenAI 对 IP 行为有监控**： + +``` +❌ 危险： + Windows 国内 IP 登录 → 拿 cookie → 贴到海外 VPS → chat2api 从美国 IP 调 + → OpenAI 看到"10 秒前在中国，现在在美国"→ 🚨 异地登录邮件 / 临时冻结 + +✅ 安全： + 登录拿 cookie 的 IP ≈ chat2api 运行的 IP（同国或同大洲） +``` + +--- + +## 🟢 方法 A：SSH 动态端口转发（推荐 ⭐⭐⭐⭐⭐） + +让你 Windows/Mac 浏览器**临时走云服务器出口**访问 chatgpt，使用 IP 和登录 IP 完全一致，零风险。 + +### 步骤 + +#### 1. 建立 SSH 隧道 + +**Windows**（需要 Git Bash / WSL / OpenSSH）： +```bash +ssh -D 1080 -N -q root@你的云服务器IP +# 保持这个窗口开着，别关 +``` + +**Mac / Linux** Terminal 一样： +```bash +ssh -D 1080 -N -q root@你的云服务器IP +``` + +**PuTTY 用户**： +- Connection → SSH → Tunnels +- Source port: `1080` +- 勾选 `Dynamic` +- Add → 保存连接后开启 + +#### 2. 浏览器挂代理 + +**Chrome / Edge**： +1. 装扩展 [SwitchyOmega](https://chrome.google.com/webstore/detail/proxy-switchyomega) +2. 新建 profile： + - Protocol: `SOCKS5` + - Server: `127.0.0.1` + - Port: `1080` +3. 启用该 profile + +**Firefox**： +- 设置 → 网络设置 → 手动代理 → SOCKS 主机 `127.0.0.1:1080` → SOCKS v5 + +#### 3. 验证出口 IP + +新标签页访问 `https://ifconfig.me`，显示的应该是**你云服务器的 IP**。 + +#### 4. 登录 chatgpt.com 抓 cookie + +1. 访问 `https://chatgpt.com` +2. 登录（这次 OpenAI 看到的登录 IP = 云服务器 IP） +3. F12 → Console，输入： + ```javascript + document.cookie.split(';').filter(x=>x.includes('session-token')).join('; ') + ``` +4. 回车 → 右键复制输出 +5. 关掉 SSH 隧道或关 SwitchyOmega + +#### 5. 粘贴到 chat2api + +- 管理后台 → "账号采集 Harvester" → 新增账号 → 🍪 粘贴 Cookie +- 粘贴 → [验证并导入] → 成功 + +--- + +## 🟡 方法 B：Clash 挂海外节点 + +如果不方便 SSH，但 Clash 有和云服务器**同地区**的节点： + +1. Clash 切到云服务器同地区节点（日本 / 美国 / 新加坡） +2. 确认系统代理生效（访问 ifconfig.me 看 IP 地区对） +3. 按方法 A 的第 4-5 步抓 cookie + +**不如方法 A 精确**（仍是两个不同 IP，只是地区一致），但触发风控的概率大幅降低。 + +--- + +## 🔴 方法 C：直接在本地抓（国内 IP → 海外 VPS） + +- 第一次一定收到 OpenAI 异地登录警告邮件 +- 点"是我本人" → 后续可用 +- 不推荐但**可行** +- 适用场景：一次性测试账号，不介意邮件验证 + +--- + +## 正确粘贴格式 + +所有姿势都支持，后端自动识别： + +### 姿势 1：F12 Console 一行抓（最推荐） + +```javascript +document.cookie.split(';').filter(x=>x.includes('session-token')).join('; ') +``` + +输出： +``` + __Secure-next-auth.session-token.0=; __Secure-next-auth.session-token.1= +``` + +直接粘贴到 UI。 + +### 姿势 2：Application → Cookies 手工拼 Name=Value + +``` +__Secure-next-auth.session-token.0=;__Secure-next-auth.session-token.1= +``` + +### 姿势 3：只粘 Value（英文分号分隔） + +如果只复制 Value 列（没 name）： + +``` +; +``` + +### 姿势 4：整段 document.cookie（含其他 cookie） + +直接 Console 输入 `document.cookie` 复制整串，后端自动过滤噪音 cookie。 + +--- + +## ⚠️ 常见错误 + +| 错误信息 | 原因 | 解决 | +|---|---|---| +| `'latin-1' codec can't encode character '\uff1b'` | 粘贴里含**中文全角分号 `；`** | 系统会自动归一化；如仍报错，用 [替换工具](https://www.baidu.com/s?wd=全角半角转换) 转半角 | +| `未识别到有效的 session-token cookie` | 只粘了值但长度不够 / 格式错乱 | 用姿势 1 重新抓 | +| `500: Failed to connect to 127.0.0.1 port 7890` | 云服务器想走 Clash 但代理在你 Mac | 清空 `PROXY_URL` 或在 UI "代理与路由" 配**可达的**代理 | +| `session cookie 无效或过期` | Cookie 真过期了，或少抓了 `.1` | F12 看看是不是有 `.1`，全部抓 | +| `401 / cf_chl_opt` | IP 触发 CF 挑战 | 换代理、换 VPS 地区、或等 15 分钟再试 | + +--- + +## 验证抓取成功 + +在 chat2api 后台点 "🍪 粘贴 Cookie" → 验证导入后，应看到： + +``` +✅ 成功导入 yours@example.com +access_token 预览: eyJhbGciOi...xxx +``` + +去"账号与令牌"页，新账号的"类型"列应显示 **SessionToken**。 + +测试真正能用： +```bash +curl -H "Authorization: Bearer $AUTHORIZATION" \ + -H "Content-Type: application/json" \ + -d '{"model":"gpt-4o","messages":[{"role":"user","content":"say hi"}]}' \ + http://your.server/your_prefix/v1/chat/completions +``` + +--- + +## Cookie 过期了怎么办 + +**Cookie 通常能用数月**。当你发现聊天失败时： + +1. 管理后台 → "系统日志" 看到 `[sess2ac] status=401` +2. 说明 cookie 失效，需要重抓 +3. 回到你 Windows 浏览器重复上面流程抓新 cookie +4. 管理后台 → 账号采集 → 对应邮箱点"编辑"或"🍪 粘贴 Cookie"（粘新的即可覆盖） + +chat2api 的 `SCHEDULED_REFRESH` 只能刷新 **access_token**（短 JWT），**不能**帮你刷新 cookie 本身——cookie 过期必须人工重抓。 + +--- + +## 账号生命周期管理 + +推荐节奏： + +``` + 首次导入 + ↓ + [持续数月] ────→ 自动续 access_token（每 8 分钟，chat2api 自己做） + ↓ + Cookie 过期 / 被强制登出 + ↓ + 管理后台发现异常 → 重抓 cookie → 更新 +``` + +如果你有 **多个账号池**（比如 20+），建议： +- 每月固定时间检查一次系统日志 +- 用 `docker compose logs chat2api | grep -E "401|sess2ac.*fail"` 批量看哪些失效 +- 一批一起重抓（SSH 隧道下同时开多个浏览器 Profile 登录） diff --git a/docs/FEATURES.md b/docs/FEATURES.md new file mode 100644 index 00000000..1fee5326 --- /dev/null +++ b/docs/FEATURES.md @@ -0,0 +1,311 @@ +# 新功能总览（nanashiwang 分支） + +> 本文档汇总 nanashiwang 分支相比上游 `LanQian528/chat2api` 的增强功能。 +> 按能力分类，每节包含：功能简介 → 配置方式 → 使用方式。 + +--- + +## 目录 + +1. [Antiban 风控规避层](#1-antiban-风控规避层) +2. [Harvester 账号采集](#2-harvester-账号采集) +3. [管理后台增强](#3-管理后台增强) +4. [系统日志 UI](#4-系统日志-ui) +5. [安全加固](#5-安全加固) +6. [新版 Token 支持](#6-新版-token-支持) +7. [一键部署](#7-一键部署) + +--- + +## 1. Antiban 风控规避层 + +> 针对 OpenAI 对机房 IP / 批量登录 / 异常请求的风控，提供自动化保护。 + +### 功能 + +- **IP-账号粘性桶**：每个住宅 IP 绑定固定 N 个账号，账号**永不跨桶漂移**（OpenAI 对账号历史 IP 突变极敏感） +- **账号级冷却**：单账号两次请求间自动保持最小间隔（默认 60s）+ 抖动 +- **IP 地域一致性**：根据代理 IP 自动调整 `Accept-Language` / `timezone` 等 header +- **熔断自愈**：403/cf_chl_opt 自动降级 IP 桶、429 自动账号退避（指数 60→300→1800s） +- **指纹持久化**：每账号独立 UA + screen + cores，永不漂移 + +### 配置（`.env` 或 docker-compose environment） + +```yaml +ENABLE_ANTIBAN: 'true' # 总开关 +STRICT_IP_BINDING: 'true' # 严格 IP 绑定（有代理池时开） +BUCKET_MAX_ACCOUNTS_PER_IP: '5' # 每个 IP 容纳的账号数 +ACCOUNT_MIN_INTERVAL_SECONDS: '60' # Team/Plus 最小间隔 +FREE_ACCOUNT_MIN_INTERVAL_SECONDS: '180' # 免费账号最小间隔 +ACCOUNT_COOLDOWN_JITTER: '0.3' # 冷却抖动 ±30% +ACCOUNT_MAX_WAIT_SECONDS: '30' # 账号排队最长等待 +IP_GEO_PROVIDER: 'ip-api' # 地域查询提供商 +CIRCUIT_429_COOLDOWN: '1800' # 429 初始退避 +CIRCUIT_403_COOLDOWN: '3600' # 403 IP 桶冷冻时长 +CIRCUIT_BUCKET_HEAL_MINUTES: '30' # 桶自愈扫描间隔 +``` + +### 观察 + +启动日志会看到： +``` +[antiban] enabled | buckets=N accounts=M healthy=N degraded=0 +``` + +运行中：管理后台 → "代理与路由" 可看到每个桶的状态（healthy/degraded/dead）。 + +--- + +## 2. Harvester 账号采集 + +> 从浏览器 cookie 采集 ChatGPT 网页会话，一次导入后 chat2api 自动续期数月。 + +### 核心能力 + +| 能力 | 实现 | +|---|---| +| 可视化 email 清单 | 管理后台 → "账号采集 Harvester" 页面 | +| 一键粘贴 Cookie | 🍪 按钮 + 多种粘贴格式自动识别（整段 document.cookie / 裸 value / 分片 .0 .1 .2） | +| 全角分号归一化 | 用户粘贴里意外的 `；`（中文）自动转 `;`（英文） | +| 每账号状态看板 | fresh / stale / failed / pending 颜色标识 | +| 代理绑定 | CSV 里的 proxy_name 自动反查并绑定代理 | +| 批量导入 | 支持上传 CSV（email,note,proxy_name 三列）| +| 自动上报 | cookie 验证成功自动更新看板 last_rt_prefix | + +### 使用流程 + +详见 [`docs/COOKIE_HARVEST.md`](./COOKIE_HARVEST.md)。 + +1. 浏览器（走代理）登录 `https://chatgpt.com` +2. F12 Console: + ```javascript + document.cookie.split(';').filter(x=>x.includes('session-token')).join('; ') + ``` +3. 管理后台 → Harvester → 新增 email → 🍪 粘贴 Cookie → 完成 + +### 存储 + +- 元数据 `data/harvester_accounts.json`：email / note / proxy_name / last_rt_prefix +- **不存密码**，cookie 本身即凭证 +- 权限：建议挂载 `./data` 为 600，单机部署 + +--- + +## 3. 管理后台增强 + +### 分页式布局 + +之前的滚动锚点式 nav 改为真正页面切换： + +- 控制台总览 +- 账号与令牌 +- 代理与路由 +- 账号采集 Harvester（新） +- 运行日志（新） + +### 导入账号：支持文件上传 + 自动解析 + +- `.txt` 纯文本（每行 token，`#` 注释） +- `.json` 配置导出（递归扫描，识别 `refresh_token` / `access_token` 字段） +- 预览识别结果（SessionToken / RefreshToken / AccessToken 分桶） +- 复选后才写入（避免误操作） +- 2 MB 文件大小限制 + 后缀白名单 + +### 代理热加载 + +- UI "代理与路由" → 添加 / 编辑 / 删除 → **立即生效，无需重启** +- 保存后 antiban 桶自动重建（dead/orphaned 桶清理 + 未分配账号重分） +- 允许清空代理池（切到直连模式） + +--- + +## 4. 系统日志 UI + +> 把 `docker logs` 搬到 UI，方便远程运维。 + +### 功能 + +- **实时**：默认 3s 轮询增量（基于 `since_id`） +- **级别筛选**：ALL / DEBUG+ / INFO+ / WARNING+ / ERROR+ +- **关键字搜索**：400ms 防抖 +- **一键下载**：当前筛选结果 或 全部缓冲区 +- **自动滚底** / **自动刷新** 开关 +- **ANSI 颜色自动剥离**（日志里的 `\x1b[...]` 不会污染 UI） + +### 配置 + +```yaml +LOG_BUFFER_SIZE: '3000' # 内存环形缓冲条数，默认 2000 +``` + +### 使用 + +管理后台 → "运行日志" + +- `Ctrl+F` 前端查找支持 +- 想定位某次 chat_refresh 失败 → 关键字输入 `chat_refresh` 即可 + +--- + +## 5. 安全加固 + +详见 [`docs/SECURITY.md`](./SECURITY.md)。 + +### 已修复的历史漏洞 + +| # | 漏洞 | 修复 | +|---|---|---| +| P0 | ADMIN_PASSWORD 未配时回退到 AUTHORIZATION 或空放行 | 未配置 → 整个后台返回 503 | +| P1 | `gateway/gpts.py` 未登录访问触发 `len(None)` 崩溃 | 加判空，重定向登录页 | +| P2 | admin_token 被服务端注入前端 JS 绕开 HttpOnly | 移除注入，HttpOnly + SameSite=Strict + Secure | + +### 新增防护 + +| 防护 | 配置 | 效果 | +|---|---|---| +| 管理后台 IP 白名单 | `ADMIN_IP_WHITELIST=1.2.3.4,10.0.0.0/8` | 非白名单 IP 访问 `/admin/*` 直接 403 | +| 反代 IP 识别 | `ADMIN_TRUST_PROXY=true` | 走 CF/Nginx 后读真实 X-Forwarded-For | +| HttpOnly + SameSite=Strict | 默认 | 防 XSS 偷 admin cookie | +| 登录失败锁定 | 默认 | 同 IP 5 次错误锁 10 分钟 | +| 分级速率限制 | 默认 | 管理接口 120/min，登录接口 10/5min | + +### 推荐链路 + +``` +[公网用户] + ↓ +[Cloudflare 免费 WAF + 自动 HTTPS] + ↓ +[云服务器 UFW 只允许 CF IP 段 + 你的运维 IP] + ↓ +[chat2api IP 白名单 (ADMIN_IP_WHITELIST)] + ↓ +[ADMIN_PASSWORD 鉴权 + HttpOnly Cookie] + ↓ +[业务] +``` + +--- + +## 6. 新版 Token 支持 + +### 识别规则 + +`utils/routing.py::detect_token_type`： + +| 前缀/长度 | 类型 | 刷新机制 | +|---|---|---| +| `sess-*` | **SessionToken**（新）| 调 `chatgpt.com/api/auth/session`，8 分钟缓存 | +| `rt_*`（长度 ≥ 60）| RefreshToken（Auth0 新格式）| 调 `auth.openai.com/oauth/token` | +| 长度正好 45 | RefreshToken（老格式） | 调 `auth.openai.com/oauth/token` | +| `eyJhbGciOi*` | AccessToken (JWT) | 不刷新（2 小时后过期） | +| `fk-*` | AccessToken (fakeopen) | 不刷新 | +| 其他 | CustomToken | 直接透传 | + +### chat_refresh 现代化 + +- 端点从废弃的 `auth0.openai.com` 切到 **`auth.openai.com`** +- Content-Type 从 `application/json` → `application/x-www-form-urlencoded` +- User-Agent 从 iOS app → `Codex_CLI/0.1.0` +- 可通过环境变量完全覆盖： + ```yaml + OPENAI_AUTH_CLIENT_ID: 'app_EMoamEEZ73f0CkXaXp7hrann' + OPENAI_AUTH_TOKEN_URL: 'https://auth.openai.com/oauth/token' + OPENAI_AUTH_REDIRECT_URI: 'http://localhost:1455/auth/callback' + OPENAI_AUTH_SCOPE: 'openid profile email offline_access' + ``` + +### SessionToken 续期 + +每 8 分钟自动续一次 access_token。cookie 本身可用数月。流程： + +``` +请求进来 → verify_token 看 token 前缀 + ├── sess- → sess2ac() 查缓存 → 命中返回 / 失效重新调 /api/auth/session + ├── rt_ → rt2ac() 同上逻辑，调 auth.openai.com + └── eyJ → 直接用 +``` + +--- + +## 7. 一键部署 + +> 在新云服务器上一行命令完成部署。 + +### 使用 + +```bash +# 方式 1：远程一行启动 +curl -fsSL https://raw.githubusercontent.com/nanashiwang/chat2api/main/deploy/install.sh | bash + +# 方式 2：clone 后本地跑 +git clone https://github.com/nanashiwang/chat2api.git +cd chat2api/deploy +bash install.sh + +# 交互模式（让脚本问你密码/前缀） +INTERACTIVE=1 bash install.sh + +# 自定义安装目录 +INSTALL_DIR=/opt/chat2api bash install.sh +``` + +### 脚本做的事 + +1. 检测 OS（Ubuntu / Debian / CentOS / RHEL / Rocky / Alma）和架构（amd64 / arm64） +2. 自动装 Docker + docker-compose 插件（如缺） +3. 下载 `docker-compose.template.yml` 到安装目录 +4. 生成强随机凭据： + - `ADMIN_PASSWORD`：24 位字母数字 + - `AUTHORIZATION`：`sk-` + 32 位 + - `API_PREFIX`：`api-` + 12 位 +5. 写入 `.env`（chmod 600） +6. `docker compose up -d` +7. 自动安装宿主管理命令 `chat2api` +8. 等待健康检查通过 +9. 打印访问 URL + 凭据 + 下一步操作指引 + +### 凭据安全 + +- `.env` 永远不进镜像（通过 `env_file` 挂载） +- 部署完自动 `chmod 600` +- 终端打印一次后就只能从 `.env` 看 + +### 升级 + +```bash +chat2api update +``` + +旧机器没有该命令时，重新跑一键部署脚本会沿用现有配置并补装。 + +--- + +## 配置速查表 + +| 变量 | 默认 | 作用 | +|---|---|---| +| `ADMIN_PASSWORD` | (必填) | 管理后台登录密码 | +| `AUTHORIZATION` | (必填) | API 调用方 Bearer token | +| `API_PREFIX` | (必填) | URL 路径前缀，建议随机 | +| `ADMIN_IP_WHITELIST` | 空 | 管理后台 IP 白名单（逗号 / CIDR） | +| `ADMIN_TRUST_PROXY` | false | 走 CF/Nginx 时设 true 读 XFF | +| `ENABLE_ANTIBAN` | false | Antiban 风控层总开关 | +| `STRICT_IP_BINDING` | true | 账号严格绑定 IP | +| `SCHEDULED_REFRESH` | false | 每 4 天定时刷新 RT | +| `LOG_BUFFER_SIZE` | 2000 | 日志面板内存条数 | +| `OPENAI_AUTH_CLIENT_ID` | Codex CLI | OAuth client_id 覆盖 | +| `OPENAI_AUTH_TOKEN_URL` | auth.openai.com | token 端点覆盖 | + +--- + +## 常见问题速索引 + +| 问题 | 文档位置 | +|---|---| +| Cookie 怎么抓？ | [COOKIE_HARVEST.md](./COOKIE_HARVEST.md) | +| 公网部署安全？ | [SECURITY.md](./SECURITY.md) | +| chat_refresh 报 404 | 已修复，使用 `auth.openai.com` | +| 粘贴 cookie 报 latin-1 编码错 | 全角分号自动归一化，详见 COOKIE_HARVEST.md | +| 要不要用 Playwright 方案 | 已废弃，harvester/ 目录保留供参考 | +| 账号池怎么绑代理 | UI "代理与路由" → 编辑账号 → 选代理名 | diff --git a/docs/SECURITY.md b/docs/SECURITY.md new file mode 100644 index 00000000..92a59805 --- /dev/null +++ b/docs/SECURITY.md @@ -0,0 +1,176 @@ +# chat2api 生产环境安全加固指南 + +> 把管理后台暴露在公网是个严肃的安全问题。本文档给出分层防护建议。 + +## 威胁模型 + +| 威胁 | 描述 | 缓解层 | +|---|---|---| +| **暴力破解密码** | 攻击者拿字典撞 `ADMIN_PASSWORD` | 应用层（已有 5 次失败锁 10 分钟） | +| **DDoS / CC** | 刷登录接口打满带宽 | CF 边缘过滤 | +| **未授权访问** | 扫 80/443 端口发现后台入口 | CF + IP 白名单 | +| **XSS / CSRF** | 通过前端漏洞偷 admin cookie | HttpOnly cookie（已有） | +| **MITM** | 明文 HTTP 传输密码被窃听 | CF 自动 HTTPS | +| **配置泄露** | docker-compose.yml 里的密码入 git | 用 .env 文件（已推荐） | + +--- + +## 防护分层（按效果/成本排序） + +### 🥇 第 0 层：Cloudflare 免费版（10 分钟，0 代码） + +1. 把你服务器的域名（例如 `api.your-domain.com`）DNS 解析指向服务器 IP +2. 在 Cloudflare 把该域名 **接入**（免费套餐即可） +3. 开启 **橙色云朵 proxied** 图标 +4. SSL/TLS → 设为 `Full (strict)` +5. 防火墙规则 → 根据需要开启： + - **Country block**：只允许你的国家访问（可选） + - **Challenge (I'm Under Attack)**：被攻击时一键启用 + - **Rate limiting**：对 `/admin/login` 10 次/分钟限流 +6. 服务器侧 + - 防火墙只允许 Cloudflare IP 段连 443，其他全拒绝 + - CF 官方 IP 段：https://www.cloudflare.com/ips/ + +**效果**： +- 服务器真实 IP 隐藏 +- HTTPS 自动（Let's Encrypt 类似） +- CC 防护 +- WAF 基础规则 +- 访问日志更详细 + +如果配了 CF，在 chat2api 设置： +```yaml +environment: + ADMIN_TRUST_PROXY: 'true' # 信任 X-Forwarded-For（IP 白名单会读真实客户端 IP） +``` + +--- + +### 🥈 第 1 层：IP 白名单（5 行配置） + +只允许**你家宽带 IP + VPN 出口 IP**访问管理后台： + +```yaml +environment: + ADMIN_IP_WHITELIST: '1.2.3.4,5.6.7.0/24,2001:db8::/32' + ADMIN_TRUST_PROXY: 'true' # 如果走了 CF / Nginx 反代才开 +``` + +格式： +- 单 IP：`1.2.3.4` +- CIDR 子网：`192.168.1.0/24` +- IPv6：`2001:db8::1` 或 `2001:db8::/32` +- 多条逗号分隔 + +**行为**： +- 白名单**内** IP：正常看到登录页 +- 白名单**外** IP：所有 `/admin/*` 路径 **403**，连登录表单都看不到 + +**注意**： +- 如果你的 IP 是动态的（家用宽带），可用 DDNS + 脚本每小时更新 CIDR +- 调用 OpenAI API 的 `/v1/chat/completions` **不受白名单限制**（只保护后台） + +--- + +### 🥉 第 2 层：应用层（已默认开启，无需配置） + +| 功能 | 说明 | +|---|---| +| 登录失败锁定 | 同 IP 连续 5 次错误 → 10 分钟不能重试 | +| HttpOnly Cookie | 防 XSS 偷 admin token | +| SameSite=Strict | 防 CSRF | +| 速率限制 | 管理接口 120 次/分钟 | +| 密码隔离 | `ADMIN_PASSWORD` 必须独立，不再允许回退到 `AUTHORIZATION` | +| 分级错误响应 | 未配置 ADMIN_PASSWORD 时整个后台返回 503，而不是误放行 | + +--- + +## Cloudflare 配置详细步骤 + +### 1. 域名接入 CF + +1. 注册 cloudflare.com 免费账号 +2. Add Site → 输入你的域名 → Free 套餐 +3. CF 会给你两个 NS，到你的域名注册商把 NS 改成 CF 的 +4. 等待生效（10 分钟到 24 小时） + +### 2. DNS 记录 + +A 记录 `api.your-domain.com` → 你的服务器公网 IP +- **代理状态选 Proxied（橙色云）** + +### 3. SSL/TLS 配置 + +- SSL/TLS → Overview → 设为 `Full (strict)`（推荐） +- Edge Certificates → Always Use HTTPS: ON +- Edge Certificates → Min TLS Version: TLS 1.2 + +### 4. 防火墙规则（WAF → Custom rules） + +**规则 1：保护管理后台** +``` +Expression: (http.request.uri.path contains "/admin") and (ip.src ne 1.2.3.4) +Action: Block +``` + +**规则 2：限流登录接口** +``` +Expression: (http.request.uri.path eq "/your_prefix/admin/login") and (http.request.method eq "POST") +Action: Managed Challenge +Rate: 5 requests per 10 seconds +``` + +### 5. 服务器防火墙（UFW / iptables） + +只允许 Cloudflare IP 段 + 你自己的运维 IP： + +```bash +# Ubuntu UFW 示例 +sudo ufw default deny incoming +sudo ufw allow 22/tcp # SSH（建议改非默认端口） +# CF IPv4 段（复制自 https://www.cloudflare.com/ips-v4） +for ip in 173.245.48.0/20 103.21.244.0/22 103.22.200.0/22 \ + 103.31.4.0/22 141.101.64.0/18 108.162.192.0/18 \ + 190.93.240.0/20 188.114.96.0/20 197.234.240.0/22 \ + 198.41.128.0/17 162.158.0.0/15 104.16.0.0/13 \ + 104.24.0.0/14 172.64.0.0/13 131.0.72.0/22; do + sudo ufw allow from $ip to any port 60403 +done +sudo ufw enable +``` + +这样服务器 60403 端口只接受 CF 转发的流量，攻击者扫描你真实 IP 无法直连。 + +--- + +## 验证 + +配完后测试： + +```bash +# 1. 从白名单外 IP 访问 +curl -I https://api.your-domain.com/nanapi-2026-a1/admin/login +# 预期: 403 Forbidden (IP 白名单拦截) + +# 2. 从白名单 IP 访问 +curl -I https://api.your-domain.com/nanapi-2026-a1/admin/login +# 预期: 200 HTML 登录页 + +# 3. 服务器日志应能看到真实客户端 IP（ADMIN_TRUST_PROXY=true 生效） +docker compose logs chat2api --tail 20 | grep admin-ipwl +# 预期: [admin-ipwl] 拒绝 IP: xxx.xxx.xxx.xxx （而不是 CF 的 IP） + +# 4. 尝试直接用服务器 IP 访问（绕过 CF） +curl -I http://你的服务器IP:60403/ +# 预期: 超时或拒绝连接（UFW 拦截） +``` + +--- + +## 额外建议 + +- **备份 data/ 目录**：每天一次，放 S3 或另一台机器 +- **限制 SSH**：改端口 + 公钥登录 + Fail2ban +- **监控 /admin/logs**：出现大量 401 / 拒绝 IP 时立即警觉 +- **定期轮换 ADMIN_PASSWORD**：每 3 个月换一次 +- **审计 data/harvester_accounts.json**：发现不认识的 email 立即查 diff --git a/gateway/admin.py b/gateway/admin.py index e69de29b..87d886ee 100644 --- a/gateway/admin.py +++ b/gateway/admin.py @@ -0,0 +1,1214 @@ +import ipaddress +import json +import time +from collections import defaultdict, deque + +from fastapi import File, HTTPException, Request, UploadFile +from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse, RedirectResponse + +from app import app, templates +from utils.Client import Client +from utils.configs import admin_password, admin_ip_whitelist, admin_trust_proxy, api_prefix, authorization_list +from utils.Logger import logger +from utils.routing import ( + build_group_assignments, + detect_token_type, + get_dashboard_payload, + get_routing_config, + remove_account_binding, + save_routing_config, + sync_bindings_to_fp, + update_account_meta, + update_single_binding, +) +import utils.globals as globals +from chatgpt.refreshToken import rt2ac + +ADMIN_COOKIE_NAME = "admin_auth" +ADMIN_COOKIE_MAX_AGE = 8 * 60 * 60 +MAX_UPLOAD_BYTES = 2 * 1024 * 1024 # 2MB +ALLOWED_IMPORT_EXTENSIONS = {"txt", "json"} +rate_limit_buckets = defaultdict(deque) +failed_login_buckets = defaultdict(deque) + +if not admin_password: + logger.warning( + "[admin] ADMIN_PASSWORD is NOT configured. " + "Admin backend endpoints are disabled for safety. " + "Set ADMIN_PASSWORD to a strong independent secret (do NOT reuse AUTHORIZATION)." + ) +elif authorization_list and admin_password in authorization_list: + logger.warning( + "[admin] ADMIN_PASSWORD is identical to one of AUTHORIZATION entries. " + "Use a distinct secret to avoid privilege escalation via API key leakage." + ) + + +def admin_login_path(): + if api_prefix: + return f"/{api_prefix}/admin/login" + return "/admin/login" + + +def get_admin_secrets(): + """管理后台专用密码集合。 + + 安全要求： + - 必须独立配置 ADMIN_PASSWORD，不再回退到 AUTHORIZATION； + - 未配置时返回空列表 → require_admin_auth / 登录接口一律 403。 + """ + if admin_password: + return [admin_password] + return [] + + +def get_client_key(request: Request): + """获取客户端 IP。 + + 仅在 ADMIN_TRUST_PROXY=true 时读 X-Forwarded-For，否则用真实 TCP 连接 IP。 + 避免攻击者伪造 XFF 头绕过白名单。 + """ + if admin_trust_proxy: + forwarded = request.headers.get("x-forwarded-for", "").split(",")[0].strip() + if forwarded: + return forwarded + if request.client: + return request.client.host + return "unknown" + + +def _is_ip_whitelisted(request: Request) -> bool: + """检查客户端 IP 是否在白名单。空白名单 = 全放行。""" + if not admin_ip_whitelist: + return True + client_ip = get_client_key(request) + if client_ip in ("unknown", ""): + return False + try: + ip_obj = ipaddress.ip_address(client_ip) + except ValueError: + logger.warning(f"[admin-ipwl] 无法解析客户端 IP: {client_ip}") + return False + for rule in admin_ip_whitelist: + rule = rule.strip() + if not rule: + continue + try: + if "/" in rule: + if ip_obj in ipaddress.ip_network(rule, strict=False): + return True + else: + if client_ip == rule or ip_obj == ipaddress.ip_address(rule): + return True + except ValueError: + logger.warning(f"[admin-ipwl] 白名单规则无效: {rule}") + continue + return False + + +def require_ip_whitelist(request: Request): + """IP 白名单检查：不在白名单的直接 403，连登录页都看不到。""" + if not _is_ip_whitelisted(request): + client_ip = get_client_key(request) + logger.warning(f"[admin-ipwl] 拒绝 IP: {client_ip}") + raise HTTPException( + status_code=403, + detail="Forbidden: your IP is not whitelisted for admin backend", + ) + + +def check_rate_limit(bucket_key, limit, window_seconds): + now = time.time() + bucket = rate_limit_buckets[bucket_key] + while bucket and now - bucket[0] > window_seconds: + bucket.popleft() + if len(bucket) >= limit: + raise HTTPException(status_code=429, detail="Too many admin requests") + bucket.append(now) + + +def record_failed_login(client_key): + now = time.time() + bucket = failed_login_buckets[client_key] + while bucket and now - bucket[0] > 600: + bucket.popleft() + bucket.append(now) + + +def ensure_login_not_locked(client_key): + now = time.time() + bucket = failed_login_buckets[client_key] + while bucket and now - bucket[0] > 600: + bucket.popleft() + if len(bucket) >= 5: + raise HTTPException(status_code=429, detail="Too many failed login attempts") + + +def is_admin_authorized(request: Request): + cookie_token = request.cookies.get(ADMIN_COOKIE_NAME, "") + header_token = request.headers.get("authorization", "").replace("Bearer ", "").strip() + token = header_token or cookie_token + return bool(token and token in get_admin_secrets()) + + +def get_current_admin_token(request: Request): + cookie_token = request.cookies.get(ADMIN_COOKIE_NAME, "") + header_token = request.headers.get("authorization", "").replace("Bearer ", "").strip() + token = header_token or cookie_token + if token and token in get_admin_secrets(): + return token + return "" + + +def require_admin_auth(request: Request): + # 第一道：IP 白名单（若开启） + require_ip_whitelist(request) + # 第二道：未配置 ADMIN_PASSWORD 时，后台接口一律拒绝（不再放行） + if not get_admin_secrets(): + raise HTTPException( + status_code=503, + detail="Admin backend disabled: ADMIN_PASSWORD is not configured.", + ) + check_rate_limit(f"admin:{get_client_key(request)}", 120, 60) + if not is_admin_authorized(request): + raise HTTPException(status_code=401, detail="Admin authorization required") + + +async def routing_admin_login_page(request: Request): + require_ip_whitelist(request) + check_rate_limit(f"admin-page:{get_client_key(request)}", 60, 60) + if not get_admin_secrets(): + raise HTTPException( + status_code=503, + detail="Admin backend disabled: ADMIN_PASSWORD is not configured.", + ) + if is_admin_authorized(request): + return RedirectResponse(url=f"/{api_prefix}/admin/routing" if api_prefix else "/admin/routing", status_code=302) + return templates.TemplateResponse( + "admin_login.html", + { + "request": request, + "api_prefix": api_prefix, + }, + ) + + +async def routing_admin_login_submit(request: Request): + require_ip_whitelist(request) + client_key = get_client_key(request) + ensure_login_not_locked(client_key) + check_rate_limit(f"admin-login:{client_key}", 10, 300) + if not get_admin_secrets(): + raise HTTPException( + status_code=503, + detail="Admin backend disabled: ADMIN_PASSWORD is not configured.", + ) + form = await request.form() + password = (form.get("password") or "").strip() + if not password or password not in get_admin_secrets(): + record_failed_login(client_key) + return templates.TemplateResponse( + "admin_login.html", + { + "request": request, + "api_prefix": api_prefix, + "error": "授权码无效", + }, + status_code=401, + ) + + response = RedirectResponse( + url=f"/{api_prefix}/admin/routing" if api_prefix else "/admin/routing", + status_code=302, + ) + failed_login_buckets.pop(client_key, None) + # Cookie 安全：HttpOnly 阻止 JS 读取；SameSite=Strict 防 CSRF； + # Secure 在 HTTPS 下生效（HTTP 反代内网环境 request.url.scheme 为 http，自动不发 Secure） + is_https = request.url.scheme == "https" or request.headers.get("x-forwarded-proto") == "https" + cookie_path = f"/{api_prefix}/admin" if api_prefix else "/admin" + response.set_cookie( + ADMIN_COOKIE_NAME, + value=password, + httponly=True, + secure=is_https, + samesite="strict", + max_age=ADMIN_COOKIE_MAX_AGE, + path=cookie_path, + ) + return response + + +async def routing_admin_logout(request: Request): + response = RedirectResponse(url=admin_login_path(), status_code=302) + cookie_path = f"/{api_prefix}/admin" if api_prefix else "/admin" + response.delete_cookie(ADMIN_COOKIE_NAME, path=cookie_path) + return response + + +async def routing_admin_page(request: Request): + require_ip_whitelist(request) + check_rate_limit(f"admin-page:{get_client_key(request)}", 60, 60) + if not get_admin_secrets(): + raise HTTPException( + status_code=503, + detail="Admin backend disabled: ADMIN_PASSWORD is not configured.", + ) + if not is_admin_authorized(request): + return RedirectResponse(url=admin_login_path(), status_code=302) + return templates.TemplateResponse( + "account_proxy_bindings.html", + { + "request": request, + "api_prefix": api_prefix, + }, + ) + + +async def routing_admin_data(request: Request): + require_admin_auth(request) + payload = get_dashboard_payload() + payload["routing_config"] = get_routing_config() + payload["proxy_options"] = get_routing_config().get("proxies", []) + return JSONResponse(payload) + + +async def routing_admin_save(request: Request): + require_admin_auth(request) + try: + body = await request.json() + except Exception: + raise HTTPException(status_code=400, detail="Invalid JSON body") + + proxies = body.get("proxies", []) + group_size = body.get("group_size", 25) + if not isinstance(proxies, list): + raise HTTPException(status_code=400, detail="proxies must be a list") + # 允许清空代理池（直连模式）：传 [] 时清理所有 bindings + if not proxies: + logger.info("[admin] clearing all proxies (direct mode)") + empty_config = { + "proxies": [], + "groups": [], + "bindings": {}, + "account_meta": get_routing_config().get("account_meta", {}), + } + save_routing_config(empty_config) + sync_bindings_to_fp({}) + else: + result = build_group_assignments(list(globals.token_list), proxies, group_size) + save_routing_config(result) + sync_bindings_to_fp(result["bindings"]) + + # 热同步 antiban 桶：不重启即生效 + try: + from utils.antiban import bucket as _bucket + _bucket.resync_from_routing() + except Exception as e: # pragma: no cover + logger.warning(f"[admin] antiban resync failed: {e}") + + return JSONResponse( + { + "status": "success", + "message": "Routing config saved" if proxies else "Proxies cleared (direct mode)", + "summary": get_dashboard_payload()["summary"], + } + ) + + +async def routing_admin_bind_account(request: Request): + require_admin_auth(request) + try: + body = await request.json() + except Exception: + raise HTTPException(status_code=400, detail="Invalid JSON body") + + token = (body.get("token") or "").strip() + proxy_url = (body.get("proxy_url") or "").strip() + proxy_name = (body.get("proxy_name") or "").strip() + if not token or not proxy_url: + raise HTTPException(status_code=400, detail="token and proxy_url are required") + + config = get_routing_config() + if not proxy_name: + proxy = next((item for item in config.get("proxies", []) if item.get("proxy_url") == proxy_url), None) + proxy_name = proxy.get("name") if proxy else "Custom Proxy" + + binding = update_single_binding(token, proxy_name, proxy_url) + # 热同步 antiban 桶 + try: + from utils.antiban import bucket as _bucket + _bucket.resync_from_routing() + except Exception as e: + logger.warning(f"[admin] antiban resync failed: {e}") + return JSONResponse({"status": "success", "binding": binding}) + + +async def routing_admin_import_accounts(request: Request): + require_admin_auth(request) + try: + body = await request.json() + except Exception: + raise HTTPException(status_code=400, detail="Invalid JSON body") + + text = (body.get("text") or "").strip() + note = (body.get("note") or "").strip() + group_name = (body.get("group_name") or "").strip() + proxy_url = (body.get("proxy_url") or "").strip() + proxy_name = (body.get("proxy_name") or "").strip() + overwrite_existing = bool(body.get("overwrite_existing")) + if not text: + raise HTTPException(status_code=400, detail="text is required") + + incoming_tokens = [] + for line in text.splitlines(): + token = line.strip() + if token and not token.startswith("#"): + incoming_tokens.append(token) + + if not incoming_tokens: + raise HTTPException(status_code=400, detail="No valid tokens found") + + existing = set(globals.token_list) + added = [] + updated = [] + for token in incoming_tokens: + if token not in existing: + globals.token_list.append(token) + existing.add(token) + added.append(token) + elif overwrite_existing: + updated.append(token) + + if added: + with open(globals.TOKENS_FILE, "a", encoding="utf-8") as f: + for token in added: + f.write(token + "\n") + + config = get_routing_config() + if proxy_url and not proxy_name: + proxy = next((item for item in config.get("proxies", []) if item.get("proxy_url") == proxy_url), None) + proxy_name = proxy.get("name") if proxy else "Custom Proxy" + + for token in added + updated: + update_account_meta( + token, + note=note, + group_name=group_name or None, + proxy_name=proxy_name or None, + proxy_url=proxy_url or None, + ) + + return JSONResponse( + { + "status": "success", + "added_count": len(added), + "updated_count": len(updated), + "skipped_count": len(incoming_tokens) - len(added) - len(updated), + "message": "账号已保存", + } + ) + + +async def routing_admin_parse_file(request: Request, file: UploadFile = File(...)): + """上传文件并解析其中的 token，仅返回预览，不写入。前端确认后再调用 import 路由。""" + require_admin_auth(request) + + filename = (file.filename or "").strip() + if not filename: + raise HTTPException(status_code=400, detail="缺少文件名") + + ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else "" + if ext not in ALLOWED_IMPORT_EXTENSIONS: + raise HTTPException( + status_code=400, + detail=f"不支持的文件后缀: .{ext}；仅允许 {sorted(ALLOWED_IMPORT_EXTENSIONS)}", + ) + + content = await file.read() + if not content: + raise HTTPException(status_code=400, detail="文件为空") + if len(content) > MAX_UPLOAD_BYTES: + raise HTTPException( + status_code=413, + detail=f"文件过大：最大 {MAX_UPLOAD_BYTES // 1024} KB", + ) + + from utils.token_parser import mask_token, parse_file + result = parse_file(filename, content) + + # 为前端预览附加 masked 版本，避免整屏泄漏 token + result["filename"] = filename + result["masked"] = { + "refresh_tokens": [mask_token(t) for t in result["refresh_tokens"]], + "access_tokens": [mask_token(t) for t in result["access_tokens"]], + "unknown": [mask_token(t) for t in result["unknown"]], + } + return JSONResponse(result) + + +async def routing_admin_delete_account(request: Request): + require_admin_auth(request) + try: + body = await request.json() + except Exception: + raise HTTPException(status_code=400, detail="Invalid JSON body") + + token = (body.get("token") or "").strip() + if not token: + raise HTTPException(status_code=400, detail="token is required") + + if token not in globals.token_list: + raise HTTPException(status_code=404, detail="token not found") + + globals.token_list[:] = [item for item in globals.token_list if item != token] + with open(globals.TOKENS_FILE, "w", encoding="utf-8") as f: + for item in globals.token_list: + f.write(item + "\n") + + remove_account_binding(token) + if token in globals.refresh_map: + globals.refresh_map.pop(token, None) + with open(globals.REFRESH_MAP_FILE, "w", encoding="utf-8") as f: + json.dump(globals.refresh_map, f, indent=4, ensure_ascii=False) + if token in globals.error_token_list: + globals.error_token_list[:] = [item for item in globals.error_token_list if item != token] + with open(globals.ERROR_TOKENS_FILE, "w", encoding="utf-8") as f: + for item in globals.error_token_list: + f.write(item + "\n") + + return JSONResponse( + { + "status": "success", + "message": "账号已删除", + } + ) + + +async def routing_admin_refresh_account(request: Request): + require_admin_auth(request) + try: + body = await request.json() + except Exception: + raise HTTPException(status_code=400, detail="Invalid JSON body") + + token = (body.get("token") or "").strip() + if not token: + raise HTTPException(status_code=400, detail="token is required") + if token not in globals.token_list: + raise HTTPException(status_code=404, detail="token not found") + if detect_token_type(token) != "RefreshToken": + raise HTTPException(status_code=400, detail="Only RefreshToken supports manual refresh") + + try: + access_token = await rt2ac(token, force_refresh=True) + except HTTPException as exc: + raise HTTPException(status_code=exc.status_code, detail=exc.detail) + + refresh_info = globals.refresh_map.get(token, {}) + return JSONResponse( + { + "status": "success", + "message": "RefreshToken 刷新成功", + "token_masked": f"{access_token[:6]}...{access_token[-4:]}" if access_token else "", + "refresh_updated_at": refresh_info.get("last_success_at", refresh_info.get("timestamp", 0)), + } + ) + + +async def routing_admin_refresh_all_accounts(request: Request): + require_admin_auth(request) + refresh_tokens = [token for token in globals.token_list if detect_token_type(token) == "RefreshToken"] + if not refresh_tokens: + return JSONResponse( + { + "status": "success", + "message": "当前没有可刷新的 RefreshToken", + "success_count": 0, + "failed_count": 0, + } + ) + + success_count = 0 + failed_tokens = [] + for token in refresh_tokens: + try: + await rt2ac(token, force_refresh=True) + success_count += 1 + except HTTPException: + failed_tokens.append(token) + + message = f"批量刷新完成：成功 {success_count}，失败 {len(failed_tokens)}" + return JSONResponse( + { + "status": "success" if not failed_tokens else "partial", + "message": message, + "success_count": success_count, + "failed_count": len(failed_tokens), + "failed_tokens": failed_tokens, + } + ) + + +async def routing_admin_test_proxy(request: Request): + require_admin_auth(request) + try: + body = await request.json() + except Exception: + raise HTTPException(status_code=400, detail="Invalid JSON body") + + proxy_url = (body.get("proxy_url") or "").strip() + if not proxy_url: + raise HTTPException(status_code=400, detail="proxy_url is required") + + client = Client(proxy=proxy_url, timeout=15) + results = [] + try: + targets = [ + "https://chatgpt.com", + "https://auth0.openai.com", + ] + for target in targets: + try: + response = await client.get(target, timeout=10) + results.append({ + "target": target, + "ok": 200 <= response.status_code < 500, + "status_code": response.status_code, + }) + except Exception as exc: + results.append({ + "target": target, + "ok": False, + "error": str(exc), + }) + + overall_ok = all(item.get("ok") for item in results) + return JSONResponse( + { + "status": "success" if overall_ok else "partial", + "proxy_url": proxy_url, + "results": results, + } + ) + finally: + await client.close() + + +async def routing_admin_logs_tail(request: Request): + """轮询日志：支持增量 since_id、级别、关键字筛选。""" + require_admin_auth(request) + + from utils.log_buffer import log_buffer + + params = request.query_params + since_id = params.get("since_id") + try: + since_id_val = int(since_id) if since_id not in (None, "") else None + except ValueError: + since_id_val = None + + level = (params.get("level") or "").strip().upper() or None + keyword = (params.get("keyword") or "").strip() or None + try: + limit = int(params.get("limit") or 500) + except ValueError: + limit = 500 + limit = max(1, min(limit, 2000)) + + items = log_buffer.snapshot( + since_id=since_id_val, + level=level, + keyword=keyword, + limit=limit, + ) + return JSONResponse( + { + "items": items, + "latest_id": log_buffer.latest_id, + "capacity": log_buffer.capacity, + "total": len(log_buffer), + } + ) + + +async def routing_admin_logs_download(request: Request): + """下载日志文本；scope=all 下载全部，否则按筛选下载当前视图。""" + require_admin_auth(request) + + from utils.log_buffer import log_buffer, render_plaintext + + params = request.query_params + scope = (params.get("scope") or "filtered").lower() + + if scope == "all": + records = log_buffer.snapshot_all() + else: + level = (params.get("level") or "").strip().upper() or None + keyword = (params.get("keyword") or "").strip() or None + records = log_buffer.snapshot(level=level, keyword=keyword, limit=2000) + + text = render_plaintext(records) + filename = f"chat2api-{int(time.time())}-{scope}.log" + return PlainTextResponse( + content=text, + headers={ + "Content-Disposition": f'attachment; filename="{filename}"', + "Content-Type": "text/plain; charset=utf-8", + }, + ) + + +# ============ Harvester 账号元数据 ============ + +async def routing_admin_harvester_list(request: Request): + """返回所有 Harvester 账号元数据及统计。""" + require_admin_auth(request) + from utils import harvester_meta + return JSONResponse({ + "accounts": harvester_meta.list_all(), + "stats": harvester_meta.stats(), + }) + + +async def routing_admin_harvester_upsert(request: Request): + """新增或编辑账号元数据（不含密码）。""" + require_admin_auth(request) + try: + body = await request.json() + except Exception: + raise HTTPException(status_code=400, detail="Invalid JSON body") + + email = (body.get("email") or "").strip() + note = (body.get("note") or "").strip() + proxy_name = (body.get("proxy_name") or "").strip() + if not email or "@" not in email: + raise HTTPException(status_code=400, detail="email 不合法") + + from utils import harvester_meta + try: + rec = harvester_meta.upsert(email, note=note, proxy_name=proxy_name) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) + return JSONResponse({"status": "success", "account": rec}) + + +async def routing_admin_harvester_delete(request: Request): + """删除元数据（不动 token.txt）。""" + require_admin_auth(request) + try: + body = await request.json() + except Exception: + raise HTTPException(status_code=400, detail="Invalid JSON body") + email = (body.get("email") or "").strip() + if not email: + raise HTTPException(status_code=400, detail="email 不能为空") + from utils import harvester_meta + ok = harvester_meta.delete(email) + return JSONResponse({"status": "success" if ok else "not_found"}) + + +async def routing_admin_harvester_bulk_import(request: Request): + """批量导入 email 清单。接受两种输入： + 1) JSON: {"rows": [{"email":"...","note":"...","proxy_name":"..."}]} + 2) multipart: file= CSV 文件（表头 email,note,proxy_name） + """ + require_admin_auth(request) + rows = [] + content_type = request.headers.get("content-type", "") + + if "multipart/form-data" in content_type: + form = await request.form() + file = form.get("file") + if file is None: + raise HTTPException(status_code=400, detail="缺少 file 字段") + raw = (await file.read()).decode("utf-8", errors="replace") + import csv + import io + reader = csv.DictReader(io.StringIO(raw)) + for r in reader: + rows.append({ + "email": (r.get("email") or "").strip(), + "note": (r.get("note") or "").strip(), + "proxy_name": (r.get("proxy_name") or "").strip(), + }) + else: + try: + body = await request.json() + except Exception: + raise HTTPException(status_code=400, detail="需要 JSON 或 CSV 文件") + rows = body.get("rows") or [] + if not isinstance(rows, list): + raise HTTPException(status_code=400, detail="rows 必须是数组") + + if not rows: + raise HTTPException(status_code=400, detail="没有可导入的行") + + from utils import harvester_meta + result = harvester_meta.bulk_upsert(rows) + return JSONResponse({"status": "success", **result, "total": len(rows)}) + + +async def routing_admin_harvester_report(request: Request): + """Harvester 采集成功/失败后回调此接口上报状态。""" + require_admin_auth(request) + try: + body = await request.json() + except Exception: + raise HTTPException(status_code=400, detail="Invalid JSON body") + + email = (body.get("email") or "").strip() + if not email: + raise HTTPException(status_code=400, detail="email 不能为空") + success = bool(body.get("success", True)) + rt_prefix = (body.get("rt_prefix") or "").strip() + error = (body.get("error") or "").strip() + imported_token = (body.get("imported_token") or "").strip() + + from utils import harvester_meta + rec = harvester_meta.report_harvest( + email=email, + rt_prefix=rt_prefix, + success=success, + error=error, + imported_token=imported_token, + ) + return JSONResponse({"status": "success", "account": rec}) + + +# ============ Harvester 浏览器登录（OAuth PKCE，用户在本地浏览器完成）============ + +async def routing_admin_harvester_authorize_start(request: Request): + """启动一次 OAuth 授权会话，返回 authorize_url 供前端展示给用户复制。""" + require_admin_auth(request) + try: + body = await request.json() + except Exception: + raise HTTPException(status_code=400, detail="Invalid JSON body") + + email = (body.get("email") or "").strip() + note = (body.get("note") or "").strip() + proxy_name = (body.get("proxy_name") or "").strip() + if not email or "@" not in email: + raise HTTPException(status_code=400, detail="email 不合法") + + from utils import oauth_session + try: + result = oauth_session.start_session(email, note=note, proxy_name=proxy_name) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) + logger.info(f"[harvester-auth] start session for {email}") + return JSONResponse({"status": "success", **result}) + + +async def routing_admin_harvester_authorize_exchange(request: Request): + """用户粘贴浏览器地址栏的 com.openai.chat://...?code=X&state=Y 过来。 + + 步骤： + 1. pop session（验证 session_id 有效且未过期，一次性消费） + 2. 解析 callback URL 拿 code / state + 3. 校验 state 防 CSRF + 4. 用 verifier + code 调 Auth0 /oauth/token 换 refresh_token + 5. 调已有 routing_admin_import_accounts 的内部逻辑写入 chat2api 账号池 + 6. 通过 harvester_meta.report_harvest 更新看板 + """ + require_admin_auth(request) + try: + body = await request.json() + except Exception: + raise HTTPException(status_code=400, detail="Invalid JSON body") + + session_id = (body.get("session_id") or "").strip() + callback_url = (body.get("callback_url") or "").strip() + if not session_id or not callback_url: + raise HTTPException(status_code=400, detail="session_id 和 callback_url 必填") + + from urllib.parse import parse_qs, urlparse + from utils import harvester_meta, oauth_session + + sess = oauth_session.pop_session(session_id) + if sess is None: + raise HTTPException(status_code=404, detail="会话不存在或已过期，请重新开始") + + # 解析回调 URL + parsed = urlparse(callback_url) + qs = parse_qs(parsed.query) + + # error 优先 + if qs.get("error"): + err = qs.get("error", ["unknown"])[0] + desc = qs.get("error_description", [""])[0] + harvester_meta.report_harvest( + email=sess.email, success=False, error=f"{err}: {desc}"[:200] + ) + raise HTTPException(status_code=400, detail=f"OAuth 错误: {err} {desc}") + + code_list = qs.get("code") + state_list = qs.get("state") + if not code_list or not state_list: + harvester_meta.report_harvest( + email=sess.email, success=False, error="callback URL 缺 code/state" + ) + raise HTTPException(status_code=400, detail="回调 URL 中未找到 code 或 state") + + code = code_list[0] + returned_state = state_list[0] + if returned_state != sess.state: + harvester_meta.report_harvest( + email=sess.email, success=False, error="state mismatch (CSRF?)" + ) + raise HTTPException(status_code=400, detail="state 不匹配，可能是 CSRF 或会话错配") + + # 换 token + token_set = await _exchange_code_for_tokens(code, sess) + rt = token_set.get("refresh_token", "") + if not rt: + harvester_meta.report_harvest( + email=sess.email, success=False, error="Auth0 未返回 refresh_token" + ) + raise HTTPException(status_code=502, detail="Auth0 响应缺 refresh_token") + + rt_prefix = rt[:12] + + # 复用现有的 import_accounts 业务逻辑：这里为了避免构造假 request，直接调用底层 + try: + await _harvester_import_rt(sess, rt) + except Exception as e: + harvester_meta.report_harvest( + email=sess.email, success=False, error=f"import failed: {e}"[:200] + ) + raise HTTPException(status_code=500, detail=f"写入 chat2api 失败: {e}") + + # 更新看板 + harvester_meta.report_harvest( + email=sess.email, + rt_prefix=rt_prefix, + success=True, + imported_token=rt, + ) + logger.info(f"[harvester-auth] ✓ {sess.email} → rt_prefix={rt_prefix[:8]}...") + return JSONResponse({ + "status": "success", + "email": sess.email, + "rt_prefix": rt_prefix, + }) + + +async def routing_admin_harvester_import_cookie(request: Request): + """从浏览器粘贴的 NextAuth session cookie 导入账号。 + + 支持 3 种粘贴格式（自动识别）： + 1. 单一 cookie value: "eyJxxxxx..." + 2. 完整 cookie 串（多片）: "__Secure-next-auth.session-token.0=xxx; __Secure-next-auth.session-token.1=yyy" + 3. 完整 document.cookie 字符串: "_ga=xxx; __Secure-next-auth.session-token.0=xxx; __Secure-next-auth.session-token.1=yyy; cf_clearance=xxx" + + 自动提取 `__Secure-next-auth.session-token.N` 的所有片段，按顺序拼接。 + + 前端传递 {email, session_token, note?, proxy_name?}。 + """ + require_admin_auth(request) + try: + body = await request.json() + except Exception: + raise HTTPException(status_code=400, detail="Invalid JSON body") + + email = (body.get("email") or "").strip() + raw_input = (body.get("session_token") or "").strip() + note = (body.get("note") or "").strip() + proxy_name = (body.get("proxy_name") or "").strip() + + if not email or "@" not in email: + raise HTTPException(status_code=400, detail="email 不合法") + if not raw_input: + raise HTTPException(status_code=400, detail="session_token 不能为空") + + # ----- 智能解析分片 cookie ----- + from chatgpt.refreshToken import SESS_CHUNK_SEPARATOR, sess2ac + session_cookie = _parse_session_cookie_input(raw_input) + if not session_cookie or len(session_cookie) < 20: + raise HTTPException( + status_code=400, + detail="未识别到有效的 session-token cookie。" + "请从浏览器 F12 → Application → Cookies 复制整段 Cookie 字符串粘贴。", + ) + + storage_key = "sess-" + session_cookie + chunk_count = session_cookie.count(SESS_CHUNK_SEPARATOR) + 1 if SESS_CHUNK_SEPARATOR in session_cookie else 1 + logger.info( + f"[harvester-cookie] parsed {chunk_count} chunk(s) for {email}" + ) + + # 验证 cookie 是否真能换出 access_token + from utils import harvester_meta + from utils.routing import get_routing_config, update_account_meta + + try: + access_token = await sess2ac(storage_key, force_refresh=True) + except Exception as e: + harvester_meta.report_harvest( + email=email, success=False, error=f"cookie 验证失败: {e}" + ) + raise HTTPException(status_code=400, detail=f"Cookie 验证失败：{str(e)[:200]}") + + if not access_token: + raise HTTPException(status_code=400, detail="Cookie 有效但未拿到 access_token") + + # 写入 token 池 + if storage_key not in globals.token_list: + globals.token_list.append(storage_key) + with open(globals.TOKENS_FILE, "a", encoding="utf-8") as f: + f.write(storage_key + "\n") + + # 绑定代理 / 备注 + proxy_url = "" + if proxy_name: + cfg = get_routing_config() + match = next( + (p for p in cfg.get("proxies", []) if p.get("name") == proxy_name), + None, + ) + if match: + proxy_url = match.get("proxy_url", "") or "" + else: + logger.warning( + f"[harvester-cookie] proxy_name='{proxy_name}' 未找到" + ) + final_note = f"{email} [chunks={chunk_count}]" + (f" · {note}" if note else "") + update_account_meta( + storage_key, + note=final_note, + group_name=None, + proxy_name=proxy_name if proxy_url else None, + proxy_url=proxy_url if proxy_url else None, + ) + + # 更新看板 + harvester_meta.report_harvest( + email=email, + rt_prefix=f"sess({chunk_count})...", + success=True, + imported_token=storage_key, + ) + + logger.info(f"[harvester-cookie] ✓ {email} → chunks={chunk_count}, access_token 已换出") + return JSONResponse({ + "status": "success", + "email": email, + "token_type": "SessionToken", + "chunks": chunk_count, + "access_token_preview": access_token[:16] + "...", + }) + + +def _parse_session_cookie_input(raw: str) -> str: + """从用户粘贴的原始输入中提取 NextAuth session-token，按分片顺序拼接。 + + 支持 5 种粘贴姿势： + 1. 整段 document.cookie "_ga=x; __Secure-next-auth.session-token.0=v0; __Secure-next-auth.session-token.1=v1; ..." + 2. 只带 session 的 cookie 串 "__Secure-next-auth.session-token.0=v0; __Secure-next-auth.session-token.1=v1" + 3. 单片（老版） "__Secure-next-auth.session-token=value" + 4. 裸 value 分号拼接（F12 Application 逐个复制后拼） "value0;value1" + 5. 单个裸 value（未分片时） "eyJxxx.yyy.zzz..." + + 返回内部存储用的字符串： + - 单片：cookie 原值 + - 多片：chunk0|||chunk1|||chunk2... + """ + import re + from chatgpt.refreshToken import SESS_CHUNK_SEPARATOR + + txt = raw.strip() + # 去掉 "Cookie:" 前缀 + if txt.lower().startswith("cookie:"): + txt = txt[7:].strip() + + # 归一化全角标点（用户可能用输入法不小心敲成全角） + txt = txt.translate(str.maketrans({ + "\uff1b": ";", # ； 全角分号 + "\uff1d": "=", # ＝ 全角等号 + "\u3000": " ", # 　 全角空格 + "\uff1a": ":", # ： 全角冒号 + })) + + # 情况 1：整段只是一个裸值（无 ; 无 =） + if "=" not in txt and ";" not in txt: + return txt + + indexed_chunks = {} # {.N 下标: value}（带 name.N） + single_match = None # 带 name 的单片 + positional_chunks = [] # 没 name 的裸 value（按顺序收集） + + for part in re.split(r";\s*", txt): + part = part.strip() + if not part: + continue + + if "=" not in part: + # 没 name 的裸 value：只接受看起来像 JWE/base64url 的长片段 + # 避免把 cookie 噪音碎片误吞 + if len(part) >= 50 and re.match(r"^[A-Za-z0-9_\-\.=+/]+$", part): + positional_chunks.append(part) + continue + + key, _, value = part.partition("=") + key = key.strip() + value = value.strip() + # 带下标的 __Secure-next-auth.session-token.N + m = re.match(r"^__Secure-next-auth\.session-token\.(\d+)$", key) + if m: + indexed_chunks[int(m.group(1))] = value + continue + # 不带下标的单片（老版或非分片情况） + if key in ("__Secure-next-auth.session-token", + "__Host-next-auth.session-token", + "next-auth.session-token"): + single_match = value + + # 优先级：带 name 的下标 chunk > 带 name 的单片 > 裸 value 顺序拼接 > 整段兜底 + + if indexed_chunks: + ordered = [indexed_chunks[i] for i in sorted(indexed_chunks.keys())] + if len(ordered) == 1: + return ordered[0] + return SESS_CHUNK_SEPARATOR.join(ordered) + + if single_match: + return single_match + + # 姿势 4：裸 value 用分号拼接的情况 + if positional_chunks: + if len(positional_chunks) == 1: + return positional_chunks[0] + return SESS_CHUNK_SEPARATOR.join(positional_chunks) + + # 兜底：整段看起来是合法 JWE（长，字符集对） + if len(txt) >= 100 and re.match(r"^[A-Za-z0-9_\-\.=+/]+$", txt): + return txt + + return "" + + +async def _exchange_code_for_tokens(code: str, sess) -> dict: + """调用 OpenAI /oauth/token 换 refresh_token。 + + 使用 Codex CLI 风格：application/x-www-form-urlencoded。 + 端点从 oauth_session._get_oauth_config() 取（auth.openai.com/oauth/token）。 + """ + from urllib.parse import urlencode + from utils.Client import Client + from utils import oauth_session as _oauth + + client_id, redirect_uri, _audience, _scope, _auth_url, token_url = _oauth._get_oauth_config() + form = urlencode({ + "grant_type": "authorization_code", + "client_id": client_id, + "redirect_uri": redirect_uri, + "code": code, + "code_verifier": sess.verifier, + }) + headers = { + "Accept": "application/json", + "Content-Type": "application/x-www-form-urlencoded", + "User-Agent": "Codex_CLI/0.1.0", + } + + c = Client(impersonate=None) + try: + # curl_cffi 的 post 接受 data= 作为 form body + r = await c.post(token_url, data=form, headers=headers, timeout=20) + raw = (r.text or "").strip() + if r.status_code != 200: + raise RuntimeError(f"OpenAI token status={r.status_code} body={raw[:300]}") + import json as _json + payload = _json.loads(raw) + if "refresh_token" not in payload: + raise RuntimeError(f"响应缺 refresh_token: keys={list(payload.keys())}") + return payload + finally: + await c.close() + + +async def _harvester_import_rt(sess, refresh_token: str) -> None: + """把 rt 写入 chat2api 账号池（复用现有 import 流程的底层调用）。""" + from utils.routing import ( + get_routing_config, + update_account_meta, + ) + # 加到 globals.token_list + token.txt + if refresh_token not in globals.token_list: + globals.token_list.append(refresh_token) + with open(globals.TOKENS_FILE, "a", encoding="utf-8") as f: + f.write(refresh_token + "\n") + + # 绑定代理 / 备注 + proxy_url = "" + proxy_name = sess.proxy_name or "" + if proxy_name: + cfg = get_routing_config() + match = next( + (p for p in cfg.get("proxies", []) if p.get("name") == proxy_name), + None, + ) + if match: + proxy_url = match.get("proxy_url", "") or "" + else: + logger.warning( + f"[harvester-auth] proxy_name='{proxy_name}' 未找到，rt 已导入但不绑定代理" + ) + note = f"{sess.email}" + (f" · {sess.note}" if sess.note else "") + update_account_meta( + refresh_token, + note=note, + group_name=None, + proxy_name=proxy_name if proxy_url else None, + proxy_url=proxy_url if proxy_url else None, + ) + + +app.add_api_route("/admin/routing", routing_admin_page, methods=["GET"], response_class=HTMLResponse) +app.add_api_route("/admin/routing/data", routing_admin_data, methods=["GET"]) +app.add_api_route("/admin/routing/save", routing_admin_save, methods=["POST"]) +app.add_api_route("/admin/login", routing_admin_login_page, methods=["GET"], response_class=HTMLResponse) +app.add_api_route("/admin/login", routing_admin_login_submit, methods=["POST"]) +app.add_api_route("/admin/logout", routing_admin_logout, methods=["POST"]) +app.add_api_route("/admin/routing/account-bind", routing_admin_bind_account, methods=["POST"]) +app.add_api_route("/admin/routing/accounts/import", routing_admin_import_accounts, methods=["POST"]) +app.add_api_route("/admin/routing/accounts/parse-file", routing_admin_parse_file, methods=["POST"]) +app.add_api_route("/admin/routing/accounts/delete", routing_admin_delete_account, methods=["POST"]) +app.add_api_route("/admin/routing/accounts/refresh", routing_admin_refresh_account, methods=["POST"]) +app.add_api_route("/admin/routing/accounts/refresh-all", routing_admin_refresh_all_accounts, methods=["POST"]) +app.add_api_route("/admin/routing/test-proxy", routing_admin_test_proxy, methods=["POST"]) +app.add_api_route("/admin/logs/tail", routing_admin_logs_tail, methods=["GET"]) +app.add_api_route("/admin/logs/download", routing_admin_logs_download, methods=["GET"]) +app.add_api_route("/admin/harvester/accounts", routing_admin_harvester_list, methods=["GET"]) +app.add_api_route("/admin/harvester/accounts", routing_admin_harvester_upsert, methods=["POST"]) +app.add_api_route("/admin/harvester/accounts/delete", routing_admin_harvester_delete, methods=["POST"]) +app.add_api_route("/admin/harvester/accounts/bulk-import", routing_admin_harvester_bulk_import, methods=["POST"]) +app.add_api_route("/admin/harvester/report", routing_admin_harvester_report, methods=["POST"]) +app.add_api_route("/admin/harvester/authorize/start", routing_admin_harvester_authorize_start, methods=["POST"]) +app.add_api_route("/admin/harvester/authorize/exchange", routing_admin_harvester_authorize_exchange, methods=["POST"]) +app.add_api_route("/admin/harvester/import-cookie", routing_admin_harvester_import_cookie, methods=["POST"]) + +if api_prefix: + app.add_api_route(f"/{api_prefix}/admin/routing", routing_admin_page, methods=["GET"], response_class=HTMLResponse) + app.add_api_route(f"/{api_prefix}/admin/routing/data", routing_admin_data, methods=["GET"]) + app.add_api_route(f"/{api_prefix}/admin/routing/save", routing_admin_save, methods=["POST"]) + app.add_api_route(f"/{api_prefix}/admin/login", routing_admin_login_page, methods=["GET"], response_class=HTMLResponse) + app.add_api_route(f"/{api_prefix}/admin/login", routing_admin_login_submit, methods=["POST"]) + app.add_api_route(f"/{api_prefix}/admin/logout", routing_admin_logout, methods=["POST"]) + app.add_api_route(f"/{api_prefix}/admin/routing/account-bind", routing_admin_bind_account, methods=["POST"]) + app.add_api_route(f"/{api_prefix}/admin/routing/accounts/import", routing_admin_import_accounts, methods=["POST"]) + app.add_api_route(f"/{api_prefix}/admin/routing/accounts/parse-file", routing_admin_parse_file, methods=["POST"]) + app.add_api_route(f"/{api_prefix}/admin/routing/accounts/delete", routing_admin_delete_account, methods=["POST"]) + app.add_api_route(f"/{api_prefix}/admin/routing/accounts/refresh", routing_admin_refresh_account, methods=["POST"]) + app.add_api_route(f"/{api_prefix}/admin/routing/accounts/refresh-all", routing_admin_refresh_all_accounts, methods=["POST"]) + app.add_api_route(f"/{api_prefix}/admin/routing/test-proxy", routing_admin_test_proxy, methods=["POST"]) + app.add_api_route(f"/{api_prefix}/admin/logs/tail", routing_admin_logs_tail, methods=["GET"]) + app.add_api_route(f"/{api_prefix}/admin/logs/download", routing_admin_logs_download, methods=["GET"]) + app.add_api_route(f"/{api_prefix}/admin/harvester/accounts", routing_admin_harvester_list, methods=["GET"]) + app.add_api_route(f"/{api_prefix}/admin/harvester/accounts", routing_admin_harvester_upsert, methods=["POST"]) + app.add_api_route(f"/{api_prefix}/admin/harvester/accounts/delete", routing_admin_harvester_delete, methods=["POST"]) + app.add_api_route(f"/{api_prefix}/admin/harvester/accounts/bulk-import", routing_admin_harvester_bulk_import, methods=["POST"]) + app.add_api_route(f"/{api_prefix}/admin/harvester/report", routing_admin_harvester_report, methods=["POST"]) + app.add_api_route(f"/{api_prefix}/admin/harvester/authorize/start", routing_admin_harvester_authorize_start, methods=["POST"]) + app.add_api_route(f"/{api_prefix}/admin/harvester/authorize/exchange", routing_admin_harvester_authorize_exchange, methods=["POST"]) + app.add_api_route(f"/{api_prefix}/admin/harvester/import-cookie", routing_admin_harvester_import_cookie, methods=["POST"]) diff --git a/gateway/backend.py b/gateway/backend.py index ee9e5fad..1a4b037b 100644 --- a/gateway/backend.py +++ b/gateway/backend.py @@ -40,11 +40,15 @@ chatgpt_paths = ["c/[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"] +def has_direct_access_token(token: str) -> bool: + return len(token) == 45 or token.startswith("eyJhbGciOi") + + @app.get("/backend-api/accounts/check/v4-2023-04-27") async def check_account(request: Request): - token = request.headers.get("Authorization").replace("Bearer ", "") + token = request.headers.get("Authorization", "").replace("Bearer ", "") check_account_response = await chatgpt_reverse_proxy(request, "backend-api/accounts/check/v4-2023-04-27") - if len(token) == 45 or token.startswith("eyJhbGciOi"): + if has_direct_access_token(token): return check_account_response else: check_account_str = check_account_response.body.decode('utf-8') @@ -62,7 +66,7 @@ async def check_account(request: Request): @app.get("/backend-api/gizmos/bootstrap") async def get_gizmos_bootstrap(request: Request): token = request.headers.get("Authorization", "").replace("Bearer ", "") - if len(token) == 45 or token.startswith("eyJhbGciOi"): + if has_direct_access_token(token): return await chatgpt_reverse_proxy(request, "backend-api/gizmos/bootstrap") else: return {"gizmos": []} @@ -71,7 +75,7 @@ async def get_gizmos_bootstrap(request: Request): @app.get("/backend-api/gizmos/pinned") async def get_gizmos_pinned(request: Request): token = request.headers.get("Authorization", "").replace("Bearer ", "") - if len(token) == 45 or token.startswith("eyJhbGciOi"): + if has_direct_access_token(token): return await chatgpt_reverse_proxy(request, "backend-api/gizmos/pinned") else: return {"items": [], "cursor": None} @@ -80,7 +84,7 @@ async def get_gizmos_pinned(request: Request): @app.get("/public-api/gizmos/discovery/recent") async def get_gizmos_discovery_recent(request: Request): token = request.headers.get("Authorization", "").replace("Bearer ", "") - if len(token) == 45 or token.startswith("eyJhbGciOi"): + if has_direct_access_token(token): return await chatgpt_reverse_proxy(request, "public-api/gizmos/discovery/recent") else: return { @@ -98,7 +102,7 @@ async def get_gizmos_discovery_recent(request: Request): @app.get("/backend-api/gizmos/snorlax/sidebar") async def get_gizmos_snorlax_sidebar(request: Request): token = request.headers.get("Authorization", "").replace("Bearer ", "") - if len(token) == 45 or token.startswith: + if has_direct_access_token(token): return await chatgpt_reverse_proxy(request, "backend-api/gizmos/snorlax/sidebar") else: return {"items": [], "cursor": None} @@ -107,7 +111,7 @@ async def get_gizmos_snorlax_sidebar(request: Request): @app.post("/backend-api/gizmos/snorlax/upsert") async def get_gizmos_snorlax_upsert(request: Request): token = request.headers.get("Authorization", "").replace("Bearer ", "") - if len(token) == 45 or token.startswith: + if has_direct_access_token(token): return await chatgpt_reverse_proxy(request, "backend-api/gizmos/snorlax/upsert") else: raise HTTPException(status_code=403, detail="Forbidden") @@ -673,11 +677,19 @@ async def c_close(client, clients): @app.api_route("/{path:path}", methods=["GET", "POST", "PUT", "DELETE", "OPTIONS", "HEAD", "PATCH", "TRACE"]) async def reverse_proxy(request: Request, path: str): token = request.headers.get("Authorization", "").replace("Bearer ", "") + normalized_path = "/" + path.lstrip("/") if len(token) != 45 and not token.startswith("eyJhbGciOi"): for banned_path in banned_paths: if re.match(banned_path, path): raise HTTPException(status_code=403, detail="Forbidden") + # 如果后台路径没有命中显式 admin 路由，说明 nginx / API_PREFIX 映射大概率错了。 + if "/admin/" in normalized_path or normalized_path.endswith("/admin"): + raise HTTPException( + status_code=404, + detail="Admin route mismatch: check API_PREFIX and nginx slug mapping.", + ) + for chatgpt_path in chatgpt_paths: if re.match(chatgpt_path, path): return await chatgpt_html(request) diff --git a/gateway/gpts.py b/gateway/gpts.py index 25f93d24..ec90e24a 100644 --- a/gateway/gpts.py +++ b/gateway/gpts.py @@ -2,10 +2,11 @@ from urllib.parse import quote from fastapi import Request -from fastapi.responses import Response +from fastapi.responses import RedirectResponse, Response from app import app from gateway.chatgpt import chatgpt_html +from gateway.login import login_html from utils.kv_utils import set_value_for_key_list with open("templates/gpts_context.json", "r", encoding="utf-8") as f: @@ -28,7 +29,11 @@ async def get_gpts(request: Request): async def get_gizmo_json(request: Request, gizmo_id: str): params = request.query_params if params.get("_routes") == "routes/g.$gizmoId._index": - token = request.cookies.get("token") + # 安全：未登录访问应返回登录页而非 500。 + # 原实现直接 len(token)/token.startswith(...) 对 None 调用会抛 TypeError。 + token = request.cookies.get("token") or "" + if not token: + return await login_html(request) if len(token) != 45 and not token.startswith("eyJhbGciOi"): token = quote(token) user_gpts_context = gpts_context.copy() diff --git a/gateway/reverseProxy.py b/gateway/reverseProxy.py index 510640e3..4a69b9dd 100644 --- a/gateway/reverseProxy.py +++ b/gateway/reverseProxy.py @@ -13,7 +13,7 @@ from chatgpt.fp import get_fp from utils.Client import Client from utils.Logger import logger -from utils.configs import chatgpt_base_url_list, sentinel_proxy_url_list, force_no_history, file_host, voice_host +from utils.configs import chatgpt_base_url_list, sentinel_proxy_url_list, force_no_history, file_host, voice_host, accept_language def generate_current_time(): @@ -220,7 +220,7 @@ async def chatgpt_reverse_proxy(request: Request, path: str): headers.update(fp) headers.update({ - "accept-language": "en-US,en;q=0.9", + "accept-language": accept_language, "host": base_url.replace("https://", "").replace("http://", ""), "origin": base_url, "referer": f"{base_url}/" @@ -323,8 +323,13 @@ async def chatgpt_reverse_proxy(request: Request, path: str): response = Response(content=content, headers=rheaders, status_code=r.status_code, background=background) return response + except HTTPException as e: + await client.close() + raise HTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: await client.close() + logger.error(f"Reverse proxy failed for {path}: {str(e)}") + raise HTTPException(status_code=502, detail="Upstream request failed") except HTTPException as e: raise e except Exception as e: diff --git a/gateway/share.py b/gateway/share.py index aaf910ed..e6aeeae6 100644 --- a/gateway/share.py +++ b/gateway/share.py @@ -15,14 +15,15 @@ from gateway.reverseProxy import get_real_req_token from utils.Client import Client from utils.Logger import logger -from utils.configs import proxy_url_list, chatgpt_base_url_list, authorization_list +from utils.configs import proxy_url_list, chatgpt_base_url_list, authorization_list, accept_language, oai_language +from utils.routing import get_bound_proxy base_headers = { 'accept': '*/*', 'accept-encoding': 'gzip, deflate, br, zstd', - 'accept-language': 'en-US,en;q=0.9', + 'accept-language': accept_language, 'content-type': 'application/json', - 'oai-language': 'en-US', + 'oai-language': oai_language, 'priority': 'u=1, i', 'sec-fetch-dest': 'empty', 'sec-fetch-mode': 'cors', @@ -138,7 +139,7 @@ async def chatgpt_account_check(access_token): headers.update({"authorization": f"Bearer {access_token}"}) session_id = hashlib.md5(access_token.encode()).hexdigest() - proxy_url = random.choice(proxy_url_list).replace("{}", session_id) if proxy_url_list else None + proxy_url = proxy_url.replace("{}", session_id) if proxy_url else None client = Client(proxy=proxy_url, impersonate=impersonate) r = await client.get(f"{host_url}/backend-api/models?history_and_training_disabled=false", headers=headers, timeout=10) @@ -186,7 +187,10 @@ async def chatgpt_account_check(access_token): async def chatgpt_refresh(refresh_token): session_id = hashlib.md5(refresh_token.encode()).hexdigest() - proxy_url = random.choice(proxy_url_list).replace("{}", session_id) if proxy_url_list else None + bound_proxy = get_bound_proxy(refresh_token) + proxy_url = bound_proxy or (random.choice(proxy_url_list).replace("{}", session_id) if proxy_url_list else None) + if proxy_url: + proxy_url = proxy_url.replace("{}", session_id) client = Client(proxy=proxy_url) try: data = { @@ -252,5 +256,3 @@ async def refresh(request: Request): return Response(content=json.dumps(auth_info), media_type="application/json") raise HTTPException(status_code=401, detail="Unauthorized") - - diff --git a/handoff.md b/handoff.md new file mode 100644 index 00000000..cf30132e --- /dev/null +++ b/handoff.md @@ -0,0 +1,95 @@ +# 协作交接 + +最后更新：2026-04-17 +负责人：主调度线 + +## 使用规则 + +- 每条工作线开始前先读 `memory.md` 与本文件。 +- 任何工作线只修改自己认领的文件；跨线修改前先在本文件登记。 +- 不记录真实 Token、Cookie、代理账号、私有域名或任何敏感信息。 +- 优先做稳定性、兼容性、可观测性、资源释放、退避、缓存、限流与错误语义收敛。 + +## 当前工作线 + +### 线路 A：模型解析与模型校验 + +- 负责人：现有工作线 +- 目标：收敛模型解析、缓存上游模型列表、统一 `model_not_found` 语义 +- 主要文件：`api/models.py`、`chatgpt/ChatService.py` +- 进入条件：修改前先确认 `/v1/models` 兼容面不会被破坏 +- 交付标准： + - 未登录默认模型路径稳定 + - `gizmo` 路径不回归 + - 未知模型名、`CHECK_MODEL=true` 行为可解释 + +### 线路 B：OpenAI 兼容 API + +- 负责人：现有工作线 +- 目标：收敛 `/v1/models`、`/v1/chat/completions` 的错误处理与响应形态 +- 主要文件：`api/chat2api.py` +- 依赖：线路 A +- 交付标准： + - 资源释放明确 + - `JSONResponse` / `StreamingResponse` 路径一致 + - `/v1/models` 返回结构固定 + +### 线路 C：网关反代稳定性 + +- 负责人：现有工作线 +- 目标：统一上游异常到网关错误语义，补足可观测性 +- 主要文件：`gateway/reverseProxy.py`、`gateway/backend.py` +- 依赖：线路 D +- 交付标准： + - 上游失败时错误码稳定 + - 客户端总能关闭 + - 日志能定位到路径与失败类型 + +### 线路 D：重试与低风险运行 + +- 负责人：现有工作线 +- 目标：控制重试范围、加入退避、避免雪崩式重复请求 +- 主要文件：`utils/retry.py`、`chatgpt/chatLimit.py` +- 依赖：线路 B、线路 C +- 交付标准： + - 只对可重试错误重试 + - 退避参数可配置 + - 不放大 4xx 语义错误 + +### 线路 E：主调度线新增 + +- 负责人：主调度线 +- 目标：统一任务优先级、合并交付、补充共享记忆、做安全边界把控 +- 主要文件：`memory.md`、`handoff.md` +- 交付标准： + - 每条线边界清晰 + - 交付顺序明确 + - 风险项被记录 + +## 当前优先级 + +1. 先稳定 A + B：确保模型解析与 `/v1/models` 不互相打架 +2. 再稳定 C + D：确保 502/超时/瞬时 5xx 不触发雪崩重试 +3. 最后处理 E：把回归结果和遗留风险回写到 `memory.md` + +## 主调度线判定标准 + +- 允许推进： + - 缓存、队列、限流、退避、熔断、日志、指标、错误码收敛 + - 兼容性修复、资源释放、重复逻辑收敛 + - 文档、配置、部署与运行时保护 + +## 回归清单 + +- A/B 联合： + - `/v1/chat/completions` + - `/v1/models` + - 未登录默认模型 + - `gizmo` 模型 + - 未知模型名 +- C/D 联合： + - 上游超时 + - 上游 502 + - 上游瞬时 500/503 + - 非可重试 4xx + - 客户端关闭与日志输出 diff --git a/harvester/.env.example b/harvester/.env.example new file mode 100644 index 00000000..22070ade --- /dev/null +++ b/harvester/.env.example @@ -0,0 +1,21 @@ +# ============ chat2api 后端（必填）============ +CHAT2API_BASE_URL=http://localhost:60403 +CHAT2API_API_PREFIX=nanapi-2026-a1 +CHAT2API_ADMIN_PASSWORD=your_admin_password + +# ============ 行为（可选）============ +HEADLESS=false +PAUSE_FOR_ARKOSE_SECONDS=90 +TIMEOUT_PER_ACCOUNT_SECONDS=180 +PARALLEL=1 +RETRY_ON_FAIL=1 +INTERVAL_BETWEEN_ACCOUNTS_SECONDS=30 + +# ============ OAuth 参数（一般不需要改）============ +OAUTH_CLIENT_ID=pdlLIX2Y72MIl2rhLhTE9VV9bN905kBh +OAUTH_REDIRECT_URI=com.openai.chat://auth0.openai.com/ios/com.openai.chat/callback +OAUTH_AUDIENCE=https://api.openai.com/v1 +OAUTH_SCOPE=openid email profile offline_access model.request model.read organization.read organization.write + +# ============ 可选：Playwright 走代理 ============ +# PLAYWRIGHT_PROXY=socks5://127.0.0.1:7890 diff --git a/harvester/.gitignore b/harvester/.gitignore new file mode 100644 index 00000000..b16cf3ee --- /dev/null +++ b/harvester/.gitignore @@ -0,0 +1,29 @@ +# Runtime data +profiles/ +state/ +logs/ +failed_imports.json +tokens.json + +# Secrets - 禁止入库 +.env +accounts.csv + +# Python +__pycache__/ +*.pyc +*.pyo +*.egg-info/ +.pytest_cache/ +.venv/ +venv/ +env/ + +# OS +.DS_Store +Thumbs.db + +# Editor +.vscode/ +.idea/ +*.swp diff --git a/harvester/README.md b/harvester/README.md new file mode 100644 index 00000000..99e20f4c --- /dev/null +++ b/harvester/README.md @@ -0,0 +1,222 @@ +# Chat2API Harvester + +> ⚠️ **DEPRECATED（已废弃，保留供参考）** +> +> 本 Playwright CLI 工具已被 chat2api 管理后台内置的"**🌐 浏览器登录**"功能取代。 +> +> 新方案优势： +> - 不用在本地装 Playwright / chromium（省 250MB） +> - 不用维护 `accounts.csv` 密码文件 +> - 不用本地 venv / Python 环境 +> - Arkose 挑战由真实浏览器处理，触发概率更低 +> - 完全 UI 驱动，一个账号 2 次粘贴完成 +> +> **迁移方法**： +> 1. 打开 `http://你的服务器:60403/{API_PREFIX}/admin/routing` +> 2. 左侧 → "账号采集 Harvester" +> 3. 点某账号行的 **[🌐 浏览器登录]** 按钮，按向导 2 次粘贴 URL 即可 +> +> 本目录代码仅作为历史参考保留。**无需运行本目录任何命令**。 +> +> 确认新方案无问题后可以直接 `rm -rf harvester/` 整个目录。 + +--- + +## （以下为原文档，仅供参考） + +> 自家 ChatGPT 账号池 → RefreshToken 自动采集 → 写入 chat2api + +本地 Python + Playwright 工具，用 iOS ChatGPT app 的 OAuth2 PKCE 流程，从自家账号登录并拿到 `rt_*` RefreshToken，自动通过管理后台 API 写入 chat2api 账号池。**一次人工辅助跑完，后续 2-3 个月由 chat2api 内建 `SCHEDULED_REFRESH` 接管刷新**。 + +--- + +## 适用场景 + +- 你有 20+ 个**自家** Team/Plus 账号（email + password） +- 希望彻底摆脱共享渠道 token 频繁失效 +- 能接受"首次约 30 分钟人工辅助登录，之后全自动"的节奏 + +--- + +## 前置要求 + +- Python 3.9+（Mac 系统自带 `python3` 即可） +- chat2api 已在本机或局域网运行（管理后台可通） +- chat2api 的 `ADMIN_PASSWORD` 已配置 +- 账号凭据：email + password（混合 2FA 可以） + +--- + +## 安装 + +```bash +cd /Users/nanashiwang/Documents/Projects/chat2api/harvester + +# 创建虚拟环境（推荐） +python3 -m venv .venv +source .venv/bin/activate + +# 安装依赖 +pip install -r requirements.txt + +# 安装 Playwright chromium（首次约 250MB，只需一次） +playwright install chromium +``` + +--- + +## 配置 + +### 1. `.env` + +```bash +cp .env.example .env +``` + +编辑 `.env`，至少填： + +```ini +CHAT2API_BASE_URL=http://localhost:60403 +CHAT2API_API_PREFIX=nanapi-2026-a1 +CHAT2API_ADMIN_PASSWORD=与docker-compose里一致 +``` + +其他参数（超时、并发、OAuth client_id）一般不需要动。 + +### 2. `accounts.csv` + +```bash +cp accounts.csv.example accounts.csv +``` + +编辑 `accounts.csv`，字段： + +| 列名 | 必填 | 说明 | +|---|---|---| +| `email` | ✓ | 登录邮箱 | +| `password` | ✓ | 登录密码（含特殊字符用双引号包裹） | +| `totp_secret` | | Google Authenticator 的 base32 secret，有 2FA 时填 | +| `note` | | 导入 chat2api 后的备注，如 "Team-01 HK" | +| `proxy_name` | | chat2api 里已存在的代理名，会自动绑定 | + +示例： +```csv +email,password,totp_secret,note,proxy_name +alice@example.com,Pwd@12345,,Team-01 HK,HK-Resi-01 +bob@example.com,"Pwd,with,comma",JBSWY3DPEHPK3PXP,Team-02 JP,JP-Resi-02 +``` + +**⚠️ `.env` 和 `accounts.csv` 已在 `.gitignore` 中，不会被提交。** + +--- + +## 运行 + +### 全量（跳过已成功） + +```bash +python -m src.harvest +``` + +浏览器会弹出，脚本自动填邮箱+密码+TOTP。若遇到 **Arkose 人机验证**，你需要手工点一下（脚本等 90s）。 + +首次建议： + +```bash +# 只跑一个测试账号 +python -m src.harvest --only alice@example.com +``` + +跑通之后再全量。 + +### 按需运行 + +```bash +python -m src.harvest --only a@b.com,c@d.com # 指定账号 +python -m src.harvest --failed # 只重试上次失败 +python -m src.harvest --force # 忽略 state，全量重跑 +python -m src.harvest --export tokens.json # 不调 chat2api，导出完整 rt 到 JSON +``` + +### 什么时候跑？ + +- **首次部署**：跑 1 次全量 +- **rt 寿命到期**（约 60-90 天后 chat2api 日志会报大量 `refresh_token_expired`）：跑 1 次 `--failed` +- **新增账号**：把新账号追加到 `accounts.csv`，跑 `--only 新邮箱` + +--- + +## 工作原理 + +``` +1. PKCE 生成 code_verifier + code_challenge +2. Playwright 打开 https://auth0.openai.com/authorize?client_id= +3. 自动填 email → password → TOTP（如果有） +4. Arkose 挑战时暂停 90s 等人工点击 +5. 登录成功 → Auth0 重定向到 com.openai.chat:// 回调 +6. 从 URL 抽取 authorization code +7. POST /oauth/token 用 code + verifier 换 refresh_token +8. 立即调用 chat2api /admin/routing/accounts/import 写入 +``` + +- 每账号一个独立 Playwright profile（`profiles//`）。第二次登录同账号可能因为 cookie 保留而免过 Arkose。 +- rt 完整值只在内存里短暂出现（oauth_flow.harvest_one），`on_success` 回调立即写入 chat2api 后即被释放。 +- `state/.json` 记录最近一次成功/失败时间，重启脚本会按 `--force` 选项决定是否跳过。 + +--- + +## 故障排查 + +| 症状 | 可能原因 | 处理 | +|---|---|---| +| `chat2api 鉴权失败` | `.env` 里 ADMIN_PASSWORD 错 | 对照 chat2api 的 `docker-compose.yml` | +| `chat2api 后台已禁用` | chat2api 未配 ADMIN_PASSWORD | 先修好 chat2api | +| `chat2api 不可达` | 端口/地址错 | 确认 `CHAT2API_BASE_URL` 能 curl 通 | +| 浏览器打开后长时间卡住 | Arkose 未过 / 页面改版 | headful 观察；若选择器失效联系维护 | +| 某账号一直密码错 | 真的错了 / 被锁定 | 检查是否需要解锁（登录网页一次） | +| Playwright 装不上 chromium | 网络问题 | 配 `HTTPS_PROXY` 后重跑 `playwright install` | + +查日志：`logs/harvester.log`（密码和 rt 已自动脱敏）。 + +--- + +## 安全提醒 + +- `accounts.csv` 里是你账号的**原文密码**——请： + - 不要放在会同步到云盘的目录 + - 不要 commit 到任何 git（`.gitignore` 已防护） + - 跑完可选 `rm accounts.csv` 只保留 state +- Playwright profiles 含 cookie/session，`chmod 700 profiles/` 限制权限 +- 日志文件做了 rt/JWT 截断，但别分享整个 `logs/` 给他人 + +--- + +## 目录结构 + +``` +harvester/ +├── .env # 你的配置（gitignored） +├── .env.example +├── accounts.csv # 你的账号（gitignored） +├── accounts.csv.example +├── requirements.txt +├── README.md +├── src/ +│ ├── harvest.py # CLI 入口 +│ ├── config.py # .env + CSV 解析 +│ ├── models.py # Account / TokenSet / HarvestResult +│ ├── oauth_flow.py # PKCE + Playwright + token 交换 +│ ├── totp.py # 2FA +│ ├── chat2api_client.py +│ ├── cache.py # per-account state +│ └── log_setup.py +├── profiles/ # Playwright persistent contexts (runtime) +├── state/ # per-account 状态 JSON (runtime) +└── logs/ # 执行日志 (runtime) +``` + +--- + +## License + +MIT（与主项目一致） diff --git a/harvester/accounts.csv.example b/harvester/accounts.csv.example new file mode 100644 index 00000000..6a0d3bdb --- /dev/null +++ b/harvester/accounts.csv.example @@ -0,0 +1,4 @@ +email,password,totp_secret,note,proxy_name +alice@example.com,Pwd@12345,,Team-01 HK,HK-Resi-01 +bob@example.com,"Pwd,with,comma",JBSWY3DPEHPK3PXP,Team-02 JP,JP-Resi-02 +charlie@example.com,Pwd@abcde,,, diff --git a/harvester/requirements.txt b/harvester/requirements.txt new file mode 100644 index 00000000..2b7232fa --- /dev/null +++ b/harvester/requirements.txt @@ -0,0 +1,5 @@ +playwright==1.47.0 +pyotp==2.9.0 +httpx==0.27.0 +click==8.1.7 +python-dotenv==1.0.1 diff --git a/harvester/src/__init__.py b/harvester/src/__init__.py new file mode 100644 index 00000000..1470b6db --- /dev/null +++ b/harvester/src/__init__.py @@ -0,0 +1,3 @@ +"""harvester: 自家账号池 RefreshToken 自动采集工具。""" + +__version__ = "0.1.0" diff --git a/harvester/src/cache.py b/harvester/src/cache.py new file mode 100644 index 00000000..10b182ae --- /dev/null +++ b/harvester/src/cache.py @@ -0,0 +1,84 @@ +"""Per-account 持久化状态：成功/失败/重试计数。""" + +import hashlib +import json +import time +from pathlib import Path +from typing import Dict, Optional + + +def _email_hash(email: str) -> str: + return hashlib.sha256(email.encode("utf-8")).hexdigest()[:16] + + +class StateStore: + """每账号一个 json 文件，记录最近一次成功/失败信息。""" + + def __init__(self, state_dir: Path): + self.state_dir = state_dir + self.state_dir.mkdir(parents=True, exist_ok=True) + + def _path(self, email: str) -> Path: + return self.state_dir / f"{_email_hash(email)}.json" + + def get(self, email: str) -> Dict: + p = self._path(email) + if not p.exists(): + return {} + try: + return json.loads(p.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {} + + def is_recently_success(self, email: str, within_seconds: int = 7 * 24 * 3600) -> bool: + rec = self.get(email) + last = rec.get("last_success_at", 0) + return bool(last and (time.time() - last) < within_seconds) + + def mark_success(self, email: str, rt_prefix: str, imported: bool) -> None: + now = int(time.time()) + rec = self.get(email) + rec.update({ + "email": email, + "last_success_at": now, + "last_attempt_at": now, + "last_rt_prefix": rt_prefix, + "imported": imported, + "last_error": "", + "fail_count": 0, + "banned": False, + }) + self._write(email, rec) + + def mark_failure(self, email: str, error: str, banned: bool = False) -> None: + now = int(time.time()) + rec = self.get(email) + rec.update({ + "email": email, + "last_attempt_at": now, + "last_error": error[:300], + "last_error_at": now, + "fail_count": int(rec.get("fail_count", 0)) + 1, + "banned": banned or rec.get("banned", False), + }) + self._write(email, rec) + + def list_failed(self) -> list: + emails = [] + for p in self.state_dir.glob("*.json"): + try: + rec = json.loads(p.read_text(encoding="utf-8")) + except json.JSONDecodeError: + continue + if not rec.get("last_success_at") and rec.get("email"): + emails.append(rec["email"]) + return emails + + def is_banned(self, email: str) -> bool: + return bool(self.get(email).get("banned")) + + def _write(self, email: str, data: Dict) -> None: + self._path(email).write_text( + json.dumps(data, indent=2, ensure_ascii=False), + encoding="utf-8", + ) diff --git a/harvester/src/chat2api_client.py b/harvester/src/chat2api_client.py new file mode 100644 index 00000000..3a29d323 --- /dev/null +++ b/harvester/src/chat2api_client.py @@ -0,0 +1,134 @@ +"""chat2api 管理后台 API 客户端。 + +鉴权：直接带 Cookie `admin_auth=`（chat2api 的 cookie 值就是密码本身， +见 gateway/admin.py::routing_admin_login_submit）。省去登录表单提交一步。 +""" + +import logging +from typing import Dict, List, Optional + +import httpx + + +logger = logging.getLogger("harvester") + + +class Chat2ApiClient: + def __init__(self, base_url: str, api_prefix: str, admin_password: str): + self.base = base_url.rstrip("/") + self.prefix = f"/{api_prefix}" if api_prefix else "" + self._cookie_header = f"admin_auth={admin_password}" + + def _url(self, path: str) -> str: + return f"{self.base}{self.prefix}{path}" + + async def healthcheck(self) -> bool: + """调 /admin/routing/data 探测：鉴权 + 可达双重验证。""" + url = self._url("/admin/routing/data") + try: + async with httpx.AsyncClient(timeout=10) as c: + r = await c.get(url, headers={"Cookie": self._cookie_header}) + if r.status_code == 401: + logger.error("chat2api 鉴权失败：请检查 CHAT2API_ADMIN_PASSWORD") + return False + if r.status_code == 503: + logger.error( + "chat2api 后台已禁用：请在 docker-compose 里配置 ADMIN_PASSWORD" + ) + return False + r.raise_for_status() + return True + except httpx.RequestError as e: + logger.error(f"chat2api 不可达: {e}") + return False + + async def list_proxies(self) -> List[Dict]: + """返回 [{'name': 'HK-Resi-01', 'proxy_url': '...'}]。""" + url = self._url("/admin/routing/data") + async with httpx.AsyncClient(timeout=15) as c: + r = await c.get(url, headers={"Cookie": self._cookie_header}) + r.raise_for_status() + return r.json().get("proxy_options", []) + + async def resolve_proxy(self, proxy_name: str) -> Optional[str]: + """根据 name 反查 proxy_url；找不到返回 None。""" + if not proxy_name: + return None + for item in await self.list_proxies(): + if item.get("name") == proxy_name: + return item.get("proxy_url") + return None + + async def import_token( + self, + refresh_token: str, + note: str = "", + proxy_name: str = "", + proxy_url: str = "", + group_name: str = "", + ) -> Dict: + """把单个 rt 写入 chat2api 账号池。 + + 注意：这里默认 overwrite_existing=False，重复 rt 会被 chat2api 静默跳过。 + """ + url = self._url("/admin/routing/accounts/import") + payload = { + "text": refresh_token, + "note": note, + "group_name": group_name, + "proxy_name": proxy_name, + "proxy_url": proxy_url, + "overwrite_existing": False, + } + async with httpx.AsyncClient(timeout=30) as c: + r = await c.post( + url, + json=payload, + headers={ + "Cookie": self._cookie_header, + "Content-Type": "application/json", + }, + ) + if r.status_code == 401: + raise RuntimeError("chat2api 鉴权失败（ADMIN_PASSWORD 不对）") + if r.status_code == 503: + raise RuntimeError("chat2api 后台未开启（未配置 ADMIN_PASSWORD）") + r.raise_for_status() + return r.json() + + async def report_harvest( + self, + email: str, + rt_prefix: str = "", + success: bool = True, + error: str = "", + imported_token: str = "", + ) -> None: + """把采集结果上报给 chat2api，更新 Harvester 看板的 last_rt_prefix / 状态。 + + 失败不抛异常，避免因元数据回传问题污染主流程。 + """ + url = self._url("/admin/harvester/report") + payload = { + "email": email, + "success": success, + "rt_prefix": rt_prefix, + "error": error, + "imported_token": imported_token, + } + try: + async with httpx.AsyncClient(timeout=10) as c: + r = await c.post( + url, + json=payload, + headers={ + "Cookie": self._cookie_header, + "Content-Type": "application/json", + }, + ) + if r.status_code != 200: + logger.warning( + f"report_harvest 返回 {r.status_code}: {r.text[:200]}" + ) + except Exception as e: + logger.warning(f"report_harvest 失败（忽略）: {e}") diff --git a/harvester/src/config.py b/harvester/src/config.py new file mode 100644 index 00000000..a35ee268 --- /dev/null +++ b/harvester/src/config.py @@ -0,0 +1,136 @@ +"""配置加载：.env + accounts.csv。""" + +import csv +import os +from dataclasses import dataclass, field +from pathlib import Path +from typing import List, Optional + +from dotenv import load_dotenv + +from .models import Account + + +def _here() -> Path: + return Path(__file__).resolve().parent.parent + + +@dataclass +class Config: + # chat2api + chat2api_base_url: str + chat2api_api_prefix: str + chat2api_admin_password: str + + # Playwright / 行为 + headless: bool = False + pause_for_arkose_seconds: int = 90 + timeout_per_account_seconds: int = 180 + parallel: int = 1 + retry_on_fail: int = 1 + interval_between_accounts_seconds: int = 30 + playwright_proxy: Optional[str] = None + + # OAuth + oauth_client_id: str = "pdlLIX2Y72MIl2rhLhTE9VV9bN905kBh" + oauth_redirect_uri: str = "com.openai.chat://auth0.openai.com/ios/com.openai.chat/callback" + oauth_audience: str = "https://api.openai.com/v1" + oauth_scope: str = ( + "openai email profile offline_access model.request model.read " + "organization.read organization.write" + ) + + # 路径 + root_dir: Path = field(default_factory=_here) + + @property + def profiles_dir(self) -> Path: + return self.root_dir / "profiles" + + @property + def state_dir(self) -> Path: + return self.root_dir / "state" + + @property + def logs_dir(self) -> Path: + return self.root_dir / "logs" + + def ensure_dirs(self) -> None: + for d in (self.profiles_dir, self.state_dir, self.logs_dir): + d.mkdir(parents=True, exist_ok=True) + + +def _is_true(val: str) -> bool: + return (val or "").strip().lower() in ("true", "1", "yes", "y", "t") + + +def load_config(env_path: Optional[Path] = None) -> Config: + root = _here() + load_dotenv(dotenv_path=env_path or (root / ".env")) + + base_url = os.getenv("CHAT2API_BASE_URL", "").strip() + api_prefix = os.getenv("CHAT2API_API_PREFIX", "").strip() + admin_pwd = os.getenv("CHAT2API_ADMIN_PASSWORD", "").strip() + + if not base_url or not admin_pwd: + raise RuntimeError( + "缺少必要环境变量：请在 harvester/.env 里配置 " + "CHAT2API_BASE_URL + CHAT2API_ADMIN_PASSWORD。" + ) + + scope_default = ( + "openid email profile offline_access model.request model.read " + "organization.read organization.write" + ) + + return Config( + chat2api_base_url=base_url, + chat2api_api_prefix=api_prefix, + chat2api_admin_password=admin_pwd, + headless=_is_true(os.getenv("HEADLESS", "false")), + pause_for_arkose_seconds=int(os.getenv("PAUSE_FOR_ARKOSE_SECONDS", "90")), + timeout_per_account_seconds=int(os.getenv("TIMEOUT_PER_ACCOUNT_SECONDS", "180")), + parallel=int(os.getenv("PARALLEL", "1")), + retry_on_fail=int(os.getenv("RETRY_ON_FAIL", "1")), + interval_between_accounts_seconds=int(os.getenv("INTERVAL_BETWEEN_ACCOUNTS_SECONDS", "30")), + playwright_proxy=os.getenv("PLAYWRIGHT_PROXY", "").strip() or None, + oauth_client_id=os.getenv("OAUTH_CLIENT_ID", "pdlLIX2Y72MIl2rhLhTE9VV9bN905kBh").strip(), + oauth_redirect_uri=os.getenv( + "OAUTH_REDIRECT_URI", + "com.openai.chat://auth0.openai.com/ios/com.openai.chat/callback", + ).strip(), + oauth_audience=os.getenv("OAUTH_AUDIENCE", "https://api.openai.com/v1").strip(), + oauth_scope=os.getenv("OAUTH_SCOPE", scope_default).strip(), + root_dir=root, + ) + + +def load_accounts(csv_path: Optional[Path] = None) -> List[Account]: + path = csv_path or (_here() / "accounts.csv") + if not path.exists(): + raise FileNotFoundError(f"账号文件不存在: {path}；请参照 accounts.csv.example 创建。") + + accounts: List[Account] = [] + with open(path, encoding="utf-8") as f: + reader = csv.DictReader(f) + required = {"email", "password"} + missing = required - set(h.strip().lower() for h in (reader.fieldnames or [])) + if missing: + raise RuntimeError(f"accounts.csv 缺少必填列: {sorted(missing)}") + for row in reader: + email = (row.get("email") or "").strip() + password = (row.get("password") or "").strip() + if not email or not password: + continue + accounts.append( + Account( + email=email, + password=password, + totp_secret=(row.get("totp_secret") or "").strip(), + note=(row.get("note") or "").strip(), + proxy_name=(row.get("proxy_name") or "").strip(), + ) + ) + if not accounts: + raise RuntimeError(f"accounts.csv 为空或所有行缺失 email/password") + return accounts diff --git a/harvester/src/harvest.py b/harvester/src/harvest.py new file mode 100644 index 00000000..a78d6dcc --- /dev/null +++ b/harvester/src/harvest.py @@ -0,0 +1,202 @@ +"""harvester CLI 入口。 + +用法示例： + python -m src.harvest # 全量（跳过已成功） + python -m src.harvest --only a@b.com,c@d.com + python -m src.harvest --failed + python -m src.harvest --force + python -m src.harvest --headless + python -m src.harvest --export tokens.json # 不调 chat2api，只导出 +""" + +import asyncio +import json +import sys +from pathlib import Path +from typing import List, Optional + +import click + +from .cache import StateStore +from .chat2api_client import Chat2ApiClient +from .config import load_accounts, load_config +from .log_setup import setup_logging +from .models import Account, HarvestResult, TokenSet +from .oauth_flow import harvest_one + + +def _filter_accounts( + accounts: List[Account], + only: Optional[str], + failed_only: bool, + force: bool, + state: StateStore, +) -> List[Account]: + if only: + wanted = {e.strip().lower() for e in only.split(",") if e.strip()} + accounts = [a for a in accounts if a.email.lower() in wanted] + if not accounts: + raise click.ClickException("--only 指定的邮箱在 accounts.csv 中找不到") + + if failed_only: + fails = set(e.lower() for e in state.list_failed()) + accounts = [a for a in accounts if a.email.lower() in fails] + + if not force: + filtered = [] + for a in accounts: + if state.is_banned(a.email): + continue + if state.is_recently_success(a.email): + continue + filtered.append(a) + accounts = filtered + + return accounts + + +async def _run( + only: Optional[str], + failed_only: bool, + force: bool, + headless_override: bool, + export_path: Optional[str], +) -> int: + config = load_config() + if headless_override: + config.headless = True + config.ensure_dirs() + + logger = setup_logging(config.logs_dir) + logger.info(f"harvester 启动 base={config.chat2api_base_url} headless={config.headless}") + + all_accounts = load_accounts() + state = StateStore(config.state_dir) + accounts = _filter_accounts(all_accounts, only, failed_only, force, state) + + if not accounts: + logger.info("没有需要处理的账号（全部已成功 / 已禁用）") + return 0 + logger.info(f"共 {len(accounts)} 个账号待处理（总 {len(all_accounts)}）") + + # 模式分叉：导出 vs 写回 chat2api + exported_tokens: List[dict] = [] + client: Optional[Chat2ApiClient] = None + + if export_path: + async def on_success(account: Account, token_set: TokenSet) -> None: + # 注意：rt 完整值此处会写入 JSON 文件，仅用于 --export 模式 + exported_tokens.append({ + "email": account.email, + "note": account.note, + "proxy_name": account.proxy_name, + "refresh_token": token_set.refresh_token, + }) + else: + client = Chat2ApiClient( + config.chat2api_base_url, + config.chat2api_api_prefix, + config.chat2api_admin_password, + ) + if not await client.healthcheck(): + logger.error("chat2api 健康检查失败，中止。请先启动 chat2api 或检查 .env") + return 2 + + async def on_success(account: Account, token_set: TokenSet) -> None: + proxy_url = "" + if account.proxy_name: + proxy_url = await client.resolve_proxy(account.proxy_name) or "" + if not proxy_url: + logger.warning( + f"[{account.masked_email()}] proxy_name='{account.proxy_name}' " + f"在 chat2api 中未找到，将导入但不绑定代理" + ) + await client.import_token( + refresh_token=token_set.refresh_token, + note=account.note, + proxy_name=account.proxy_name if proxy_url else "", + proxy_url=proxy_url, + ) + # 额外：上报给 Harvester 看板，更新 last_rt_prefix / 状态 + await client.report_harvest( + email=account.email, + rt_prefix=token_set.rt_prefix, + success=True, + imported_token=token_set.refresh_token, + ) + logger.info(f"[{account.masked_email()}] → chat2api import OK + reported") + + # 串行执行（并行留作未来增强） + results: List[HarvestResult] = [] + for idx, acc in enumerate(accounts, start=1): + masked = acc.masked_email() + logger.info(f"--- [{idx}/{len(accounts)}] {masked} ---") + + if idx > 1 and config.interval_between_accounts_seconds > 0: + await asyncio.sleep(config.interval_between_accounts_seconds) + + attempts = max(config.retry_on_fail, 0) + 1 + last: Optional[HarvestResult] = None + for attempt in range(1, attempts + 1): + last = await harvest_one(acc, config, on_success=on_success) + if last.ok: + break + if attempt < attempts: + logger.warning(f"[{masked}] 第 {attempt} 次失败，将重试：{last.error[:120]}") + await asyncio.sleep(5) + + assert last is not None + results.append(last) + + if last.ok: + state.mark_success(acc.email, last.rt_prefix, imported=last.imported) + else: + banned = any(k in (last.error or "").lower() for k in ("banned", "blocked", "suspended")) + state.mark_failure(acc.email, last.error, banned=banned) + # 非 export 模式下，把失败也上报给 chat2api 看板 + if client is not None: + try: + await client.report_harvest( + email=acc.email, + success=False, + error=last.error, + ) + except Exception: + pass + + # 汇总 + ok_count = sum(1 for r in results if r.ok) + failed_count = len(results) - ok_count + logger.info(f"====== 完成 | 成功 {ok_count} | 失败 {failed_count} ======") + for r in results: + if not r.ok: + logger.info(f" ❌ {r.email}: {r.error[:200]}") + + if export_path and exported_tokens: + out = Path(export_path) + out.write_text( + json.dumps(exported_tokens, indent=2, ensure_ascii=False), + encoding="utf-8", + ) + logger.info(f"已导出 {len(exported_tokens)} 条 rt 到 {out.resolve()}") + + return 0 if failed_count == 0 else 1 + + +@click.command() +@click.option("--only", default=None, help="仅处理这些邮箱，英文逗号分隔") +@click.option("--failed", "failed_only", is_flag=True, help="只处理上次失败的账号") +@click.option("--force", is_flag=True, help="忽略 state，强制全量重跑") +@click.option("--headless", is_flag=True, help="覆盖 .env 的 HEADLESS=true（仅在确认无 Arkose 时用）") +@click.option("--export", "export_path", default=None, help="不调 chat2api，导出完整 rt 到 JSON 文件") +def main(only, failed_only, force, headless, export_path): + try: + code = asyncio.run(_run(only, failed_only, force, headless, export_path)) + except KeyboardInterrupt: + click.echo("用户中断") + sys.exit(130) + sys.exit(code) + + +if __name__ == "__main__": + main() diff --git a/harvester/src/log_setup.py b/harvester/src/log_setup.py new file mode 100644 index 00000000..676a39ce --- /dev/null +++ b/harvester/src/log_setup.py @@ -0,0 +1,52 @@ +"""日志配置：控制台 + 文件。密码/token 自动脱敏。""" + +import logging +import re +import sys +from pathlib import Path +from typing import Optional + + +_PASSWORD_HINTS = ("password", "passwd", "secret", "token") + + +class SensitiveFilter(logging.Filter): + """在日志输出前对关键字段做 mask。简单启发式，不保证万能。""" + + # refresh token 前缀 rt_ 后紧跟字符 → 保留前 8 char + _RT_PATTERN = re.compile(r"(rt_[A-Za-z0-9\-_\.]{5})[A-Za-z0-9\-_\.]{20,}") + _JWT_PATTERN = re.compile(r"(eyJ[A-Za-z0-9\-_\.]{10})[A-Za-z0-9\-_\.]{40,}") + + def filter(self, record: logging.LogRecord) -> bool: + try: + msg = record.getMessage() + msg = self._RT_PATTERN.sub(r"\1***", msg) + msg = self._JWT_PATTERN.sub(r"\1***", msg) + # args 里的 password 如果被显式 %s 了一般也会在 msg 里，已处理 + record.msg = msg + record.args = () + except Exception: + pass + return True + + +def setup_logging(logs_dir: Path, level: str = "INFO") -> logging.Logger: + logs_dir.mkdir(parents=True, exist_ok=True) + logger = logging.getLogger("harvester") + logger.setLevel(level) + logger.handlers.clear() + + fmt = logging.Formatter("%(asctime)s | %(levelname)s | %(message)s") + + ch = logging.StreamHandler(sys.stdout) + ch.setFormatter(fmt) + ch.addFilter(SensitiveFilter()) + logger.addHandler(ch) + + fh = logging.FileHandler(logs_dir / "harvester.log", encoding="utf-8") + fh.setFormatter(fmt) + fh.addFilter(SensitiveFilter()) + logger.addHandler(fh) + + logger.propagate = False + return logger diff --git a/harvester/src/models.py b/harvester/src/models.py new file mode 100644 index 00000000..e5019d22 --- /dev/null +++ b/harvester/src/models.py @@ -0,0 +1,51 @@ +"""数据模型。""" + +from dataclasses import dataclass, field +from typing import Optional + + +@dataclass +class Account: + email: str + password: str + totp_secret: str = "" + note: str = "" + proxy_name: str = "" + + def masked_email(self) -> str: + local, _, domain = self.email.partition("@") + if len(local) <= 2: + return f"{local[0]}*@{domain}" + return f"{local[0]}***{local[-1]}@{domain}" + + +@dataclass +class TokenSet: + """Auth0 /oauth/token 返回的 token 集合。""" + access_token: str + refresh_token: str + id_token: str = "" + expires_in: int = 0 + token_type: str = "Bearer" + scope: str = "" + + @property + def rt_prefix(self) -> str: + return self.refresh_token[:8] if self.refresh_token else "" + + +@dataclass +class HarvestResult: + email: str + ok: bool + rt_prefix: str = "" + error: str = "" + imported: bool = False # 是否已写入 chat2api + + @classmethod + def success(cls, email: str, token_set: TokenSet, imported: bool = False) -> "HarvestResult": + return cls(email=email, ok=True, rt_prefix=token_set.rt_prefix, imported=imported) + + @classmethod + def failure(cls, email: str, error: str) -> "HarvestResult": + return cls(email=email, ok=False, error=error[:300]) diff --git a/harvester/src/oauth_flow.py b/harvester/src/oauth_flow.py new file mode 100644 index 00000000..fe32e7c3 --- /dev/null +++ b/harvester/src/oauth_flow.py @@ -0,0 +1,366 @@ +"""iOS ChatGPT app 的 OAuth2 PKCE 流程。""" + +import asyncio +import base64 +import hashlib +import hmac +import logging +import secrets +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Awaitable, Callable, Optional +from urllib.parse import parse_qs, urlencode, urlparse + +import httpx + +from .models import Account, HarvestResult, TokenSet +from .totp import current_code as totp_code + + +logger = logging.getLogger("harvester") + +# 成功回调类型：拿到 TokenSet 后立即调用，rt 完整值只在这里短暂出现 +OnSuccessCallback = Callable[[Account, TokenSet], Awaitable[None]] + + +# ========================= PKCE ========================= + +def _base64url(data: bytes) -> str: + return base64.urlsafe_b64encode(data).rstrip(b"=").decode("ascii") + + +def generate_pkce_pair() -> tuple: + """RFC 7636 PKCE pair。返回 (verifier, challenge)。""" + verifier = _base64url(secrets.token_bytes(32)) + challenge = _base64url(hashlib.sha256(verifier.encode("ascii")).digest()) + return verifier, challenge + + +def verify_challenge(verifier: str, challenge: str) -> bool: + return hmac.compare_digest( + _base64url(hashlib.sha256(verifier.encode("ascii")).digest()), + challenge, + ) + + +def generate_state() -> str: + return _base64url(secrets.token_bytes(16)) + + +# ========================= Authorize URL ========================= + +AUTH_BASE = "https://auth0.openai.com/authorize" +TOKEN_ENDPOINT = "https://auth0.openai.com/oauth/token" + + +def build_authorize_url(challenge: str, state: str, config) -> str: + params = { + "client_id": config.oauth_client_id, + "audience": config.oauth_audience, + "redirect_uri": config.oauth_redirect_uri, + "response_type": "code", + "scope": config.oauth_scope, + "code_challenge": challenge, + "code_challenge_method": "S256", + "state": state, + "prompt": "login", + } + return f"{AUTH_BASE}?{urlencode(params)}" + + +# ========================= 核心登录流程 ========================= + +@dataclass +class _LoginContext: + account: Account + config: "object" + challenge: str + verifier: str + state: str + + +# Auth0 登录页的选择器集中定义，便于未来维护 +SEL_EMAIL_INPUT = 'input[name="username"]' +SEL_PASSWORD_INPUT = 'input[name="password"]' +SEL_SUBMIT_BUTTON = 'button[type="submit"]' +SEL_TOTP_INPUT = 'input[name="code"], input[autocomplete="one-time-code"], input[type="tel"][maxlength="6"]' +SEL_ERROR_MESSAGE = '[class*="error"], [class*="alert"], .ulp-alert' + + +async def harvest_one( + account: Account, + config, + on_success: Optional[OnSuccessCallback] = None, +) -> HarvestResult: + """单账号完整 PKCE 流程。 + + 成功拿到 TokenSet 后，会立即调用 on_success（若传入）写入外部存储。 + rt 完整值不会出现在 HarvestResult 里（只保留前缀），以减少泄露风险。 + """ + from playwright.async_api import async_playwright # 延迟导入，减少 CLI 冷启动 + + verifier, challenge = generate_pkce_pair() + state = generate_state() + ctx = _LoginContext(account=account, config=config, challenge=challenge, verifier=verifier, state=state) + + profile_dir = _profile_dir_for(account.email, config) + profile_dir.mkdir(parents=True, exist_ok=True) + + masked = account.masked_email() + logger.info(f"[{masked}] 启动浏览器 (headless={config.headless}, profile={profile_dir.name})") + + try: + async with async_playwright() as pw: + launch_kwargs = { + "user_data_dir": str(profile_dir), + "headless": config.headless, + "viewport": {"width": 1280, "height": 800}, + "locale": "en-US", + "args": ["--disable-blink-features=AutomationControlled"], + } + if config.playwright_proxy: + launch_kwargs["proxy"] = {"server": config.playwright_proxy} + + browser = await pw.chromium.launch_persistent_context(**launch_kwargs) + try: + page = await browser.new_page() + code = await asyncio.wait_for( + _run_login_flow(page, ctx), + timeout=config.timeout_per_account_seconds, + ) + finally: + await browser.close() + + token_set = await exchange_code_for_tokens(code, ctx) + logger.info(f"[{masked}] ✅ 成功 rt={token_set.rt_prefix}***") + + # 成功回调：让调用方立即处理 rt，oauth_flow 不保留 + if on_success: + try: + await on_success(account, token_set) + except Exception as e: + logger.error(f"[{masked}] on_success 回调异常: {e}") + return HarvestResult.failure(account.email, f"post-import error: {e}") + + return HarvestResult.success(account.email, token_set, imported=on_success is not None) + + except asyncio.TimeoutError: + logger.error(f"[{masked}] 超时（{config.timeout_per_account_seconds}s）") + return HarvestResult.failure(account.email, "timeout") + except HarvestError as e: + logger.error(f"[{masked}] 失败：{e}") + return HarvestResult.failure(account.email, str(e)) + except Exception as e: + logger.exception(f"[{masked}] 未预期异常") + return HarvestResult.failure(account.email, f"unexpected: {e}") + + +def _profile_dir_for(email: str, config) -> Path: + h = hashlib.sha256(email.encode("utf-8")).hexdigest()[:16] + return config.profiles_dir / h + + +class HarvestError(Exception): + pass + + +# ========================= 登录步骤 ========================= + +async def _run_login_flow(page, ctx: _LoginContext) -> str: + """执行完整登录流程，返回 authorization code。""" + config = ctx.config + account = ctx.account + masked = account.masked_email() + + auth_url = build_authorize_url(ctx.challenge, ctx.state, config) + logger.info(f"[{masked}] 打开 /authorize") + await page.goto(auth_url, wait_until="domcontentloaded") + + # Step 1: email + await _maybe_fill_email(page, account.email, masked) + + # Step 2: password + await _maybe_fill_password(page, account.password, masked) + + # Step 3: 等待登录完成 (可能中途出现 Arkose / TOTP) + callback_url = await _wait_for_callback_or_challenges(page, ctx) + + # Step 4: 从 URL 提取 code + parsed = urlparse(callback_url) + qs = parse_qs(parsed.query) + code_list = qs.get("code") + if not code_list: + error = qs.get("error", ["unknown"])[0] + desc = qs.get("error_description", [""])[0] + raise HarvestError(f"未捕获到 authorization code: error={error} desc={desc}") + return code_list[0] + + +async def _maybe_fill_email(page, email: str, masked: str) -> None: + try: + await page.wait_for_selector(SEL_EMAIL_INPUT, timeout=15000) + except Exception: + # 有时直接走到密码页（profile 里记住了邮箱） + logger.info(f"[{masked}] 跳过邮箱输入（可能已记住）") + return + await page.fill(SEL_EMAIL_INPUT, email) + logger.info(f"[{masked}] 已填邮箱") + await page.click(SEL_SUBMIT_BUTTON) + + +async def _maybe_fill_password(page, password: str, masked: str) -> None: + try: + await page.wait_for_selector(SEL_PASSWORD_INPUT, timeout=15000) + except Exception: + raise HarvestError("未找到密码输入框（可能 Auth0 改版或账号不存在）") + await page.fill(SEL_PASSWORD_INPUT, password) + logger.info(f"[{masked}] 已填密码") + await page.click(SEL_SUBMIT_BUTTON) + + +async def _wait_for_callback_or_challenges(page, ctx: _LoginContext) -> str: + """同时监听：回调 URL / TOTP 输入框 / Arkose 挑战。""" + config = ctx.config + account = ctx.account + masked = account.masked_email() + + callback_future = asyncio.Future() + + def on_framenav(frame): + url = frame.url or "" + if url.startswith(config.oauth_redirect_uri): + if not callback_future.done(): + callback_future.set_result(url) + + page.on("framenavigated", on_framenav) + + deadline = time.time() + config.timeout_per_account_seconds + totp_handled = False + arkose_prompted_at = None + + while True: + remaining = deadline - time.time() + if remaining <= 0: + raise HarvestError("等待登录完成超时") + if callback_future.done(): + return callback_future.result() + + # 检查 TOTP 输入框（仅账号配了 totp_secret 时自动填） + if not totp_handled and account.totp_secret: + if await _try_fill_totp(page, account.totp_secret, masked): + totp_handled = True + continue + + # 检查 Arkose iframe + arkose_visible = await _is_arkose_visible(page) + if arkose_visible: + if arkose_prompted_at is None: + arkose_prompted_at = time.time() + logger.warning( + f"[{masked}] ⚠️ 检测到 Arkose 人机挑战，请在浏览器中手工完成；" + f"最多等待 {config.pause_for_arkose_seconds}s..." + ) + elif time.time() - arkose_prompted_at > config.pause_for_arkose_seconds: + raise HarvestError("Arkose 挑战未在规定时间内完成") + + # 检查错误提示（密码错、账号不存在等） + err = await _read_error_text(page) + if err: + raise HarvestError(f"登录页报错：{err[:200]}") + + # 等一小会儿再轮询 + try: + await asyncio.wait_for(asyncio.shield(callback_future), timeout=1.0) + except asyncio.TimeoutError: + continue + + +async def _try_fill_totp(page, secret: str, masked: str) -> bool: + """检测到 TOTP 输入框则填入当前 6 位码，返回 True。""" + try: + input_handle = await page.query_selector(SEL_TOTP_INPUT) + except Exception: + return False + if not input_handle: + return False + try: + code = totp_code(secret) + except Exception as e: + logger.error(f"[{masked}] TOTP 生成失败: {e}") + return False + logger.info(f"[{masked}] 检测到 2FA 输入框，已自动填入 TOTP") + await input_handle.fill(code) + # 尝试找提交按钮（有些页面需手动按回车） + try: + await page.click(SEL_SUBMIT_BUTTON, timeout=2000) + except Exception: + await input_handle.press("Enter") + return True + + +async def _is_arkose_visible(page) -> bool: + """检测页面是否嵌入 arkoselabs iframe。""" + try: + frames = page.frames + except Exception: + return False + for f in frames: + url = (f.url or "").lower() + if "arkoselabs" in url or "funcaptcha" in url: + return True + return False + + +async def _read_error_text(page) -> str: + try: + handle = await page.query_selector(SEL_ERROR_MESSAGE) + if not handle: + return "" + text = (await handle.inner_text()).strip() + return text + except Exception: + return "" + + +# ========================= Code → Token ========================= + +async def exchange_code_for_tokens(code: str, ctx: _LoginContext) -> TokenSet: + """用 authorization code 向 Auth0 换 access/refresh token。""" + config = ctx.config + data = { + "grant_type": "authorization_code", + "client_id": config.oauth_client_id, + "redirect_uri": config.oauth_redirect_uri, + "code": code, + "code_verifier": ctx.verifier, + } + logger.info(f"[{ctx.account.masked_email()}] 交换 code → tokens") + + proxies = config.playwright_proxy if config.playwright_proxy else None + async with httpx.AsyncClient(timeout=30, proxies=proxies) as c: + r = await c.post( + TOKEN_ENDPOINT, + json=data, + headers={ + "Accept": "application/json", + "Content-Type": "application/json", + "User-Agent": "ChatGPT/1.2025.084 (iOS 17.5.1; iPhone15,3; build 1402)", + }, + ) + if r.status_code != 200: + raise HarvestError( + f"Auth0 /oauth/token 失败 status={r.status_code}: {r.text[:300]}" + ) + payload = r.json() + if "refresh_token" not in payload: + raise HarvestError(f"Auth0 响应缺 refresh_token: keys={list(payload.keys())}") + return TokenSet( + access_token=payload.get("access_token", ""), + refresh_token=payload["refresh_token"], + id_token=payload.get("id_token", ""), + expires_in=int(payload.get("expires_in", 0)), + token_type=payload.get("token_type", "Bearer"), + scope=payload.get("scope", ""), + ) + diff --git a/harvester/src/totp.py b/harvester/src/totp.py new file mode 100644 index 00000000..786369b8 --- /dev/null +++ b/harvester/src/totp.py @@ -0,0 +1,19 @@ +"""TOTP 2FA：由 pyotp 封装。""" + +import pyotp + + +def current_code(secret: str) -> str: + """根据 base32 secret 生成当前 6 位 TOTP 码。""" + clean = (secret or "").replace(" ", "").upper() + if not clean: + raise ValueError("TOTP secret is empty") + return pyotp.TOTP(clean).now() + + +def is_valid_secret(secret: str) -> bool: + try: + current_code(secret) + return True + except Exception: + return False diff --git a/memory.md b/memory.md new file mode 100644 index 00000000..f88081c7 --- /dev/null +++ b/memory.md @@ -0,0 +1,186 @@ +# 共享记忆 + +最后更新：2026-04-17 +编码：UTF-8 with BOM（为 Windows 终端和编辑器兼容性显式设置） +维护方式：任何线路开始动手前先读本文件；完成一个可交付点后立即回写本文件。 + +## 主调度策略 + +- 主调度目标：先提升稳定性、兼容性、可观测性和资源控制，再考虑更大范围的架构调整。 +- 当前协作文档：高层事实记录在 `memory.md`，逐线交接记录在 `handoff.md`。 +- 短期优先级：A/B 联合稳定模型与 API 兼容，C/D 联合稳定反代与重试。 +- 中期优先级：统一请求上下文、错误映射合同、缓存/状态存储和结构化日志。 +- 长期方向：如果要继续降低运行风险，优先考虑迁移到官方 API/正式接入方案。 + +## 合规边界 + +- 可以做：缓存、限流、并发保护、退避、熔断、日志、指标、错误语义收敛、协议兼容修复、状态存储改造。 + +## 目标 + +- 为当前仓库内的多条并行工作线提供统一上下文。 +- 记录事实、边界、风险和当前认领，减少重复实现与相互覆盖。 +- 本文件只记录协作信息，不存放任何密钥、Token、Cookie 或敏感配置值。 + +## 项目概览 + +- 项目类型：基于 FastAPI 的 ChatGPT -> OpenAI 兼容 API 代理，同时支持官网镜像网关能力。 +- 入口文件：`app.py` +- API 主入口：`api/chat2api.py` +- 核心会话编排：`chatgpt/ChatService.py` +- 模型映射与模型辅助逻辑：`api/models.py` +- 反向代理主逻辑：`gateway/reverseProxy.py` +- 重试策略：`utils/retry.py` +- 运行时配置：`utils/configs.py` +- 网关相关逻辑：`gateway/*.py` +- 模板与静态上下文：`templates/*` + +## 当前结构认知 + +### 事实 + +- `app.py` 负责创建 FastAPI 应用、注册 CORS，并根据 `ENABLE_GATEWAY` 决定是否加载 `gateway/*` 路由。 +- `api/chat2api.py` 暴露 `/v1/chat/completions`、`/v1/models`、`/tokens` 等接口，并负责调度 `ChatService`。 +- `chatgpt/ChatService.py` 负责请求上下文初始化、鉴权、模型选择、上游会话准备与发送。 +- `utils/configs.py` 负责读取环境变量；当前已存在 `CHECK_MODEL` 与 `HISTORY_DISABLED` 配置项。 +- `gateway/backend.py`、`gateway/v1.py`、`gateway/reverseProxy.py` 承担官网镜像侧的反向代理与响应修正。 +- `utils/retry.py` 是 API 路径上的通用重试入口，行为变化会影响多个调用链。 + +### 建议并行工作面 + +- 线路 A：模型解析、模型可用性校验、模型错误语义一致性。 +- 线路 B：OpenAI 兼容 API 路由、请求/响应格式稳定性。 +- 线路 C：官网镜像网关、反向代理行为、上游错误映射。 +- 线路 D：重试策略、瞬时故障恢复、异常放大控制。 +- 线路 E：配置、部署、文档与环境变量说明。 + +## 当前已纳入的本地进行中改动 + +### 线路 A：模型解析与上游模型校验 + +涉及文件：`api/models.py`、`chatgpt/ChatService.py` + +当前未提交改动的意图如下： + +- 抽取 `MODEL_REQUEST_RULES`，统一维护“外部模型名 -> 上游请求模型名”的映射规则。 +- 新增 `get_response_model(origin_model)`，统一响应模型名回写逻辑。 +- 新增 `resolve_request_model(origin_model)`，把请求模型解析、`gizmo` 识别、动态模型判定集中到一个地方。 +- 新增 `extract_model_slugs(models_payload)`，用于从上游 `/backend-api/models` 返回中提取可用模型集合。 +- 在 `ChatService` 中引入 `resolve_auth_context()` 与 `initialize_request_context()`，把原本耦合在 `set_dynamic_data()` 中的职责拆开。 +- 增加 `model_not_found()`、`fetch_available_models()`、`validate_model_access()`，统一 404 语义并增加模型可用性校验。 +- 增加按 `host_url + account_id + token_hash` 划分的模型缓存，TTL 为 300 秒。 + +当前判断： + +- 这是一次明显的“去重复 + 规则下沉 + 职责拆分”的重构，符合 DRY / SRP / KISS。 +- 该改动会直接影响模型选择、鉴权上下文初始化、上游请求前校验，属于核心链路改动。 + +### 线路 B：OpenAI 兼容 API 路由补强 + +涉及文件：`api/chat2api.py` + +当前未提交改动的意图如下： + +- 给 `process()` 增加异常关闭与统一错误处理，避免准备发送阶段异常时泄漏客户端资源。 +- 新增 `parse_bool_query()`，统一布尔查询参数解析。 +- 新增 `format_models_response()`，以 OpenAI 风格返回模型列表。 +- 新增 `/v1/models` 路由，复用 `ChatService.fetch_available_models()` 暴露上游模型列表。 +- 路由层显式接入 `history_disabled` 作为模型列表请求的默认值。 + +当前判断： + +- 这是围绕“模型可见性”补齐 OpenAI 兼容面的改动，和线路 A 存在中等耦合。 +- 如果后续继续扩展 `/v1/models` 兼容性，优先在该文件收口，不要把响应格式散落到 `ChatService`。 + +### 线路 C：官网镜像反向代理错误语义收敛 + +涉及文件：`gateway/reverseProxy.py`、`gateway/backend.py` + +当前未提交改动的意图如下： + +- 在内部请求块中单独捕获 `HTTPException`，关闭客户端后按原状态继续抛出。 +- 对未知异常增加日志 `Reverse proxy failed for {path}`。 +- 对通用异常统一返回 `502 Upstream request failed`，避免把底层异常直接泄露到调用方。 +- 修复 `gateway/backend.py` 中两处 `token.startswith` 被误写为方法对象判断的问题。 +- 修复 `check_account` 在缺少 `Authorization` 请求头时直接触发 500 的问题。 + +当前判断： + +- 这是一次“资源清理 + 错误语义收敛”的稳定性修正。 +- 该改动会直接影响网关错误码、观测性和调用方重试行为，属于网关主链路改动。 + +### 线路 D：重试策略收敛与退避 + +涉及文件：`utils/retry.py` + +当前未提交改动的意图如下： + +- 增加 `RETRYABLE_STATUS_CODES`，把可重试状态码限制为 `408/500/502/503/504`。 +- 增加 `get_retry_delay()`，采用指数退避，基础延迟 0.5 秒，最大 4 秒。 +- `async_retry()` 与 `retry()` 改为只在可重试状态码上重试，其余异常立即抛出。 +- 同步补充异步/同步两条路径的等待逻辑，避免无差别快速重试。 + +当前判断： + +- 这是一次“限制盲目重试 + 控制重试节奏”的通用基础设施改动，符合 KISS / YAGNI。 +- 该文件属于横切关注点，一旦继续修改，必须同步考虑 API 路径与网关路径的影响。 + +## 当前协作边界 + +### 已被占用/应谨慎操作的文件 + +- `api/models.py` +- `chatgpt/ChatService.py` +- `api/chat2api.py` +- `gateway/reverseProxy.py` +- `gateway/backend.py` +- `utils/retry.py` + +说明： + +- 上述文件均已有本地未提交改动；后续任何线路进入前都必须先读最新内容，再决定是否继续叠加修改。 +- 线路 A 与线路 B 通过 `/v1/models` 和 `fetch_available_models()` 形成耦合。 +- 线路 C 与线路 D 在“上游失败 -> 错误码 -> 是否重试”的路径上存在耦合。 + +### 推荐文件归属 + +- 模型相关：`api/models.py`、`chatgpt/ChatService.py` +- API 路由相关：`api/chat2api.py` +- 网关相关：`gateway/backend.py`、`gateway/v1.py`、`gateway/chatgpt.py`、`gateway/login.py`、`gateway/reverseProxy.py` +- 重试/容错相关：`utils/retry.py` +- 配置与部署：`utils/configs.py`、`.env.example`、`README.md`、`docker-compose*.yml`、`Dockerfile` + +## 当前风险与待验证项 + +- `README.md` 在当前 PowerShell 输出中出现乱码，推测是终端编码展示问题；不要基于控制台乱码直接改写文档内容。 +- `memory.md` 已按 UTF-8 with BOM 保存，以提高 Windows 本地打开时的稳定性；如果终端仍乱码，优先检查控制台输出编码，而不是重写文件内容。 +- `/v1/models` 依赖上游 `/backend-api/models`；需要验证不同账户态、代理态、异常态下的返回是否稳定。 +- `resolve_request_model()` 的默认回退策略已从“未知模型强制回退到 `gpt-4o`”转向“保留原模型并标记为动态模型”；这会影响未知模型名的兼容性，应重点回归。 +- `gizmo` 模型路径当前绕过模型可用性校验；如后续要增强校验，需要确认 GPTs 的真实上游约束。 +- `gateway/reverseProxy.py` 现在把通用异常统一映射为 502；需要确认上层调用链是否会因此触发新的重试分支。 +- `utils/retry.py` 现在只重试部分状态码；需要确认历史上依赖“所有 HTTPException 都重试”的调用场景是否存在行为变化。 +- 当前缓存 Key 使用 `host_url + account_id + token_hash`；如果后续多网关/多账户轮询策略变化，需要重新确认缓存粒度是否足够。 + +## 协作规则 + +- 开始工作前：先读本文件，再读自己要改的目标文件。 +- 开始修改时：先在“当前认领”中登记负责范围。 +- 修改完成后：同步更新“当前状态 / 风险 / 下一步”。 +- 不要在本文件记录敏感信息、真实 Token、代理账号、Cookie、私有域名。 +- 避免多条线同时修改同一文件；若不可避免，先在本文件明确主导线路。 + +## 当前认领 + +- 线路 A（模型解析与校验）：进行中；主文件 `api/models.py`、`chatgpt/ChatService.py` +- 线路 B（API 路由与 `/v1/models`）：进行中；主文件 `api/chat2api.py` +- 线路 C（网关反向代理错误语义）：进行中；主文件 `gateway/reverseProxy.py` + 补充：`gateway/backend.py` 已纳入该线 +- 线路 D（重试策略与退避）：进行中；主文件 `utils/retry.py` +- 线路 E（配置/部署/文档）：未认领 + +## 下一步建议 + +- 先做一轮 A + B 联合回归：验证模型选择、未登录访问、`gizmo`、未知模型名、`CHECK_MODEL=true`、`/v1/models` 返回结构。 +- 再做一轮 C + D 联合回归：验证上游 502、超时、瞬时 500/503 时的反代错误码与重试次数是否符合预期。 +- 其余线路如需并行，优先避开上面 5 个已占用文件，先处理文档、部署或其他网关路由。 +- 如果后续要长期并行协作，建议新增 `handoff.md` 专门记录每条线的交接事项，本文件只保留高层事实。 diff --git a/templates/account_proxy_bindings.html b/templates/account_proxy_bindings.html new file mode 100644 index 00000000..d4f190fe --- /dev/null +++ b/templates/account_proxy_bindings.html @@ -0,0 +1,2156 @@ + + + + + + Chat2API Admin + + + + + +
+ + +
+
+
+
+ 生产环境 + 服务器时间 -- + 就绪 +
+
+ + + +
+
+
+ +
+
+
+
+

Account Instances

+

账号实例

+

集中查看账号健康、代理绑定和刷新状态。

+
+
+ 最后发布：尚未发布 + + +
+
+ +
+ +
+
+
+
+

实例列表

+

--

+
+
+ 全部 + Healthy + Degraded +
+
+
+ + + + + + + + + + + + +
账号标签/分组代理状态最近刷新操作
+
+
+ + +
+
+
+ + + + + + + + +
+
+ + + + + + + + + + + + + + + + diff --git a/templates/admin_login.html b/templates/admin_login.html new file mode 100644 index 00000000..ea701552 --- /dev/null +++ b/templates/admin_login.html @@ -0,0 +1,39 @@ + + + + + + Admin Login + + + +
+
+

管理后台登录

+

优先使用 `ADMIN_PASSWORD`，未配置时回退到 `AUTHORIZATION`。

+
+ + {% if error %} +
+ {{ error }} +
+ {% endif %} + +
+
+ + +
+ +
+
+ + diff --git a/utils/Logger.py b/utils/Logger.py index c0f424f3..c620cf73 100644 --- a/utils/Logger.py +++ b/utils/Logger.py @@ -1,7 +1,15 @@ import logging +from utils.log_buffer import log_buffer + logging.basicConfig(level=logging.INFO, format='%(asctime)s | %(levelname)s | %(message)s') +# 附加：把所有日志同时送进内存环形缓冲，供管理后台 UI 读取 +# 不替换 stdout handler，仅增加一份内存副本 +_root = logging.getLogger() +if not any(isinstance(h, type(log_buffer)) for h in _root.handlers): + _root.addHandler(log_buffer) + class Logger: @staticmethod diff --git a/utils/antiban/__init__.py b/utils/antiban/__init__.py new file mode 100644 index 00000000..863a0407 --- /dev/null +++ b/utils/antiban/__init__.py @@ -0,0 +1,14 @@ +"""Antiban: chat2api 风控规避与账号保护层（骨架）。 + +模块分工： + bucket IP-账号终身粘性桶 + cooldown 账号级冷却与请求节奏 + geo 代理 IP → 地域/时区/语言 + fingerprint 指纹扩展与持久化 + circuit 熔断与黑名单自愈 + guard 统一对外入口 + +总开关：ENABLE_ANTIBAN（默认 False）。未启用时本模块不影响任何现有流程。 +""" + +from utils.antiban.guard import acquire_context, report_error, report_network_error, report_success, init, sniff_account_warning # noqa: F401 diff --git a/utils/antiban/account_risk.py b/utils/antiban/account_risk.py new file mode 100644 index 00000000..e6a3dd5c --- /dev/null +++ b/utils/antiban/account_risk.py @@ -0,0 +1,215 @@ +"""账号风险嗅探（Step A：仅记录，不联动 dead/cooldown）。 + +目标：在流式响应中检测 ChatGPT 服务端下发的"账号使用异常"软警告，记录到 +data/account_warnings.json 供后续校准。命中后**不**修改 token 状态、**不**降级桶、 +**不**中断响应，只写日志 + 持久化。 + +接入点：chatgpt/chatFormat.py 的 system 角色分支 + moderation 类型 chunk。 + +校准建议： + 1. 上线观察 3-7 天 + 2. 看 data/account_warnings.json 里的 snippet 是否都是真警告（避免误判正常 system 提示） + 3. 根据真实样本调整 WARNING_PATTERNS / METADATA_FLAG_KEYS + 4. 校准后再启用 Step B（联动 cooldown / mark_dead） +""" + +import json +import re +import threading +import time +from typing import Any, Dict, Optional + +import utils.globals as globals +from utils import configs +from utils.Logger import logger + + +# 关键词：英文官方文案 + 中文常见文案 + 通用近义词 +# 多语言覆盖，宽松匹配（precision 低没关系，先用样本校准） +WARNING_PATTERNS = [ + # 英文标准文案 + r"unusual\s+activity", + r"detected\s+automated", + r"automated\s+behavior", + r"flagged\s+for", + r"flagged\s+as", + r"violat\w+\s+(our|the)\s+(usage\s+)?polic", + r"suspicious\s+activity", + r"abnormal\s+(use|activity)", + r"account\s+has\s+been\s+(temporarily\s+)?(suspended|restricted|flagged)", + r"please\s+slow\s+down", + r"sharing\s+(your\s+)?account", + # 中文常见文案 + r"账[号户].*?(异常|被限制|被封)", + r"检测到.*?(异常|可疑|自动化)", + r"涉嫌.*?违反", + r"使用频率过高", + r"账[号户]共享", +] +_COMPILED_PATTERNS = [re.compile(p, re.IGNORECASE) for p in WARNING_PATTERNS] + + +# metadata 中可能携带的风险标记字段（OpenAI 经常调整字段名，按发现增减） +METADATA_FLAG_KEYS = ( + "is_user_system_message", + "user_system_message", + "warning", + "warning_type", + "moderation_response", + "message_safety_events", + "safety_event", + "risk_event", + "abuse_event", +) + + +_write_lock = threading.Lock() +_MAX_RECORDS_PER_TOKEN = 50 # 单 token 最多保留 50 条最新警告，避免文件无限增长 + + +def _persist() -> None: + """写盘（持锁调用）。失败仅打日志，不向上抛。""" + try: + with open(globals.ACCOUNT_WARNINGS_FILE, "w", encoding="utf-8") as f: + json.dump(globals.account_warnings, f, indent=2, ensure_ascii=False) + except Exception as e: + logger.error(f"[account_risk] persist failed: {e}") + + +def _extract_text(message: Dict[str, Any]) -> str: + """从 ChatGPT 消息体里提取所有可能含警告文本的字段，拼成单条字符串供 regex 扫描。""" + fragments = [] + + content = message.get("content") or {} + if isinstance(content, dict): + # 主要文本：content.parts[] + parts = content.get("parts") or [] + for p in parts: + if isinstance(p, str): + fragments.append(p) + elif isinstance(p, dict): + # multimodal_text 等场景，部分 part 是 dict + for v in p.values(): + if isinstance(v, str): + fragments.append(v) + # 备用字段 + for k in ("text", "result"): + v = content.get(k) + if isinstance(v, str): + fragments.append(v) + + # metadata 里的 banner/warning 文案 + meta = message.get("metadata") or {} + for k in ("warning_text", "banner_text", "rate_limit_reached_text", "reason"): + v = meta.get(k) + if isinstance(v, str): + fragments.append(v) + + return "\n".join(fragments).strip() + + +def _check_metadata_flags(meta: Dict[str, Any]) -> Optional[str]: + """返回命中的 metadata flag 名（用作 pattern 字段）；未命中返回 None。""" + if not isinstance(meta, dict): + return None + for k in METADATA_FLAG_KEYS: + v = meta.get(k) + if v in (None, False, "", 0, [], {}): + continue + return f"metadata.{k}" + return None + + +def _match_text(text: str) -> Optional[str]: + """返回命中的 pattern 源串；未命中返回 None。""" + if not text: + return None + for pat, src in zip(_COMPILED_PATTERNS, WARNING_PATTERNS): + if pat.search(text): + return src + return None + + +def sniff(token: Optional[str], message: Dict[str, Any], raw_chunk: Optional[Dict[str, Any]] = None) -> None: + """主入口：检查一条 SSE chunk 中的 message 是否含账号风险信号。 + + 设计为非阻塞、强容错：任何异常都吞掉只打日志，确保不影响主响应流。 + + Args: + token: 当前请求 token（refresh_token / sess- / access_token 均可，仅作 key） + message: chunk_old_data.get("message", {}) 对应的 dict + raw_chunk: chunk_old_data 整体，用于读取顶层 type / safe_urls 等 + """ + if not configs.enable_antiban: + return + if not token: + return + + try: + # 1) 文本嗅探 + text = _extract_text(message) + pattern_hit = _match_text(text) + + # 2) metadata 标记嗅探 + if not pattern_hit: + meta_hit = _check_metadata_flags(message.get("metadata") or {}) + if meta_hit: + pattern_hit = meta_hit + + # 3) 顶层 type 嗅探（如 type=="warning" / "safety_event"） + if not pattern_hit and isinstance(raw_chunk, dict): + top_type = (raw_chunk.get("type") or "").lower() + if top_type in ("warning", "safety_event", "abuse_event", "account_warning"): + pattern_hit = f"chunk.type={top_type}" + + if not pattern_hit: + return + + # 命中：构造记录 + snippet = (text or json.dumps(message.get("metadata") or {}, ensure_ascii=False))[:300] + record = { + "hit_at": int(time.time()), + "pattern": pattern_hit, + "snippet": snippet, + "conversation_id": (raw_chunk or {}).get("conversation_id") if isinstance(raw_chunk, dict) else None, + "message_id": message.get("id"), + "role": (message.get("author") or {}).get("role"), + } + + with _write_lock: + bucket = globals.account_warnings.setdefault(token, []) + bucket.append(record) + # 截断到最大保留条数 + if len(bucket) > _MAX_RECORDS_PER_TOKEN: + del bucket[: len(bucket) - _MAX_RECORDS_PER_TOKEN] + _persist() + + logger.warning( + f"[account_risk] HIT token={token[:12]}... pattern={pattern_hit} " + f"snippet={snippet[:120]!r}" + ) + except Exception as e: + # 任何异常都不能影响主流程 + logger.error(f"[account_risk] sniff error (suppressed): {e}") + + +def get_warnings(token: str) -> list: + """供管理后台读取某 token 的历史警告。""" + if not token: + return [] + return list(globals.account_warnings.get(token, [])) + + +def get_warning_summary() -> Dict[str, Dict[str, Any]]: + """供后台批量展示：token -> {count, last_hit_at, last_pattern}。""" + summary = {} + for token, records in globals.account_warnings.items(): + if not records: + continue + last = records[-1] + summary[token] = { + "count": len(records), + "last_hit_at": last.get("hit_at"), + "last_pattern": last.get("pattern"), + } + return summary diff --git a/utils/antiban/bucket.py b/utils/antiban/bucket.py new file mode 100644 index 00000000..369093bb --- /dev/null +++ b/utils/antiban/bucket.py @@ -0,0 +1,286 @@ +"""IP-账号终身粘性桶。 + +核心不变量： + 1. 一个账号 token 一旦首次绑定到某个桶（IP），**永不漂移**到其他桶； + 2. 桶可在状态 healthy/degraded/dead 间切换，桶内账号一律跟随桶，不会在运行时被重分配； + 3. 分配策略：新账号选择"已有账号最少 & healthy"的桶，容量上限由 BUCKET_MAX_ACCOUNTS_PER_IP 控制； + 4. routing.py 的 bindings 作为桶的唯一数据源；antiban_bucket.json 只保存运行期状态（last_request_at、status）。 +""" + +import json +import threading +import time +from typing import Dict, List, Optional, Tuple + +import utils.globals as globals +from utils import configs +from utils.Logger import logger +from utils.routing import get_routing_config, update_single_binding + +_write_lock = threading.Lock() + + +def _persist() -> None: + with _write_lock: + with open(globals.ANTIBAN_BUCKET_FILE, "w", encoding="utf-8") as f: + json.dump(globals.antiban_bucket, f, indent=2, ensure_ascii=False) + + +def _ensure_structure() -> None: + globals.antiban_bucket.setdefault("buckets", {}) + globals.antiban_bucket.setdefault("account_index", {}) + + +def _now() -> int: + return int(time.time()) + + +def _bucket_id_for_proxy(proxy_url: str) -> str: + return f"bkt::{proxy_url}" + + +def _sync_from_routing() -> None: + """把 routing_config.json 中已存在的 bindings 吸收到桶里（幂等）。""" + _ensure_structure() + routing = get_routing_config() + bindings = routing.get("bindings", {}) + if not bindings: + return + changed = False + for token, binding in bindings.items(): + proxy_url = binding.get("proxy_url") + if not proxy_url: + continue + bucket_id = _bucket_id_for_proxy(proxy_url) + bucket = globals.antiban_bucket["buckets"].setdefault(bucket_id, { + "proxy_url": proxy_url, + "proxy_name": binding.get("proxy_name", ""), + "group": binding.get("group", ""), + "accounts": [], + "last_request_at": {}, + "status": "healthy", + "degraded_until": 0, + "created_at": _now(), + }) + if token not in bucket["accounts"]: + bucket["accounts"].append(token) + changed = True + if globals.antiban_bucket["account_index"].get(token) != bucket_id: + globals.antiban_bucket["account_index"][token] = bucket_id + changed = True + if changed: + _persist() + logger.info(f"[antiban] synced {len(bindings)} routing bindings into buckets") + + +def _pick_least_loaded_healthy() -> Optional[Tuple[str, Dict]]: + """返回 (bucket_id, bucket) 中账号最少且 healthy、未满的桶。""" + cap = configs.bucket_max_accounts_per_ip + candidates = [] + for bucket_id, bucket in globals.antiban_bucket["buckets"].items(): + if bucket.get("status") != "healthy": + continue + size = len(bucket.get("accounts", [])) + if size >= cap: + continue + candidates.append((size, bucket_id, bucket)) + if not candidates: + return None + candidates.sort(key=lambda x: x[0]) + _, bucket_id, bucket = candidates[0] + return bucket_id, bucket + + +def assign_account(token: str) -> Optional[str]: + """核心分配函数。已绑定 → 直接返回桶 id；未绑定 → 选最空 healthy 桶 & 写入 routing_config。""" + if not configs.enable_antiban or not token: + return None + _ensure_structure() + + existing = globals.antiban_bucket["account_index"].get(token) + if existing and existing in globals.antiban_bucket["buckets"]: + return existing + + # 首次同步 routing 作为冷启动保底 + if not globals.antiban_bucket["buckets"]: + _sync_from_routing() + existing = globals.antiban_bucket["account_index"].get(token) + if existing: + return existing + + pick = _pick_least_loaded_healthy() + if not pick: + # 所有桶都满了/不健康；严格模式拒绝漂移；宽松模式 → 拒绝分配让上游走默认 + if configs.strict_ip_binding: + logger.warning( + f"[antiban] no healthy bucket for token {token[:12]}... " + f"(strict_ip_binding=True); caller must handle" + ) + return None + + bucket_id, bucket = pick + bucket["accounts"].append(token) + bucket.setdefault("last_request_at", {})[token] = 0 + globals.antiban_bucket["account_index"][token] = bucket_id + _persist() + + # 同步回 routing_config.json，保持跨重启一致 + try: + update_single_binding( + token, + proxy_name=bucket.get("proxy_name", ""), + proxy_url=bucket.get("proxy_url", ""), + group_name=bucket.get("group") or None, + ) + except Exception as e: # pragma: no cover - defensive + logger.error(f"[antiban] failed to persist binding to routing_config: {e}") + + logger.info( + f"[antiban] token {token[:12]}... assigned to bucket " + f"{bucket.get('proxy_name','?')} ({len(bucket['accounts'])}/{configs.bucket_max_accounts_per_ip})" + ) + return bucket_id + + +def get_bucket_proxy(token: str) -> Optional[str]: + if not configs.enable_antiban or not token: + return None + _ensure_structure() + bucket_id = globals.antiban_bucket["account_index"].get(token) + if not bucket_id: + return None + bucket = globals.antiban_bucket["buckets"].get(bucket_id, {}) + return bucket.get("proxy_url") + + +def get_bucket_meta(bucket_id: Optional[str]) -> Dict: + if not bucket_id: + return {} + return globals.antiban_bucket.get("buckets", {}).get(bucket_id, {}) + + +def mark_used(token: str) -> None: + if not configs.enable_antiban or not token: + return + _ensure_structure() + bucket_id = globals.antiban_bucket["account_index"].get(token) + if not bucket_id: + return + bucket = globals.antiban_bucket["buckets"].get(bucket_id) + if not bucket: + return + bucket.setdefault("last_request_at", {})[token] = _now() + # 写盘频率控制：每 5 次请求批量刷一次，避免 I/O 瓶颈 + bucket["_dirty_count"] = bucket.get("_dirty_count", 0) + 1 + if bucket["_dirty_count"] >= 5: + bucket["_dirty_count"] = 0 + _persist() + + +def degrade_bucket(bucket_id: str, seconds: int) -> None: + if not bucket_id or bucket_id not in globals.antiban_bucket.get("buckets", {}): + return + bucket = globals.antiban_bucket["buckets"][bucket_id] + bucket["status"] = "degraded" + bucket["degraded_until"] = _now() + seconds + _persist() + logger.warning(f"[antiban] bucket {bucket_id} degraded for {seconds}s") + + +def heal_buckets() -> int: + """定时任务：把已过冷却时间的 degraded 桶恢复为 healthy。返回恢复数量。""" + _ensure_structure() + restored = 0 + now = _now() + for bucket in globals.antiban_bucket["buckets"].values(): + if bucket.get("status") == "degraded" and bucket.get("degraded_until", 0) <= now: + bucket["status"] = "healthy" + bucket["degraded_until"] = 0 + restored += 1 + if restored: + _persist() + logger.info(f"[antiban] healed {restored} degraded bucket(s)") + return restored + + +def get_bucket_stats() -> Dict[str, int]: + _ensure_structure() + buckets = globals.antiban_bucket["buckets"] + return { + "bucket_count": len(buckets), + "account_total": sum(len(b.get("accounts", [])) for b in buckets.values()), + "healthy": sum(1 for b in buckets.values() if b.get("status") == "healthy"), + "degraded": sum(1 for b in buckets.values() if b.get("status") == "degraded"), + } + + +def bulk_assign(tokens: List[str]) -> Dict[str, int]: + """应用启动时批量分配已加载的 tokens。""" + if not configs.enable_antiban: + return {"assigned": 0, "skipped": 0} + _ensure_structure() + _sync_from_routing() + + assigned = skipped = 0 + for token in tokens: + if not token: + continue + if globals.antiban_bucket["account_index"].get(token): + skipped += 1 + continue + if assign_account(token): + assigned += 1 + else: + skipped += 1 + logger.info(f"[antiban] bulk_assign result: assigned={assigned} skipped={skipped}") + return {"assigned": assigned, "skipped": skipped} + + +def resync_from_routing() -> Dict[str, int]: + """热同步：routing_config.json 改动后调用，重建桶索引并重新分配未绑定账号。 + + 清除已不存在的代理对应的桶（保留桶历史 metadata 但标记为 dead）。 + """ + if not configs.enable_antiban: + return {"synced": 0} + _ensure_structure() + + # 1. 从 routing 重新导入 bindings + _sync_from_routing() + + # 2. 清理：routing 里已不存在的 proxy_url 对应的桶标记为 dead + routing = get_routing_config() + valid_proxy_urls = {p.get("proxy_url") for p in routing.get("proxies", []) if p.get("proxy_url")} + dead_count = 0 + for bucket_id, bucket in globals.antiban_bucket["buckets"].items(): + if bucket.get("proxy_url") not in valid_proxy_urls: + if bucket.get("status") != "dead": + bucket["status"] = "dead" + dead_count += 1 + + # 3. 清理孤儿 account_index（对应的桶已消失） + orphaned = [tk for tk, bid in globals.antiban_bucket["account_index"].items() + if bid not in globals.antiban_bucket["buckets"]] + for tk in orphaned: + globals.antiban_bucket["account_index"].pop(tk, None) + + # 4. 已存在 token 但未在任何桶 → 尝试重新分配到 healthy 桶 + reassigned = 0 + for token in list(globals.token_list): + if not token: + continue + if globals.antiban_bucket["account_index"].get(token): + continue + if assign_account(token): + reassigned += 1 + + _persist() + logger.info( + f"[antiban] resync_from_routing: " + f"dead_buckets={dead_count}, orphaned={len(orphaned)}, reassigned={reassigned}" + ) + return { + "dead_buckets": dead_count, + "orphaned": len(orphaned), + "reassigned": reassigned, + } diff --git a/utils/antiban/circuit.py b/utils/antiban/circuit.py new file mode 100644 index 00000000..ca0366f0 --- /dev/null +++ b/utils/antiban/circuit.py @@ -0,0 +1,153 @@ +"""熔断与黑名单自愈。 + +错误分级： + 403 + cf_chl_opt → bucket 降级 CIRCUIT_403_COOLDOWN；桶内账号一并延长冷却 + 429 rate-limit → 账号指数退避冷却（60→300→1800→7200s 封顶） + 401 invalid_grant → 加入 error_token_list，等 refreshToken 恢复 + account_deactivated→ 永久黑名单 antiban_dead.json + 200 成功 → 重置账号退避等级 +""" + +import json +import threading +import time +from typing import Optional + +import utils.globals as globals +from utils import configs +from utils.antiban import bucket as _bucket +from utils.antiban import cooldown +from utils.Logger import logger + +_write_lock = threading.Lock() + +_account_backoff_level = {} # token -> 0..N +_bucket_network_errors = {} # bucket_id -> 连续网络层错误计数（连接超时/拒绝/DNS） + +# 同一桶连续网络层错误阈值；达到后降级 300s（代理本身可能挂了） +_NETWORK_ERROR_THRESHOLD = 3 +_NETWORK_ERROR_COOLDOWN = 300 + + +def _persist_dead() -> None: + with _write_lock: + with open(globals.ANTIBAN_DEAD_FILE, "w", encoding="utf-8") as f: + json.dump(globals.antiban_dead_tokens, f, indent=2, ensure_ascii=False) + + +def is_bucket_allowed(bucket_id: Optional[str]) -> bool: + if not bucket_id: + return True + meta = _bucket.get_bucket_meta(bucket_id) + if meta.get("status") == "degraded": + if meta.get("degraded_until", 0) > int(time.time()): + return False + return True + + +def is_token_dead(token: str) -> bool: + return token in globals.antiban_dead_tokens + + +def mark_dead(token: str, reason: str = "") -> None: + if not token: + return + globals.antiban_dead_tokens[token] = { + "reason": reason, + "dead_at": int(time.time()), + } + try: + _persist_dead() + except Exception as e: # pragma: no cover + logger.error(f"[antiban] failed to persist dead token: {e}") + logger.error(f"[antiban] token {token[:12]}... marked dead: {reason}") + + +def reset_backoff(token: str) -> None: + _account_backoff_level.pop(token, None) + + +def bump_backoff(token: str) -> int: + level = _account_backoff_level.get(token, 0) + 1 + _account_backoff_level[token] = level + return level + + +def _cooldown_bucket_accounts(bucket_id: str, seconds: int) -> None: + """IP 桶降级时，给桶内所有账号加一次冷却延长，避免下次还用该桶相关账号立刻命中。""" + meta = _bucket.get_bucket_meta(bucket_id) + for token in meta.get("accounts", []): + cooldown.extend_cooldown(token, seconds) + + +def handle_response_error(token: str, bucket_id: Optional[str], status_code: int, detail) -> None: + if not configs.enable_antiban: + return + detail_str = (str(detail) if detail is not None else "").lower() + + # Cloudflare 挑战 → 整桶降级 + if status_code == 403 and "cf_chl_opt" in detail_str: + if bucket_id: + _bucket.degrade_bucket(bucket_id, configs.circuit_403_cooldown) + _cooldown_bucket_accounts(bucket_id, configs.circuit_403_cooldown) + return + + # 频控 → 账号指数退避 + if status_code == 429 or "rate-limit" in detail_str: + level = bump_backoff(token) + base = configs.circuit_429_cooldown + cooldown.extend_cooldown(token, min(base * (2 ** (level - 1)), 7200)) + return + + # 刷新凭据失效 → error_token_list（refreshToken 流程会尝试恢复） + if status_code == 401 and ("invalid_grant" in detail_str or "unauthorized" in detail_str): + if token and token not in globals.error_token_list: + globals.error_token_list.append(token) + return + + # 账号停用/封禁 → 永久黑名单 + if "account_deactivated" in detail_str or "banned" in detail_str: + mark_dead(token, detail_str[:120]) + return + + # 其他 5xx：不立即降级，但轻度退避（避免风暴） + if 500 <= status_code < 600 and token: + cooldown.extend_cooldown(token, 30) + + +def handle_response_success(token: str) -> None: + if not configs.enable_antiban: + return + reset_backoff(token) + + +def handle_network_error(token: str, bucket_id: Optional[str], error_kind: str = "") -> None: + """M3: 网络层错误（连接被拒/超时/DNS 失败）连续 N 次 → 标记代理桶不健康。 + + 与 handle_response_error 区分：那是 HTTP 状态码，这是 transport 失败。 + """ + if not configs.enable_antiban or not bucket_id: + return + count = _bucket_network_errors.get(bucket_id, 0) + 1 + _bucket_network_errors[bucket_id] = count + if count >= _NETWORK_ERROR_THRESHOLD: + _bucket.degrade_bucket(bucket_id, _NETWORK_ERROR_COOLDOWN) + _cooldown_bucket_accounts(bucket_id, _NETWORK_ERROR_COOLDOWN) + _bucket_network_errors[bucket_id] = 0 + logger.warning( + f"[antiban] bucket {bucket_id} degraded due to {_NETWORK_ERROR_THRESHOLD}x network errors " + f"(last={error_kind or 'unknown'})" + ) + + +def reset_network_errors(bucket_id: Optional[str]) -> None: + """成功响应后清空网络错误计数。""" + if bucket_id and bucket_id in _bucket_network_errors: + _bucket_network_errors.pop(bucket_id, None) + + +async def scheduled_heal() -> None: + """定时任务入口：由 APScheduler 调用。""" + restored = _bucket.heal_buckets() + if restored: + logger.info(f"[antiban] scheduled_heal restored {restored} bucket(s)") diff --git a/utils/antiban/cooldown.py b/utils/antiban/cooldown.py new file mode 100644 index 00000000..df8753cb --- /dev/null +++ b/utils/antiban/cooldown.py @@ -0,0 +1,87 @@ +"""账号级冷却与请求节奏。 + +行为： + wait_or_skip(token) → 若 next_available > now + max_wait, 立即返回 False； + 否则 asyncio.sleep 到可用并返回 True； + 同 token 并发请求会串行（通过 _get_lock）。 + record_request(token) → 把 next_available 设到 now + interval * (1 ± jitter)； + 在请求"成功"后调用（antiban.guard.report_success 内部触发）。 + extend_cooldown(token, sec) → 429/403 时强制延长冷却。 +""" + +import asyncio +import random +import time +from typing import Dict, Optional + +from utils import configs +from utils.Logger import logger + +_account_next_available: Dict[str, float] = {} +_account_locks: Dict[str, asyncio.Lock] = {} + + +def _get_lock(token: str) -> asyncio.Lock: + lock = _account_locks.get(token) + if lock is None: + lock = asyncio.Lock() + _account_locks[token] = lock + return lock + + +def _resolve_interval(token: str, persona: Optional[str] = None) -> int: + """Team/Plus persona → 较短间隔；free/未知 → 较长间隔。""" + if persona == "chatgpt-freeaccount": + return configs.free_account_min_interval_seconds + return configs.account_min_interval_seconds + + +async def wait_or_skip(token: str, persona: Optional[str] = None, max_wait: Optional[int] = None) -> bool: + if not configs.enable_antiban or not token: + return True + + max_wait_seconds = max_wait if max_wait is not None else configs.account_max_wait_seconds + now = time.time() + next_at = _account_next_available.get(token, 0.0) + remaining = next_at - now + + if remaining <= 0: + return True + + if remaining > max_wait_seconds: + logger.info( + f"[antiban] cooldown skip token={token[:12]}... " + f"remaining={remaining:.1f}s > max_wait={max_wait_seconds}s" + ) + return False + + # 串行化同 token 的并发请求，避免"3 个协程同时读到未冷却" + async with _get_lock(token): + now = time.time() + remaining = _account_next_available.get(token, 0.0) - now + if remaining > 0: + logger.info(f"[antiban] cooldown sleep token={token[:12]}... {remaining:.1f}s") + await asyncio.sleep(remaining) + return True + + +def record_request(token: str, persona: Optional[str] = None, min_interval: Optional[int] = None) -> None: + if not configs.enable_antiban or not token: + return + interval = min_interval if min_interval is not None else _resolve_interval(token, persona) + jitter = configs.account_cooldown_jitter + delta = interval * (1 + random.uniform(-jitter, jitter)) + _account_next_available[token] = time.time() + max(0.0, delta) + + +def extend_cooldown(token: str, seconds: int) -> None: + if not token or seconds <= 0: + return + now = time.time() + current = _account_next_available.get(token, now) + _account_next_available[token] = max(current, now + seconds) + logger.info(f"[antiban] cooldown extended token={token[:12]}... +{seconds}s") + + +def get_next_available(token: str) -> float: + return _account_next_available.get(token, 0.0) diff --git a/utils/antiban/fingerprint.py b/utils/antiban/fingerprint.py new file mode 100644 index 00000000..8801aed7 --- /dev/null +++ b/utils/antiban/fingerprint.py @@ -0,0 +1,617 @@ +"""指纹持久化强化。 + +职责： + 1. 以 fp_map 为底，扩展附加字段并持久化（screen、cores、device_memory 等）； + 2. 保证同一 token/账号在多次请求间指纹不变（防漂移）； + 3. 提供只读访问给 proofofWork/ChatService，避免直接修改 fp_map 污染。 + +注意：UA/impersonate/oai-device-id 仍由 chatgpt/fp.py 创建； +本模块只在其基础上补齐字段，不替代 fp.py。 +""" + +import hashlib +import json +import random +import threading +import time +import uuid +from typing import Dict, Optional + +import utils.globals as globals +from utils import configs +from utils.Logger import logger + +_write_lock = threading.Lock() + +_SCREEN_POOL = [ + {"width": 1920, "height": 1080, "color_depth": 24}, + {"width": 2560, "height": 1440, "color_depth": 24}, + {"width": 1920, "height": 1200, "color_depth": 24}, + {"width": 2560, "height": 1600, "color_depth": 30}, + {"width": 1680, "height": 1050, "color_depth": 24}, +] +_CORES_POOL = [4, 8, 8, 8, 12, 16] +_DEVICE_MEMORY_POOL = [4, 8, 8, 16] +# 真实设备 devicePixelRatio：绝大多数设备是 1.0（普通屏）或 2.0（Retina/HiDPI） +# 1.5、1.25、2.5 等占比极小；硬编码 1.5 是典型自动化客户端特征 +_PIXEL_RATIO_POOL = [1.0, 1.0, 2.0, 2.0, 2.0, 1.5] +# 浏览器窗口实际可视区（page_height/width）：真实用户通常最大化 / 大部分屏幕 +# page_height 必然 < screen_height，page_width <= screen_width +_VIEWPORT_RATIO_POOL = [0.95, 0.92, 0.85, 0.78] # 占屏比例 + +# ============================================================ +# 扩展指纹池（PR-补强：覆盖 sentinel / CF 常用判别维度） +# 设计原则：所有派生字段从 fp 内已有字段推导，token 级稳定，与 screen/cores 同样持久化。 +# ============================================================ + +# navigator.platform 字符串（强一致性：必须与 sec-ch-ua-platform 对应） +_NAV_PLATFORM_MAP = { + "macos": "MacIntel", + "windows": "Win32", + "linux": "Linux x86_64", + "chromeos": "Linux x86_64", + "chrome os": "Linux x86_64", +} +# 各系统任务栏/Dock 占用的高度（screen.height - avail_height） +_TASKBAR_OFFSET = {"MacIntel": 25, "Win32": 40, "Linux x86_64": 27} +# WebGL vendor/renderer 必须与 platform 一致（macOS=Apple/Intel，Win=ANGLE，Linux=Mesa） +_WEBGL_BY_PLATFORM = { + "MacIntel": [ + ("Apple Inc.", "Apple M1"), + ("Apple Inc.", "Apple M2"), + ("Apple Inc.", "Apple M3"), + ("Apple Inc.", "Apple M1 Pro"), + ("Intel Inc.", "Intel Iris OpenGL Engine"), + ], + "Win32": [ + ("Google Inc. (Intel)", "ANGLE (Intel, Intel(R) UHD Graphics 630 Direct3D11 vs_5_0 ps_5_0, D3D11)"), + ("Google Inc. (NVIDIA)", "ANGLE (NVIDIA, NVIDIA GeForce RTX 3060 Direct3D11 vs_5_0 ps_5_0, D3D11)"), + ("Google Inc. (NVIDIA)", "ANGLE (NVIDIA, NVIDIA GeForce GTX 1650 Direct3D11 vs_5_0 ps_5_0, D3D11)"), + ("Google Inc. (NVIDIA)", "ANGLE (NVIDIA, NVIDIA GeForce RTX 4060 Direct3D11 vs_5_0 ps_5_0, D3D11)"), + ("Google Inc. (AMD)", "ANGLE (AMD, AMD Radeon RX 6700 XT Direct3D11 vs_5_0 ps_5_0, D3D11)"), + ], + "Linux x86_64": [ + ("Mesa", "Mesa Intel(R) Graphics (ADL-P GT2)"), + ("Mesa/X.org", "llvmpipe (LLVM 15.0.7, 256 bits)"), + ], +} +# navigator.languages：与 accept-language 大致对齐；多数英语用户为 ["en-US","en"] +_LANGUAGES_POOL = [ + ["en-US", "en"], + ["en-US", "en"], + ["en-US", "en"], + ["en-US", "en", "zh-CN"], + ["en-GB", "en"], + ["zh-CN", "zh", "en-US", "en"], +] +# matchMedia prefers-color-scheme：浅色占 70% +_COLOR_SCHEME_POOL = ["light"] * 7 + ["dark"] * 3 +# matchMedia prefers-reduced-motion：no-preference 占 95% +_REDUCED_MOTION_POOL = ["no-preference"] * 19 + ["reduce"] * 1 +# AudioContext.sampleRate：48000 占主流；少量 macOS/USB DAC 用 44100 +_AUDIO_SAMPLE_POOL = [48000] * 9 + [44100] * 1 +# NetworkInformation.effectiveType：宽带绝大多数为 4g +_CONNECTION_EFFECTIVE_TYPE_POOL = ["4g"] * 19 + ["3g"] * 1 + +# 国家 → languages 优先列表（H2: 与 antiban geo 级联，避免 IP/语言矛盾） +_COUNTRY_LANGUAGES = { + "US": [["en-US", "en"], ["en-US", "en"], ["en-US", "en", "es"]], + "GB": [["en-GB", "en"], ["en-GB", "en", "en-US"]], + "CA": [["en-CA", "en"], ["en-CA", "en", "fr-CA"], ["fr-CA", "fr", "en-CA", "en"]], + "AU": [["en-AU", "en"]], + "JP": [["ja-JP", "ja", "en-US", "en"], ["ja", "en-US", "en"]], + "KR": [["ko-KR", "ko", "en-US", "en"]], + "SG": [["en-SG", "en", "zh-CN", "zh"], ["en-SG", "en"]], + "HK": [["zh-HK", "zh", "en"], ["en-US", "en", "zh-HK"]], + "TW": [["zh-TW", "zh", "en-US", "en"]], + "DE": [["de-DE", "de", "en"], ["de", "en-US", "en"]], + "FR": [["fr-FR", "fr", "en"], ["fr", "en-US", "en"]], +} + +# 用户节奏画像（M1: time_since_loaded 不再纯随机；同账号保持一致的"用户性格"） +# 真实用户：快速浏览者 ~1-5s 提问；普通 ~5-30s；深思 ~30-120s（含读题/思考/打字） +_USER_PACE_POOL = ["fast"] * 3 + ["normal"] * 5 + ["slow"] * 2 +_PACE_TIME_RANGE = { + "fast": (1500, 8000), + "normal": (5000, 35000), + "slow": (20000, 120000), +} + +# ============================================================ +# T3: WebGL 完整指纹扩展池（按 platform 分桶，保持物理一致性） +# ============================================================ +# max_texture_size：现代独显常见 16384；集显 8192；老旧 4096 +_WEBGL_MAX_TEXTURE_BY_RENDERER = { + "Apple": 16384, + "NVIDIA": 16384, + "AMD": 16384, + "Intel(R) UHD": 8192, + "Intel Iris": 16384, + "Mesa": 8192, + "llvmpipe": 8192, +} +# WebGL extensions：现代 Chrome 上每个 GPU 各自常见 ~30-35 项 +# 用集合的哈希值作为指纹（同账号稳定，不同账号略有差异） +_WEBGL_EXTENSIONS_TEMPLATE = [ + "ANGLE_instanced_arrays", "EXT_blend_minmax", "EXT_color_buffer_half_float", + "EXT_disjoint_timer_query", "EXT_float_blend", "EXT_frag_depth", + "EXT_shader_texture_lod", "EXT_texture_compression_bptc", + "EXT_texture_compression_rgtc", "EXT_texture_filter_anisotropic", + "EXT_sRGB", "KHR_parallel_shader_compile", "OES_element_index_uint", + "OES_fbo_render_mipmap", "OES_standard_derivatives", "OES_texture_float", + "OES_texture_float_linear", "OES_texture_half_float", + "OES_texture_half_float_linear", "OES_vertex_array_object", + "WEBGL_color_buffer_float", "WEBGL_compressed_texture_s3tc", + "WEBGL_compressed_texture_s3tc_srgb", "WEBGL_debug_renderer_info", + "WEBGL_debug_shaders", "WEBGL_depth_texture", "WEBGL_draw_buffers", + "WEBGL_lose_context", "WEBGL_multi_draw", +] + +# ============================================================ +# T2: Canvas / Font / AudioContext 指纹哈希池 +# 真实环境下这些都是 sentinel.js 通过执行 JS 主动计算的哈希值。 +# 我们作为 API 代理无法实际渲染 canvas/采样 audio，但**持久化 token 级稳定的代理哈希**： +# - 同账号多次请求一致 → 不触发"指纹漂移" +# - 不同账号分散 → 不触发"批量自动化" +# 这些值在未来 sentinel 主动采集时可注入相关字段。 +# ============================================================ +# 字体集合按 OS 分桶（真实系统默认安装字体的子集） +_FONTS_BY_PLATFORM = { + "MacIntel": [ + "Apple SD Gothic Neo", "AppleMyungjo", "Arial", "Arial Black", "Arial Hebrew", + "Avenir", "Avenir Next", "Charter", "Comic Sans MS", "Courier", "Courier New", + "Geneva", "Georgia", "Helvetica", "Helvetica Neue", "Hiragino Sans", + "Lucida Grande", "Menlo", "Monaco", "Optima", "PT Sans", "Palatino", + "SF Mono", "SF Pro", "STIX", "Symbol", "Tahoma", "Times", "Times New Roman", + "Trebuchet MS", "Verdana", "Zapfino", + ], + "Win32": [ + "Arial", "Arial Black", "Arial Narrow", "Cambria", "Cambria Math", "Candara", + "Comic Sans MS", "Consolas", "Constantia", "Corbel", "Courier New", "Ebrima", + "Franklin Gothic Medium", "Gabriola", "Gadugi", "Georgia", "Impact", + "Lucida Console", "Lucida Sans Unicode", "Malgun Gothic", "Marlett", + "Microsoft Sans Serif", "Microsoft YaHei", "MingLiU-ExtB", "Mongolian Baiti", + "MS Gothic", "Palatino Linotype", "Segoe Print", "Segoe Script", "Segoe UI", + "SimSun", "Sylfaen", "Symbol", "Tahoma", "Times New Roman", "Trebuchet MS", + "Verdana", "Webdings", "Wingdings", + ], + "Linux x86_64": [ + "Bitstream Vera Sans", "Bitstream Vera Sans Mono", "Bitstream Vera Serif", + "Courier 10 Pitch", "DejaVu Sans", "DejaVu Sans Mono", "DejaVu Serif", + "Liberation Mono", "Liberation Sans", "Liberation Serif", "Nimbus Mono L", + "Nimbus Roman No9 L", "Nimbus Sans L", "Noto Color Emoji", "Noto Mono", + "Noto Sans", "Noto Sans Mono", "Noto Serif", "Source Code Pro", "Ubuntu", + "Ubuntu Mono", "URW Bookman L", "URW Chancery L", "URW Gothic L", + "URW Palladio L", + ], +} + +# AudioContext oscillator 输出基线值（不同 OS/硬件浮点 mantissa 略有差异） +# 真实 sentinel.js 采集 OSC + Analyser 后 sum() 取 4 位小数；我们模拟为按 platform 分桶的 base+jitter +_AUDIO_FP_BASE_BY_PLATFORM = { + "MacIntel": 35.749561, + "Win32": 35.748905, + "Linux x86_64": 35.748734, +} + +# T5: 国家 → 默认 IANA timezone（用于 fp 持久化，与 geo 模块 _GEO_DEFAULTS 保持一致） +_COUNTRY_TIMEZONE = { + "US": "America/Los_Angeles", "JP": "Asia/Tokyo", "SG": "Asia/Singapore", + "HK": "Asia/Hong_Kong", "TW": "Asia/Taipei", "KR": "Asia/Seoul", + "DE": "Europe/Berlin", "GB": "Europe/London", "CA": "America/Toronto", + "FR": "Europe/Paris", "AU": "Australia/Sydney", +} + +# ============================================================ +# D2: WebGPU adapter 持久化（与 webgl renderer 强一致） +# Chrome 113+ 暴露 navigator.gpu.requestAdapter() → architecture/vendor/device/description +# ============================================================ +# renderer 关键词 → (architecture, vendor) +_WEBGPU_ARCH_BY_RENDERER = { + "Apple M1": ("apple-silicon", "apple"), + "Apple M2": ("apple-silicon", "apple"), + "Apple M3": ("apple-silicon", "apple"), + "Apple M1 Pro": ("apple-silicon", "apple"), + "Intel Iris": ("haswell", "intel"), + "Intel(R) UHD": ("kabylake", "intel"), + "NVIDIA": ("ampere", "nvidia"), + "AMD": ("rdna-2", "amd"), + "Mesa": ("gen-9", "mesa"), + "llvmpipe": ("software", "mesa"), +} + +# ============================================================ +# D3: WebRTC ICE 候选模拟（持久化模拟值；curl 不发但 sentinel.js 可主动检测） +# 真实浏览器通过 RTCPeerConnection 收集本地 IP；现代 Chrome 默认 mDNS 隐藏（.local） +# ============================================================ +_WEBRTC_LOCAL_IP_POOLS = [ + # 主流家庭/办公私网段 + lambda r: f"192.168.{r.randint(0, 9)}.{r.randint(2, 254)}", + lambda r: f"192.168.1.{r.randint(2, 254)}", + lambda r: f"10.0.{r.randint(0, 255)}.{r.randint(2, 254)}", + lambda r: f"172.{r.randint(16, 31)}.{r.randint(0, 255)}.{r.randint(2, 254)}", +] +# Chrome 默认 STUN 服务器（无显式配置时使用 Google STUN） +_WEBRTC_STUN_SERVERS = [ + "stun:stun.l.google.com:19302", + "stun:stun1.l.google.com:19302", +] + + +def _platform_key_from_ch(sec_ch_ua_platform): + """从 sec-ch-ua-platform 头值（形如 '"macOS"'）提取小写键。""" + if not sec_ch_ua_platform: + return None + return str(sec_ch_ua_platform).strip().strip('"').lower() + + +def _derive_nav_platform(fp): + """根据 sec-ch-ua-platform 推导 navigator.platform；缺失时回退 Win32。""" + key = _platform_key_from_ch(fp.get("sec-ch-ua-platform")) + return _NAV_PLATFORM_MAP.get(key, "Win32") + + +def _persist_fp() -> None: + with _write_lock: + with open(globals.FP_FILE, "w", encoding="utf-8") as f: + json.dump(globals.fp_map, f, indent=4, ensure_ascii=False) + + +def ensure_extended(token: str) -> Dict: + """确保 fp_map[token] 含扩展字段；缺失则补齐并持久化。返回 fp 副本。""" + if not token: + return {} + fp = globals.fp_map.setdefault(token, {}) + dirty = False + + if "screen" not in fp: + fp["screen"] = random.choice(_SCREEN_POOL) + dirty = True + if "hardware_concurrency" not in fp: + fp["hardware_concurrency"] = random.choice(_CORES_POOL) + dirty = True + if "device_memory" not in fp: + fp["device_memory"] = random.choice(_DEVICE_MEMORY_POOL) + dirty = True + if "pixel_ratio" not in fp: + fp["pixel_ratio"] = random.choice(_PIXEL_RATIO_POOL) + dirty = True + # 浏览器实际可视区：基于 screen 推导，token 级稳定（同账号多次请求不再抖动） + if "viewport" not in fp: + screen = fp.get("screen") or {} + sw = int(screen.get("width") or 1920) + sh = int(screen.get("height") or 1080) + ratio_w = random.choice(_VIEWPORT_RATIO_POOL) + ratio_h = random.choice(_VIEWPORT_RATIO_POOL) + fp["viewport"] = { + "page_width": int(sw * ratio_w), + "page_height": int(sh * ratio_h - 120), # 减去浏览器 chrome 高度（地址栏+标签栏） + "screen_width": sw, + "screen_height": sh, + } + dirty = True + + # ====== 扩展指纹（sentinel / CF 常用维度） ====== + # navigator.platform：与 sec-ch-ua-platform 一致；老 fp 缺 CH 头时回退 Win32 + if "nav_platform" not in fp: + fp["nav_platform"] = _derive_nav_platform(fp) + dirty = True + + # screen 内部补强：avail_width/avail_height/pixel_depth + screen = fp.get("screen") or {} + if isinstance(screen, dict): + screen_dirty = False + if "avail_width" not in screen: + screen["avail_width"] = int(screen.get("width") or 1920) + screen_dirty = True + if "avail_height" not in screen: + offset = _TASKBAR_OFFSET.get(fp.get("nav_platform", "Win32"), 40) + screen["avail_height"] = int((screen.get("height") or 1080)) - offset + screen_dirty = True + if "pixel_depth" not in screen: + screen["pixel_depth"] = int(screen.get("color_depth") or 24) + screen_dirty = True + if screen_dirty: + fp["screen"] = screen + dirty = True + + # WebGL vendor/renderer + 完整扩展指纹（T3）：按 nav_platform 分桶；不匹配时回退 Win32 + if "webgl" not in fp: + pool = _WEBGL_BY_PLATFORM.get(fp.get("nav_platform"), _WEBGL_BY_PLATFORM["Win32"]) + vendor, renderer = random.choice(pool) + # 按 renderer 关键词推断 max_texture_size + max_tex = 8192 + for key, val in _WEBGL_MAX_TEXTURE_BY_RENDERER.items(): + if key in renderer: + max_tex = val + break + # extensions：从模板池中随机选 24-32 项（真实数量范围），稳定排序后哈希 + ext_count = random.randint(24, 32) + ext_subset = sorted(random.sample(_WEBGL_EXTENSIONS_TEMPLATE, + min(ext_count, len(_WEBGL_EXTENSIONS_TEMPLATE)))) + ext_hash = hashlib.sha256("|".join(ext_subset).encode()).hexdigest()[:16] + # unmasked_* 是 WEBGL_debug_renderer_info 暴露的真实值（Chrome 已限制为 generic） + # 现代 Chrome 138+ 默认隐藏后返回与普通 vendor/renderer 相同；老版本暴露驱动细节 + unmasked_vendor = vendor + unmasked_renderer = renderer + fp["webgl"] = { + "vendor": vendor, + "renderer": renderer, + "max_texture_size": max_tex, + "max_viewport_dims": [max_tex, max_tex], + "shading_language_version": "WebGL GLSL ES 1.0 (OpenGL ES GLSL ES 1.0 Chromium)", + "extensions_hash": ext_hash, + "extensions_count": len(ext_subset), + "unmasked_vendor": unmasked_vendor, + "unmasked_renderer": unmasked_renderer, + } + dirty = True + + # navigator.languages：与 antiban geo 级联（H2）→ 与 accept-language / IP 地域一致 + if "languages" not in fp: + languages = None + try: + # 通过桶绑定的 proxy_url 反查地域 + from utils.antiban import bucket as _bucket + from utils.antiban import geo as _geo + proxy_url = _bucket.get_bucket_proxy(token) + geo_info = _geo.get_geo(proxy_url) if proxy_url else None + country = (geo_info or {}).get("country") + if country and country in _COUNTRY_LANGUAGES: + languages = random.choice(_COUNTRY_LANGUAGES[country]) + except Exception: + languages = None + fp["languages"] = languages or random.choice(_LANGUAGES_POOL) + dirty = True + + # maxTouchPoints：桌面=0；当前主流场景 device_tuple 都是 desktop + if "max_touch_points" not in fp: + try: + from utils import configs as _configs + device_tuple = getattr(_configs, "device_tuple", None) + except Exception: + device_tuple = None + is_mobile = bool(device_tuple) and any(d in ("mobile", "tablet") for d in device_tuple) + fp["max_touch_points"] = 5 if is_mobile else 0 + dirty = True + + # color_scheme：替代 ChatService 中硬编码的 is_dark_mode + if "color_scheme" not in fp: + fp["color_scheme"] = random.choice(_COLOR_SCHEME_POOL) + dirty = True + + # prefers-reduced-motion：95% no-preference + if "prefers_reduced_motion" not in fp: + fp["prefers_reduced_motion"] = random.choice(_REDUCED_MOTION_POOL) + dirty = True + + # color_gamut：与 pixel_ratio + platform 协同（Retina Mac 才有 p3） + if "color_gamut" not in fp: + pr = float(fp.get("pixel_ratio") or 1.0) + is_retina_mac = fp.get("nav_platform") == "MacIntel" and pr >= 2.0 + fp["color_gamut"] = random.choice(["p3", "srgb"]) if is_retina_mac else "srgb" + dirty = True + + # NetworkInformation：模拟宽带 4g + if "connection" not in fp: + fp["connection"] = { + "effective_type": random.choice(_CONNECTION_EFFECTIVE_TYPE_POOL), + "downlink": round(random.uniform(8.0, 15.0), 2), + "rtt": random.randint(50, 150), + "save_data": False, + } + dirty = True + + # AudioContext 指纹 + if "audio" not in fp: + fp["audio"] = { + "sample_rate": random.choice(_AUDIO_SAMPLE_POOL), + "base_latency": round(random.uniform(0.005, 0.012), 6), + } + dirty = True + + # 用户节奏画像（M1）：token 级稳定，决定 time_since_loaded 抽样分布 + if "user_pace" not in fp: + fp["user_pace"] = random.choice(_USER_PACE_POOL) + dirty = True + + # 虚拟页面加载偏移（M2）：让 perf_counter / time_since_loaded 起点 token 级稳定 + # 真实用户进入聊天页后通常停留 30s-30min，PoW 用 (now - load) 而非进程级 perf_counter + if "virtual_page_load_ms" not in fp: + # 存储相对偏移 (ms)：模拟"用户已在页面停留 30s-1800s" + fp["virtual_page_load_ms"] = round(random.uniform(30_000, 1_800_000), 3) + dirty = True + + # ====== T2: Canvas / Font / Audio FP 哈希持久化（sentinel.js 主采项的代理值） ====== + # token 级稳定：同账号永不变；不同账号自然分散（基于 token + platform 派生） + if "canvas_hash" not in fp: + # 真实 sentinel.js 计算的是 canvas.toDataURL() 的 MD5/SHA1 截断哈希 + # 我们用 token + nav_platform + screen 派生：保证 token 级稳定且系统级有差异 + nav = fp.get("nav_platform", "Win32") + screen = fp.get("screen", {}) + seed = f"canvas|{token}|{nav}|{screen.get('width')}x{screen.get('height')}" + fp["canvas_hash"] = hashlib.sha256(seed.encode()).hexdigest()[:32] + dirty = True + + if "font_list_hash" not in fp: + # 按 platform 选典型字体集，hash 之 + nav = fp.get("nav_platform", "Win32") + fonts = _FONTS_BY_PLATFORM.get(nav, _FONTS_BY_PLATFORM["Win32"]) + # 同 OS 内部稍作 jitter（个别字体存在性差异）：取池子的 80%-100% 子集，token 决定 + rng = random.Random(token + "|fonts") + keep_count = rng.randint(int(len(fonts) * 0.8), len(fonts)) + subset = sorted(rng.sample(fonts, keep_count)) + fp["font_list_hash"] = hashlib.sha256("|".join(subset).encode()).hexdigest()[:32] + fp["font_list_count"] = len(subset) + dirty = True + + if "audio_fp_hash" not in fp: + nav = fp.get("nav_platform", "Win32") + base = _AUDIO_FP_BASE_BY_PLATFORM.get(nav, _AUDIO_FP_BASE_BY_PLATFORM["Win32"]) + # token 决定微观 jitter（10^-6 量级，模拟硬件浮点指纹差异） + rng = random.Random(token + "|audio") + jitter = rng.uniform(-0.0001, 0.0001) + fp["audio_fp_hash"] = round(base + jitter, 6) + dirty = True + + # ====== T5: Intl / IANA timezone 持久化（与 antiban geo 级联） ====== + if "timezone" not in fp or "intl_locale" not in fp: + timezone_val = None + locale_val = None + try: + from utils.antiban import bucket as _bucket + from utils.antiban import geo as _geo + proxy_url = _bucket.get_bucket_proxy(token) + geo_info = _geo.get_geo(proxy_url) if proxy_url else None + if geo_info: + timezone_val = geo_info.get("timezone") + locale_val = geo_info.get("oai_language") # 如 "en-US"/"ja-JP" + except Exception: + pass + # 兜底：configs 默认值 + fp["timezone"] = timezone_val or configs.client_timezone + # locale 优先用 languages[0]（已与 geo 级联） + if not locale_val and isinstance(fp.get("languages"), list) and fp["languages"]: + locale_val = fp["languages"][0] + fp["intl_locale"] = locale_val or configs.oai_language + dirty = True + + # ====== D2: WebGPU adapter 持久化（与 webgl renderer 强一致） ====== + if "webgpu" not in fp: + wg = fp.get("webgl") or {} + renderer = wg.get("renderer", "") + arch, vendor_kw = "x86-64", "intel" + for kw, (a, v) in _WEBGPU_ARCH_BY_RENDERER.items(): + if kw in renderer: + arch, vendor_kw = a, v + break + # device 取 webgl renderer 中的可读 GPU 名（去除 ANGLE 包装） + device_name = renderer + if "ANGLE (" in renderer: + # ANGLE (NVIDIA, NVIDIA GeForce RTX 3060 Direct3D11 ..., D3D11) → NVIDIA GeForce RTX 3060 + import re as _re + m = _re.search(r"ANGLE \([^,]+,\s*([^,]+?)\s+Direct3D", renderer) + if m: + device_name = m.group(1) + fp["webgpu"] = { + "architecture": arch, + "vendor": vendor_kw, + "device": device_name[:80], # 截断防止过长 + "description": f"{vendor_kw.capitalize()} {device_name[:60]}", + } + dirty = True + + # ====== D3: WebRTC ICE 候选模拟（local_ip + stun + candidate_type） ====== + if "webrtc" not in fp: + rng = random.Random(token + "|webrtc") + local_ip_gen = rng.choice(_WEBRTC_LOCAL_IP_POOLS) + # mDNS 模式：现代 Chrome 默认开启，本地 IP 被替换为 .local 唯一标识符 + mdns_id = uuid.UUID(bytes=hashlib.sha256(("mdns|" + token).encode()).digest()[:16]) + fp["webrtc"] = { + "local_ip": local_ip_gen(rng), + "mdns_hostname": f"{mdns_id}.local", + "stun_server": rng.choice(_WEBRTC_STUN_SERVERS), + "ice_candidate_type": "host", # mDNS 后真实 IP 不外泄，类型仍为 host + } + dirty = True + + if dirty: + try: + _persist_fp() + logger.info(f"[antiban] fingerprint extended for token={token[:12]}...") + except Exception as e: # pragma: no cover + logger.error(f"[antiban] failed to persist extended fp: {e}") + + return dict(fp) + + +def get_stable_fp(token: str) -> Optional[Dict]: + if not configs.enable_antiban or not token: + return None + fp = globals.fp_map.get(token) + if not fp: + return None + return dict(fp) + + +def get_screen_resolution_sum(token: str) -> Optional[int]: + """供 proofofWork.get_config 使用：返回 width+height 总和。""" + if not configs.enable_antiban or not token: + return None + fp = globals.fp_map.get(token, {}) + screen = fp.get("screen") + if isinstance(screen, dict) and "width" in screen and "height" in screen: + return int(screen["width"]) + int(screen["height"]) + return None + + +def get_hardware_concurrency(token: str) -> Optional[int]: + if not configs.enable_antiban or not token: + return None + val = globals.fp_map.get(token, {}).get("hardware_concurrency") + return int(val) if val else None + + +def get_contextual_info(token: str) -> Optional[Dict]: + """返回 token 级稳定的 client_contextual_info 数据，供 ChatService 注入到 chat_request。 + + 未启用 antiban 或 fp 未持久化 → 返回 None，调用方走老逻辑（随机）。 + """ + if not configs.enable_antiban or not token: + return None + fp = globals.fp_map.get(token, {}) + viewport = fp.get("viewport") + pixel_ratio = fp.get("pixel_ratio") + if not viewport: + return None + return { + "page_width": int(viewport.get("page_width") or 1820), + "page_height": int(viewport.get("page_height") or 960), + "screen_width": int(viewport.get("screen_width") or 1920), + "screen_height": int(viewport.get("screen_height") or 1080), + "pixel_ratio": float(pixel_ratio) if pixel_ratio else 2.0, + # token 级稳定的 color_scheme，供 is_dark_mode 派生 + "color_scheme": fp.get("color_scheme") or "light", + # T5: Intl 字段供未来注入 chat_request / payload + "timezone": fp.get("timezone"), + "intl_locale": fp.get("intl_locale"), + } + + +def get_timezone(token: str) -> Optional[str]: + """供 proofofWork.get_parse_time 使用：返回 token 级稳定的 IANA timezone。""" + if not configs.enable_antiban or not token: + return None + val = globals.fp_map.get(token, {}).get("timezone") + return str(val) if val else None + + +def get_color_scheme(token: str) -> Optional[str]: + """返回 token 级稳定的 color_scheme（"light"/"dark"）；未启用或缺失 → None。""" + if not configs.enable_antiban or not token: + return None + val = globals.fp_map.get(token, {}).get("color_scheme") + return val if val in ("light", "dark") else None + + +def get_user_pace_range(token: str) -> Optional[tuple]: + """供 ChatService 派生 time_since_loaded：返回该 token 的 (min_ms, max_ms) 范围。 + + 未启用或缺失 → None（调用方走默认随机区间）。 + """ + if not configs.enable_antiban or not token: + return None + pace = globals.fp_map.get(token, {}).get("user_pace") + return _PACE_TIME_RANGE.get(pace) if pace else None + + +def get_virtual_page_load_ms(token: str) -> Optional[float]: + """供 proofofWork 派生 perf_counter：返回 token 级稳定的页面加载偏移（毫秒）。""" + if not configs.enable_antiban or not token: + return None + val = globals.fp_map.get(token, {}).get("virtual_page_load_ms") + return float(val) if val is not None else None + + +def is_fingerprint_locked(token: str) -> bool: + return bool(globals.fp_map.get(token, {}).get("user-agent")) diff --git a/utils/antiban/geo.py b/utils/antiban/geo.py new file mode 100644 index 00000000..63366ecd --- /dev/null +++ b/utils/antiban/geo.py @@ -0,0 +1,135 @@ +"""代理 IP 地域一致性查询与缓存。 + +策略： + 1. 从 proxy_url 解析 host； + 2. 查询 _GEO_DEFAULTS（手工配置的主流地区表）作为回退； + 3. 若 IP_GEO_PROVIDER=ip-api 且允许联网，通过 ip-api.com/json/{ip} 查询； + 失败/超时即用回退表，不阻塞请求； + 4. 结果缓存到 data/antiban_geo.json，TTL=IP_GEO_CACHE_TTL_DAYS。 + +输出结构： + { + "country": "JP", + "timezone": "Asia/Tokyo", + "tz_offset_min": 540, + "accept_language": "ja-JP,ja;q=0.9,en-US;q=0.8,en;q=0.7", + "oai_language": "ja-JP", + "_ts": + } +""" + +import json +import re +import socket +import threading +import time +from typing import Dict, Optional +from urllib import request as urllib_request +from urllib.error import URLError + +import utils.globals as globals +from utils import configs +from utils.Logger import logger + +_write_lock = threading.Lock() + +_GEO_DEFAULTS = { + "US": ("en-US,en;q=0.9", "en-US", -480, "America/Los_Angeles"), + "JP": ("ja-JP,ja;q=0.9,en-US;q=0.8,en;q=0.7", "ja-JP", 540, "Asia/Tokyo"), + "SG": ("en-SG,en;q=0.9,zh-CN;q=0.7,zh;q=0.6", "en-SG", 480, "Asia/Singapore"), + "HK": ("zh-HK,zh;q=0.9,en;q=0.8", "zh-HK", 480, "Asia/Hong_Kong"), + "TW": ("zh-TW,zh;q=0.9,en-US;q=0.8,en;q=0.7", "zh-TW", 480, "Asia/Taipei"), + "KR": ("ko-KR,ko;q=0.9,en;q=0.7", "ko-KR", 540, "Asia/Seoul"), + "DE": ("de-DE,de;q=0.9,en;q=0.7", "de-DE", 60, "Europe/Berlin"), + "GB": ("en-GB,en;q=0.9", "en-GB", 0, "Europe/London"), + "CA": ("en-CA,en;q=0.9,fr-CA;q=0.7", "en-CA", -300, "America/Toronto"), + "FR": ("fr-FR,fr;q=0.9,en;q=0.7", "fr-FR", 60, "Europe/Paris"), + "AU": ("en-AU,en;q=0.9", "en-AU", 600, "Australia/Sydney"), +} + + +def _extract_host(proxy_url: str) -> Optional[str]: + if not proxy_url: + return None + m = re.match(r"^(?:https?|socks5h?)://(?:[^@]+@)?([^:/?#]+)", proxy_url, re.I) + return m.group(1) if m else None + + +def _resolve_host(host: str) -> Optional[str]: + try: + return socket.gethostbyname(host) + except OSError: + return None + + +def _persist() -> None: + with _write_lock: + with open(globals.ANTIBAN_GEO_FILE, "w", encoding="utf-8") as f: + json.dump(globals.antiban_geo_cache, f, indent=2, ensure_ascii=False) + + +def _build_geo_from_country(country: str) -> Dict: + row = _GEO_DEFAULTS.get(country.upper()) + if not row: + return { + "country": country.upper(), + "timezone": configs.client_timezone, + "tz_offset_min": configs.client_timezone_offset_min, + "accept_language": configs.accept_language, + "oai_language": configs.oai_language, + } + return { + "country": country.upper(), + "accept_language": row[0], + "oai_language": row[1], + "tz_offset_min": row[2], + "timezone": row[3], + } + + +def _query_ip_api(ip: str, timeout: int = 3) -> Optional[str]: + """返回 country code，失败返回 None。不走代理，查询本机看到的 IP。""" + try: + url = f"http://ip-api.com/json/{ip}?fields=status,countryCode" + with urllib_request.urlopen(url, timeout=timeout) as resp: + data = json.loads(resp.read().decode("utf-8")) + if data.get("status") == "success": + return data.get("countryCode") + except (URLError, socket.timeout, json.JSONDecodeError) as e: + logger.info(f"[antiban] ip-api query failed for {ip}: {e}") + except Exception as e: # pragma: no cover + logger.warning(f"[antiban] ip-api unexpected error: {e}") + return None + + +def get_geo(proxy_url: Optional[str]) -> Optional[Dict]: + if not configs.enable_antiban or not proxy_url: + return None + + host = _extract_host(proxy_url) + if not host: + return None + + cache = globals.antiban_geo_cache + ttl = configs.ip_geo_cache_ttl_days * 86400 + cached = cache.get(host) + if cached and time.time() - cached.get("_ts", 0) < ttl: + return cached + + ip = _resolve_host(host) + if not ip: + return None + + country = _query_ip_api(ip) if configs.ip_geo_provider == "ip-api" else None + if not country: + return None + + geo_info = _build_geo_from_country(country) + geo_info["_ts"] = int(time.time()) + cache[host] = geo_info + try: + _persist() + except Exception as e: # pragma: no cover + logger.error(f"[antiban] geo persist failed: {e}") + logger.info(f"[antiban] geo resolved {host} → {country}") + return geo_info diff --git a/utils/antiban/guard.py b/utils/antiban/guard.py new file mode 100644 index 00000000..64fc4a4a --- /dev/null +++ b/utils/antiban/guard.py @@ -0,0 +1,125 @@ +"""Antiban 对外统一入口（PR-1 骨架：不拦截不改动，仅日志）。 + +后续 PR 按顺序充实： + PR-2 bucket 粘性 + PR-3 cooldown/geo + PR-4 circuit 报错上报 + PR-5 fingerprint 扩展 +""" + +from dataclasses import dataclass, field +from typing import Any, Dict, Optional + +from utils import configs +from utils.antiban import account_risk, bucket, circuit, cooldown, fingerprint, geo +from utils.Logger import logger + + +@dataclass +class AntibanContext: + token: str = "" + bucket_id: Optional[str] = None + proxy_url: Optional[str] = None + header_overrides: Dict[str, str] = field(default_factory=dict) + tz_offset_min: Optional[int] = None + fp_overrides: Dict[str, Any] = field(default_factory=dict) + enabled: bool = False + + +async def init() -> None: + """应用启动时调用。PR-1 骨架仅打印开关状态。""" + if not configs.enable_antiban: + logger.info("[antiban] disabled; original behavior preserved") + return + + # 冷启动：把已加载 tokens 批量分配到桶（已绑定则跳过） + try: + import utils.globals as _globals # 避免循环 + bucket.bulk_assign(list(_globals.token_list)) + except Exception as e: + logger.error(f"[antiban] bulk_assign on startup failed: {e}") + + stats = bucket.get_bucket_stats() + logger.info( + f"[antiban] enabled | buckets={stats['bucket_count']} " + f"accounts={stats['account_total']} healthy={stats['healthy']} degraded={stats.get('degraded', 0)}" + ) + + # D4: 后台异步探测 oai-client-version 是否过旧（不阻塞启动） + try: + import asyncio + from utils.antiban import version_check + asyncio.create_task(version_check.probe_and_compare()) + except Exception as e: + logger.info(f"[antiban] version_check schedule failed: {e}") + + +async def acquire_context(req_token: Optional[str]) -> AntibanContext: + """在 ChatService.initialize_request_context() 中调用。 + + PR-1 骨架：返回空 context，不改变现有行为。 + """ + ctx = AntibanContext(token=req_token or "", enabled=configs.enable_antiban) + if not configs.enable_antiban: + return ctx + + # 已绑定即复用；未绑定暂不分配（PR-2 实现） + ctx.bucket_id = bucket.assign_account(ctx.token) + ctx.proxy_url = bucket.get_bucket_proxy(ctx.token) + + # 冷却放行检查（PR-3 真正生效） + await cooldown.wait_or_skip(ctx.token) + + # 熔断检查 + if not circuit.is_bucket_allowed(ctx.bucket_id): + logger.warning(f"[antiban] bucket {ctx.bucket_id} still degraded") + + # 地域头（若未查到则保持默认） + geo_info = geo.get_geo(ctx.proxy_url) + if geo_info: + ctx.header_overrides = { + "accept-language": geo_info.get("accept_language", ""), + "oai-language": geo_info.get("oai_language", ""), + "_timezone_name": geo_info.get("timezone", ""), # 非 header，仅透传给 chat_request + } + ctx.tz_offset_min = geo_info.get("tz_offset_min") + + # 指纹扩展（PR-5：按 token 维度补齐 screen/cores/device_memory，并返回拷贝） + fp = fingerprint.ensure_extended(ctx.token) + if fp: + ctx.fp_overrides = fp + + return ctx + + +async def report_error(ctx: AntibanContext, status_code: int, detail: Any = None) -> None: + if not ctx.enabled: + return + circuit.handle_response_error(ctx.token, ctx.bucket_id, status_code, detail) + + +async def report_network_error(ctx: AntibanContext, error_kind: str = "") -> None: + """M3: transport 层失败（ConnectionError/超时/DNS）→ 累计后降级桶。""" + if not ctx.enabled: + return + circuit.handle_network_error(ctx.token, ctx.bucket_id, error_kind) + + +async def report_success(ctx: AntibanContext) -> None: + if not ctx.enabled: + return + cooldown.record_request(ctx.token) + circuit.handle_response_success(ctx.token) + circuit.reset_network_errors(ctx.bucket_id) + bucket.mark_used(ctx.token) + + +def sniff_account_warning(ctx: AntibanContext, message: dict, raw_chunk: dict = None) -> None: + """流式响应中的账号风险嗅探（Step A：仅记录，不联动 cooldown/dead）。 + + 调用方：chatgpt/chatFormat.py 的 stream_response，在 system 角色 continue 之前。 + 设计为同步非阻塞，任何异常都吞掉，确保不影响主流程。 + """ + if not ctx or not ctx.enabled: + return + account_risk.sniff(ctx.token, message or {}, raw_chunk or {}) diff --git a/utils/antiban/version_check.py b/utils/antiban/version_check.py new file mode 100644 index 00000000..d95956c4 --- /dev/null +++ b/utils/antiban/version_check.py @@ -0,0 +1,115 @@ +"""D4: 客户端版本启动时自检。 + +策略： + 1. 启动时 GET https://chatgpt.com/，从 HTML 中提取 data-build 属性； + 2. 与本地 configs.oai_client_version / oai_client_build_number 比对； + 3. 偏差大（前缀完全不同 或 build 号差距 > 阈值）→ 日志告警， + 提示用户手工同步避免"客户端版本过旧"风控。 + +不强制更新，仅告警，避免影响主流程。 +""" + +import re +from typing import Optional, Tuple + +from utils import configs +from utils.Logger import logger + +# 启动时使用的最小请求超时（避免阻塞） +_PROBE_TIMEOUT = 5 + +# build number 偏差阈值：超过则告警 +_BUILD_NUMBER_GAP_THRESHOLD = 200_000 + + +def _extract_data_build(html: str) -> Optional[str]: + """从 HTML 中解析 字符串。""" + m = re.search(r']*data-build="([^"]+)"', html) + return m.group(1) if m else None + + +def _extract_build_number(html: str) -> Optional[int]: + """从 ChatGPT HTML 中解析 buildNumber（嵌入在 __NEXT_DATA__ 或 script 中）。""" + # buildNumber 通常出现在 _buildManifest.js 或全局变量中；做宽匹配 + m = re.search(r'"buildNumber"\s*:\s*(\d+)', html) + return int(m.group(1)) if m else None + + +def _build_prefix(version: str) -> str: + """提取 build 字符串的"前缀稳定段"用于粗比对。 + + 例: "prod-f501fe933b3edf57aea882da888e1a544df99840" → "prod" + """ + if not version: + return "" + if "-" in version: + return version.split("-", 1)[0] + return version[:8] + + +async def probe_and_compare() -> Tuple[bool, str]: + """探测官网 data-build，与本地配置比对。 + + 返回 (is_drift, message)： + is_drift=True 表示偏差大，已发告警；False 表示同步或网络不可达（跳过告警）。 + """ + import httpx + local_version = configs.oai_client_version or "" + local_build_num = configs.oai_client_build_number + + base_urls = configs.chatgpt_base_url_list or ["https://chatgpt.com"] + target = (base_urls[0] if isinstance(base_urls, list) else base_urls).rstrip("/") + + try: + async with httpx.AsyncClient(timeout=_PROBE_TIMEOUT, follow_redirects=False) as client: + resp = await client.get(target + "/", headers={ + "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "accept-language": configs.accept_language or "en-US,en;q=0.9", + "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", + }) + except Exception as e: + # 启动期网络问题不阻断主流程，仅 info 级日志 + logger.info(f"[antiban] version_check skipped (network: {e})") + return False, "skipped" + + if resp.status_code >= 400: + logger.info(f"[antiban] version_check skipped (status {resp.status_code})") + return False, "skipped" + + remote_build = _extract_data_build(resp.text) + remote_build_num = _extract_build_number(resp.text) + + if not remote_build: + logger.info("[antiban] version_check: data-build not found in HTML; sentinel may have changed") + return False, "no-build" + + # 比对 1: 前缀（prod- / dev- 等） + if _build_prefix(local_version) != _build_prefix(remote_build): + msg = ( + f"oai-client-version prefix mismatch: local={local_version[:30]}... " + f"remote={remote_build[:30]}..." + ) + logger.warning(f"[antiban] version_check DRIFT: {msg}") + return True, msg + + # 比对 2: build number 偏差 + if remote_build_num and local_build_num: + try: + gap = abs(int(remote_build_num) - int(local_build_num)) + if gap > _BUILD_NUMBER_GAP_THRESHOLD: + msg = ( + f"oai-client-build-number gap={gap} exceeds " + f"threshold={_BUILD_NUMBER_GAP_THRESHOLD}; consider updating configs.py " + f"(local={local_build_num} remote={remote_build_num})" + ) + logger.warning(f"[antiban] version_check DRIFT: {msg}") + return True, msg + except (TypeError, ValueError): + pass + + logger.info( + f"[antiban] version_check OK: local={_build_prefix(local_version)}... " + f"remote build_num={remote_build_num}" + ) + return False, "in-sync" diff --git a/utils/bootstrap.py b/utils/bootstrap.py new file mode 100644 index 00000000..9cc1763d --- /dev/null +++ b/utils/bootstrap.py @@ -0,0 +1,93 @@ +from utils.Logger import logger +from utils.configs import ( + init_apply_on_empty, + init_force, + init_group_size, + init_proxies, + init_tokens, +) +import utils.globals as globals +from utils.routing import build_group_assignments, save_routing_config, sync_bindings_to_fp + + +def _split_items(raw): + if not raw: + return [] + if "\n" in raw: + parts = raw.splitlines() + else: + parts = raw.split(",") + items = [] + for part in parts: + item = part.strip() + if item: + items.append(item) + return items + + +def _parse_proxies(raw): + proxies = [] + for index, item in enumerate(_split_items(raw), start=1): + if "|" in item: + name, proxy_url = item.split("|", 1) + name = name.strip() or f"IP-{index}" + proxy_url = proxy_url.strip() + else: + name = f"IP-{index}" + proxy_url = item + if proxy_url: + proxies.append({"name": name, "proxy_url": proxy_url}) + return proxies + + +def initialize_tokens(): + tokens = _split_items(init_tokens) + if not tokens: + return False + has_existing = bool(globals.token_list) + if has_existing and init_apply_on_empty and not init_force: + logger.info("Bootstrap tokens skipped: token.txt already populated") + return False + + seen = set() + normalized = [] + for token in tokens: + if token not in seen: + normalized.append(token) + seen.add(token) + + globals.token_list[:] = normalized + with open(globals.TOKENS_FILE, "w", encoding="utf-8") as f: + for token in normalized: + f.write(token + "\n") + logger.info(f"Bootstrap tokens initialized: {len(normalized)} accounts") + return True + + +def initialize_routing(): + proxies = _parse_proxies(init_proxies) + if not proxies: + return False + if not globals.token_list: + logger.info("Bootstrap routing skipped: no tokens available") + return False + + has_existing = bool(globals.routing_config.get("bindings")) + if has_existing and init_apply_on_empty and not init_force: + logger.info("Bootstrap routing skipped: routing_config.json already populated") + return False + + result = build_group_assignments(list(globals.token_list), proxies, init_group_size) + save_routing_config(result) + sync_bindings_to_fp(result["bindings"]) + logger.info( + f"Bootstrap routing initialized: {len(result['proxies'])} proxies, " + f"{len(result['bindings'])} bindings, group size {init_group_size}" + ) + return True + + +def initialize_from_env(): + changed_tokens = initialize_tokens() + changed_routing = initialize_routing() + return changed_tokens or changed_routing diff --git a/utils/configs.py b/utils/configs.py index b88c5be0..455c0cdf 100644 --- a/utils/configs.py +++ b/utils/configs.py @@ -21,6 +21,14 @@ def is_true(x): api_prefix = os.getenv('API_PREFIX', None) authorization = os.getenv('AUTHORIZATION', '').replace(' ', '') +admin_password = os.getenv('ADMIN_PASSWORD', None) +# 管理后台 IP 白名单（逗号分隔，支持单 IP / CIDR / 'trust_proxy'） +# 空串 = 不启用（允许所有 IP 访问登录页；真正 API 仍受 ADMIN_PASSWORD 保护） +# 示例: ADMIN_IP_WHITELIST="1.2.3.4,10.0.0.0/8,192.168.1.0/24" +admin_ip_whitelist_raw = os.getenv('ADMIN_IP_WHITELIST', '').replace(' ', '') +admin_ip_whitelist = [x for x in admin_ip_whitelist_raw.split(',') if x] +# 是否信任 X-Forwarded-For 头（仅在 CF / Nginx 反代场景开启，否则可被伪造绕过） +admin_trust_proxy = os.getenv('ADMIN_TRUST_PROXY', '').lower() in ('true', '1', 'yes') chatgpt_base_url = os.getenv('CHATGPT_BASE_URL', 'https://chatgpt.com').replace(' ', '') auth_key = os.getenv('AUTH_KEY', None) x_sign = os.getenv('X_SIGN', None) @@ -51,7 +59,12 @@ def is_true(x): check_model = is_true(os.getenv('CHECK_MODEL', False)) scheduled_refresh = is_true(os.getenv('SCHEDULED_REFRESH', False)) random_token = is_true(os.getenv('RANDOM_TOKEN', True)) -oai_language = os.getenv('OAI_LANGUAGE', 'zh-CN') +oai_language = os.getenv('OAI_LANGUAGE', 'en-US') +chat_requirements_timeout = int(os.getenv('CHAT_REQUIREMENTS_TIMEOUT', 15)) +chat_request_timeout = int(os.getenv('CHAT_REQUEST_TIMEOUT', 30)) +accept_language = os.getenv('ACCEPT_LANGUAGE', 'en-US,en;q=0.9') +client_timezone = os.getenv('CLIENT_TIMEZONE', 'America/Los_Angeles') +client_timezone_offset_min = int(os.getenv('CLIENT_TIMEZONE_OFFSET_MIN', -480)) authorization_list = authorization.split(',') if authorization else [] chatgpt_base_url_list = chatgpt_base_url.split(',') if chatgpt_base_url else [] @@ -68,6 +81,87 @@ def is_true(x): auto_seed = is_true(os.getenv('AUTO_SEED', True)) force_no_history = is_true(os.getenv('FORCE_NO_HISTORY', False)) no_sentinel = is_true(os.getenv('NO_SENTINEL', False)) +init_tokens = os.getenv('INIT_TOKENS', '') +init_proxies = os.getenv('INIT_PROXIES', '') +init_group_size = int(os.getenv('INIT_GROUP_SIZE', 25)) +init_apply_on_empty = is_true(os.getenv('INIT_APPLY_ON_EMPTY', True)) +init_force = is_true(os.getenv('INIT_FORCE', False)) + +# ========================= OpenAI 前端版本指纹（反降智） ========================= +# 这两个值需要定期（1-2 周）从 chatgpt.com 的真实请求中刷新，否则会被风控识别 +# 抓取方法：浏览器登录 chatgpt.com → F12 Network → 任一 /backend-api/* 请求 → Headers +oai_client_version = os.getenv( + 'OAI_CLIENT_VERSION', + 'prod-767c16cfce2fbcbdd1ae079fcf0b43838ff1b3ed', +) +oai_client_build_number = os.getenv( + 'OAI_CLIENT_BUILD_NUMBER', + '6549031', +) + +# ========================= OpenAI Auth0 凭据刷新 ========================= +# 默认 Codex CLI client_id（新版 OpenAI 登录流程，适用于 auth.openai.com 端点） +# 老版 iOS app client_id `pdlLIX2Y72MIl2rhLhTE9VV9bN905kBh` + auth0.openai.com 已失效（返回 404） +openai_auth_client_id = os.getenv( + 'OPENAI_AUTH_CLIENT_ID', + 'app_EMoamEEZ73f0CkXaXp7hrann', +) +# 默认 localhost HTTP 回调（Codex CLI 风格，浏览器能正常识别） +openai_auth_redirect_uri = os.getenv( + 'OPENAI_AUTH_REDIRECT_URI', + 'http://localhost:1455/auth/callback', +) +# 新版 Authorize / Token 端点（去掉了 0） +openai_auth_authorize_url = os.getenv( + 'OPENAI_AUTH_AUTHORIZE_URL', + 'https://auth.openai.com/oauth/authorize', +) +openai_auth_token_url = os.getenv( + 'OPENAI_AUTH_TOKEN_URL', + 'https://auth.openai.com/oauth/token', +) +openai_auth_scope = os.getenv( + 'OPENAI_AUTH_SCOPE', + 'openid profile email offline_access', +) + +# ========================= Antiban (风控规避层) ========================= +# 总开关；默认关闭，保持向后兼容 +enable_antiban = is_true(os.getenv('ENABLE_ANTIBAN', False)) +# IP 粘性桶：每个代理最多容纳的账号数 +bucket_max_accounts_per_ip = int(os.getenv('BUCKET_MAX_ACCOUNTS_PER_IP', 5)) +# 严格 IP 绑定：开启后账号一旦绑定 IP 即永不漂移 +strict_ip_binding = is_true(os.getenv('STRICT_IP_BINDING', True)) +# 账号级最小请求间隔秒数（Team/Plus 默认 60s） +account_min_interval_seconds = int(os.getenv('ACCOUNT_MIN_INTERVAL_SECONDS', 60)) +# 免费账号最小请求间隔秒数（通常需更长） +free_account_min_interval_seconds = int(os.getenv('FREE_ACCOUNT_MIN_INTERVAL_SECONDS', 180)) +# 冷却抖动比例（±jitter） +account_cooldown_jitter = float(os.getenv('ACCOUNT_COOLDOWN_JITTER', 0.3)) +# 账号排队最长等待秒数；超过则返回 503 让上游切换 +account_max_wait_seconds = int(os.getenv('ACCOUNT_MAX_WAIT_SECONDS', 30)) +# Geo 查询服务提供商：ip-api | ipinfo +ip_geo_provider = os.getenv('IP_GEO_PROVIDER', 'ip-api') +# Geo 缓存 TTL（天） +ip_geo_cache_ttl_days = int(os.getenv('IP_GEO_CACHE_TTL_DAYS', 30)) +# 熔断参数 +circuit_429_cooldown = int(os.getenv('CIRCUIT_429_COOLDOWN', 1800)) +circuit_403_cooldown = int(os.getenv('CIRCUIT_403_COOLDOWN', 3600)) +circuit_dead_account_recheck_hours = int(os.getenv('CIRCUIT_DEAD_ACCOUNT_RECHECK_HOURS', 24)) +circuit_bucket_heal_minutes = int(os.getenv('CIRCUIT_BUCKET_HEAL_MINUTES', 30)) + +# ========================= Session Sticky (LibreChat 会话粘性) ========================= +# 用于 LibreChat → New-API → chat2api 链路；将 LibreChat 端 conversationId +# 翻译为 ChatGPT 服务端 conversation_id，实现窗口级会话连续。默认关闭。 +enable_session_sticky = is_true(os.getenv('ENABLE_SESSION_STICKY', False)) +# SQLite 文件路径（默认在 data 卷内，跟随实例数据；与 utils/globals.DATA_FOLDER 保持一致） +session_db_path = os.getenv('SESSION_DB_PATH', os.path.join('data', 'sessions.db')) +# 多少天未更新的映射会被清理（cleanup_expired 调用时生效） +session_ttl_days = int(os.getenv('SESSION_TTL_DAYS', 30)) +# request body 中携带 LibreChat conversationId 的字段名（默认与 librechat.yaml addParams 对齐） +session_lc_field = os.getenv('SESSION_LC_FIELD', 'librechat_conversation_id') +# 命中映射时是否把 messages[] 截短到最后一条 user message（依赖 ChatGPT 服务端续接历史，省 token） +session_trim_to_last_user = is_true(os.getenv('SESSION_TRIM_TO_LAST_USER', True)) with open('version.txt') as f: version = f.read().strip() @@ -79,6 +173,8 @@ def is_true(x): logger.info("------------------------- Security -------------------------") logger.info("API_PREFIX: " + str(api_prefix)) logger.info("AUTHORIZATION: " + str(authorization_list)) +logger.info("ADMIN_PASSWORD: " + str(bool(admin_password))) +logger.info("ADMIN_IP_WHITELIST:" + (f" {len(admin_ip_whitelist)} rule(s) [{'trust_proxy' if admin_trust_proxy else 'no_proxy'}]" if admin_ip_whitelist else " (disabled)")) logger.info("AUTH_KEY: " + str(auth_key)) logger.info("------------------------- Request --------------------------") logger.info("CHATGPT_BASE_URL: " + str(chatgpt_base_url_list)) @@ -99,8 +195,35 @@ def is_true(x): logger.info("SCHEDULED_REFRESH: " + str(scheduled_refresh)) logger.info("RANDOM_TOKEN: " + str(random_token)) logger.info("OAI_LANGUAGE: " + str(oai_language)) +logger.info("ACCEPT_LANGUAGE: " + str(accept_language)) +logger.info("CLIENT_TIMEZONE: " + str(client_timezone)) +logger.info("CLIENT_TZ_OFFSET: " + str(client_timezone_offset_min)) +logger.info("CHAT_REQUIREMENTS_TIMEOUT: " + str(chat_requirements_timeout)) +logger.info("CHAT_REQUEST_TIMEOUT: " + str(chat_request_timeout)) +logger.info("OAI_CLIENT_VERSION: " + str(oai_client_version)) +logger.info("OAI_CLIENT_BUILD_NUMBER: " + str(oai_client_build_number)) logger.info("------------------------- Gateway --------------------------") logger.info("ENABLE_GATEWAY: " + str(enable_gateway)) logger.info("AUTO_SEED: " + str(auto_seed)) logger.info("FORCE_NO_HISTORY: " + str(force_no_history)) +logger.info("INIT_TOKENS: " + str(bool(init_tokens))) +logger.info("INIT_PROXIES: " + str(bool(init_proxies))) +logger.info("INIT_GROUP_SIZE: " + str(init_group_size)) +logger.info("INIT_FORCE: " + str(init_force)) +logger.info("------------------------- Antiban --------------------------") +logger.info("ENABLE_ANTIBAN: " + str(enable_antiban)) +logger.info("STRICT_IP_BINDING: " + str(strict_ip_binding)) +logger.info("BUCKET_MAX_ACCOUNTS_PER_IP: " + str(bucket_max_accounts_per_ip)) +logger.info("ACCOUNT_MIN_INTERVAL_SECONDS: " + str(account_min_interval_seconds)) +logger.info("ACCOUNT_MAX_WAIT_SECONDS: " + str(account_max_wait_seconds)) +logger.info("IP_GEO_PROVIDER: " + str(ip_geo_provider)) +logger.info("CIRCUIT_429_COOLDOWN: " + str(circuit_429_cooldown)) +logger.info("CIRCUIT_403_COOLDOWN: " + str(circuit_403_cooldown)) +logger.info("--------------------- Session Sticky -----------------------") +logger.info("ENABLE_SESSION_STICKY: " + str(enable_session_sticky)) +if enable_session_sticky: + logger.info("SESSION_DB_PATH: " + str(session_db_path)) + logger.info("SESSION_TTL_DAYS: " + str(session_ttl_days)) + logger.info("SESSION_LC_FIELD: " + str(session_lc_field)) + logger.info("SESSION_TRIM_TO_LAST_USER: " + str(session_trim_to_last_user)) logger.info("-" * 60) diff --git a/utils/globals.py b/utils/globals.py index fd4bf574..21390f5a 100644 --- a/utils/globals.py +++ b/utils/globals.py @@ -10,8 +10,17 @@ ERROR_TOKENS_FILE = os.path.join(DATA_FOLDER, "error_token.txt") WSS_MAP_FILE = os.path.join(DATA_FOLDER, "wss_map.json") FP_FILE = os.path.join(DATA_FOLDER, "fp_map.json") +ROUTING_CONFIG_FILE = os.path.join(DATA_FOLDER, "routing_config.json") SEED_MAP_FILE = os.path.join(DATA_FOLDER, "seed_map.json") CONVERSATION_MAP_FILE = os.path.join(DATA_FOLDER, "conversation_map.json") +# Antiban 持久化文件（PR-1 骨架） +ANTIBAN_BUCKET_FILE = os.path.join(DATA_FOLDER, "antiban_bucket.json") +ANTIBAN_GEO_FILE = os.path.join(DATA_FOLDER, "antiban_geo.json") +ANTIBAN_DEAD_FILE = os.path.join(DATA_FOLDER, "antiban_dead.json") +# 账号风险嗅探：仅记录命中的软警告，不立即标 dead（Step A：观察期，校准关键词） +ACCOUNT_WARNINGS_FILE = os.path.join(DATA_FOLDER, "account_warnings.json") +# Harvester 账号元数据（不含密码，仅 email+note+proxy_name+采集历史） +HARVESTER_ACCOUNTS_FILE = os.path.join(DATA_FOLDER, "harvester_accounts.json") count = 0 token_list = [] @@ -19,21 +28,19 @@ refresh_map = {} wss_map = {} fp_map = {} +routing_config = {} seed_map = {} conversation_map = {} +# Antiban 内存状态（PR-1 骨架，后续 PR 填充） +antiban_bucket = {"buckets": {}, "account_index": {}} +antiban_geo_cache = {} +antiban_dead_tokens = {} +# 账号风险嗅探：token -> [{hit_at, snippet, pattern, conversation_id}, ...] +account_warnings = {} impersonate_list = [ - "chrome99", - "chrome100", - "chrome101", - "chrome104", - "chrome107", - "chrome110", - "chrome116", "chrome119", "chrome120", "chrome123", - "edge99", - "edge101", ] if not configs.impersonate_list else configs.impersonate_list if not os.path.exists(DATA_FOLDER): @@ -66,6 +73,15 @@ else: fp_map = {} +if os.path.exists(ROUTING_CONFIG_FILE): + with open(ROUTING_CONFIG_FILE, "r", encoding="utf-8") as f: + try: + routing_config = json.load(f) + except: + routing_config = {} +else: + routing_config = {} + if os.path.exists(SEED_MAP_FILE): with open(SEED_MAP_FILE, "r") as f: try: @@ -84,6 +100,37 @@ else: conversation_map = {} +# Antiban 冷启动加载（骨架：无数据时保持默认空结构） +if os.path.exists(ANTIBAN_BUCKET_FILE): + with open(ANTIBAN_BUCKET_FILE, "r", encoding="utf-8") as f: + try: + antiban_bucket = json.load(f) + antiban_bucket.setdefault("buckets", {}) + antiban_bucket.setdefault("account_index", {}) + except: + antiban_bucket = {"buckets": {}, "account_index": {}} + +if os.path.exists(ANTIBAN_GEO_FILE): + with open(ANTIBAN_GEO_FILE, "r", encoding="utf-8") as f: + try: + antiban_geo_cache = json.load(f) + except: + antiban_geo_cache = {} + +if os.path.exists(ANTIBAN_DEAD_FILE): + with open(ANTIBAN_DEAD_FILE, "r", encoding="utf-8") as f: + try: + antiban_dead_tokens = json.load(f) + except: + antiban_dead_tokens = {} + +if os.path.exists(ACCOUNT_WARNINGS_FILE): + with open(ACCOUNT_WARNINGS_FILE, "r", encoding="utf-8") as f: + try: + account_warnings = json.load(f) + except: + account_warnings = {} + if os.path.exists(TOKENS_FILE): with open(TOKENS_FILE, "r", encoding="utf-8") as f: for line in f: @@ -105,3 +152,16 @@ if token_list: logger.info(f"Token list count: {len(token_list)}, Error token list count: {len(error_token_list)}") logger.info("-" * 60) + + +def persist_token_list(): + """全量重写 data/token.txt（用于 cookie 滚动续期后同步磁盘）。""" + with open(TOKENS_FILE, "w", encoding="utf-8") as f: + for t in token_list: + f.write(t + "\n") + + +def persist_fp_map(): + """全量重写 data/fp_map.json。""" + with open(FP_FILE, "w", encoding="utf-8") as f: + json.dump(fp_map, f, indent=2, ensure_ascii=False) diff --git a/utils/harvester_meta.py b/utils/harvester_meta.py new file mode 100644 index 00000000..98728070 --- /dev/null +++ b/utils/harvester_meta.py @@ -0,0 +1,209 @@ +"""Harvester 账号元数据：chat2api 后台与 Harvester 之间共享的"账号清单"。 + +只存**非敏感字段**： + - email + - note + - proxy_name + - last_rt_prefix (最近一次采集到的 rt 前缀 8 字符) + - last_harvest_at (unix ts) + - last_error (最近一次失败原因) + - fail_count + - imported_token (data/token.txt 中对应的完整 rt；用于 UI 查找) + +**绝不存密码 / TOTP secret**。密码始终在用户 Mac 本地 harvester/accounts.csv。 + +持久化文件：data/harvester_accounts.json +""" + +import json +import threading +import time +from pathlib import Path +from typing import Dict, List, Optional + +import utils.globals as globals +from utils.Logger import logger + + +_write_lock = threading.Lock() + +# 状态阈值（秒） +FRESH_WITHIN = 7 * 24 * 3600 # 最近 7 天内采集 → fresh +STALE_WITHIN = 45 * 24 * 3600 # 超过 7 天 <= 45 天 → stale +# 超过 45 天未采集或 last_error → failed + + +def _load() -> Dict: + path = Path(globals.HARVESTER_ACCOUNTS_FILE) + if not path.exists(): + return {"accounts": {}, "updated_at": 0} + try: + data = json.loads(path.read_text(encoding="utf-8")) + data.setdefault("accounts", {}) + return data + except (json.JSONDecodeError, OSError): + return {"accounts": {}, "updated_at": 0} + + +def _save(data: Dict) -> None: + data["updated_at"] = int(time.time()) + with _write_lock: + Path(globals.HARVESTER_ACCOUNTS_FILE).write_text( + json.dumps(data, indent=2, ensure_ascii=False), + encoding="utf-8", + ) + + +def _email_key(email: str) -> str: + return (email or "").strip().lower() + + +def _compute_status(rec: Dict) -> str: + last_ts = rec.get("last_harvest_at", 0) or 0 + if rec.get("last_error") and not rec.get("last_rt_prefix"): + return "failed" + if not last_ts: + return "pending" + age = time.time() - last_ts + if age < FRESH_WITHIN: + return "fresh" + if age < STALE_WITHIN: + return "stale" + return "failed" + + +def list_all() -> List[Dict]: + """返回账号列表，按 email 排序，带计算字段 status。""" + data = _load() + items = [] + for email_lower, rec in data.get("accounts", {}).items(): + out = dict(rec) + out["email"] = rec.get("email", email_lower) + out["status"] = _compute_status(rec) + items.append(out) + items.sort(key=lambda x: x["email"].lower()) + return items + + +def get(email: str) -> Optional[Dict]: + key = _email_key(email) + if not key: + return None + data = _load() + rec = data.get("accounts", {}).get(key) + if not rec: + return None + out = dict(rec) + out["status"] = _compute_status(rec) + return out + + +def upsert(email: str, note: str = "", proxy_name: str = "") -> Dict: + """新增或更新账号元数据（只改 email/note/proxy_name，不动采集历史）。""" + key = _email_key(email) + if not key or "@" not in key: + raise ValueError("invalid email") + data = _load() + rec = data["accounts"].get(key, {"created_at": int(time.time())}) + rec["email"] = email.strip() + rec["note"] = (note or "").strip() + rec["proxy_name"] = (proxy_name or "").strip() + rec.setdefault("last_rt_prefix", "") + rec.setdefault("last_harvest_at", 0) + rec.setdefault("last_error", "") + rec.setdefault("fail_count", 0) + rec.setdefault("imported_token", "") + data["accounts"][key] = rec + _save(data) + return dict(rec) + + +def bulk_upsert(rows: List[Dict]) -> Dict[str, int]: + """批量导入 [{email, note?, proxy_name?}]，返回 {added, updated}。""" + data = _load() + added = updated = 0 + for row in rows: + email = (row.get("email") or "").strip() + key = _email_key(email) + if not key or "@" not in key: + continue + existed = key in data["accounts"] + rec = data["accounts"].get(key, {"created_at": int(time.time())}) + rec["email"] = email + rec["note"] = (row.get("note") or rec.get("note", "")).strip() + rec["proxy_name"] = (row.get("proxy_name") or rec.get("proxy_name", "")).strip() + rec.setdefault("last_rt_prefix", "") + rec.setdefault("last_harvest_at", 0) + rec.setdefault("last_error", "") + rec.setdefault("fail_count", 0) + rec.setdefault("imported_token", "") + data["accounts"][key] = rec + if existed: + updated += 1 + else: + added += 1 + _save(data) + return {"added": added, "updated": updated} + + +def delete(email: str) -> bool: + """从元数据中删除账号（不影响 data/token.txt 中的 token）。""" + key = _email_key(email) + data = _load() + if key in data["accounts"]: + data["accounts"].pop(key) + _save(data) + return True + return False + + +def report_harvest( + email: str, + rt_prefix: str = "", + success: bool = True, + error: str = "", + imported_token: str = "", +) -> Dict: + """Harvester 成功/失败后上报。若 email 未知则自动创建。""" + key = _email_key(email) + if not key or "@" not in key: + raise ValueError("invalid email") + data = _load() + rec = data["accounts"].get(key, { + "email": email.strip(), + "note": "", + "proxy_name": "", + "created_at": int(time.time()), + }) + rec["email"] = email.strip() + now = int(time.time()) + if success: + rec["last_rt_prefix"] = (rt_prefix or "")[:12] + rec["last_harvest_at"] = now + rec["last_error"] = "" + rec["fail_count"] = 0 + if imported_token: + rec["imported_token"] = imported_token + else: + rec["last_error"] = (error or "unknown")[:200] + rec["last_error_at"] = now + rec["fail_count"] = int(rec.get("fail_count", 0)) + 1 + data["accounts"][key] = rec + _save(data) + logger.info( + f"[harvester-meta] report email={email} success={success} " + f"rt_prefix={rt_prefix[:8] if rt_prefix else ''}..." + ) + return dict(rec) + + +def stats() -> Dict: + """总览数据，供 UI 顶部卡片使用。""" + items = list_all() + return { + "total": len(items), + "fresh": sum(1 for x in items if x["status"] == "fresh"), + "stale": sum(1 for x in items if x["status"] == "stale"), + "failed": sum(1 for x in items if x["status"] == "failed"), + "pending": sum(1 for x in items if x["status"] == "pending"), + } diff --git a/utils/log_buffer.py b/utils/log_buffer.py new file mode 100644 index 00000000..7fcf3330 --- /dev/null +++ b/utils/log_buffer.py @@ -0,0 +1,118 @@ +"""日志环形缓冲：内存保存最近 N 条日志，供管理后台 UI 读取。 + +设计要点： + - 线程安全（logging 从多个线程/协程写入） + - 每条日志带递增 id，方便前端轮询增量获取（since_id） + - 不替代 stdout 输出，仅附加一份内存副本 + - 缓冲区大小可配（LOG_BUFFER_SIZE，默认 2000） +""" + +import io +import logging +import os +import re +import threading +import time +from collections import deque +from typing import Dict, Iterable, List, Optional + + +_DEFAULT_SIZE = int(os.getenv("LOG_BUFFER_SIZE", 2000)) +_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m") + + +def _strip_ansi(text: str) -> str: + return _ANSI_RE.sub("", text or "") + + +class RingBufferHandler(logging.Handler): + """线程安全的环形缓冲 handler。""" + + def __init__(self, capacity: int = _DEFAULT_SIZE): + super().__init__(level=logging.DEBUG) + self._buf: deque = deque(maxlen=max(int(capacity), 100)) + self._lock = threading.Lock() + self._seq = 0 + + def emit(self, record: logging.LogRecord) -> None: + try: + raw_msg = record.getMessage() + # 去除项目 Logger 中混入的 ANSI 颜色码，避免 UI 显示乱码 + clean_msg = _strip_ansi(raw_msg).strip("\n\r\t ") + item = { + "ts": record.created, + "level": record.levelname, + "logger": record.name, + "msg": clean_msg, + } + with self._lock: + self._seq += 1 + item["id"] = self._seq + self._buf.append(item) + except Exception: # pragma: no cover + # logging 内部出错不得反过来影响业务 + self.handleError(record) + + def snapshot( + self, + since_id: Optional[int] = None, + level: Optional[str] = None, + keyword: Optional[str] = None, + limit: Optional[int] = None, + ) -> List[Dict]: + with self._lock: + items = list(self._buf) + + if since_id is not None: + items = [x for x in items if x["id"] > since_id] + + if level: + level_up = level.upper() + if level_up != "ALL": + # 级别层级：DEBUG < INFO < WARNING < ERROR < CRITICAL + threshold = logging.getLevelName(level_up) + if isinstance(threshold, int): + items = [ + x for x in items + if logging.getLevelName(x["level"]) >= threshold + ] + + if keyword: + k = keyword.lower() + items = [x for x in items if k in x["msg"].lower()] + + if limit and limit > 0: + items = items[-int(limit):] + + return items + + def snapshot_all(self) -> List[Dict]: + """一键下载全部时使用，不走筛选。""" + with self._lock: + return list(self._buf) + + @property + def latest_id(self) -> int: + with self._lock: + return self._seq + + @property + def capacity(self) -> int: + return self._buf.maxlen or 0 + + def __len__(self) -> int: + with self._lock: + return len(self._buf) + + +def render_plaintext(records: Iterable[Dict]) -> str: + """把 records 渲染为可下载的纯文本。""" + buf = io.StringIO() + for rec in records: + ts = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(rec.get("ts", 0))) + buf.write(f"{ts} | {rec.get('level','INFO'):<8} | {rec.get('msg','')}\n") + return buf.getvalue() + + +# 全局单例，Logger 注册、admin 路由、外部诊断都用同一个 +log_buffer = RingBufferHandler() diff --git a/utils/oauth_session.py b/utils/oauth_session.py new file mode 100644 index 00000000..3b0e71f4 --- /dev/null +++ b/utils/oauth_session.py @@ -0,0 +1,182 @@ +"""OAuth PKCE session 管理（纯内存 + TTL）。 + +用于 Harvester 的"浏览器登录"流程： + 1. start(email, ...) → 生成 PKCE + state + session_id，存入内存，返回 authorize_url + 2. 用户在真实浏览器完成 OAuth，拿到 com.openai.chat://...?code=X&state=Y 回调 + 3. exchange(session_id, callback_url) → 校验 state + 用 verifier 换 token + +设计： + - 内存 dict，不持久化（服务重启即失效，符合用户选择） + - 15 分钟 TTL（Auth0 的 code 也是 ~10 分钟过期） + - 线程安全（threading.Lock） + - 自动清理过期会话（每次写操作时机会性清理） +""" + +import base64 +import hashlib +import secrets +import threading +import time +from dataclasses import dataclass, field +from typing import Dict, Optional +from urllib.parse import urlencode + + +# 与 Codex CLI 一致的新版配置（claude-relay-service 已验证可用） +# 老版 iOS app (pdlLIX... + auth0.openai.com) 已失效，返回 404 +_DEFAULT_CLIENT_ID = "app_EMoamEEZ73f0CkXaXp7hrann" +_DEFAULT_REDIRECT_URI = "http://localhost:1455/auth/callback" +_DEFAULT_AUDIENCE = "https://api.openai.com/v1" +_DEFAULT_SCOPE = "openid profile email offline_access" +_DEFAULT_AUTHORIZE_URL = "https://auth.openai.com/oauth/authorize" +_DEFAULT_TOKEN_ENDPOINT = "https://auth.openai.com/oauth/token" + +# 向后兼容：保留老常量名（TOKEN_ENDPOINT 被外部引用） +AUTH_BASE = _DEFAULT_AUTHORIZE_URL +TOKEN_ENDPOINT = _DEFAULT_TOKEN_ENDPOINT + +SESSION_TTL_SECONDS = 15 * 60 + + +@dataclass +class OAuthSession: + session_id: str + verifier: str + challenge: str + state: str + email: str + note: str = "" + proxy_name: str = "" + created_at: int = field(default_factory=lambda: int(time.time())) + + def expired(self) -> bool: + return time.time() - self.created_at > SESSION_TTL_SECONDS + + +# ========================= 内存存储 ========================= + +_sessions: Dict[str, OAuthSession] = {} +_lock = threading.Lock() + + +def _base64url(data: bytes) -> str: + return base64.urlsafe_b64encode(data).rstrip(b"=").decode("ascii") + + +def _gen_pkce_pair() -> tuple: + verifier = _base64url(secrets.token_bytes(32)) + challenge = _base64url(hashlib.sha256(verifier.encode("ascii")).digest()) + return verifier, challenge + + +def _gen_state() -> str: + return _base64url(secrets.token_bytes(16)) + + +def _gen_session_id() -> str: + return _base64url(secrets.token_bytes(12)) + + +def _gc_expired() -> None: + """机会性清理过期会话，调用方需已持锁。""" + now = time.time() + expired = [ + sid for sid, s in _sessions.items() + if now - s.created_at > SESSION_TTL_SECONDS + ] + for sid in expired: + _sessions.pop(sid, None) + + +# ========================= 对外 API ========================= + +def _get_oauth_config(): + """允许通过 configs 覆盖 client_id / redirect_uri / endpoints。返回 7-tuple。""" + try: + from utils import configs + client_id = getattr(configs, "openai_auth_client_id", None) or _DEFAULT_CLIENT_ID + redirect_uri = getattr(configs, "openai_auth_redirect_uri", None) or _DEFAULT_REDIRECT_URI + audience = getattr(configs, "openai_auth_audience", None) or _DEFAULT_AUDIENCE + scope = getattr(configs, "openai_auth_scope", None) or _DEFAULT_SCOPE + authorize_url = getattr(configs, "openai_auth_authorize_url", None) or _DEFAULT_AUTHORIZE_URL + token_url = getattr(configs, "openai_auth_token_url", None) or _DEFAULT_TOKEN_ENDPOINT + except Exception: + client_id = _DEFAULT_CLIENT_ID + redirect_uri = _DEFAULT_REDIRECT_URI + audience = _DEFAULT_AUDIENCE + scope = _DEFAULT_SCOPE + authorize_url = _DEFAULT_AUTHORIZE_URL + token_url = _DEFAULT_TOKEN_ENDPOINT + return client_id, redirect_uri, audience, scope, authorize_url, token_url + + +def start_session(email: str, note: str = "", proxy_name: str = "") -> Dict: + """生成一次性 OAuth 授权会话，返回前端需要的 URL + session_id。""" + if not email or "@" not in email: + raise ValueError("email 不合法") + + client_id, redirect_uri, audience, scope, authorize_url, _token_url = _get_oauth_config() + + verifier, challenge = _gen_pkce_pair() + state = _gen_state() + session_id = _gen_session_id() + + sess = OAuthSession( + session_id=session_id, + verifier=verifier, + challenge=challenge, + state=state, + email=email.strip(), + note=(note or "").strip(), + proxy_name=(proxy_name or "").strip(), + ) + with _lock: + _gc_expired() + _sessions[session_id] = sess + + # Codex CLI 风格的 query 参数（额外参数让 OpenAI 返回 organization 信息） + params = { + "response_type": "code", + "client_id": client_id, + "redirect_uri": redirect_uri, + "scope": scope, + "code_challenge": challenge, + "code_challenge_method": "S256", + "state": state, + "id_token_add_organizations": "true", + "codex_cli_simplified_flow": "true", + } + full_url = f"{authorize_url}?{urlencode(params)}" + return { + "session_id": session_id, + "authorize_url": full_url, + "redirect_uri": redirect_uri, + "expires_in": SESSION_TTL_SECONDS, + } + + +def pop_session(session_id: str) -> Optional[OAuthSession]: + """取出并删除会话（一次性使用）。过期返回 None。""" + with _lock: + sess = _sessions.pop(session_id, None) + if not sess: + return None + if sess.expired(): + return None + return sess + + +def peek_session(session_id: str) -> Optional[OAuthSession]: + """不删除地查看会话（供调试）。""" + with _lock: + sess = _sessions.get(session_id) + if sess and sess.expired(): + return None + return sess + + +def stats() -> Dict: + with _lock: + _gc_expired() + total = len(_sessions) + return {"active_sessions": total, "ttl_seconds": SESSION_TTL_SECONDS} diff --git a/utils/retry.py b/utils/retry.py index ac488d1a..36726906 100644 --- a/utils/retry.py +++ b/utils/retry.py @@ -1,8 +1,24 @@ +import asyncio +import time + from fastapi import HTTPException from utils.Logger import logger from utils.configs import retry_times +RETRYABLE_STATUS_CODES = {408, 500, 502, 503, 504} +BASE_RETRY_DELAY_SECONDS = 0.5 +MAX_RETRY_DELAY_SECONDS = 4 + + +def should_retry_http_exception(status_code): + return status_code in RETRYABLE_STATUS_CODES + + +def get_retry_delay(attempt): + delay = BASE_RETRY_DELAY_SECONDS * (2 ** attempt) + return min(delay, MAX_RETRY_DELAY_SECONDS) + async def async_retry(func, *args, max_retries=retry_times, **kwargs): for attempt in range(max_retries + 1): @@ -10,12 +26,18 @@ async def async_retry(func, *args, max_retries=retry_times, **kwargs): result = await func(*args, **kwargs) return result except HTTPException as e: - if attempt == max_retries: + should_retry = should_retry_http_exception(e.status_code) + if attempt == max_retries or not should_retry: logger.error(f"Throw an exception {e.status_code}, {e.detail}") if e.status_code == 500: raise HTTPException(status_code=500, detail="Server error") raise HTTPException(status_code=e.status_code, detail=e.detail) - logger.info(f"Retry {attempt + 1} status code {e.status_code}, {e.detail}. Retrying...") + delay = get_retry_delay(attempt) + logger.info( + f"Retry {attempt + 1} status code {e.status_code}, {e.detail}. " + f"Retrying in {delay:.1f}s..." + ) + await asyncio.sleep(delay) def retry(func, *args, max_retries=retry_times, **kwargs): @@ -24,9 +46,15 @@ def retry(func, *args, max_retries=retry_times, **kwargs): result = func(*args, **kwargs) return result except HTTPException as e: - if attempt == max_retries: + should_retry = should_retry_http_exception(e.status_code) + if attempt == max_retries or not should_retry: logger.error(f"Throw an exception {e.status_code}, {e.detail}") if e.status_code == 500: raise HTTPException(status_code=500, detail="Server error") raise HTTPException(status_code=e.status_code, detail=e.detail) - logger.error(f"Retry {attempt + 1} status code {e.status_code}, {e.detail}. Retrying...") + delay = get_retry_delay(attempt) + logger.error( + f"Retry {attempt + 1} status code {e.status_code}, {e.detail}. " + f"Retrying in {delay:.1f}s..." + ) + time.sleep(delay) diff --git a/utils/routing.py b/utils/routing.py new file mode 100644 index 00000000..42b63959 --- /dev/null +++ b/utils/routing.py @@ -0,0 +1,346 @@ +import json +from datetime import datetime, timezone + +import utils.globals as globals +from utils.Logger import logger + + +def utc_now(): + return datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") + + +def mask_token(token): + if not token: + return "" + if len(token) <= 12: + return token + return f"{token[:6]}...{token[-4:]}" + + +def detect_token_type(token): + """识别 token 类型。 + + 规则： + - SessionToken: 以 'sess-' 开头（chatgpt.com 网页 session cookie，带前缀存储） + - AccessToken: 以 'eyJhbGciOi' 或 'fk-' 开头（JWT / fakeopen token） + - RefreshToken: 老版 45 字符；或新版 Auth0 'rt_' 前缀（长度通常 80-100+） + - CustomToken: 其他 + """ + if not token: + return "Unknown" + if token.startswith("sess-"): + return "SessionToken" + if token.startswith("eyJhbGciOi") or token.startswith("fk-"): + return "AccessToken" + # 新版 Auth0 RefreshToken: rt_.，长度 ≥ 60 才算有效 + if token.startswith("rt_") and len(token) >= 60: + return "RefreshToken" + if len(token) == 45: + return "RefreshToken" + return "CustomToken" + + +def format_refresh_time(timestamp): + if not timestamp: + return "-" + return datetime.fromtimestamp(timestamp, tz=timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") + + +def get_routing_config(): + config = globals.routing_config or {} + config.setdefault("proxies", []) + config.setdefault("groups", []) + config.setdefault("bindings", {}) + config.setdefault("account_meta", {}) + config.setdefault("updated_at", None) + return config + + +def save_routing_config(config): + config["updated_at"] = utc_now() + globals.routing_config = config + with open(globals.ROUTING_CONFIG_FILE, "w", encoding="utf-8") as f: + json.dump(config, f, indent=2, ensure_ascii=False) + + +def resolve_group_name(config, proxy_name, proxy_url): + for group in config.get("groups", []): + if group.get("proxy_url") == proxy_url: + return group.get("name") + return f"Group {proxy_name}" + + +def build_group_assignments(tokens, proxies, group_size): + group_size = max(int(group_size or 1), 1) + existing_config = get_routing_config() + bindings = {} + groups = [] + normalized_proxies = [] + + for index, proxy in enumerate(proxies): + proxy_name = (proxy.get("name") or f"IP-{index + 1}").strip() + proxy_url = (proxy.get("proxy_url") or "").strip() + if not proxy_url: + continue + normalized_proxies.append({ + "id": f"proxy-{index + 1}", + "name": proxy_name, + "proxy_url": proxy_url, + }) + + for proxy_index, proxy in enumerate(normalized_proxies): + start = proxy_index * group_size + end = min(start + group_size, len(tokens)) + group_tokens = tokens[start:end] + if not group_tokens: + break + group_name = f"Group {chr(65 + proxy_index)}" + groups.append({ + "id": f"group-{proxy_index + 1}", + "name": group_name, + "proxy_name": proxy["name"], + "proxy_url": proxy["proxy_url"], + "size": len(group_tokens), + "strategy": "fixed", + "status": "enabled", + }) + for token in group_tokens: + previous_meta = existing_config.get("account_meta", {}).get(token, {}) + bindings[token] = { + "group": group_name, + "proxy_name": proxy["name"], + "proxy_url": proxy["proxy_url"], + "updated_at": utc_now(), + "note": previous_meta.get("note", ""), + } + + return { + "proxies": normalized_proxies, + "groups": groups, + "bindings": bindings, + "account_meta": existing_config.get("account_meta", {}), + } + + +def sync_bindings_to_fp(bindings): + changed = False + for token, binding in bindings.items(): + if not token: + continue + fp = globals.fp_map.get(token, {}) + proxy_url = binding.get("proxy_url") + if fp.get("proxy_url") != proxy_url: + fp["proxy_url"] = proxy_url + changed = True + if binding.get("group"): + fp["group"] = binding["group"] + if binding.get("proxy_name"): + fp["proxy_name"] = binding["proxy_name"] + fp["updated_at"] = binding.get("updated_at", utc_now()) + globals.fp_map[token] = fp + if changed: + logger.info("Routing bindings synced to fp_map.json") + with open(globals.FP_FILE, "w", encoding="utf-8") as f: + json.dump(globals.fp_map, f, indent=2, ensure_ascii=False) + + +def update_single_binding(token, proxy_name, proxy_url, group_name=None): + config = get_routing_config() + existing_proxy = next((item for item in config.get("proxies", []) if item.get("proxy_url") == proxy_url), None) + if not existing_proxy: + config.setdefault("proxies", []).append({ + "id": f"proxy-{len(config.get('proxies', [])) + 1}", + "name": proxy_name, + "proxy_url": proxy_url, + }) + + if not group_name: + group_name = resolve_group_name(config, proxy_name, proxy_url) + + config.setdefault("bindings", {})[token] = { + "group": group_name, + "proxy_name": proxy_name, + "proxy_url": proxy_url, + "updated_at": utc_now(), + "note": config.get("account_meta", {}).get(token, {}).get("note", ""), + } + save_routing_config(config) + sync_bindings_to_fp({token: config["bindings"][token]}) + return config["bindings"][token] + + +def update_account_meta(token, note="", group_name=None, proxy_name=None, proxy_url=None): + config = get_routing_config() + meta = config.setdefault("account_meta", {}).get(token, {}) + meta["note"] = (note or "").strip() + meta["updated_at"] = utc_now() + config["account_meta"][token] = meta + + if proxy_url: + binding = config.setdefault("bindings", {}).get(token, {}) + binding.update({ + "group": (group_name or binding.get("group") or resolve_group_name(config, proxy_name or "Custom Proxy", proxy_url)), + "proxy_name": proxy_name or binding.get("proxy_name") or "Custom Proxy", + "proxy_url": proxy_url, + "updated_at": utc_now(), + "note": meta["note"], + }) + config["bindings"][token] = binding + elif token in config.get("bindings", {}): + config["bindings"][token]["note"] = meta["note"] + config["bindings"][token]["updated_at"] = utc_now() + + save_routing_config(config) + if token in config.get("bindings", {}): + sync_bindings_to_fp({token: config["bindings"][token]}) + return { + "meta": config["account_meta"].get(token, {}), + "binding": config.get("bindings", {}).get(token), + } + + +def remove_account_binding(token): + config = get_routing_config() + removed_binding = config.get("bindings", {}).pop(token, None) + config.get("account_meta", {}).pop(token, None) + + grouped_rules = {} + for binding_token, binding in config.get("bindings", {}).items(): + group_name = binding.get("group") or binding.get("proxy_name") or "Ungrouped" + proxy_name = binding.get("proxy_name", "-") + proxy_url = binding.get("proxy_url", "") + rule = grouped_rules.setdefault(group_name, { + "id": f"group-{len(grouped_rules) + 1}", + "name": group_name, + "proxy_name": proxy_name, + "proxy_url": proxy_url, + "size": 0, + "strategy": "fixed", + "status": "enabled", + }) + rule["size"] += 1 + config["groups"] = list(grouped_rules.values()) + save_routing_config(config) + + if token in globals.fp_map: + globals.fp_map.pop(token, None) + with open(globals.FP_FILE, "w", encoding="utf-8") as f: + json.dump(globals.fp_map, f, indent=2, ensure_ascii=False) + + return removed_binding + + +def get_bound_proxy(req_token): + binding = get_routing_config().get("bindings", {}).get(req_token) + if binding: + return binding.get("proxy_url") + return None + + +def get_dashboard_payload(): + config = get_routing_config() + bindings = config.get("bindings", {}) + proxies = config.get("proxies", []) + tokens = list(globals.token_list) + error_tokens = set(globals.error_token_list) + active_tokens = len([token for token in tokens if token not in error_tokens]) + grouped_rules = {} + + proxy_stats = [] + for proxy in proxies: + matched_tokens = [token for token, binding in bindings.items() if binding.get("proxy_url") == proxy["proxy_url"]] + bad_count = len([token for token in matched_tokens if token in error_tokens]) + rule_name = None + if matched_tokens: + rule_name = bindings[matched_tokens[0]].get("group") + grouped_rules[rule_name or proxy["name"]] = { + "name": rule_name or proxy["name"], + "proxy_name": proxy["name"], + "proxy_url": proxy["proxy_url"], + "size": len(matched_tokens), + "strategy": "fixed", + "status": "enabled", + } + proxy_stats.append({ + "name": proxy["name"], + "proxy_url": proxy["proxy_url"], + "accounts": len(matched_tokens), + "ok": len(matched_tokens) - bad_count, + "bad": bad_count, + "group": rule_name or "-", + }) + + accounts = [] + for index, token in enumerate(tokens, start=1): + binding = bindings.get(token, {}) + account_meta = config.get("account_meta", {}).get(token, {}) + status = "异常" if token in error_tokens else "正常" + proxy_name = binding.get("proxy_name", "-") + proxy_url = binding.get("proxy_url", "") + group_name = binding.get("group", "-") + token_type = detect_token_type(token) + refresh_info = globals.refresh_map.get(token, {}) if token_type == "RefreshToken" else {} + refresh_status = "-" + if token_type == "RefreshToken": + if token in error_tokens: + refresh_status = "刷新异常" + elif refresh_info.get("last_success_at") or refresh_info.get("timestamp"): + refresh_status = "已刷新" + else: + refresh_status = "待刷新" + accounts.append({ + "id": f"acct-{index:03d}", + "token": token, + "token_masked": mask_token(token), + "token_type": token_type, + "status": status, + "proxy_name": proxy_name, + "proxy_url": proxy_url, + "group": group_name, + "note": account_meta.get("note", binding.get("note", "")), + "updated_at": binding.get("updated_at") or globals.fp_map.get(token, {}).get("updated_at") or "-", + "refresh_status": refresh_status, + "refresh_updated_at": format_refresh_time( + refresh_info.get("last_success_at") or refresh_info.get("timestamp") + ), + "refresh_error": refresh_info.get("last_error", ""), + "refresh_fail_count": refresh_info.get("fail_count", 0), + "can_refresh": token_type == "RefreshToken", + }) + + alerts = [] + if error_tokens: + alerts.append(f"当前有 {len(error_tokens)} 个异常账号，建议优先检查刷新状态。") + unbound_count = max(len(tokens) - len(bindings), 0) + if unbound_count: + alerts.append(f"还有 {unbound_count} 个账号未绑定固定代理。") + if not alerts: + alerts.append("当前未发现异常告警。") + + refresh_token_count = len([token for token in tokens if detect_token_type(token) == "RefreshToken"]) + stale_refresh_count = len([ + token for token in tokens + if detect_token_type(token) == "RefreshToken" + and not globals.refresh_map.get(token, {}).get("last_success_at") + and not globals.refresh_map.get(token, {}).get("timestamp") + ]) + if refresh_token_count: + alerts.append(f"当前共导入 {refresh_token_count} 个 RefreshToken。") + if stale_refresh_count: + alerts.append(f"其中有 {stale_refresh_count} 个 RefreshToken 还没有成功刷新过。") + + return { + "summary": { + "accounts_total": len(tokens), + "accounts_ok": active_tokens, + "accounts_bad": len(error_tokens), + "proxy_total": len(proxies), + "group_total": len(grouped_rules), + "bound_total": len(bindings), + }, + "ip_cards": proxy_stats, + "accounts": accounts, + "rules": list(grouped_rules.values()), + "alerts": alerts, + "updated_at": config.get("updated_at"), + } diff --git a/utils/token_parser.py b/utils/token_parser.py new file mode 100644 index 00000000..e076e171 --- /dev/null +++ b/utils/token_parser.py @@ -0,0 +1,209 @@ +"""Token 解析器：从文件/文本中识别 ChatGPT 账号凭据。 + +职责：纯函数模块，不依赖 FastAPI / globals，易于单测。 + +识别规则（与 utils/routing.detect_token_type 一致）： + - AccessToken: 以 'eyJhbGciOi' 或 'fk-' 开头 + - RefreshToken: 长度恰好 45 字符 + - 其他：归为 unknown + +返回结构 ParseResult: + { + "refresh_tokens": [str, ...], # 去重后 + "access_tokens": [str, ...], + "unknown": [str, ...], + "stats": {refresh_count, access_count, unknown_count, total, source}, + "warnings": [str, ...], + } +""" + +import json +import re +from typing import Dict, List, Optional, Set + + +# 候选字符串扫描：长度 ≥ 20 的字母数字 / 下划线 / 连字符 / 点 连串 +# JWT 含 '.'，RefreshToken 含字母数字 +_TOKEN_CANDIDATE_RE = re.compile(r"[A-Za-z0-9_\-\.]{20,}") + +# JSON 中的 token 字段名（大小写不敏感） +_TOKEN_KEY_NAMES = { + "refresh_token", "refreshtoken", "refresh", + "access_token", "accesstoken", "accesstoken", + "token", "session_token", "sessiontoken", +} + + +def _classify(token: str) -> str: + """返回 'session' | 'access' | 'refresh' | 'unknown'。 + + 与 utils/routing.detect_token_type 规则保持一致： + - session: 'sess-' 前缀（chatgpt.com 网页 session cookie，带前缀存储） + - access: 'eyJhbGciOi' / 'fk-' 开头 + - refresh: 'rt_' 前缀且长度 ≥ 60（新版 Auth0 格式）或 长度 45（老版） + """ + if not token: + return "unknown" + if token.startswith("sess-"): + return "session" + if token.startswith("eyJhbGciOi") or token.startswith("fk-"): + return "access" + if token.startswith("rt_") and len(token) >= 60: + return "refresh" + if len(token) == 45: + return "refresh" + return "unknown" + + +def _collect_from_json(obj, bucket: Set[str]) -> None: + """递归扫描 JSON，抽取所有看起来像 token 的字符串。""" + if isinstance(obj, dict): + for k, v in obj.items(): + key_lower = str(k).lower() + if isinstance(v, str): + # 命名约定优先：字段名命中 token 关键词 → 整值入库 + if key_lower in _TOKEN_KEY_NAMES: + stripped = v.strip() + if stripped: + bucket.add(stripped) + else: + # 普通字段：按候选正则扫描 + for m in _TOKEN_CANDIDATE_RE.findall(v): + bucket.add(m) + else: + _collect_from_json(v, bucket) + elif isinstance(obj, list): + for item in obj: + _collect_from_json(item, bucket) + elif isinstance(obj, str): + for m in _TOKEN_CANDIDATE_RE.findall(obj): + bucket.add(m) + # 其他类型（数字、布尔、None）忽略 + + +def _build_result(raw_tokens: Set[str], source: str, warnings: Optional[List[str]] = None) -> Dict: + session: List[str] = [] + refresh: List[str] = [] + access: List[str] = [] + unknown: List[str] = [] + for t in raw_tokens: + t = (t or "").strip() + if not t: + continue + kind = _classify(t) + if kind == "session": + session.append(t) + elif kind == "refresh": + refresh.append(t) + elif kind == "access": + access.append(t) + else: + unknown.append(t) + + session_set = sorted(set(session)) + refresh_set = sorted(set(refresh)) + access_set = sorted(set(access)) + unknown_set = sorted(set(unknown)) + + total = len({*session_set, *refresh_set, *access_set, *unknown_set}) + + return { + "session_tokens": session_set, + "refresh_tokens": refresh_set, + "access_tokens": access_set, + "unknown": unknown_set, + "stats": { + "session_count": len(session_set), + "refresh_count": len(refresh_set), + "access_count": len(access_set), + "unknown_count": len(unknown_set), + "total": total, + "source": source, + }, + "warnings": warnings or [], + } + + +def parse_text(content: str) -> Dict: + """按行扫描纯文本。 + + 规则： + - 空行跳过 + - '#' 开头行视为注释跳过 + - 支持 "token,account_id" 格式：取逗号前部分 + """ + if not content: + return _build_result(set(), source="text", warnings=["文件内容为空"]) + + tokens: Set[str] = set() + for line in content.splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + # 形如 "token,account_id" 取首段；无逗号则整行 + primary = line.split(",", 1)[0].strip() + if primary: + tokens.add(primary) + + warnings: List[str] = [] + if not tokens: + warnings.append("未从文件中识别到任何 token 候选") + + return _build_result(tokens, source="text", warnings=warnings) + + +def parse_json(content: str) -> Dict: + """尝试解析 JSON；失败则回退按纯文本扫描。""" + if not content: + return _build_result(set(), source="json", warnings=["文件内容为空"]) + + try: + obj = json.loads(content) + except json.JSONDecodeError as e: + # 退化为文本扫描（避免一个括号错误就丢掉整个文件） + fallback = parse_text(content) + fallback["stats"]["source"] = "json-fallback" + fallback.setdefault("warnings", []).append(f"JSON 解析失败，已回退文本扫描: {e}") + return fallback + + tokens: Set[str] = set() + _collect_from_json(obj, tokens) + + warnings: List[str] = [] + if not tokens: + warnings.append("JSON 中未识别到任何 token 候选") + + return _build_result(tokens, source="json", warnings=warnings) + + +def parse_file(filename: str, content: bytes) -> Dict: + """根据扩展名路由到对应解析器。""" + ext = "" + if filename and "." in filename: + ext = filename.rsplit(".", 1)[-1].lower() + + # 解码：UTF-8 优先，GBK 兜底 + try: + text = content.decode("utf-8") + except UnicodeDecodeError: + try: + text = content.decode("gbk") + except UnicodeDecodeError: + return _build_result( + set(), + source=ext or "bin", + warnings=["文件编码不支持（需 UTF-8 或 GBK）"], + ) + + if ext == "json": + return parse_json(text) + return parse_text(text) + + +def mask_token(token: str) -> str: + """前端预览用：显示前 6 + 后 4，防止整屏泄漏。""" + if not token: + return "" + if len(token) <= 12: + return token[:3] + "***" + token[-2:] + return f"{token[:6]}...{token[-4:]}"