diff --git a/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml b/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml index 06752f53..7a642bfc 100644 --- a/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml +++ b/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml @@ -79,3 +79,9 @@ body: label: 复现链接（可选） description: | 请提供能复现此问题的链接。 + - type: textarea + id: aigenerated + attributes: + label: AI生成内容（可选） + description: | + 如果此问题是由AI辅助您发现的，请提供全部聊天记录，包括使用的模型信息。 diff --git a/.github/ISSUE_TEMPLATE/01-bug_report_en.yml b/.github/ISSUE_TEMPLATE/01-bug_report_en.yml index 5d263c3c..322bbf96 100644 --- a/.github/ISSUE_TEMPLATE/01-bug_report_en.yml +++ b/.github/ISSUE_TEMPLATE/01-bug_report_en.yml @@ -79,3 +79,9 @@ body: label: Reproduction Link (optional) description: | Please provide a link to a repo or page that can reproduce this issue. + - type: textarea + id: aigenerated + attributes: + label: AI Generated Content (optional) + description: | + If this issue was identified with the assistance of AI, please provide the complete chat log, including information about the model used. diff --git a/.github/ISSUE_TEMPLATE/02-feature_request_zh.yml b/.github/ISSUE_TEMPLATE/02-feature_request_zh.yml index 821b2c44..76ef66d3 100644 --- a/.github/ISSUE_TEMPLATE/02-feature_request_zh.yml +++ b/.github/ISSUE_TEMPLATE/02-feature_request_zh.yml @@ -46,3 +46,9 @@ body: label: 附加信息 description: | 相关的任何其他上下文或截图，或者你觉得有帮助的信息 + - type: textarea + id: aigenerated + attributes: + label: AI生成内容（可选） + description: | + 如果此请求是由AI辅助您提交的，请提供全部聊天记录，包括使用的模型信息。 diff --git a/.github/ISSUE_TEMPLATE/03-feature_request_en.yml b/.github/ISSUE_TEMPLATE/03-feature_request_en.yml index 85b02488..521a4a2f 100644 --- a/.github/ISSUE_TEMPLATE/03-feature_request_en.yml +++ b/.github/ISSUE_TEMPLATE/03-feature_request_en.yml @@ -46,3 +46,9 @@ body: label: Additional Information description: | Any other context or screenshots related to this feature request, or information you find helpful. + - type: textarea + id: aigenerated + attributes: + label: AI Generated Content (optional) + description: | + If this request was submitted with the assistance of an AI, please provide the complete chat log, including information about the model used. diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index c8344eb6..6d4a6ef2 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -106,7 +106,7 @@ Violating these terms may lead to a permanent ban. ### 4. Permanent Ban **Community Impact**: Demonstrating a pattern of violation of community -standards, including sustained inappropriate behavior, harassment of an +standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. **Consequence**: A permanent ban from any sort of public interaction within @@ -116,7 +116,7 @@ the community. This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.0, available at -https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. +. Community Impact Guidelines were inspired by [Mozilla's code of conduct enforcement ladder](https://github.com/mozilla/diversity). @@ -124,5 +124,5 @@ enforcement ladder](https://github.com/mozilla/diversity). [homepage]: https://www.contributor-covenant.org For answers to common questions about this code of conduct, see the FAQ at -https://www.contributor-covenant.org/faq. Translations are available at -https://www.contributor-covenant.org/translations. +. Translations are available at +. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6064b03c..57317c62 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,7 +7,7 @@ Prerequisites: - [git](https://git-scm.com) -- [Go 1.24+](https://golang.org/doc/install) +- [Go](https://golang.org/doc/install) version declared in [`go.mod`](./go.mod) - [gcc](https://gcc.gnu.org/) - [nodejs](https://nodejs.org/) @@ -16,8 +16,8 @@ Prerequisites: Fork and clone `OpenList` and `OpenList-Frontend` anywhere: ```shell -$ git clone https://github.com//OpenList.git -$ git clone --recurse-submodules https://github.com//OpenList-Frontend.git +git clone https://github.com//OpenList.git +git clone --recurse-submodules https://github.com//OpenList-Frontend.git ``` ## Creating a branch @@ -25,7 +25,7 @@ $ git clone --recurse-submodules https://github.com//OpenList-Fro Create a new branch from the `main` branch, with an appropriate name. ```shell -$ git checkout -b +git checkout -b ``` ## Preview your change @@ -33,26 +33,36 @@ $ git checkout -b ### backend ```shell -$ go run main.go +go run main.go ``` ### frontend ```shell -$ pnpm dev +pnpm dev ``` ## Add a new driver Copy `drivers/template` folder and rename it, and follow the comments in it. +## Community and policies + +By contributing, you agree to follow the repository's code of conduct and license terms. + +- Code of conduct: [CODE_OF_CONDUCT.md](./CODE_OF_CONDUCT.md) +- License: [LICENSE](./LICENSE) +- Security issues: please report privately according to [SECURITY.md](./SECURITY.md) + +If your contribution includes substantial AI-assisted content, disclose the tools used and the scope of assistance in the pull request. + ## Create a commit Commit messages should be well formatted, and to make that "standardized". Submit your pull request. For PR titles, follow [Conventional Commits](https://www.conventionalcommits.org). -https://github.com/OpenListTeam/OpenList/issues/376 + It's suggested to sign your commits. See: [How to sign commits](https://docs.github.com/en/authentication/managing-commit-signature-verification/signing-commits) @@ -72,6 +82,19 @@ At least 1 approving review is required by reviewers with write access. You can (Optional) After your pull request is merged, you can delete your branch. +## AI Disclosure + +If your pull request includes substantial AI-assisted content, disclose it in the PR description. + +Please include: + +- Tools used, such as ChatGPT, GitHub Copilot, Claude, Cursor, or other AI tools. +- Usage scope, such as code generation, refactoring, documentation, tests, translation, or review assistance. +- Confirmation that you have reviewed and validated all AI-assisted content before submission. +- Confirmation that the submitted content complies with this repository's license and contribution policies. + +Minor AI assistance, such as typo fixes, autocomplete, formatting suggestions, or wording polish, does not need to be disclosed. + --- Thank you for your contribution! Let's make OpenList better together! diff --git a/README.md b/README.md index c4f29462..4a94dbfc 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@
- logo + logo

OpenList is a resilient, long-term governance, community-driven fork of AList — built to defend open source against trust-based attacks.

@@ -116,9 +116,10 @@ Thank you for your support and understanding of the OpenList project. ## Document -- 📘 [Global Site](https://doc.oplist.org) -- 📚 [Backup Site](https://doc.openlist.team) -- 🌏 [CN Site](https://doc.oplist.org.cn) +- 📘 [Docs](https://doc.oplist.org) +- 🌏 [CN Mirror](https://doc.oplist.org.cn) +- ⚖️ [Terms of Use](https://doc.oplist.org/terms) +- 🔒 [Privacy Policy](https://doc.oplist.org/privacy) ## Demo diff --git a/README_cn.md b/README_cn.md index adac3d0b..55fe2210 100644 --- a/README_cn.md +++ b/README_cn.md @@ -1,5 +1,5 @@
- logo + logo

OpenList 是一个有韧性、长期治理、社区驱动的 AList 分支，旨在防御基于信任的开源攻击。

@@ -116,14 +116,15 @@ OpenList 是一个由 OpenList 团队独立维护的开源项目，遵循 AGPL-3 ## 文档 -- 🌏 [国内站点](https://doc.oplist.org.cn) -- 📘 [海外站点](https://doc.oplist.org) -- 📚 [备用站点](https://doc.openlist.team) +- 📘 [文档](https://doc.oplist.org) +- 🌏 [中国镜像](https://doc.oplist.org.cn) +- ⚖️ [使用条款](https://doc.oplist.org/terms) +- 🔒 [隐私政策](https://doc.oplist.org/privacy) -## 演示 +## Demo -- 🇨🇳 [国内演示站](https://demo.oplist.org.cn) -- 🌎 [海外演示站](https://demo.oplist.org) +- 🌎 [全球 Demo](https://demo.oplist.org) +- 🇨🇳 [中国 Demo](https://demo.oplist.org.cn) ## 讨论 diff --git a/README_ja.md b/README_ja.md index 52c1a01c..261223de 100644 --- a/README_ja.md +++ b/README_ja.md @@ -1,5 +1,5 @@
- logo + logo

OpenList は、信頼ベースの攻撃からオープンソースを守るために構築された、レジリエントで長期ガバナンス、コミュニティ主導の AList フォークです。

@@ -116,14 +116,15 @@ OpenListプロジェクトへのご支援とご理解をありがとうござい ## ドキュメント -- 📘 [グローバルサイト](https://doc.oplist.org) -- 📚 [バックアップサイト](https://doc.openlist.team) -- 🌏 [CNサイト](https://doc.oplist.org.cn) +- 📘 [ドキュメント](https://doc.oplist.org) +- 🌏 [中国ミラー](https://doc.oplist.org.cn) +- ⚖️ [利用規約](https://doc.oplist.org/terms) +- 🔒 [プライバシーポリシー](https://doc.oplist.org/privacy) -## デモ +## Demo -- 🌎 [グローバルデモ](https://demo.oplist.org) -- 🇨🇳 [CNデモ](https://demo.oplist.org.cn) +- 🌎 [グローバル Demo](https://demo.oplist.org) +- 🇨🇳 [中国 Demo](https://demo.oplist.org.cn) ## ディスカッション diff --git a/README_nl.md b/README_nl.md index 8b9e62ec..d3be2703 100644 --- a/README_nl.md +++ b/README_nl.md @@ -1,5 +1,5 @@
- logo + logo

OpenList is een veerkrachtige, langetermijn, door de gemeenschap geleide fork van AList — gebouwd om open source te beschermen tegen op vertrouwen gebaseerde aanvallen.

@@ -116,9 +116,10 @@ Dank u voor uw ondersteuning en begrip ## Documentatie -- 📘 [Global Site](https://doc.oplist.org) -- 📚 [Backup Site](https://doc.openlist.team) -- 🌏 [CN Site](https://doc.oplist.org.cn) +- 📘 [Documentatie](https://doc.oplist.org) +- 🌏 [CN Mirror](https://doc.oplist.org.cn) +- ⚖️ [Gebruiksvoorwaarden](https://doc.oplist.org/terms) +- 🔒 [Privacybeleid](https://doc.oplist.org/privacy) ## Demo diff --git a/build.sh b/build.sh index d0f70a8a..e42a99f1 100644 --- a/build.sh +++ b/build.sh @@ -31,7 +31,7 @@ else # webVersion=$(eval "curl -fsSL --max-time 2 $githubAuthArgs \"https://api.github.com/repos/$frontendRepo/releases/latest\"" | grep "tag_name" | head -n 1 | awk -F ":" '{print $2}' | sed 's/\"//g;s/,//g;s/ //g') fi -webVersion=4.2.1 +webVersion=4.2.2 echo "backend version: $version" echo "frontend version: $webVersion" diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go index 7bb58890..0ea1270d 100644 --- a/drivers/115_open/driver.go +++ b/drivers/115_open/driver.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "net/http" + stdpath "path" + "slices" "strconv" "strings" "time" @@ -25,8 +27,9 @@ import ( type Open115 struct { model.Storage Addition - client *sdk.Client - limiter *rate.Limiter + client *sdk.Client + limiter *rate.Limiter + parentPath string } func (d *Open115) Config() driver.Config { @@ -62,6 +65,28 @@ func (d *Open115) Init(ctx context.Context) error { d.PageSize = 1150 } + // add parent path + d.parentPath = "/" + if d.GetRootId() != d.Config().DefaultRoot { + folderInfo, err := d.client.GetFolderInfo(ctx, d.GetRootId()) + if err != nil { + return err + } + + if folderInfo.FileID != d.Config().DefaultRoot { + d.parentPath = stdpath.Join(d.parentPath, folderInfo.FileName) + } + + parentPaths := folderInfo.Paths + slices.Reverse(parentPaths) + for _, parentPathInfo := range parentPaths { + if parentPathInfo.FileID == d.Config().DefaultRoot { + d.parentPath = stdpath.Join("/", d.parentPath) + } else { + d.parentPath = stdpath.Join("/", parentPathInfo.FileName, d.parentPath) + } + } + } return nil } @@ -144,6 +169,24 @@ func (d *Open115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) }, nil } +func (d *Open115) Get(ctx context.Context, path string) (model.Obj, error) { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } + path = stdpath.Join(d.parentPath, path) + resp, err := d.client.GetFolderInfoByPath(ctx, path) + if err != nil { + return nil, err + } + return &Obj{ + Fid: resp.FileID, + Fn: resp.FileName, + Fc: resp.FileCategory, + Sha1: resp.Sha1, + Pc: resp.PickCode, + }, nil +} + func (d *Open115) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { if err := d.WaitLimit(ctx); err != nil { return nil, err diff --git a/drivers/115_share/driver.go b/drivers/115_share/driver.go index 56d75b18..2af753e9 100644 --- a/drivers/115_share/driver.go +++ b/drivers/115_share/driver.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "net/http" _115 "github.com/OpenListTeam/OpenList/v4/drivers/115" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/op" @@ -133,12 +134,14 @@ func (d *Pan115Share) link(ctx context.Context, file model.Obj, args model.LinkA if err != nil { return nil, err } - go delayDelete115(pan115, sha1) exp := 4 * time.Hour + header := http.Header{} + header.Set("User-Agent", conf.UA115Browser) return &model.Link{ URL: downloadInfo.URL.URL + fmt.Sprintf("#storageId=%d", pan115.ID), Expiration: &exp, + Header: header, Concurrency: pan115.Concurrency, PartSize: pan115.ChunkSize * utils.KB, }, nil diff --git a/drivers/139/util.go b/drivers/139/util.go index d61cf0b4..26694633 100644 --- a/drivers/139/util.go +++ b/drivers/139/util.go @@ -658,10 +658,12 @@ func (d *Yun139) personalGetLink(fileId string) (string, error) { } cdnUrl := jsoniter.Get(res, "data", "cdnUrl").ToString() if cdnUrl != "" { - return cdnUrl, nil - } else { - return jsoniter.Get(res, "data", "url").ToString(), nil + cdnSwitch := jsoniter.Get(res, "data", "cdnSwitch").ToBool() + if cdnSwitch { + return cdnUrl, nil + } } + return jsoniter.Get(res, "data", "url").ToString(), nil } func (d *Yun139) getAuthorization() string { @@ -1181,7 +1183,6 @@ func (d *Yun139) step3_third_party_login(dycpwd string) (string, error) { "x-DeviceInfo": "4|127.0.0.1|5|1.2.6|Xiaomi|23116PN5BC||02-00-00-00-00-00|android 15|1440x3200|android|||", "Content-Type": "text/plain;charset=UTF-8", "Host": "user-njs.yun.139.com", - "Connection": "Keep-Alive", "Accept-Encoding": "gzip", "User-Agent": "okhttp/3.12.2", } diff --git a/drivers/189/torrent.go b/drivers/189/torrent.go new file mode 100644 index 00000000..4e41bf49 --- /dev/null +++ b/drivers/189/torrent.go @@ -0,0 +1,149 @@ +package _189 + +import ( + "context" + "crypto/sha1" + "encoding/hex" + "fmt" + "io" + "strings" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/torrent" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +// GenerateTorrent 根据上传过程中收集的哈希信息生成包含 CAS 扩展的 torrent 文件 +func GenerateTorrent(fileName string, fileSize int64, fileMD5 string, sliceMD5s []string, sliceSize int64, pieceHashes []byte) ([]byte, error) { + // 计算 sliceMD5 + sliceMD5 := fileMD5 + if len(sliceMD5s) > 1 { + joined := strings.Join(sliceMD5s, "\n") + sliceMD5 = strings.ToUpper(torrent.GetMD5Str(joined)) + } + + t := torrent.NewTorrent(fileName, fileSize, fileMD5) + t.Info.PieceLength = sliceSize + t.SetPieces(pieceHashes) + t.SetCASInfo(&torrent.CASInfo{ + FileMD5: fileMD5, + SliceMD5: sliceMD5, + SliceMD5s: sliceMD5s, + SliceSize: sliceSize, + Cloud: "189", + }) + + return t.Encode() +} + +// RapidUploadFromTorrent 从 torrent 文件中提取 CAS 信息进行秒传 +func (d *Cloud189) RapidUploadFromTorrent(ctx context.Context, dstDir model.Obj, torrentData []byte) error { + // 解析 torrent + t, err := torrent.Decode(torrentData) + if err != nil { + return fmt.Errorf("解析 torrent 失败: %w", err) + } + + // 检查是否包含 CAS 扩展信息 + if !t.HasCASInfo() { + return fmt.Errorf("torrent 不包含 CAS 扩展信息，无法秒传") + } + + cas := t.CAS + fileName := t.Info.Name + fileSize := t.GetTotalSize() + + // 获取 sessionKey + sessionKey, err := d.getSessionKey() + if err != nil { + return err + } + d.sessionKey = sessionKey + + // 初始化上传 + res, err := d.uploadRequest("/person/initMultiUpload", map[string]string{ + "parentFolderId": dstDir.GetID(), + "fileName": encode(fileName), + "fileSize": fmt.Sprint(fileSize), + "sliceSize": fmt.Sprint(cas.SliceSize), + "lazyCheck": "1", + }, nil) + if err != nil { + return fmt.Errorf("初始化上传失败: %w", err) + } + + uploadFileId := utils.Json.Get(res, "data", "uploadFileId").ToString() + + // 提交上传（使用 CAS 信息秒传） + _, err = d.uploadRequest("/person/commitMultiUploadFile", map[string]string{ + "uploadFileId": uploadFileId, + "fileMd5": cas.FileMD5, + "sliceMd5": cas.SliceMD5, + "lazyCheck": "1", + "opertype": "3", + }, nil) + if err != nil { + return fmt.Errorf("秒传提交失败: %w", err) + } + + return nil +} + +// ComputeTorrentFromReader 从 io.Reader 计算并生成 torrent 文件 +func ComputeTorrentFromReader(reader io.Reader, fileName string, fileSize int64, sliceSize int64) ([]byte, error) { + if sliceSize <= 0 { + sliceSize = torrent.DefaultPieceSize + } + + hw := torrent.NewHashWriter(sliceSize, sliceSize) + + buf := make([]byte, 32*1024) + for { + n, err := reader.Read(buf) + if n > 0 { + hw.Write(buf[:n]) + } + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + } + hw.Finish() + + fileMD5 := hw.GetFileMD5() + sliceMD5s := hw.GetSliceMD5s() + pieceHashes := hw.GetPieceHashes() + + return GenerateTorrent(fileName, fileSize, fileMD5, sliceMD5s, sliceSize, pieceHashes) +} + +// ComputePieceSHA1 计算单个分片的 SHA-1 哈希 +func ComputePieceSHA1(data []byte) []byte { + h := sha1.Sum(data) + return h[:] +} + +// ExtractCASFromTorrent 从 torrent 数据中提取 CAS 信息 +func ExtractCASFromTorrent(torrentData []byte) (*torrent.CASInfo, string, int64, error) { + t, err := torrent.Decode(torrentData) + if err != nil { + return nil, "", 0, fmt.Errorf("解析 torrent 失败: %w", err) + } + + if !t.HasCASInfo() { + return nil, "", 0, fmt.Errorf("torrent 不包含 CAS 扩展信息") + } + + return t.CAS, t.Info.Name, t.GetTotalSize(), nil +} + +// GetInfoHashHex 获取 torrent 的 info_hash（十六进制字符串） +func GetInfoHashHex(torrentData []byte) (string, error) { + t, err := torrent.Decode(torrentData) + if err != nil { + return "", err + } + return hex.EncodeToString(t.InfoHash), nil +} diff --git a/drivers/189/util.go b/drivers/189/util.go index 1a14b3dd..2211f266 100644 --- a/drivers/189/util.go +++ b/drivers/189/util.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "crypto/md5" + sha1Pkg "crypto/sha1" "encoding/base64" "encoding/hex" "errors" @@ -18,6 +19,8 @@ import ( "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" myrand "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" "github.com/go-resty/resty/v2" @@ -311,48 +314,107 @@ func (d *Cloud189) newUpload(ctx context.Context, dstDir model.Obj, file model.F } d.sessionKey = sessionKey const DEFAULT int64 = 10485760 - count := int64(math.Ceil(float64(file.GetSize()) / float64(DEFAULT))) + fileSize := file.GetSize() + count := int64(math.Ceil(float64(fileSize) / float64(DEFAULT))) - res, err := d.uploadRequest("/person/initMultiUpload", map[string]string{ + // 先计算文件完整MD5和分片MD5，用于秒传判断 + fileMd5Hex := file.GetHash().GetHash(utils.MD5) + sliceMd5Hex := "" + md5s := make([]string, 0) + + if len(fileMd5Hex) < utils.MD5.Width { + // 没有MD5，先缓存流并同时计算文件MD5和分片MD5 + fileMd5Hash := md5.New() + sliceMd5Hash := md5.New() + var finish int64 + cache, err := file.CacheFullAndWriter(nil, io.MultiWriter(fileMd5Hash, &sliceHashWriter{ + hash: sliceMd5Hash, + md5s: &md5s, + sliceSize: DEFAULT, + finish: &finish, + fileSize: fileSize, + up: up, + ctx: ctx, + })) + if err != nil { + return nil, err + } + // 处理最后一个分片的MD5 + if finish%DEFAULT != 0 || finish == 0 { + md5s = append(md5s, strings.ToUpper(hex.EncodeToString(sliceMd5Hash.Sum(nil)))) + } + fileMd5Hex = hex.EncodeToString(fileMd5Hash.Sum(nil)) + + // seek回起始位置，供后续上传使用 + if _, err := cache.Seek(0, io.SeekStart); err != nil { + return nil, err + } + } + + // 计算sliceMd5 + if fileSize > DEFAULT && len(md5s) > 0 { + sliceMd5Hex = utils.GetMD5EncodeStr(strings.Join(md5s, "\n")) + } else { + sliceMd5Hex = fileMd5Hex + } + + // 带fileMd5调用initMultiUpload，支持秒传 + initParams := map[string]string{ "parentFolderId": dstDir.GetID(), "fileName": encode(file.GetName()), - "fileSize": strconv.FormatInt(file.GetSize(), 10), + "fileSize": strconv.FormatInt(fileSize, 10), "sliceSize": strconv.FormatInt(DEFAULT, 10), - "lazyCheck": "1", - }, nil) + "fileMd5": fileMd5Hex, + "sliceMd5": sliceMd5Hex, + } + + res, err := d.uploadRequest("/person/initMultiUpload", initParams, nil) if err != nil { return nil, err } uploadFileId := jsoniter.Get(res, "data", "uploadFileId").ToString() - //_, err = d.uploadRequest("/person/getUploadedPartsInfo", map[string]string{ - // "uploadFileId": uploadFileId, - //}, nil) + fileDataExists := jsoniter.Get(res, "data", "fileDataExists").ToInt() + + // 秒传成功，直接提交 + if fileDataExists == 1 { + _, err = d.uploadRequest("/person/commitMultiUploadFile", map[string]string{ + "uploadFileId": uploadFileId, + "fileMd5": fileMd5Hex, + "sliceMd5": sliceMd5Hex, + "lazyCheck": "1", + "opertype": "3", + }, nil) + return nil, err + } + + // 非秒传，需要上传分片 var finish int64 = 0 var i int64 var byteSize int64 - md5s := make([]string, 0) - md5Sum := md5.New() + + // 额外计算 SHA-1 piece hash 用于生成 torrent + pieceSHA1Hashes := make([]byte, 0, int(count)*20) + for i = 1; i <= count; i++ { if utils.IsCanceled(ctx) { return nil, ctx.Err() } - byteSize = file.GetSize() - finish + byteSize = fileSize - finish if DEFAULT < byteSize { byteSize = DEFAULT } - // log.Debugf("%d,%d", byteSize, finish) byteData := make([]byte, byteSize) n, err := io.ReadFull(file, byteData) - // log.Debug(err, n) if err != nil { return nil, err } finish += int64(n) md5Bytes := getMd5(byteData) - md5Hex := hex.EncodeToString(md5Bytes) md5Base64 := base64.StdEncoding.EncodeToString(md5Bytes) - md5s = append(md5s, strings.ToUpper(md5Hex)) - md5Sum.Write(byteData) + + // 计算 SHA-1 piece hash + sha1Hash := sha1Pkg.Sum(byteData) + pieceSHA1Hashes = append(pieceSHA1Hashes, sha1Hash[:]...) var resp UploadUrlsResp res, err = d.uploadRequest("/person/getMultiUploadUrls", map[string]string{ "partInfo": fmt.Sprintf("%s-%s", strconv.FormatInt(i, 10), md5Base64), @@ -379,28 +441,62 @@ func (d *Cloud189) newUpload(ctx context.Context, dstDir model.Obj, file model.F } log.Debugf("%+v %+v", r, r.Request.Header) _ = r.Body.Close() - up(float64(i) * 100 / float64(count)) - } - fileMd5 := hex.EncodeToString(md5Sum.Sum(nil)) - sliceMd5 := fileMd5 - if file.GetSize() > DEFAULT { - sliceMd5 = utils.GetMD5EncodeStr(strings.Join(md5s, "\n")) + up(50 + float64(i)*50/float64(count)) } res, err = d.uploadRequest("/person/commitMultiUploadFile", map[string]string{ "uploadFileId": uploadFileId, - "fileMd5": fileMd5, - "sliceMd5": sliceMd5, + "fileMd5": fileMd5Hex, + "sliceMd5": sliceMd5Hex, "lazyCheck": "1", "opertype": "3", }, nil) if err != nil { return nil, err } + + // 生成 torrent 文件（异步，不影响上传结果） + capturedDstDir := dstDir + capturedFileName := file.GetName() + capturedFileSize := fileSize + capturedFileMd5Hex := fileMd5Hex + capturedMd5s := md5s + go func() { + fileMD5Upper := strings.ToUpper(capturedFileMd5Hex) + torrentData, err := GenerateTorrent(capturedFileName, capturedFileSize, fileMD5Upper, capturedMd5s, DEFAULT, pieceSHA1Hashes) + if err != nil { + log.Warnf("生成 torrent 失败: %v", err) + return + } + infoHash, _ := GetInfoHashHex(torrentData) + torrentName := capturedFileName + ".cas.torrent" + log.Infof("已生成 torrent: %s (info_hash: %s, size: %d bytes)", + torrentName, infoHash, len(torrentData)) + + // 将 torrent 文件上传到同一目录 + torrentFileStream := &stream.FileStream{ + Ctx: context.Background(), + Obj: &model.Object{ + Name: torrentName, + Size: int64(len(torrentData)), + IsFolder: false, + }, + Reader: bytes.NewReader(torrentData), + Mimetype: "application/x-bittorrent", + } + uploadErr := d.oldUpload(capturedDstDir, torrentFileStream) + if uploadErr != nil { + log.Warnf("上传 torrent 文件失败: %v", uploadErr) + } else { + log.Infof("torrent 文件已上传: %s", torrentName) + op.Cache.DeleteDirectory(d, capturedDstDir.GetPath()) + } + }() + return &casUploadInfo{ Name: file.GetName(), Size: file.GetSize(), - MD5: fileMd5, - SliceMD5: sliceMd5, + MD5: fileMd5Hex, + SliceMD5: sliceMd5Hex, }, nil } @@ -414,3 +510,52 @@ func (d *Cloud189) getCapacityInfo(ctx context.Context) (*CapacityResp, error) { } return &resp, nil } + +// sliceHashWriter 在写入过程中按分片大小自动切分并计算每个分片的MD5， +// 同时支持进度回调和取消检查。 +type sliceHashWriter struct { + hash io.Writer // 当前分片的MD5 hash + md5s *[]string // 收集每个分片的MD5十六进制字符串 + sliceSize int64 // 分片大小 + finish *int64 // 已写入的总字节数 + fileSize int64 // 文件总大小 + up driver.UpdateProgress + ctx context.Context +} + +func (w *sliceHashWriter) Write(p []byte) (int, error) { + if utils.IsCanceled(w.ctx) { + return 0, w.ctx.Err() + } + total := len(p) + written := 0 + for written < total { + // 当前分片还能写入的字节数 + sliceRemain := w.sliceSize - (*w.finish % w.sliceSize) + toWrite := int64(total - written) + if toWrite > sliceRemain { + toWrite = sliceRemain + } + n, err := w.hash.Write(p[written : written+int(toWrite)]) + if err != nil { + return written, err + } + written += n + *w.finish += int64(n) + + // 当前分片写满，记录MD5并重置 + if *w.finish%w.sliceSize == 0 { + if h, ok := w.hash.(interface{ Sum([]byte) []byte }); ok { + *w.md5s = append(*w.md5s, strings.ToUpper(hex.EncodeToString(h.Sum(nil)))) + } + if resetter, ok := w.hash.(interface{ Reset() }); ok { + resetter.Reset() + } + } + } + // 报告进度（缓存阶段占50%） + if w.fileSize > 0 && w.up != nil { + w.up(float64(*w.finish) / float64(w.fileSize) * 50) + } + return total, nil +} diff --git a/drivers/189pc/driver.go b/drivers/189pc/driver.go index f44ec306..b481b110 100644 --- a/drivers/189pc/driver.go +++ b/drivers/189pc/driver.go @@ -315,29 +315,35 @@ func (y *Cloud189PC) Rename(ctx context.Context, srcObj model.Obj, newName strin queryParam["familyId"] = y.FamilyID } - var newObj model.Obj - switch f := srcObj.(type) { + switch srcObj.(type) { case *Cloud189File: fullUrl += "/renameFile.action" queryParam["fileId"] = srcObj.GetID() queryParam["destFileName"] = newName - newObj = &Cloud189File{Icon: f.Icon} // 复用预览 case *Cloud189Folder: fullUrl += "/renameFolder.action" queryParam["folderId"] = srcObj.GetID() queryParam["destFolderName"] = newName - newObj = &Cloud189Folder{} default: return nil, errs.NotSupport } - + var resp RenameResp _, err := y.request(fullUrl, method, func(req *resty.Request) { req.SetContext(ctx).SetQueryParams(queryParam) - }, nil, newObj, isFamily) + }, nil, resp, isFamily) if err != nil { + if code, ok := resp.ResCode.(string); ok && code == "FileAlreadyExists" { + return nil, errs.ObjectAlreadyExists + } return nil, err } - return newObj, nil + switch f := srcObj.(type) { + case *Cloud189File: + return resp.toFile(f), nil + case *Cloud189Folder: + return resp.toFolder(), nil + } + return nil, errs.NotSupport } func (y *Cloud189PC) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { @@ -421,10 +427,11 @@ func (y *Cloud189PC) uploadFile(ctx context.Context, dstDir model.Obj, stream mo uploadMethod := y.UploadMethod if stream.IsForceStreamUpload() { uploadMethod = "stream" - } - - // 旧版上传家庭云也有限制 - if uploadMethod == "old" { + } else if y.Addition.RapidUpload && stream.GetFile() != nil { + // 文件流支持随机读取，走FastUpload计算MD5并尝试秒传 + uploadMethod = "rapid" + } else if uploadMethod == "old" { + // 旧版上传家庭云也有限制 return y.OldUpload(ctx, dstDir, stream, up, isFamily, overwrite) } @@ -486,6 +493,19 @@ func (y *Cloud189PC) uploadFile(ctx context.Context, dstDir model.Obj, stream mo if stream.GetSize() == 0 { return y.FastUpload(ctx, dstDir, stream, up, isFamily, overwrite) } + // 尝试秒传：如果已有MD5且启用了RapidUpload则用RapidUpload，否则走FastUpload（会计算MD5并尝试秒传） + if !stream.IsForceStreamUpload() { + fileMd5 := stream.GetHash().GetHash(utils.MD5) + if len(fileMd5) >= utils.MD5.Width && y.Addition.RapidUpload { + // 源文件已有MD5且启用了RapidUpload配置，尝试快速秒传 + if newObj, info, err := y.RapidUpload(ctx, dstDir, stream, isFamily, overwrite); err == nil { + return newObj, info, nil + } + } else if len(fileMd5) < utils.MD5.Width { + // 源文件无MD5（如从本地复制），走FastUpload计算MD5并尝试秒传 + return y.FastUpload(ctx, dstDir, stream, up, isFamily, overwrite) + } + } fallthrough default: return y.StreamUpload(ctx, dstDir, stream, up, isFamily, overwrite) diff --git a/drivers/189pc/meta.go b/drivers/189pc/meta.go index 41219f1c..9ba20b14 100644 --- a/drivers/189pc/meta.go +++ b/drivers/189pc/meta.go @@ -21,6 +21,7 @@ type Addition struct { FamilyTransfer bool `json:"family_transfer"` RapidUpload bool `json:"rapid_upload"` NoUseOcr bool `json:"no_use_ocr"` + GenerateTorrent bool `json:"generate_torrent" help:"Generate torrent file with CAS extension after upload"` GenerateCAS bool `json:"generate_cas" help:"上传文件后，在同目录生成一个同名的 .cas 元数据文件"` DeleteSource bool `json:"delete_source" help:"成功生成 .cas 文件后，自动删除原始源文件"` RestoreSourceFromCAS bool `json:"restore_source_from_cas" help:"上传 .cas 文件时，尝试根据其中的哈希信息秒传还原源文件，而不是直接上传 .cas 文件本身"` diff --git a/drivers/189pc/torrent.go b/drivers/189pc/torrent.go new file mode 100644 index 00000000..3068a0f7 --- /dev/null +++ b/drivers/189pc/torrent.go @@ -0,0 +1,296 @@ +package _189pc + +import ( + "context" + "crypto/sha1" + "encoding/hex" + "fmt" + "io" + "net/url" + "strings" + + "github.com/go-resty/resty/v2" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/torrent" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +// GenerateTorrent 根据上传过程中收集的哈希信息生成包含 CAS 扩展的 torrent 文件 +// fileMD5: 整文件 MD5（大写十六进制） +// sliceMD5s: 每个分片的 MD5 列表（大写十六进制） +// sliceSize: 分片大小 +// pieceHashes: SHA-1 piece hashes 拼接（每 20 字节一个） +// fileName: 文件名 +// fileSize: 文件大小 +func GenerateTorrent(fileName string, fileSize int64, fileMD5 string, sliceMD5s []string, sliceSize int64, pieceHashes []byte) ([]byte, error) { + // 计算 sliceMD5 + sliceMD5 := fileMD5 + if len(sliceMD5s) > 1 { + joined := strings.Join(sliceMD5s, "\n") + sliceMD5 = strings.ToUpper(torrent.GetMD5Str(joined)) + } + + t := torrent.NewTorrent(fileName, fileSize, fileMD5) + t.Info.PieceLength = sliceSize + t.SetPieces(pieceHashes) + t.SetCASInfo(&torrent.CASInfo{ + FileMD5: fileMD5, + SliceMD5: sliceMD5, + SliceMD5s: sliceMD5s, + SliceSize: sliceSize, + Cloud: "189", + }) + + return t.Encode() +} + +// RapidUploadFromTorrent 从 torrent 文件中提取 CAS 信息进行秒传 +// 返回值：上传成功的文件对象、错误 +func (y *Cloud189PC) RapidUploadFromTorrent(ctx context.Context, dstDir model.Obj, torrentData []byte, overwrite bool) (model.Obj, error) { + isFamily := y.isFamily() + + // 解析 torrent + t, err := torrent.Decode(torrentData) + if err != nil { + return nil, fmt.Errorf("解析 torrent 失败: %w", err) + } + + // 检查是否包含 CAS 扩展信息 + if !t.HasCASInfo() { + return nil, fmt.Errorf("torrent 不包含 CAS 扩展信息，无法秒传") + } + + cas := t.CAS + fileName := t.Info.Name + fileSize := t.GetTotalSize() + + // 统一 MD5 为大写（与正常上传保持一致，天翼云盘要求大写） + fileMD5Upper := strings.ToUpper(cas.FileMD5) + + // 优先使用 torrent 中嵌入的分片大小，与生成时保持一致 + sliceSize := cas.SliceSize + if sliceSize <= 0 { + sliceSize = partSize(fileSize) + } + + // 计算 sliceMd5（与上传时一致的算法） + // 优先使用 torrent 中已有的 SliceMD5；仅当有多分片列表时才重新计算 + sliceMd5Hex := strings.ToUpper(cas.SliceMD5) + if sliceMd5Hex == "" { + sliceMd5Hex = fileMD5Upper + } + if len(cas.SliceMD5s) > 1 { + // 分片 MD5 也需要统一大写后再拼接计算 + upperSliceMD5s := make([]string, len(cas.SliceMD5s)) + for i, s := range cas.SliceMD5s { + upperSliceMD5s[i] = strings.ToUpper(s) + } + sliceMd5Hex = strings.ToUpper(utils.GetMD5EncodeStr(strings.Join(upperSliceMD5s, "\n"))) + } + + + // 使用与 Web 端一致的三步秒传流程 + fullUrl := "https://upload.cloud.189.cn" + if isFamily { + fullUrl += "/family" + } else { + fullUrl += "/person" + } + + // Step 1: initMultiUpload（不传 fileMd5/sliceMd5，只传 lazyCheck） + initParams := Params{ + "parentFolderId": dstDir.GetID(), + "fileName": url.QueryEscape(fileName), + "fileSize": fmt.Sprint(fileSize), + "sliceSize": fmt.Sprint(sliceSize), + "lazyCheck": "1", + } + if isFamily { + initParams.Set("familyId", y.FamilyID) + } + + + var uploadInfo InitMultiUploadResp + _, err = y.request(fullUrl+"/initMultiUpload", "GET", func(req *resty.Request) { + req.SetContext(ctx) + }, initParams, &uploadInfo, isFamily) + if err != nil { + return nil, fmt.Errorf("initMultiUpload 失败: %w", err) + } + + + uploadFileId := uploadInfo.Data.UploadFileID + + // Step 2: checkTransSecond（用 fileMd5 + sliceMd5 + uploadFileId 检查秒传） + checkParams := Params{ + "fileMd5": fileMD5Upper, + "sliceMd5": sliceMd5Hex, + "uploadFileId": uploadFileId, + } + + + var checkResp struct { + Data struct { + FileDataExists int `json:"fileDataExists"` + } `json:"data"` + } + _, err = y.request(fullUrl+"/checkTransSecond", "GET", func(req *resty.Request) { + req.SetContext(ctx) + }, checkParams, &checkResp, isFamily) + if err != nil { + utils.Log.Errorf("[RapidUpload] checkTransSecond 失败: uploadFileId=%s, err=%v", uploadFileId, err) + return nil, fmt.Errorf("秒传检查失败: %w", err) + } + + + if checkResp.Data.FileDataExists != 1 { + return nil, fmt.Errorf("秒传失败：云端不存在该文件（fileMD5=%s, sliceMD5=%s, size=%d）", fileMD5Upper, sliceMd5Hex, fileSize) + } + + // Step 3: commitMultiUploadFile（传 fileMd5 + sliceMd5） + + var resp CommitMultiUploadFileResp + commitParams := Params{ + "uploadFileId": uploadFileId, + "fileMd5": fileMD5Upper, + "sliceMd5": sliceMd5Hex, + "lazyCheck": "1", + "opertype": IF(overwrite, "3", "1"), + } + + _, err = y.request(fullUrl+"/commitMultiUploadFile", "GET", func(req *resty.Request) { + req.SetContext(ctx) + }, commitParams, &resp, isFamily) + if err != nil { + utils.Log.Errorf("[RapidUpload] commitMultiUploadFile 失败: uploadFileId=%s, err=%v", uploadFileId, err) + return nil, fmt.Errorf("提交上传失败: %w", err) + } + + return resp.toFile(), nil +} + +// ComputeTorrentFromReader 从 io.Reader 计算并生成 torrent 文件 +// 适用于：已有文件需要生成 torrent 的场景（如下载完成后生成） +func ComputeTorrentFromReader(reader io.Reader, fileName string, fileSize int64, sliceSize int64) ([]byte, error) { + if sliceSize <= 0 { + sliceSize = torrent.DefaultPieceSize + } + + hw := torrent.NewHashWriter(sliceSize, sliceSize) + + buf := make([]byte, 32*1024) + for { + n, err := reader.Read(buf) + if n > 0 { + hw.Write(buf[:n]) + } + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + } + hw.Finish() + + fileMD5 := hw.GetFileMD5() + sliceMD5s := hw.GetSliceMD5s() + pieceHashes := hw.GetPieceHashes() + + return GenerateTorrent(fileName, fileSize, fileMD5, sliceMD5s, sliceSize, pieceHashes) +} + +// ComputePieceSHA1 计算单个分片的 SHA-1 哈希 +func ComputePieceSHA1(data []byte) []byte { + h := sha1.Sum(data) + return h[:] +} + +// ExtractCASFromTorrent 从 torrent 数据中提取 CAS 信息 +// 返回：CAS 信息、文件名、文件大小、错误 +func ExtractCASFromTorrent(torrentData []byte) (*torrent.CASInfo, string, int64, error) { + t, err := torrent.Decode(torrentData) + if err != nil { + return nil, "", 0, fmt.Errorf("解析 torrent 失败: %w", err) + } + + if !t.HasCASInfo() { + return nil, "", 0, fmt.Errorf("torrent 不包含 CAS 扩展信息") + } + + return t.CAS, t.Info.Name, t.GetTotalSize(), nil +} + +// InjectCASIntoTorrent 向已有的 torrent 文件注入 CAS 扩展信息 +// 用于：下载完成后，计算了 MD5 信息，写回到 torrent 中 +func InjectCASIntoTorrent(torrentData []byte, fileMD5 string, sliceMD5s []string, sliceSize int64) ([]byte, error) { + t, err := torrent.Decode(torrentData) + if err != nil { + return nil, fmt.Errorf("解析 torrent 失败: %w", err) + } + + // 计算 sliceMD5 + sliceMD5 := fileMD5 + if len(sliceMD5s) > 1 { + joined := strings.Join(sliceMD5s, "\n") + sliceMD5 = strings.ToUpper(torrent.GetMD5Str(joined)) + } + + // 注入 CAS 信息 + t.SetCASInfo(&torrent.CASInfo{ + FileMD5: fileMD5, + SliceMD5: sliceMD5, + SliceMD5s: sliceMD5s, + SliceSize: sliceSize, + Cloud: "189", + }) + + // 同时更新 info 中的 md5sum 字段 + if t.Info.MD5Sum == "" { + t.Info.MD5Sum = fileMD5 + } + + return t.Encode() +} + +// GetInfoHashHex 获取 torrent 的 info_hash（十六进制字符串） +func GetInfoHashHex(torrentData []byte) (string, error) { + t, err := torrent.Decode(torrentData) + if err != nil { + return "", err + } + return hex.EncodeToString(t.InfoHash), nil +} + +// ComputeSliceMD5sFromReader 从 reader 中计算每个 10MB 分片的 MD5 +// 返回：整文件 MD5、分片 MD5 列表 +func ComputeSliceMD5sFromReader(reader io.Reader, sliceSize int64) (string, []string, error) { + if sliceSize <= 0 { + sliceSize = torrent.DefaultPieceSize + } + + fileMD5Hash := utils.MD5.NewFunc() + sliceMD5s := make([]string, 0) + + buf := make([]byte, sliceSize) + for { + n, err := io.ReadFull(reader, buf) + if n > 0 { + chunk := buf[:n] + fileMD5Hash.Write(chunk) + // 计算该分片的 MD5 + sliceMD5 := strings.ToUpper(utils.HashData(utils.MD5, chunk)) + sliceMD5s = append(sliceMD5s, sliceMD5) + } + if err == io.EOF || err == io.ErrUnexpectedEOF { + break + } + if err != nil { + return "", nil, err + } + } + + fileMD5Hex := strings.ToUpper(hex.EncodeToString(fileMD5Hash.Sum(nil))) + return fileMD5Hex, sliceMD5s, nil +} diff --git a/drivers/189pc/types.go b/drivers/189pc/types.go index f05db030..faeafa52 100644 --- a/drivers/189pc/types.go +++ b/drivers/189pc/types.go @@ -429,3 +429,41 @@ type CapacityResp struct { } `json:"familyCapacityInfo"` TotalSize uint64 `json:"totalSize"` } + +type RenameResp struct { + ResMsg string `json:"res_message"` + CreateDate Time `json:"createDate"` + FileCate int `json:"fileCata"` + ID string `json:"id"` + LastOpTime Time `json:"lastOpTime"` + MD5 string `json:"md5"` + MediaType int `json:"mediaType"` + Name string `json:"name"` + Oeientation int `json:"orientation"` + ParentID int64 `json:"parentId"` + Rev string `json:"rev"` + Size int64 `json:"size"` + ResCode any `json:"res_code"` // int or string +} + +func (r *RenameResp) toFile(f *Cloud189File) *Cloud189File { + return &Cloud189File{ + ID: String(r.ID), + Name: r.Name, + Size: r.Size, + Md5: r.MD5, + LastOpTime: r.LastOpTime, + CreateDate: r.CreateDate, + Icon: f.Icon, + } +} + +func (r *RenameResp) toFolder() *Cloud189Folder { + return &Cloud189Folder{ + ID: String(r.ID), + Name: r.Name, + ParentID: r.ParentID, + LastOpTime: r.LastOpTime, + CreateDate: r.CreateDate, + } +} diff --git a/drivers/189pc/utils.go b/drivers/189pc/utils.go index e2b0995d..0e330677 100644 --- a/drivers/189pc/utils.go +++ b/drivers/189pc/utils.go @@ -3,6 +3,7 @@ package _189pc import ( "bytes" "context" + sha1Pkg "crypto/sha1" "encoding/base64" "encoding/hex" "encoding/xml" @@ -739,6 +740,10 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo silceMd5 := utils.MD5.NewFunc() var writers io.Writer = silceMd5 + // 如果启用了 torrent 生成，额外计算 SHA-1 piece hash + generateTorrent := y.Addition.GenerateTorrent + pieceSHA1Hashes := make([]byte, 0, count*20) + fileMd5Hex := file.GetHash().GetHash(utils.MD5) var fileMd5 hash.Hash if len(fileMd5Hex) != utils.MD5.Width { @@ -763,7 +768,18 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo return err } silceMd5.Reset() - w, err := utils.CopyWithBuffer(writers, reader) + + // 如果需要生成 torrent，同时计算 SHA-1 + var sha1Writer hash.Hash + var multiWriter io.Writer + if generateTorrent { + sha1Writer = sha1Pkg.New() + multiWriter = io.MultiWriter(writers, sha1Writer) + } else { + multiWriter = writers + } + + w, err := utils.CopyWithBuffer(multiWriter, reader) if w != partSize { return fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", partSize, w, err) } @@ -771,6 +787,11 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo md5Bytes := silceMd5.Sum(nil) silceMd5Hexs = append(silceMd5Hexs, strings.ToUpper(hex.EncodeToString(md5Bytes))) partInfo = fmt.Sprintf("%d-%s", i, base64.StdEncoding.EncodeToString(md5Bytes)) + + // 收集 SHA-1 piece hash + if generateTorrent && sha1Writer != nil { + pieceSHA1Hashes = append(pieceSHA1Hashes, sha1Writer.Sum(nil)...) + } return nil }, Do: func(ctx context.Context) (err error) { @@ -824,6 +845,44 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo if err != nil { return nil, nil, err } + // 生成 torrent 文件（异步，不影响上传结果） + if generateTorrent && len(pieceSHA1Hashes) > 0 { + // 捕获必要的变量 + capturedDstDir := dstDir + capturedIsFamily := isFamily + capturedFileName := file.GetName() + go func() { + torrentData, err := GenerateTorrent(capturedFileName, fileSize, fileMd5Hex, silceMd5Hexs, sliceSize, pieceSHA1Hashes) + if err != nil { + utils.Log.Warnf("生成 torrent 失败: %v", err) + return + } + infoHash, _ := GetInfoHashHex(torrentData) + torrentName := capturedFileName + ".cas.torrent" + utils.Log.Infof("已生成 torrent: %s (info_hash: %s, size: %d bytes)", + torrentName, infoHash, len(torrentData)) + + // 将 torrent 文件上传到同一目录（使用 FastUpload，因为 torrent 文件很小） + torrentFileStream := &stream.FileStream{ + Ctx: context.Background(), + Obj: &model.Object{ + Name: torrentName, + Size: int64(len(torrentData)), + IsFolder: false, + }, + Reader: bytes.NewReader(torrentData), + Mimetype: "application/x-bittorrent", + } + _, _, uploadErr := y.FastUpload(context.Background(), capturedDstDir, torrentFileStream, func(p float64) {}, capturedIsFamily, false) + if uploadErr != nil { + utils.Log.Warnf("上传 torrent 文件失败: %v", uploadErr) + } else { + utils.Log.Infof("torrent 文件已上传: %s", torrentName) + op.Cache.DeleteDirectory(y, capturedDstDir.GetPath()) + } + }() + } + return resp.toFile(), &casUploadInfo{ Name: file.GetName(), Size: fileSize, diff --git a/drivers/cloudreve/driver.go b/drivers/cloudreve/driver.go index 55462be7..12c4fc02 100644 --- a/drivers/cloudreve/driver.go +++ b/drivers/cloudreve/driver.go @@ -89,6 +89,10 @@ func (d *Cloudreve) Link(ctx context.Context, file model.Obj, args model.LinkArg } return &model.Link{ URL: dUrl, + Header: http.Header{ + "Referer": {d.Address}, + "User-Agent": {d.getUA()}, + }, }, nil } diff --git a/drivers/cloudreve_v4/driver.go b/drivers/cloudreve_v4/driver.go index 2963bf46..afd64d3d 100644 --- a/drivers/cloudreve_v4/driver.go +++ b/drivers/cloudreve_v4/driver.go @@ -167,6 +167,10 @@ func (d *CloudreveV4) Link(ctx context.Context, file model.Obj, args model.LinkA return &model.Link{ URL: url.Urls[0].URL, Expiration: &exp, + Header: http.Header{ + "Referer": {d.Address}, + "User-Agent": {d.getUA()}, + }, }, nil } diff --git a/drivers/cloudreve_v4/util.go b/drivers/cloudreve_v4/util.go index 5d0157ff..9cf8d98a 100644 --- a/drivers/cloudreve_v4/util.go +++ b/drivers/cloudreve_v4/util.go @@ -33,6 +33,7 @@ const ( CodeLoginRequired = http.StatusUnauthorized CodePathNotExist = 40016 // Path not exist CodeCredentialInvalid = 40020 // Failed to issue token + CodeObjectExisted = 40004 // Object existed // IncorrectSharePassword = 40069 // Incorrect share password ) @@ -107,6 +108,9 @@ func (d *CloudreveV4) _request(method string, path string, callback base.ReqCall if r.Code == CodePathNotExist { return errs.ObjectNotFound } + if r.Code == CodeObjectExisted { + return errs.ObjectAlreadyExists + } return fmt.Errorf("%d: %s", r.Code, r.Msg) } diff --git a/drivers/lanzou/driver.go b/drivers/lanzou/driver.go index 01d7c1ec..e143cde2 100644 --- a/drivers/lanzou/driver.go +++ b/drivers/lanzou/driver.go @@ -117,7 +117,7 @@ func (d *LanZou) Link(ctx context.Context, file model.Obj, args model.LinkArgs) return &model.Link{ URL: dfile.Url, Header: http.Header{ - "User-Agent": []string{base.UserAgent}, + "User-Agent": {d.UserAgent}, }, Expiration: &exp, }, nil diff --git a/drivers/lenovonas_share/driver.go b/drivers/lenovonas_share/driver.go index 012e2e63..526eb1f4 100644 --- a/drivers/lenovonas_share/driver.go +++ b/drivers/lenovonas_share/driver.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "net/url" + stdpath "path" "strings" "time" @@ -32,6 +33,7 @@ func (d *LenovoNasShare) GetAddition() driver.Additional { } func (d *LenovoNasShare) Init(ctx context.Context) error { + d.ShareId = stdpath.Base(d.ShareId) if err := d.getStoken(); err != nil { return err } @@ -98,9 +100,6 @@ func (d *LenovoNasShare) getStoken() error { // 获取stoken d.Host = "https://siot-share.lenovo.com.cn" } - parts := strings.Split(d.ShareId, "/") - d.ShareId = parts[len(parts)-1] - query := map[string]string{ "code": d.ShareId, "password": d.SharePwd, diff --git a/drivers/quark_uc/driver.go b/drivers/quark_uc/driver.go index d285f742..2a9630f0 100644 --- a/drivers/quark_uc/driver.go +++ b/drivers/quark_uc/driver.go @@ -85,9 +85,12 @@ func (d *QuarkOrUC) MakeDir(ctx context.Context, parentDir model.Obj, dirName st _, err := d.request("/file", http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) - if err == nil { + if err == nil || err.Error() == "file is doloading[同名冲突]" { time.Sleep(time.Second) } + if err != nil && err.Error() == "file is doloading[同名冲突]" { + return errs.ObjectAlreadyExists + } return err } diff --git a/drivers/s3/driver.go b/drivers/s3/driver.go index a9edef10..a037912f 100644 --- a/drivers/s3/driver.go +++ b/drivers/s3/driver.go @@ -16,9 +16,12 @@ import ( "github.com/OpenListTeam/OpenList/v4/pkg/cron" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) @@ -243,4 +246,74 @@ func (d *S3) GetDirectUploadInfo(ctx context.Context, _ string, dstDir model.Obj }, nil } +// implements driver.Getter interface +func (d *S3) Get(ctx context.Context, path string) (model.Obj, error) { + // try to get object as a file using HeadObject + path = stdpath.Join(d.GetRootPath(), path) + key := getKey(path, false) + headInput := &s3.HeadObjectInput{ + Bucket: &d.Bucket, + Key: &key, + } + headOutput, err := d.client.HeadObjectWithContext(ctx, headInput) + if err == nil { + // Object exists as a file + fileName := stdpath.Base(path) + return &model.Object{ + Name: fileName, + Size: *headOutput.ContentLength, + Modified: *headOutput.LastModified, + Path: path, + }, nil + } + var awsErr awserr.Error + if errors.As(err, &awsErr) && awsErr.Code() != "NotFound" { + return nil, errors.WithMessage(err, "failed to head object") + } + + // If HeadObject fails with 404, check if it's a directory + prefix := getKey(path, true) + var contents []*s3.Object + var commonPrefixes []*s3.CommonPrefix + switch d.ListObjectVersion { + case "v1": + listInput := &s3.ListObjectsInput{ + Bucket: &d.Bucket, + Prefix: &prefix, + MaxKeys: aws.Int64(1), // Only need to check if at least one object exists + } + listResult, err := d.client.ListObjectsWithContext(ctx, listInput) + if err != nil { + return nil, errors.WithMessage(err, "failed to list objects with prefix") + } + contents = listResult.Contents + commonPrefixes = listResult.CommonPrefixes + case "v2": + listInput := &s3.ListObjectsV2Input{ + Bucket: &d.Bucket, + Prefix: &prefix, + MaxKeys: aws.Int64(1), + } + listResult, err := d.client.ListObjectsV2WithContext(ctx, listInput) + if err != nil { + return nil, errors.WithMessage(err, "failed to list objects v2 with prefix") + } + contents = listResult.Contents + commonPrefixes = listResult.CommonPrefixes + default: + return nil, fmt.Errorf("unsupported ListObjectVersion: %s", d.ListObjectVersion) + } + if len(contents) > 0 || len(commonPrefixes) > 0 { + dirName := stdpath.Base(path) + return &model.Object{ + Name: dirName, + Modified: d.Modified, + IsFolder: true, + Path: path, + }, nil + } + return nil, errs.ObjectNotFound +} + var _ driver.Driver = (*S3)(nil) +var _ driver.Getter = (*S3)(nil) diff --git a/drivers/strm/driver.go b/drivers/strm/driver.go index 422f0e1b..4736ff1c 100644 --- a/drivers/strm/driver.go +++ b/drivers/strm/driver.go @@ -45,7 +45,7 @@ func (d *Strm) Init(ctx context.Context) error { return errors.New("SaveStrmLocalPath is required") } d.pathMap = make(map[string][]string) - for _, path := range strings.Split(d.Paths, "\n") { + for path := range strings.SplitSeq(d.Paths, "\n") { path = strings.TrimSpace(path) if path == "" { continue @@ -97,8 +97,8 @@ func (d *Strm) Init(ctx context.Context) error { } if d.Version != 5 { - types := strings.Split("mp4,mkv,flv,avi,wmv,ts,rmvb,webm,mp3,flac,aac,wav,ogg,m4a,wma,alac", ",") - for _, ext := range types { + types := strings.SplitSeq("mp4,mkv,flv,avi,wmv,ts,rmvb,webm,mp3,flac,aac,wav,ogg,m4a,wma,alac", ",") + for ext := range types { if _, ok := d.supportSuffix[ext]; !ok { d.supportSuffix[ext] = struct{}{} supportTypes = append(supportTypes, ext) @@ -106,8 +106,8 @@ func (d *Strm) Init(ctx context.Context) error { } d.FilterFileTypes = strings.Join(supportTypes, ",") - types = strings.Split("ass,srt,vtt,sub,strm", ",") - for _, ext := range types { + types = strings.SplitSeq("ass,srt,vtt,sub,strm", ",") + for ext := range types { if _, ok := d.downloadSuffix[ext]; !ok { d.downloadSuffix[ext] = struct{}{} downloadTypes = append(downloadTypes, ext) @@ -127,7 +127,7 @@ func (d *Strm) Drop(ctx context.Context) error { d.pathMap = nil d.downloadSuffix = nil d.supportSuffix = nil - for _, path := range strings.Split(d.Paths, "\n") { + for path := range strings.SplitSeq(d.Paths, "\n") { RemoveStrm(utils.FixAndCleanPath(strings.TrimSpace(path)), d) } return nil diff --git a/drivers/url_tree/util.go b/drivers/url_tree/util.go index 33312b83..bb7bcb87 100644 --- a/drivers/url_tree/util.go +++ b/drivers/url_tree/util.go @@ -34,10 +34,9 @@ import ( */ // if there are no name, use the last segment of url as name func BuildTree(text string, headSize bool) (*Node, error) { - lines := strings.Split(text, "\n") var root = &Node{Level: -1, Name: "root"} stack := []*Node{root} - for _, line := range lines { + for line := range strings.SplitSeq(text, "\n") { // calculate indent indent := 0 for i := 0; i < len(line); i++ { @@ -153,9 +152,7 @@ func splitPath(path string) []string { if path == "/" { return []string{"root"} } - if strings.HasSuffix(path, "/") { - path = path[:len(path)-1] - } + path = strings.TrimSuffix(path, "/") parts := strings.Split(path, "/") parts[0] = "root" return parts diff --git a/drivers/webdav/driver.go b/drivers/webdav/driver.go index 7a1b5db2..61e4a616 100644 --- a/drivers/webdav/driver.go +++ b/drivers/webdav/driver.go @@ -125,4 +125,22 @@ func (d *WebDav) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer return err } +// implements driver.Getter interface +func (d *WebDav) Get(ctx context.Context, _path string) (model.Obj, error) { + _path = path.Join(d.GetRootPath(), _path) + info, err := d.client.Stat(_path) + if err != nil { + return nil, err + } + + return &model.Object{ + Name: info.Name(), + Size: info.Size(), + Modified: info.ModTime(), + IsFolder: info.IsDir(), + Path: _path, + }, nil +} + var _ driver.Driver = (*WebDav)(nil) +var _ driver.Getter = (*WebDav)(nil) diff --git a/drivers/webdav/odrvcookie/fetch.go b/drivers/webdav/odrvcookie/fetch.go index b4eca077..1dbaba8d 100644 --- a/drivers/webdav/odrvcookie/fetch.go +++ b/drivers/webdav/odrvcookie/fetch.go @@ -151,8 +151,7 @@ func getLoginUrl(endpoint string) (string, error) { if err != nil { return "", err } - domains := strings.Split(spRoot.Host, ".") - tld := domains[len(domains)-1] + tld := spRoot.Host[strings.LastIndex(spRoot.Host, ".")+1:] loginUrl, ok := loginUrlsMap[tld] if !ok { return "", fmt.Errorf("tld %s is not supported", tld) diff --git a/drivers/wps/driver.go b/drivers/wps/driver.go index 8a3ccb6c..847425b1 100644 --- a/drivers/wps/driver.go +++ b/drivers/wps/driver.go @@ -3,16 +3,24 @@ package wps import ( "context" "fmt" + "net/http" + "strings" + "time" + "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/go-resty/resty/v2" ) type Wps struct { model.Storage Addition - companyID string + + login *loginState + client *resty.Client } func (d *Wps) Config() driver.Config { @@ -27,13 +35,84 @@ func (d *Wps) Init(ctx context.Context) error { if d.Cookie == "" { return fmt.Errorf("cookie is empty") } - return d.ensureCompanyID(ctx) + + d.client = base.NewRestyClient() + + resp, err := d.request(ctx).SetResult(&d.login).Get("https://account.kdocs.cn/api/v3/islogin") + if err != nil { + return err + } + if !resp.IsSuccess() { + return fmt.Errorf("failed to check login status, status code: %d, body: %s", resp.StatusCode(), resp.String()) + } + + return nil } func (d *Wps) Drop(ctx context.Context) error { + + if d.client != nil { + d.client = nil + } + if d.login != nil { + d.login = nil + } return nil } +func (d *Wps) GetRoot(ctx context.Context) (model.Obj, error) { + root := &Obj{ + Obj: &model.Object{ + Path: "/", + Name: "root", + Modified: d.Modified, + Ctime: d.Modified, + IsFolder: true, + }, + Kind: "root", + } + rootPath := d.RootFolderPath + if rootPath != "" && rootPath != "/" { + parts := strings.Split(strings.Trim(rootPath, "/"), "/") + groups, err := d.getGroups(ctx) + if err != nil { + return nil, err + } + var current *Obj + for _, g := range groups { + if g.Name == parts[0] { + current = g.groupToObj("/") + break + } + } + if current == nil { + return nil, fmt.Errorf("root path %q not found", rootPath) + } + parentID := int64(0) + for _, name := range parts[1:] { + files, err := d.getFiles(ctx, current.GroupID, parentID) + if err != nil { + return nil, err + } + var next *Obj + for _, f := range files { + if f.Type == "folder" && f.Name == name { + next = f.fileToObj(current.GetPath(), d.isPersonal()) + break + } + } + if next == nil { + return nil, fmt.Errorf("root path %q not found", rootPath) + } + current = next + parentID = current.FileID + } + current.Obj = &model.Object{ID: current.GetID(), Path: "/", Name: current.GetName(), IsFolder: true} + root = current + } + return root, nil +} + func (d *Wps) List(ctx context.Context, dir model.Obj, _ model.ListArgs) ([]model.Obj, error) { basePath := "/" if dir != nil { @@ -41,34 +120,261 @@ func (d *Wps) List(ctx context.Context, dir model.Obj, _ model.ListArgs) ([]mode basePath = p } } - return d.list(ctx, basePath) + node, err := unwrapWpsObj(dir) + if err != nil { + return nil, err + } + if node.Kind == "root" { + groups, err := d.getGroups(ctx) + if err != nil { + return nil, err + } + return utils.SliceConvert(groups, func(g Group) (model.Obj, error) { + return g.groupToObj(basePath), nil + }) + } + if node.Kind != "group" && node.Kind != "folder" { + return nil, nil + } + parentID := int64(0) + if node.HasFile && node.Kind == "folder" { + parentID = node.FileID + } + files, err := d.getFiles(ctx, node.GroupID, parentID) + if err != nil { + return nil, err + } + return utils.SliceConvert(files, func(f FileInfo) (model.Obj, error) { + return f.fileToObj(basePath, d.isPersonal()), nil + }) } func (d *Wps) Link(ctx context.Context, file model.Obj, _ model.LinkArgs) (*model.Link, error) { if file == nil { return nil, errs.NotSupport } - return d.link(ctx, file.GetPath()) + node, err := unwrapWpsObj(file) + if err != nil { + return nil, err + } + if node.Kind != "file" || !node.HasFile { + return nil, errs.NotSupport + } + if !node.CanDownload { + return nil, fmt.Errorf("can not download") + } + url := fmt.Sprintf("%s/api/v5/groups/%d/files/%d/download?support_checksums=sha1", d.driveHost()+d.drivePrefix(), node.GroupID, node.FileID) + var resp downloadResp + r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(url) + if err != nil { + return nil, err + } + if r != nil && r.IsError() { + return nil, fmt.Errorf("http error: %d", r.StatusCode()) + } + if resp.URL == "" { + return nil, fmt.Errorf("empty download url") + } + return &model.Link{ + URL: resp.URL, + Header: http.Header{ + "User-Agent": []string{d.getUA()}, + "Referer": []string{d.driveHost()}, + }, + }, nil } func (d *Wps) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - return d.makeDir(ctx, parentDir, dirName) + if parentDir == nil { + return errs.NotSupport + } + node, err := unwrapWpsObj(parentDir) + if err != nil { + return err + } + if node.Kind != "group" && node.Kind != "folder" { + return errs.NotSupport + } + parentID := int64(0) + if node.HasFile && node.Kind == "folder" { + parentID = node.FileID + } + body := map[string]interface{}{ + "groupid": node.GroupID, + "name": dirName, + "parentid": parentID, + } + if err := d.doJSON(ctx, http.MethodPost, d.driveURL("/api/v5/files/folder"), body); err != nil { + return err + } + return nil } func (d *Wps) Move(ctx context.Context, srcObj, dstDir model.Obj) error { - return d.move(ctx, srcObj, dstDir) + if srcObj == nil || dstDir == nil { + return errs.NotSupport + } + nodeSrc, err := unwrapWpsObj(srcObj) + if err != nil { + return fmt.Errorf("invalid source object type: %w", err) + } + nodeDst, err := unwrapWpsObj(dstDir) + if err != nil { + return fmt.Errorf("invalid destination object type: %w", err) + } + if nodeSrc.Kind != "file" && nodeSrc.Kind != "folder" { + return errs.NotSupport + } + if nodeDst.Kind != "group" && nodeDst.Kind != "folder" { + return errs.NotSupport + } + targetParentID := int64(0) + if nodeDst.HasFile && nodeDst.Kind == "folder" { + targetParentID = nodeDst.FileID + } + body := map[string]interface{}{ + "fileids": []int64{nodeSrc.FileID}, + "target_groupid": nodeDst.GroupID, + "target_parentid": targetParentID, + } + url := fmt.Sprintf("/api/v3/groups/%d/files/batch/move", nodeSrc.GroupID) + for { + var res apiResult + resp, err := d.jsonRequest(ctx). + SetBody(body). + SetResult(&res). + SetError(&res). + Post(d.driveURL(url)) + if err != nil { + return err + } + + if resp.StatusCode() == 403 && res.Result == "fileTaskDuplicated" { + time.Sleep(500 * time.Millisecond) + continue + } + + if err := checkAPI(resp, res); err != nil { + return err + } + break + } + return nil } func (d *Wps) Rename(ctx context.Context, srcObj model.Obj, newName string) error { - return d.rename(ctx, srcObj, newName) + if srcObj == nil { + return errs.NotSupport + } + node, err := unwrapWpsObj(srcObj) + if err != nil { + return err + } + if node.Kind != "file" && node.Kind != "folder" { + return errs.NotSupport + } + url := fmt.Sprintf("/api/v3/groups/%d/files/%d", node.GroupID, node.FileID) + body := map[string]string{"fname": newName} + if err := d.doJSON(ctx, http.MethodPut, d.driveURL(url), body); err != nil { + return err + } + return nil } func (d *Wps) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { - return d.copy(ctx, srcObj, dstDir) + if srcObj == nil || dstDir == nil { + return errs.NotSupport + } + nodeSrc, err := unwrapWpsObj(srcObj) + if err != nil { + return fmt.Errorf("invalid source object type: %w", err) + } + nodeDst, err := unwrapWpsObj(dstDir) + if err != nil { + return fmt.Errorf("invalid destination object type: %w", err) + } + if nodeSrc.Kind != "file" && nodeSrc.Kind != "folder" { + return errs.NotSupport + } + if nodeDst.Kind != "group" && nodeDst.Kind != "folder" { + return errs.NotSupport + } + targetParentID := int64(0) + if nodeDst.HasFile && nodeDst.Kind == "folder" { + targetParentID = nodeDst.FileID + } + body := map[string]interface{}{ + "fileids": []int64{nodeSrc.FileID}, + "groupid": nodeSrc.GroupID, + "target_groupid": nodeDst.GroupID, + "target_parentid": targetParentID, + "duplicated_name_model": 1, + } + url := fmt.Sprintf("/api/v3/groups/%d/files/batch/copy", nodeSrc.GroupID) + for { + var res apiResult + resp, err := d.jsonRequest(ctx). + SetBody(body). + SetResult(&res). + SetError(&res). + Post(d.driveURL(url)) + if err != nil { + return err + } + + if resp.StatusCode() == 403 && res.Result == "fileTaskDuplicated" { + time.Sleep(500 * time.Millisecond) + continue + } + + if err := checkAPI(resp, res); err != nil { + return err + } + break + } + return nil } func (d *Wps) Remove(ctx context.Context, obj model.Obj) error { - return d.remove(ctx, obj) + if obj == nil { + return errs.NotSupport + } + node, err := unwrapWpsObj(obj) + if err != nil { + return err + } + if node.Kind != "file" && node.Kind != "folder" { + return errs.NotSupport + } + + body := map[string]interface{}{ + "fileids": []int64{node.FileID}, + } + url := fmt.Sprintf("/api/v3/groups/%d/files/batch/delete", node.GroupID) + + for { + var res apiResult + resp, err := d.jsonRequest(ctx). + SetBody(body). + SetResult(&res). + SetError(&res). + Post(d.driveURL(url)) + if err != nil { + return err + } + + // 无法连续创建文件夹删除。如果一定要删除，每0.5s 尝试一次创建下一个删除请求，应当避免递归删除文件夹 + if resp.StatusCode() == 403 && res.Result == "fileTaskDuplicated" { + time.Sleep(500 * time.Millisecond) + continue + } + + if err := checkAPI(resp, res); err != nil { + return err + } + break + } + return nil } func (d *Wps) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { @@ -76,16 +382,48 @@ func (d *Wps) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer } func (d *Wps) GetDetails(ctx context.Context) (*model.StorageDetails, error) { - quota, err := d.spaces(ctx) + if d.isPersonal() { + url := ENDPOINT_PERSONAL + "/api/v3/spaces" + var resp spacesResp + r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(url) + if err != nil { + return nil, err + } + if r != nil && r.IsError() { + return nil, fmt.Errorf("http error: %d", r.StatusCode()) + } + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: resp.Total, + UsedSpace: resp.Used, + }, + }, nil + } + url := ENDPOINT_BUSINESS + "/3rd/plussvr/compose/v1/u/companies/batch/service-space?comp_ids=" + fmt.Sprint(d.login.CompanyID) + var resp serviceSpaceResp + r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(url) if err != nil { return nil, err } - return &model.StorageDetails{ - DiskUsage: model.DiskUsage{ - TotalSpace: quota.Total, - UsedSpace: quota.Used, - }, - }, nil + if r != nil && r.IsError() { + return nil, fmt.Errorf("http error: %d", r.StatusCode()) + } + if len(resp.Info) == 0 { + return nil, fmt.Errorf("empty service space info") + } + // info := resp.Info[0] + for _, info := range resp.Info { + if info.ID == d.login.CompanyID { + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: info.SpaceTotal, + UsedSpace: info.SpaceUsed, + }, + }, nil + } + } + return nil, fmt.Errorf("service space info not found for company ID: %d", d.login.CompanyID) } var _ driver.Driver = (*Wps)(nil) +var _ driver.GetRooter = (*Wps)(nil) diff --git a/drivers/wps/meta.go b/drivers/wps/meta.go index 7a3362f3..a520bb43 100644 --- a/drivers/wps/meta.go +++ b/drivers/wps/meta.go @@ -7,16 +7,16 @@ import ( type Addition struct { driver.RootPath - Cookie string `json:"cookie" required:"true" type:"text"` - Mode string `json:"mode" type:"select" options:"Personal,Business" default:"Business"` + Cookie string `json:"cookie" required:"true"` + Mode string `json:"mode" type:"select" options:"Personal,Business" default:"Personal"` + CustomUA string `json:"custom_ua"` } var config = driver.Config{ - Name: "WPS", - LocalSort: true, - DefaultRoot: "/", - Alert: "", - NoOverwriteUpload: true, + Name: "WPS", + LocalSort: true, + DefaultRoot: "/", + Alert: "", } func init() { diff --git a/drivers/wps/put.go b/drivers/wps/put.go new file mode 100644 index 00000000..94ba72e7 --- /dev/null +++ b/drivers/wps/put.go @@ -0,0 +1,315 @@ +package wps + +import ( + "bytes" + "context" + "crypto/sha1" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/http" + "strconv" + "strings" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +type countingWriter struct { + n *int64 +} + +func (w countingWriter) Write(p []byte) (int, error) { + *w.n += int64(len(p)) + return len(p), nil +} + +func cacheAndHash(file model.FileStreamer, up driver.UpdateProgress) (model.File, int64, string, string, error) { + h1 := sha1.New() + h256 := sha256.New() + size := file.GetSize() + var counted int64 + ws := []io.Writer{h1, h256} + if size <= 0 { + ws = append(ws, countingWriter{n: &counted}) + } + p := up + f, err := file.CacheFullAndWriter(&p, io.MultiWriter(ws...)) + if err != nil { + return nil, 0, "", "", err + } + if size <= 0 { + size = counted + } + return f, size, hex.EncodeToString(h1.Sum(nil)), hex.EncodeToString(h256.Sum(nil)), nil +} + +func (d *Wps) createUpload(ctx context.Context, groupID, parentID int64, name string, size int64, sha1Hex, sha256Hex string) (*uploadCreateUpdateResp, error) { + body := map[string]string{ + "group_id": strconv.FormatInt(groupID, 10), + "name": name, + "parent_id": strconv.FormatInt(parentID, 10), + "sha1": sha1Hex, + "sha256": sha256Hex, + "size": strconv.FormatInt(size, 10), + } + var resp uploadCreateUpdateResp + r, err := d.jsonRequest(ctx). + SetBody(body). + SetResult(&resp). + SetError(&resp). + Put(d.driveURL("/api/v5/files/upload/create_update")) + if err != nil { + return nil, err + } + if err := checkAPI(r, resp.apiResult); err != nil { + return nil, err + } + if resp.URL == "" { + return nil, fmt.Errorf("empty upload url") + } + return &resp, nil +} + +func normalizeETag(v string) string { + v = strings.TrimSpace(v) + if strings.HasPrefix(v, "W/") { + v = strings.TrimSpace(strings.TrimPrefix(v, "W/")) + } + return strings.Trim(v, `"`) +} + +func (d *Wps) commitUpload(ctx context.Context, etag, key string, groupID, parentID int64, name, sha1Hex string, size int64, store string) error { + store = strings.TrimSpace(store) + if store == "" { + store = "ks3" + } + storeKey := "" + if key != "" { + storeKey = key + } + body := map[string]interface{}{ + "etag": etag, + "groupid": groupID, + "key": key, + "name": name, + "parentid": parentID, + "sha1": sha1Hex, + "size": size, + "store": store, + "storekey": storeKey, + } + return d.doJSON(ctx, http.MethodPost, d.driveURL("/api/v5/files/file"), body) +} + +func (d *Wps) put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + if dstDir == nil || file == nil { + return errs.NotSupport + } + if up == nil { + up = func(float64) {} + } + node, err := unwrapWpsObj(dstDir) + if err != nil { + return err + } + if node.Kind != "group" && node.Kind != "folder" { + return errs.NotSupport + } + parentID := int64(0) + if node.HasFile && node.Kind == "folder" { + parentID = node.FileID + } + f, size, sha1Hex, sha256Hex, err := cacheAndHash(file, func(float64) {}) + if err != nil { + return err + } + if c, ok := f.(io.Closer); ok { + defer c.Close() + } + + // 在隐藏文件名前加_上传，这是WPS的限制，无法上传隐藏文件，也无法将任何文件重命名为隐藏文件，所有隐藏文件会被自动加上_ 上传 + // 甚至可以上传前缀是..的文件，但是单个点就是不行 + realName := file.GetName() + uploadName := realName + if strings.HasPrefix(realName, ".") { + uploadName = "_" + realName + } + + info, err := d.createUpload(ctx, node.GroupID, parentID, uploadName, size, sha1Hex, sha256Hex) + if err != nil { + return err + } + if _, err := f.Seek(0, io.SeekStart); err != nil { + return err + } + rf := driver.NewLimitedUploadFile(ctx, f) + prog := driver.NewProgress(size, model.UpdateProgressWithRange(up, 0, 1)) + + method := strings.ToUpper(strings.TrimSpace(info.Method)) + if method == "" { + method = http.MethodPut + } + + var req *http.Request + if method == http.MethodPost && len(info.Request.FormData) > 0 { + if size == 0 { + var buf bytes.Buffer + mw := multipart.NewWriter(&buf) + for k, v := range info.Request.FormData { + if err := mw.WriteField(k, v); err != nil { + return err + } + } + part, err := mw.CreateFormFile("file", uploadName) + if err != nil { + return err + } + if _, err := io.Copy(part, io.TeeReader(rf, prog)); err != nil { + return err + } + if err := mw.Close(); err != nil { + return err + } + req, err = http.NewRequestWithContext(ctx, method, info.URL, bytes.NewReader(buf.Bytes())) + if err != nil { + return err + } + for k, v := range info.Request.Headers { + req.Header.Set(k, v) + } + req.Header.Set("Content-Type", mw.FormDataContentType()) + req.ContentLength = int64(buf.Len()) + req.Header.Set("Content-Length", strconv.FormatInt(req.ContentLength, 10)) + } else { + pr, pw := io.Pipe() + mw := multipart.NewWriter(pw) + req, err = http.NewRequestWithContext(ctx, method, info.URL, pr) + if err != nil { + return err + } + for k, v := range info.Request.Headers { + req.Header.Set(k, v) + } + req.Header.Set("Content-Type", mw.FormDataContentType()) + go func() { + for k, v := range info.Request.FormData { + if err := mw.WriteField(k, v); err != nil { + pw.CloseWithError(err) + return + } + } + part, err := mw.CreateFormFile("file", uploadName) + if err != nil { + pw.CloseWithError(err) + return + } + if _, err := io.Copy(part, io.TeeReader(rf, prog)); err != nil { + pw.CloseWithError(err) + return + } + if err := mw.Close(); err != nil { + pw.CloseWithError(err) + return + } + pw.Close() + }() + } + } else { + var body = io.TeeReader(rf, prog) + if size == 0 { + body = bytes.NewReader(nil) + } + req, err = http.NewRequestWithContext(ctx, method, info.URL, body) + if err != nil { + return err + } + for k, v := range info.Request.Headers { + req.Header.Set(k, v) + } + req.ContentLength = size + req.Header.Set("Content-Length", strconv.FormatInt(size, 10)) + } + + c := *d.client.GetClient() + c.Timeout = 0 + resp, err := (&c).Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if !statusOK(resp.StatusCode, info.Response.ExpectCode) { + io.Copy(io.Discard, resp.Body) + return fmt.Errorf("http error: %d", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + + etag := normalizeETag(respArg(info.Response.ArgsETag, resp, body)) + if etag == "" { + etag = normalizeETag(resp.Header.Get("ETag")) + } + + key := strings.TrimSpace(respArg(info.Response.ArgsKey, resp, body)) + if key == "" { + key = strings.TrimSpace(resp.Header.Get("x-obs-save-key")) + } + + var pr uploadPutResp + sha1FromServer := "" + if err := json.Unmarshal(body, &pr); err == nil { + sha1FromServer = strings.TrimSpace(pr.NewFilename) + if sha1FromServer == "" { + sha1FromServer = strings.TrimSpace(pr.Sha1) + } + if etag == "" && pr.MD5 != "" { + etag = strings.TrimSpace(pr.MD5) + } + } + + if sha1FromServer == "" { + if v := extractXMLTag(string(body), "ETag"); v != "" { + sha1FromServer = v + if etag == "" { + etag = v + } + } + } + if sha1FromServer == "" && key != "" && len(key) == 40 { + sha1FromServer = key + } + if sha1FromServer == "" { + sha1FromServer = sha1Hex + } + + if etag == "" { + return fmt.Errorf("empty etag") + } + if sha1FromServer == "" { + return fmt.Errorf("empty sha1") + } + + store := strings.TrimSpace(info.Store) + commitKey := "" + if strings.TrimSpace(info.Response.ArgsKey) != "" { + commitKey = key + if commitKey == "" { + commitKey = sha1FromServer + } + } + + if err := d.commitUpload(ctx, etag, commitKey, node.GroupID, parentID, uploadName, sha1FromServer, size, store); err != nil { + return err + } + + up(1) + return nil +} diff --git a/drivers/wps/types.go b/drivers/wps/types.go index 57847539..d5329808 100644 --- a/drivers/wps/types.go +++ b/drivers/wps/types.go @@ -1,15 +1,24 @@ package wps import ( - "time" + "strconv" - "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/OpenList/v4/internal/model" ) -type workspaceResp struct { - Companies []struct { - ID int64 `json:"id"` - } `json:"companies"` +type apiResult struct { + Result string `json:"result"` + Msg string `json:"msg"` +} + +type loginState struct { + AccountNum int `json:"account_num"` + CompanyID int64 `json:"companyid"` + CurrentCompanyID int64 `json:"current_companyid"` + IsCompanyAccount bool `json:"is_company_account"` + IsPlus bool `json:"is_plus"` + LoginMode string `json:"loginmode"` + UserID int64 `json:"userid"` } type Group struct { @@ -23,6 +32,14 @@ type groupsResp struct { Groups []Group `json:"groups"` } +type personalGroupsResp struct { + apiResult + Groups []struct { + ID int64 `json:"id"` + Name string `json:"name"` + } `json:"groups"` +} + type filePerms struct { Download int `json:"download"` } @@ -40,8 +57,58 @@ type FileInfo struct { FilePerms filePerms `json:"file_perms_acl"` } +func (f *FileInfo) canDownload(isPersonal bool) bool { + if f == nil || f.Type == "folder" { + return false + } + if f.FilePerms.Download != 0 { + return true + } + return isPersonal +} + +func (f FileInfo) fileToObj(basePath string, isPersonal bool) *Obj { + name := f.Name + path := joinPath(basePath, name) + kind := "file" + if f.Type == "folder" { + kind = "folder" + } + obj := &Obj{ + Obj: &model.Object{ + ID: strconv.FormatInt(f.ID, 10), + Path: path, + Name: name, + Size: f.Size, + Modified: parseTime(f.Mtime), + Ctime: parseTime(f.Ctime), + IsFolder: f.Type == "folder", + }, + Kind: kind, + FileID: f.ID, + GroupID: f.GroupID, + HasFile: true, + CanDownload: f.canDownload(isPersonal), + } + return obj +} + +func (g Group) groupToObj(basePath string) *Obj { + return &Obj{ + Obj: &model.Object{ + ID: strconv.FormatInt(g.GroupID, 10), + Path: joinPath(basePath, g.Name), + Name: g.Name, + IsFolder: true, + }, + Kind: "group", + GroupID: g.GroupID, + } +} + type filesResp struct { - Files []FileInfo `json:"files"` + Files []FileInfo `json:"files"` + NextOffset int `json:"next_offset"` } type downloadResp struct { @@ -61,50 +128,41 @@ type spacesResp struct { } `json:"used_parts"` } -type Obj struct { - id string - name string - size int64 - ctime time.Time - mtime time.Time - isDir bool - hash utils.HashInfo - path string - canDownload bool +type serviceSpaceResp struct { + Info []struct { + ID int64 `json:"id"` + SpaceTotal int64 `json:"space_total"` + SpaceUsed int64 `json:"space_used"` + } `json:"info"` } -func (o *Obj) GetSize() int64 { - return o.size +type uploadCreateUpdateResp struct { + apiResult + Method string `json:"method"` + URL string `json:"url"` + Store string `json:"store"` + Request struct { + Headers map[string]string `json:"headers"` + FormData map[string]string `json:"formData"` + } `json:"request"` + Response struct { + ExpectCode []int `json:"expect_code"` + ArgsETag string `json:"args_etag"` + ArgsKey string `json:"args_key"` + } `json:"response"` } -func (o *Obj) GetDuration() int { - return 0 +type uploadPutResp struct { + NewFilename string `json:"newfilename"` + Sha1 string `json:"sha1"` + MD5 string `json:"md5"` } -func (o *Obj) GetName() string { - return o.name -} - -func (o *Obj) ModTime() time.Time { - return o.mtime -} - -func (o *Obj) CreateTime() time.Time { - return o.ctime -} - -func (o *Obj) IsDir() bool { - return o.isDir -} - -func (o *Obj) GetHash() utils.HashInfo { - return o.hash -} - -func (o *Obj) GetID() string { - return o.id -} - -func (o *Obj) GetPath() string { - return o.path +type Obj struct { + model.Obj + Kind string // root / group / file / folder + FileID int64 + GroupID int64 + HasFile bool // only FileInfo has file, otherwise the FileID is 0 + CanDownload bool } diff --git a/drivers/wps/util.go b/drivers/wps/util.go index 79be1ac4..10f104c9 100644 --- a/drivers/wps/util.go +++ b/drivers/wps/util.go @@ -1,102 +1,39 @@ package wps import ( - "bytes" "context" - "crypto/sha1" - "crypto/sha256" - "encoding/hex" "encoding/json" "fmt" - "io" - "mime/multipart" "net/http" "strconv" "strings" - "sync" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" - "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/go-resty/resty/v2" ) -const endpoint = "https://365.kdocs.cn" -const personalEndpoint = "https://drive.wps.cn" - -type resolvedNode struct { - kind string - group Group - file *FileInfo -} - -type resolveCacheEntry struct { - node *resolvedNode - expire time.Time -} - -type resolveCacheStore struct { - mu sync.RWMutex - m map[string]resolveCacheEntry -} - -var resolveCaches sync.Map - -type apiResult struct { - Result string `json:"result"` - Msg string `json:"msg"` -} - -type uploadCreateUpdateResp struct { - apiResult - Method string `json:"method"` - URL string `json:"url"` - Store string `json:"store"` - Request struct { - Headers map[string]string `json:"headers"` - FormData map[string]string `json:"formData"` - } `json:"request"` - Response struct { - ExpectCode []int `json:"expect_code"` - ArgsETag string `json:"args_etag"` - ArgsKey string `json:"args_key"` - } `json:"response"` -} - -type uploadPutResp struct { - NewFilename string `json:"newfilename"` - Sha1 string `json:"sha1"` - MD5 string `json:"md5"` -} - -type personalGroupsResp struct { - apiResult - Groups []struct { - ID int64 `json:"id"` - Name string `json:"name"` - } `json:"groups"` -} - -type countingWriter struct { - n *int64 -} - -func (w countingWriter) Write(p []byte) (int, error) { - *w.n += int64(len(p)) - return len(p), nil -} +const ENDPOINT_BUSINESS = "https://365.kdocs.cn" +const ENDPOINT_PERSONAL = "https://drive.wps.cn" func (d *Wps) isPersonal() bool { + // prefer d.login if available, as it may be set by islogin API + // which can determine account type more reliably + // one login session only support one type + // can not use personal and company account at the same time + if d.login != nil { + return !d.login.IsCompanyAccount + } return strings.TrimSpace(d.Mode) == "Personal" } func (d *Wps) driveHost() string { if d.isPersonal() { - return personalEndpoint + return ENDPOINT_PERSONAL } - return endpoint + return ENDPOINT_BUSINESS } func (d *Wps) drivePrefix() string { @@ -114,20 +51,18 @@ func (d *Wps) origin() string { return d.driveHost() } -func (d *Wps) canDownload(f *FileInfo) bool { - if f == nil || f.Type == "folder" { - return false - } - if f.FilePerms.Download != 0 { - return true +func (d *Wps) getUA() string { + if d.CustomUA != "" { + return d.CustomUA } - return d.isPersonal() + return base.UserAgent } func (d *Wps) request(ctx context.Context) *resty.Request { - return base.RestyClient.R(). + return d.client.R(). SetHeader("Cookie", d.Cookie). SetHeader("Accept", "application/json"). + SetHeader("User-Agent", d.getUA()). SetContext(ctx) } @@ -219,30 +154,10 @@ func checkAPI(resp *resty.Response, result apiResult) error { return nil } -func (d *Wps) ensureCompanyID(ctx context.Context) error { - if d.isPersonal() { - return nil - } - if d.companyID != "" { - return nil - } - var resp workspaceResp - r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(endpoint + "/3rd/plussvr/compose/v1/users/self/workspaces?fields=name&comp_status=active") - if err != nil { - return err - } - if r != nil && r.IsError() { - return fmt.Errorf("http error: %d", r.StatusCode()) - } - if len(resp.Companies) == 0 { - return fmt.Errorf("no company id") - } - d.companyID = strconv.FormatInt(resp.Companies[0].ID, 10) - return nil -} - func (d *Wps) getGroups(ctx context.Context) ([]Group, error) { - if d.isPersonal() { + // different APIs + switch d.Mode { + case "Personal": var resp personalGroupsResp r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(d.driveURL("/api/v3/groups")) if err != nil { @@ -256,20 +171,19 @@ func (d *Wps) getGroups(ctx context.Context) ([]Group, error) { res = append(res, Group{GroupID: g.ID, Name: g.Name}) } return res, nil + case "Business": + var resp groupsResp + url := fmt.Sprintf("%s/3rd/plus/groups/v1/companies/%d/users/self/groups/private", ENDPOINT_BUSINESS, d.login.CompanyID) + r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(url) + if err != nil { + return nil, err + } + if r != nil && r.IsError() { + return nil, fmt.Errorf("http error: %d", r.StatusCode()) + } + return resp.Groups, nil } - if err := d.ensureCompanyID(ctx); err != nil { - return nil, err - } - var resp groupsResp - url := fmt.Sprintf("%s/3rd/plus/groups/v1/companies/%s/users/self/groups/private", endpoint, d.companyID) - r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(url) - if err != nil { - return nil, err - } - if r != nil && r.IsError() { - return nil, fmt.Errorf("http error: %d", r.StatusCode()) - } - return resp.Groups, nil + return nil, fmt.Errorf("unsupported mode: %s", d.Mode) } func (d *Wps) getFiles(ctx context.Context, groupID, parentID int64) ([]FileInfo, error) { @@ -303,165 +217,6 @@ func joinPath(basePath, name string) string { return strings.TrimRight(basePath, "/") + "/" + name } -func normalizePath(path string) string { - clean := strings.TrimSpace(path) - if clean == "" || clean == "/" { - return "/" - } - return "/" + strings.Trim(clean, "/") -} - -func (d *Wps) resolveCacheStore() *resolveCacheStore { - if d == nil { - return nil - } - if v, ok := resolveCaches.Load(d); ok { - if s, ok := v.(*resolveCacheStore); ok { - return s - } - } - s := &resolveCacheStore{m: make(map[string]resolveCacheEntry)} - if v, loaded := resolveCaches.LoadOrStore(d, s); loaded { - if s2, ok := v.(*resolveCacheStore); ok { - return s2 - } - } - return s -} - -func (d *Wps) getResolveCache(path string) (*resolvedNode, bool) { - s := d.resolveCacheStore() - if s == nil { - return nil, false - } - s.mu.RLock() - e, ok := s.m[path] - s.mu.RUnlock() - if !ok || e.node == nil { - return nil, false - } - if !e.expire.IsZero() && time.Now().After(e.expire) { - s.mu.Lock() - delete(s.m, path) - s.mu.Unlock() - return nil, false - } - return e.node, true -} - -func (d *Wps) setResolveCache(path string, node *resolvedNode) { - s := d.resolveCacheStore() - if s == nil || node == nil { - return - } - s.mu.Lock() - s.m[path] = resolveCacheEntry{node: node, expire: time.Now().Add(10 * time.Minute)} - s.mu.Unlock() -} - -func (d *Wps) clearResolveCache() { - s := d.resolveCacheStore() - if s == nil { - return - } - s.mu.Lock() - if len(s.m) != 0 { - s.m = make(map[string]resolveCacheEntry) - } - s.mu.Unlock() -} - -func (d *Wps) resolvePath(ctx context.Context, path string) (*resolvedNode, error) { - cacheKey := normalizePath(path) - if n, ok := d.getResolveCache(cacheKey); ok { - return n, nil - } - clean := strings.TrimSpace(path) - if clean == "" { - clean = "/" - } - clean = strings.Trim(clean, "/") - if clean == "" { - n := &resolvedNode{kind: "root"} - d.setResolveCache("/", n) - return n, nil - } - seg := strings.Split(clean, "/") - groups, err := d.getGroups(ctx) - if err != nil { - return nil, err - } - var grp *Group - for i := range groups { - if groups[i].Name == seg[0] { - grp = &groups[i] - break - } - } - if grp == nil { - return nil, fmt.Errorf("group not found") - } - cur := "/" + seg[0] - gn := &resolvedNode{kind: "group", group: *grp} - d.setResolveCache(cur, gn) - if len(seg) == 1 { - return gn, nil - } - parentID := int64(0) - var lastNode *resolvedNode - for i := 1; i < len(seg); i++ { - files, err := d.getFiles(ctx, grp.GroupID, parentID) - if err != nil { - return nil, err - } - var found *FileInfo - for j := range files { - if files[j].Name == seg[i] { - found = &files[j] - break - } - } - if found == nil { - return nil, fmt.Errorf("path not found") - } - if i < len(seg)-1 && found.Type != "folder" { - return nil, fmt.Errorf("path not found") - } - fi := *found - parentID = fi.ID - cur = cur + "/" + seg[i] - kind := "file" - if fi.Type == "folder" { - kind = "folder" - } - n := &resolvedNode{kind: kind, group: *grp, file: &fi} - d.setResolveCache(cur, n) - lastNode = n - } - if lastNode == nil { - return nil, fmt.Errorf("path not found") - } - return lastNode, nil -} - -func (d *Wps) fileToObj(basePath string, f FileInfo) *Obj { - name := f.Name - path := joinPath(basePath, name) - obj := &Obj{ - id: path, - name: name, - size: f.Size, - ctime: parseTime(f.Ctime), - mtime: parseTime(f.Mtime), - isDir: f.Type == "folder", - path: path, - } - if !obj.isDir { - obj.canDownload = d.canDownload(&f) - } - return obj -} - func (d *Wps) doJSON(ctx context.Context, method, url string, body interface{}) error { var result apiResult req := d.jsonRequest(ctx).SetBody(body).SetResult(&result).SetError(&result) @@ -483,580 +238,16 @@ func (d *Wps) doJSON(ctx context.Context, method, url string, body interface{}) return checkAPI(resp, result) } -func (d *Wps) list(ctx context.Context, basePath string) ([]model.Obj, error) { - if strings.TrimSpace(basePath) == "" { - basePath = "/" - } - node, err := d.resolvePath(ctx, basePath) - if err != nil { - return nil, err - } - if node.kind == "root" { - groups, err := d.getGroups(ctx) - if err != nil { - return nil, err - } - res := make([]model.Obj, 0, len(groups)) - for _, g := range groups { - path := joinPath(basePath, g.Name) - obj := &Obj{ - id: path, - name: g.Name, - ctime: parseTime(0), - mtime: parseTime(0), - isDir: true, - path: path, - } - res = append(res, obj) - d.setResolveCache(normalizePath(path), &resolvedNode{kind: "group", group: g}) - } - d.setResolveCache("/", &resolvedNode{kind: "root"}) - return res, nil - } - if node.kind != "group" && node.kind != "folder" { - return nil, nil - } - parentID := int64(0) - if node.file != nil && node.kind == "folder" { - parentID = node.file.ID - } - files, err := d.getFiles(ctx, node.group.GroupID, parentID) - if err != nil { - return nil, err - } - res := make([]model.Obj, 0, len(files)) - for _, f := range files { - res = append(res, d.fileToObj(basePath, f)) - path := normalizePath(joinPath(basePath, f.Name)) - fi := f - kind := "file" - if fi.Type == "folder" { - kind = "folder" - } - d.setResolveCache(path, &resolvedNode{kind: kind, group: node.group, file: &fi}) - } - return res, nil -} - -func (d *Wps) link(ctx context.Context, path string) (*model.Link, error) { - node, err := d.resolvePath(ctx, path) - if err != nil { - return nil, err - } - if node.kind != "file" || node.file == nil { - return nil, errs.NotSupport - } - if !d.canDownload(node.file) { - return nil, fmt.Errorf("no download permission") - } - url := fmt.Sprintf("%s/api/v5/groups/%d/files/%d/download?support_checksums=sha1", d.driveHost()+d.drivePrefix(), node.group.GroupID, node.file.ID) - var resp downloadResp - r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(url) - if err != nil { - return nil, err - } - if r != nil && r.IsError() { - return nil, fmt.Errorf("http error: %d", r.StatusCode()) - } - if resp.URL == "" { - return nil, fmt.Errorf("empty download url") - } - return &model.Link{URL: resp.URL, Header: http.Header{}}, nil -} - -func (d *Wps) makeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - if parentDir == nil { - return errs.NotSupport - } - node, err := d.resolvePath(ctx, parentDir.GetPath()) - if err != nil { - return err - } - if node.kind != "group" && node.kind != "folder" { - return errs.NotSupport - } - parentID := int64(0) - if node.file != nil && node.kind == "folder" { - parentID = node.file.ID - } - body := map[string]interface{}{ - "groupid": node.group.GroupID, - "name": dirName, - "parentid": parentID, - } - if err := d.doJSON(ctx, http.MethodPost, d.driveURL("/api/v5/files/folder"), body); err != nil { - return err - } - d.clearResolveCache() - return nil -} - -func (d *Wps) move(ctx context.Context, srcObj, dstDir model.Obj) error { - if srcObj == nil || dstDir == nil { - return errs.NotSupport - } - nodeSrc, err := d.resolvePath(ctx, srcObj.GetPath()) - if err != nil { - return err - } - nodeDst, err := d.resolvePath(ctx, dstDir.GetPath()) - if err != nil { - return err - } - if nodeSrc.kind != "file" && nodeSrc.kind != "folder" { - return errs.NotSupport - } - if nodeDst.kind != "group" && nodeDst.kind != "folder" { - return errs.NotSupport - } - targetParentID := int64(0) - if nodeDst.file != nil && nodeDst.kind == "folder" { - targetParentID = nodeDst.file.ID - } - body := map[string]interface{}{ - "fileids": []int64{nodeSrc.file.ID}, - "target_groupid": nodeDst.group.GroupID, - "target_parentid": targetParentID, - } - url := fmt.Sprintf("/api/v3/groups/%d/files/batch/move", nodeSrc.group.GroupID) - for { - var res apiResult - resp, err := d.jsonRequest(ctx). - SetBody(body). - SetResult(&res). - SetError(&res). - Post(d.driveURL(url)) - if err != nil { - return err - } - - if resp.StatusCode() == 403 && res.Result == "fileTaskDuplicated" { - time.Sleep(500 * time.Millisecond) - continue - } - - if err := checkAPI(resp, res); err != nil { - return err +func unwrapWpsObj(obj model.Obj) (*Obj, error) { + for obj != nil { + if node, ok := obj.(*Obj); ok { + return node, nil } - break - } - d.clearResolveCache() - return nil -} - -func (d *Wps) rename(ctx context.Context, srcObj model.Obj, newName string) error { - if srcObj == nil { - return errs.NotSupport - } - node, err := d.resolvePath(ctx, srcObj.GetPath()) - if err != nil { - return err - } - if node.kind != "file" && node.kind != "folder" { - return errs.NotSupport - } - url := fmt.Sprintf("/api/v3/groups/%d/files/%d", node.group.GroupID, node.file.ID) - body := map[string]string{"fname": newName} - if err := d.doJSON(ctx, http.MethodPut, d.driveURL(url), body); err != nil { - return err - } - d.clearResolveCache() - return nil -} - -func (d *Wps) copy(ctx context.Context, srcObj, dstDir model.Obj) error { - if srcObj == nil || dstDir == nil { - return errs.NotSupport - } - nodeSrc, err := d.resolvePath(ctx, srcObj.GetPath()) - if err != nil { - return err - } - nodeDst, err := d.resolvePath(ctx, dstDir.GetPath()) - if err != nil { - return err - } - if nodeSrc.kind != "file" && nodeSrc.kind != "folder" { - return errs.NotSupport - } - if nodeDst.kind != "group" && nodeDst.kind != "folder" { - return errs.NotSupport - } - targetParentID := int64(0) - if nodeDst.file != nil && nodeDst.kind == "folder" { - targetParentID = nodeDst.file.ID - } - body := map[string]interface{}{ - "fileids": []int64{nodeSrc.file.ID}, - "groupid": nodeSrc.group.GroupID, - "target_groupid": nodeDst.group.GroupID, - "target_parentid": targetParentID, - "duplicated_name_model": 1, - } - url := fmt.Sprintf("/api/v3/groups/%d/files/batch/copy", nodeSrc.group.GroupID) - for { - var res apiResult - resp, err := d.jsonRequest(ctx). - SetBody(body). - SetResult(&res). - SetError(&res). - Post(d.driveURL(url)) - if err != nil { - return err - } - - if resp.StatusCode() == 403 && res.Result == "fileTaskDuplicated" { - time.Sleep(500 * time.Millisecond) - continue - } - - if err := checkAPI(resp, res); err != nil { - return err - } - break - } - d.clearResolveCache() - return nil -} - -func (d *Wps) remove(ctx context.Context, obj model.Obj) error { - if obj == nil { - return errs.NotSupport - } - node, err := d.resolvePath(ctx, obj.GetPath()) - if err != nil { - return err - } - if node.kind != "file" && node.kind != "folder" { - return errs.NotSupport - } - - body := map[string]interface{}{ - "fileids": []int64{node.file.ID}, - } - url := fmt.Sprintf("/api/v3/groups/%d/files/batch/delete", node.group.GroupID) - - for { - var res apiResult - resp, err := d.jsonRequest(ctx). - SetBody(body). - SetResult(&res). - SetError(&res). - Post(d.driveURL(url)) - if err != nil { - return err - } - - // 无法连续创建文件夹删除。如果一定要删除，每0.5s 尝试一次创建下一个删除请求，应当避免递归删除文件夹 - if resp.StatusCode() == 403 && res.Result == "fileTaskDuplicated" { - time.Sleep(500 * time.Millisecond) - continue - } - - if err := checkAPI(resp, res); err != nil { - return err - } - break - } - d.clearResolveCache() - return nil -} - -func cacheAndHash(file model.FileStreamer, up driver.UpdateProgress) (model.File, int64, string, string, error) { - h1 := sha1.New() - h256 := sha256.New() - size := file.GetSize() - var counted int64 - ws := []io.Writer{h1, h256} - if size <= 0 { - ws = append(ws, countingWriter{n: &counted}) - } - p := up - f, err := file.CacheFullAndWriter(&p, io.MultiWriter(ws...)) - if err != nil { - return nil, 0, "", "", err - } - if size <= 0 { - size = counted - } - return f, size, hex.EncodeToString(h1.Sum(nil)), hex.EncodeToString(h256.Sum(nil)), nil -} - -func (d *Wps) createUpload(ctx context.Context, groupID, parentID int64, name string, size int64, sha1Hex, sha256Hex string) (*uploadCreateUpdateResp, error) { - body := map[string]string{ - "group_id": strconv.FormatInt(groupID, 10), - "name": name, - "parent_id": strconv.FormatInt(parentID, 10), - "sha1": sha1Hex, - "sha256": sha256Hex, - "size": strconv.FormatInt(size, 10), - } - var resp uploadCreateUpdateResp - r, err := d.jsonRequest(ctx). - SetBody(body). - SetResult(&resp). - SetError(&resp). - Put(d.driveURL("/api/v5/files/upload/create_update")) - if err != nil { - return nil, err - } - if err := checkAPI(r, resp.apiResult); err != nil { - return nil, err - } - if resp.URL == "" { - return nil, fmt.Errorf("empty upload url") - } - return &resp, nil -} - -func normalizeETag(v string) string { - v = strings.TrimSpace(v) - if strings.HasPrefix(v, "W/") { - v = strings.TrimSpace(strings.TrimPrefix(v, "W/")) - } - return strings.Trim(v, `"`) -} - -func (d *Wps) commitUpload(ctx context.Context, etag, key string, groupID, parentID int64, name, sha1Hex string, size int64, store string) error { - store = strings.TrimSpace(store) - if store == "" { - store = "ks3" - } - storeKey := "" - if key != "" { - storeKey = key - } - body := map[string]interface{}{ - "etag": etag, - "groupid": groupID, - "key": key, - "name": name, - "parentid": parentID, - "sha1": sha1Hex, - "size": size, - "store": store, - "storekey": storeKey, - } - return d.doJSON(ctx, http.MethodPost, d.driveURL("/api/v5/files/file"), body) -} - -func (d *Wps) put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { - if dstDir == nil || file == nil { - return errs.NotSupport - } - if up == nil { - up = func(float64) {} - } - node, err := d.resolvePath(ctx, dstDir.GetPath()) - if err != nil { - return err - } - if node.kind != "group" && node.kind != "folder" { - return errs.NotSupport - } - parentID := int64(0) - if node.file != nil && node.kind == "folder" { - parentID = node.file.ID - } - f, size, sha1Hex, sha256Hex, err := cacheAndHash(file, func(float64) {}) - if err != nil { - return err - } - if c, ok := f.(io.Closer); ok { - defer c.Close() - } - - // 在隐藏文件名前加_上传，这是WPS的限制，无法上传隐藏文件，也无法将任何文件重命名为隐藏文件，所有隐藏文件会被自动加上_ 上传 - // 甚至可以上传前缀是..的文件，但是单个点就是不行 - realName := file.GetName() - uploadName := realName - if strings.HasPrefix(realName, ".") { - uploadName = "_" + realName - } - - info, err := d.createUpload(ctx, node.group.GroupID, parentID, uploadName, size, sha1Hex, sha256Hex) - if err != nil { - return err - } - if _, err := f.Seek(0, io.SeekStart); err != nil { - return err - } - rf := driver.NewLimitedUploadFile(ctx, f) - prog := driver.NewProgress(size, model.UpdateProgressWithRange(up, 0, 1)) - - method := strings.ToUpper(strings.TrimSpace(info.Method)) - if method == "" { - method = http.MethodPut - } - - var req *http.Request - if method == http.MethodPost && len(info.Request.FormData) > 0 { - if size == 0 { - var buf bytes.Buffer - mw := multipart.NewWriter(&buf) - for k, v := range info.Request.FormData { - if err := mw.WriteField(k, v); err != nil { - return err - } - } - part, err := mw.CreateFormFile("file", uploadName) - if err != nil { - return err - } - if _, err := io.Copy(part, io.TeeReader(rf, prog)); err != nil { - return err - } - if err := mw.Close(); err != nil { - return err - } - req, err = http.NewRequestWithContext(ctx, method, info.URL, bytes.NewReader(buf.Bytes())) - if err != nil { - return err - } - for k, v := range info.Request.Headers { - req.Header.Set(k, v) - } - req.Header.Set("Content-Type", mw.FormDataContentType()) - req.ContentLength = int64(buf.Len()) - req.Header.Set("Content-Length", strconv.FormatInt(req.ContentLength, 10)) - } else { - pr, pw := io.Pipe() - mw := multipart.NewWriter(pw) - req, err = http.NewRequestWithContext(ctx, method, info.URL, pr) - if err != nil { - return err - } - for k, v := range info.Request.Headers { - req.Header.Set(k, v) - } - req.Header.Set("Content-Type", mw.FormDataContentType()) - go func() { - for k, v := range info.Request.FormData { - if err := mw.WriteField(k, v); err != nil { - pw.CloseWithError(err) - return - } - } - part, err := mw.CreateFormFile("file", uploadName) - if err != nil { - pw.CloseWithError(err) - return - } - if _, err := io.Copy(part, io.TeeReader(rf, prog)); err != nil { - pw.CloseWithError(err) - return - } - if err := mw.Close(); err != nil { - pw.CloseWithError(err) - return - } - pw.Close() - }() - } - } else { - var body = io.TeeReader(rf, prog) - if size == 0 { - body = bytes.NewReader(nil) - } - req, err = http.NewRequestWithContext(ctx, method, info.URL, body) - if err != nil { - return err - } - for k, v := range info.Request.Headers { - req.Header.Set(k, v) - } - req.ContentLength = size - req.Header.Set("Content-Length", strconv.FormatInt(size, 10)) - } - - c := *base.RestyClient.GetClient() - c.Timeout = 0 - resp, err := (&c).Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - - if !statusOK(resp.StatusCode, info.Response.ExpectCode) { - io.Copy(io.Discard, resp.Body) - return fmt.Errorf("http error: %d", resp.StatusCode) - } - - body, err := io.ReadAll(resp.Body) - if err != nil { - return err - } - - etag := normalizeETag(respArg(info.Response.ArgsETag, resp, body)) - if etag == "" { - etag = normalizeETag(resp.Header.Get("ETag")) - } - - key := strings.TrimSpace(respArg(info.Response.ArgsKey, resp, body)) - if key == "" { - key = strings.TrimSpace(resp.Header.Get("x-obs-save-key")) - } - - var pr uploadPutResp - sha1FromServer := "" - if err := json.Unmarshal(body, &pr); err == nil { - sha1FromServer = strings.TrimSpace(pr.NewFilename) - if sha1FromServer == "" { - sha1FromServer = strings.TrimSpace(pr.Sha1) - } - if etag == "" && pr.MD5 != "" { - etag = strings.TrimSpace(pr.MD5) - } - } - - if sha1FromServer == "" { - if v := extractXMLTag(string(body), "ETag"); v != "" { - sha1FromServer = v - if etag == "" { - etag = v - } - } - } - if sha1FromServer == "" && key != "" && len(key) == 40 { - sha1FromServer = key - } - if sha1FromServer == "" { - sha1FromServer = sha1Hex - } - - if etag == "" { - return fmt.Errorf("empty etag") - } - if sha1FromServer == "" { - return fmt.Errorf("empty sha1") - } - - store := strings.TrimSpace(info.Store) - commitKey := "" - if strings.TrimSpace(info.Response.ArgsKey) != "" { - commitKey = key - if commitKey == "" { - commitKey = sha1FromServer + unwrap, ok := obj.(model.ObjUnwrap) + if !ok { + break } + obj = unwrap.Unwrap() } - - if err := d.commitUpload(ctx, etag, commitKey, node.group.GroupID, parentID, uploadName, sha1FromServer, size, store); err != nil { - return err - } - - up(1) - return nil -} - -func (d *Wps) spaces(ctx context.Context) (*spacesResp, error) { - url := fmt.Sprintf("%s/api/v3/spaces", d.driveHost()+d.drivePrefix()) - var resp spacesResp - r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(url) - if err != nil { - return nil, err - } - if r != nil && r.IsError() { - return nil, fmt.Errorf("http error: %d", r.StatusCode()) - } - return &resp, nil + return nil, fmt.Errorf("invalid object type") } diff --git a/go.mod b/go.mod index 508228f5..ec165186 100644 --- a/go.mod +++ b/go.mod @@ -167,7 +167,6 @@ require ( github.com/sorairolake/lzip-go v0.3.5 // indirect github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543 // indirect github.com/ulikunitz/xz v0.5.12 // indirect - github.com/yuin/goldmark v1.7.13 go4.org v0.0.0-20260112195520-a5071408f32f resty.dev/v3 v3.0.0-beta.2 // indirect ) @@ -313,3 +312,5 @@ replace github.com/ProtonMail/go-proton-api => github.com/henrybear327/go-proton replace github.com/cronokirby/saferith => github.com/Da3zKi7/saferith v0.33.0-fixed // replace github.com/OpenListTeam/115-sdk-go => ../../OpenListTeam/115-sdk-go + +replace github.com/KarpelesLab/reflink => github.com/OpenListTeam/reflink v0.0.0-20260520031008-ed3c0dbe8009 diff --git a/go.sum b/go.sum index fe052360..6c5b9133 100644 --- a/go.sum +++ b/go.sum @@ -21,8 +21,6 @@ github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/Da3zKi7/saferith v0.33.0-fixed h1:fnIWTk7EP9mZAICf7aQjeoAwpfrlCrkOvqmi6CbWdTk= github.com/Da3zKi7/saferith v0.33.0-fixed/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA= -github.com/KarpelesLab/reflink v1.0.2 h1:hQ1aM3TmjU2kTNUx5p/HaobDoADYk+a6AuEinG4Cv88= -github.com/KarpelesLab/reflink v1.0.2/go.mod h1:WGkTOKNjd1FsJKBw3mu4JvrPEDJyJJ+JPtxBkbPoCok= github.com/KirCute/zip v1.0.1 h1:L/tVZglOiDVKDi9Ud+fN49htgKdQ3Z0H80iX8OZk13c= github.com/KirCute/zip v1.0.1/go.mod h1:xhF7dCB+Bjvy+5a56lenYCKBsH+gxDNPZSy5Cp+nlXk= github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= @@ -35,6 +33,8 @@ github.com/OpenListTeam/go-cache v0.1.0 h1:eV2+FCP+rt+E4OCJqLUW7wGccWZNJMV0NNkh+ github.com/OpenListTeam/go-cache v0.1.0/go.mod h1:AHWjKhNK3LE4rorVdKyEALDHoeMnP8SjiNyfVlB+Pz4= github.com/OpenListTeam/gsync v0.1.0 h1:ywzGybOvA3lW8K1BUjKZ2IUlT2FSlzPO4DOazfYXjcs= github.com/OpenListTeam/gsync v0.1.0/go.mod h1:h/Rvv9aX/6CdW/7B8di3xK3xNV8dUg45Fehrd/ksZ9s= +github.com/OpenListTeam/reflink v0.0.0-20260520031008-ed3c0dbe8009 h1:qLqJPr/FAsZTJiqy65JKKuFJP0V9pRVtSaIE0kqaQ8w= +github.com/OpenListTeam/reflink v0.0.0-20260520031008-ed3c0dbe8009/go.mod h1:WGkTOKNjd1FsJKBw3mu4JvrPEDJyJJ+JPtxBkbPoCok= github.com/OpenListTeam/sftpd-openlist v1.0.1 h1:j4S3iPFOpnXCUKRPS7uCT4mF2VCl34GyqvH6lqwnkUU= github.com/OpenListTeam/sftpd-openlist v1.0.1/go.mod h1:uO/wKnbvbdq3rBLmClMTZXuCnw7XW4wlAq4dZe91a40= github.com/OpenListTeam/tache v0.2.2 h1:CWFn6sr1AIYaEjC8ONdKs+LrxHyuErheenAjEqRhh4k= @@ -669,8 +669,6 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJu github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yuin/goldmark v1.7.13 h1:GPddIs617DnBLFFVJFgpo1aBfe/4xcvMc3SB5t/D0pA= -github.com/yuin/goldmark v1.7.13/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zzzhr1990/go-common-entity v0.0.0-20250202070650-1a200048f0d3 h1:PSRwrE5QBufPnOjdgIkRs5KBV1Avq3SY8oksj2Z+k3o= diff --git a/internal/archive/iso9660/utils.go b/internal/archive/iso9660/utils.go index 0e915133..848e4966 100644 --- a/internal/archive/iso9660/utils.go +++ b/internal/archive/iso9660/utils.go @@ -29,8 +29,8 @@ func getObj(img *iso9660.Image, path string) (*iso9660.File, error) { if path == "/" { return obj, nil } - paths := strings.Split(strings.TrimPrefix(path, "/"), "/") - for _, p := range paths { + paths := strings.SplitSeq(strings.TrimPrefix(path, "/"), "/") + for p := range paths { if !obj.IsDir() { return nil, errs.ObjectNotFound } diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index cc5a6926..ce5aa0db 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -34,6 +34,13 @@ func initSettings() { } settingMap := map[string]*model.SettingItem{} for _, v := range settings { + if v.Key == "" { + err := db.DeleteSettingItemByKey(v.Key) + if err != nil { + utils.Log.Errorf("failed delete setting with empty key: %+v", err) + } + continue + } if !isActive(v.Key) && v.Flag != model.DEPRECATED { v.Flag = model.DEPRECATED err = op.SaveSettingItem(&v) @@ -153,7 +160,7 @@ func InitialSettings() []model.SettingItem { {Key: conf.SharePreviewDownloadByDefault, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, {Key: conf.SharePreviewArchivesByDefault, Value: "false", Type: conf.TypeBool, Group: model.PREVIEW}, {Key: conf.ReadMeAutoRender, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, - {Key: conf.FilterReadMeScripts, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, + {Key: conf.FilterReadMeScripts, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, // frontend {Key: conf.NonEFSZipEncoding, Value: "IBM437", Type: conf.TypeString, Group: model.PREVIEW}, // global settings {Key: conf.HideFiles, Value: "/\\/README.md/i", Type: conf.TypeText, Group: model.GLOBAL}, diff --git a/internal/db/settingitem.go b/internal/db/settingitem.go index f20e507f..0d42aca6 100644 --- a/internal/db/settingitem.go +++ b/internal/db/settingitem.go @@ -65,5 +65,5 @@ func SaveSettingItem(item *model.SettingItem) error { } func DeleteSettingItemByKey(key string) error { - return errors.WithStack(db.Delete(&model.SettingItem{Key: key}).Error) + return errors.WithStack(db.Where(fmt.Sprintf("%s = ?", columnName("key")), key).Delete(model.SettingItem{}).Error) } diff --git a/internal/db/sharing.go b/internal/db/sharing.go index 8670b15f..9c356440 100644 --- a/internal/db/sharing.go +++ b/internal/db/sharing.go @@ -64,6 +64,14 @@ func UpdateSharing(s *model.SharingDB) error { return errors.WithStack(db.Save(s).Error) } +func UpdateSharingId(oldId, newId string) error { + // Check if new ID already exists + if err := db.Where("id = ?", newId).First(&model.SharingDB{}).Error; err == nil { + return errors.New("sharing id already exists") + } + return errors.WithStack(db.Model(&model.SharingDB{}).Where("id = ?", oldId).Update("id", newId).Error) +} + func DeleteSharingById(id string) error { s := model.SharingDB{ID: id} return errors.WithStack(db.Where(s).Delete(&s).Error) diff --git a/internal/errs/object.go b/internal/errs/object.go index ab78f314..6f4befb2 100644 --- a/internal/errs/object.go +++ b/internal/errs/object.go @@ -17,3 +17,7 @@ var ( func IsObjectNotFound(err error) bool { return errors.Is(pkgerr.Cause(err), ObjectNotFound) } + +func IsObjectAlreadyExists(err error) bool { + return errors.Is(pkgerr.Cause(err), ObjectAlreadyExists) +} diff --git a/internal/model/sharing.go b/internal/model/sharing.go index c5dd95e9..c5014944 100644 --- a/internal/model/sharing.go +++ b/internal/model/sharing.go @@ -3,7 +3,7 @@ package model import "time" type SharingDB struct { - ID string `json:"id" gorm:"type:char(12);primaryKey"` + ID string `json:"id" gorm:"type:varchar(64);primaryKey"` FilesRaw string `json:"-" gorm:"type:text"` Expires *time.Time `json:"expires"` Pwd string `json:"pwd"` diff --git a/internal/model/user.go b/internal/model/user.go index 55240711..a9d766e0 100644 --- a/internal/model/user.go +++ b/internal/model/user.go @@ -55,6 +55,7 @@ type User struct { // 12: can read archives // 13: can decompress archives // 14: can share + // 15: can customize share id Permission int32 `json:"permission"` OtpSecret string `json:"-"` SsoID string `json:"sso_id"` // unique by sso platform @@ -211,6 +212,14 @@ func (u *User) CanShare() bool { return CanShare(u.Permission) } +func CanCustomizeShareID(permission int32) bool { + return (permission>>15)&1 == 1 +} + +func (u *User) CanCustomizeShareID() bool { + return CanCustomizeShareID(u.Permission) +} + func (u *User) JoinPath(reqPath string) (string, error) { return utils.JoinBasePath(u.BasePath, reqPath) } diff --git a/internal/net/request.go b/internal/net/request.go index 8d380ea4..e1f04512 100644 --- a/internal/net/request.go +++ b/internal/net/request.go @@ -6,8 +6,8 @@ import ( "fmt" "io" "net/http" + stdpath "path" "strconv" - "strings" "sync" "time" @@ -490,30 +490,27 @@ func (d *downloader) checkTotalBytes(resp *http.Response) error { totalBytes = resp.ContentLength } } else { - parts := strings.Split(contentRange, "/") - - total := int64(-1) // Checking for whether a numbered total exists // If one does not exist, we will assume the total to be -1, undefined, // and sequentially download each chunk until hitting a 416 error - totalStr := parts[len(parts)-1] + + totalStr := stdpath.Base(contentRange) if totalStr != "*" { - total, err = strconv.ParseInt(totalStr, 10, 64) - if err != nil { - err = fmt.Errorf("failed extracting file size") + if total, err := strconv.ParseInt(totalStr, 10, 64); err != nil { + err = fmt.Errorf("failed extracting file size: %s", totalStr) + } else { + totalBytes = total } } else { - err = fmt.Errorf("file size unknown") + err = fmt.Errorf("file size unknown: %s", contentRange) } - totalBytes = total } if totalBytes != d.params.Size && err == nil { err = fmt.Errorf("expect file size=%d unmatch remote report size=%d, need refresh cache", d.params.Size, totalBytes) } if err != nil { - // _ = d.interrupt() d.setErr(err) d.cancel(err) } diff --git a/internal/offline_download/aria2/aria2.go b/internal/offline_download/aria2/aria2.go index b04435ac..5c037ff4 100644 --- a/internal/offline_download/aria2/aria2.go +++ b/internal/offline_download/aria2/aria2.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "strconv" + "strings" "time" "github.com/OpenListTeam/OpenList/v4/internal/errs" @@ -61,6 +62,10 @@ func (a *Aria2) IsReady() bool { } func (a *Aria2) AddURL(args *tool.AddUrlArgs) (string, error) { + // aria2 不支持 ed2k 协议，提前检测并返回明确错误 + if strings.HasPrefix(strings.ToLower(args.Url), "ed2k://") { + return "", fmt.Errorf("aria2 does not support ed2k protocol. Please use Thunder/ThunderX/ThunderBrowser tool for ed2k links") + } options := map[string]interface{}{ "dir": args.TempDir, } diff --git a/internal/offline_download/tool/add.go b/internal/offline_download/tool/add.go index 0f574571..33128ccc 100644 --- a/internal/offline_download/tool/add.go +++ b/internal/offline_download/tool/add.go @@ -2,9 +2,11 @@ package tool import ( "context" + "fmt" "net/url" stdpath "path" "path/filepath" + "strings" _115 "github.com/OpenListTeam/OpenList/v4/drivers/115" _115_open "github.com/OpenListTeam/OpenList/v4/drivers/115_open" @@ -67,10 +69,28 @@ func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, erro } // try putting url if args.Tool == "SimpleHttp" { + if isSimpleHttpSchemeUnsupported(args.URL) { + return nil, fmt.Errorf("SimpleHttp tool does not support this URL scheme, please use aria2 or other tools for magnet/ed2k links") + } err = tryPutUrl(ctx, args.DstDirPath, args.URL) if err == nil || !errors.Is(err, errs.NotImplement) { return nil, err } + // Fallback to creating a download task when storage lacks native PutURL support. + } + + // ed2k 链接自动路由：如果当前工具不支持 ed2k，自动尝试使用迅雷系工具 + if isEd2kURL(args.URL) { + if !isEd2kCapableTool(args.Tool) { + // 尝试找到一个可用的支持 ed2k 的工具 + fallbackTool, fallbackName := findEd2kCapableTool() + if fallbackTool != nil { + // 使用找到的迅雷工具替代 + args.Tool = fallbackName + } else { + return nil, fmt.Errorf("ed2k protocol is not supported by %s. Please configure and use Thunder/ThunderX/ThunderBrowser for ed2k links", args.Tool) + } + } } // get tool @@ -171,3 +191,48 @@ func tryPutUrl(ctx context.Context, path, urlStr string) error { } return fs.PutURL(ctx, path, dstName, urlStr) } + +func isSimpleHttpSchemeUnsupported(urlStr string) bool { + u, err := url.Parse(strings.TrimSpace(urlStr)) + if err != nil || u.Scheme == "" { + return false + } + scheme := strings.ToLower(u.Scheme) + return scheme != "http" && scheme != "https" +} + +// isEd2kURL 检测 URL 是否为 ed2k 协议 +func isEd2kURL(urlStr string) bool { + return strings.HasPrefix(strings.ToLower(urlStr), "ed2k://") +} + +// ed2kCapableTools 支持 ed2k 协议的工具列表（迅雷系） +var ed2kCapableTools = []string{"Thunder", "ThunderX", "ThunderBrowser"} + +// isEd2kCapableTool 检查工具是否支持 ed2k 协议 +func isEd2kCapableTool(toolName string) bool { + for _, t := range ed2kCapableTools { + if t == toolName { + return true + } + } + return false +} + +// findEd2kCapableTool 查找一个可用的支持 ed2k 的工具 +func findEd2kCapableTool() (Tool, string) { + for _, name := range ed2kCapableTools { + t, err := Tools.Get(name) + if err != nil { + continue + } + if t.IsReady() { + return t, name + } + // 尝试初始化 + if _, err := t.Init(); err == nil && t.IsReady() { + return t, name + } + } + return nil, "" +} diff --git a/internal/offline_download/tool/transfer.go b/internal/offline_download/tool/transfer.go index fd6b8f46..7109669e 100644 --- a/internal/offline_download/tool/transfer.go +++ b/internal/offline_download/tool/transfer.go @@ -9,6 +9,7 @@ import ( "path/filepath" "time" + _189pc "github.com/OpenListTeam/OpenList/v4/drivers/189pc" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" @@ -17,6 +18,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/task" "github.com/OpenListTeam/OpenList/v4/internal/task_group" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" + "github.com/OpenListTeam/OpenList/v4/pkg/torrent" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/tache" @@ -201,8 +203,28 @@ func transferStdFile(t *TransferTask) error { } info, err := rc.Stat() if err != nil { + rc.Close() return errors.Wrapf(err, "failed to get file %s", t.SrcActualPath) } + + // 尝试对天翼云进行秒传（计算 MD5 + sliceMD5） + if rapidObj, rapidErr := tryRapidUpload189(t, rc, info.Size()); rapidErr == nil && rapidObj != nil { + rc.Close() + log.Infof("秒传成功: %s -> %s", t.SrcActualPath, t.DstStorageMp) + return nil + } + + // 秒传失败或不支持，回退到普通上传 + // 重新 seek 到文件开头 + if _, err := rc.Seek(0, 0); err != nil { + rc.Close() + // 重新打开文件 + rc, err = os.Open(t.SrcActualPath) + if err != nil { + return errors.Wrapf(err, "failed to reopen file %s", t.SrcActualPath) + } + } + mimetype := utils.GetMimeType(t.SrcActualPath) s := &stream.FileStream{ Ctx: t.Ctx(), @@ -217,7 +239,12 @@ func transferStdFile(t *TransferTask) error { Closers: utils.NewClosers(rc), } t.SetTotalBytes(info.Size()) - return op.Put(context.WithValue(t.Ctx(), conf.SkipHookKey, struct{}{}), t.DstStorage, t.DstActualPath, s, t.SetProgress) + err = op.Put(context.WithValue(t.Ctx(), conf.SkipHookKey, struct{}{}), t.DstStorage, t.DstActualPath, s, t.SetProgress) + if err != nil { + return err + } + + return nil } func removeStdTemp(t *TransferTask) { @@ -341,3 +368,47 @@ func removeObjTemp(t *TransferTask) { log.Errorf("failed to delete temp obj %s, error: %s", t.SrcActualPath, err.Error()) } } + +// tryRapidUpload189 尝试对天翼云进行秒传 +// 通过计算文件的 MD5 来尝试秒传（使用旧版接口） +// 返回上传成功的对象和错误，如果不支持秒传则返回 nil, error +func tryRapidUpload189(t *TransferTask, file *os.File, fileSize int64) (model.Obj, error) { + // 检查目标存储是否是天翼云 PC 驱动 + cloud189PC, ok := t.DstStorage.(*_189pc.Cloud189PC) + if !ok { + return nil, fmt.Errorf("not 189pc storage") + } + + // 计算整文件 MD5（旧接口只需要 fileMD5） + fileMD5, _, err := _189pc.ComputeSliceMD5sFromReader(file, torrent.DefaultPieceSize) + if err != nil { + return nil, fmt.Errorf("计算 MD5 失败: %w", err) + } + + // 获取目标目录 + dstDir, err := op.Get(t.Ctx(), t.DstStorage, t.DstActualPath) + if err != nil { + return nil, fmt.Errorf("获取目标目录失败: %w", err) + } + + // 构造文件名 + fileName := filepath.Base(t.SrcActualPath) + + // 尝试秒传（使用旧接口） + uploadInfo, err := cloud189PC.OldUploadCreate(t.Ctx(), dstDir.GetID(), fileMD5, fileName, fmt.Sprint(fileSize), false) + if err != nil { + return nil, fmt.Errorf("创建上传任务失败: %w", err) + } + + if uploadInfo.FileDataExists != 1 { + return nil, fmt.Errorf("秒传失败：云端不存在该文件") + } + + // 秒传成功，提交 + obj, err := cloud189PC.OldUploadCommit(t.Ctx(), uploadInfo.FileCommitUrl, uploadInfo.UploadFileId, false, true) + if err != nil { + return nil, fmt.Errorf("提交上传失败: %w", err) + } + + return obj, nil +} diff --git a/internal/op/fs.go b/internal/op/fs.go index 9d93318e..3d070557 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -84,8 +84,7 @@ func list(ctx context.Context, storage driver.Driver, path string, args model.Li customCachePolicies := storage.GetStorage().CustomCachePolicies if len(customCachePolicies) > 0 { - configPolicies := strings.Split(customCachePolicies, "\n") - for _, configPolicy := range configPolicies { + for configPolicy := range strings.SplitSeq(customCachePolicies, "\n") { pattern, ttlstr, ok := strings.Cut(strings.TrimSpace(configPolicy), ":") if !ok { log.Warnf("Malformed custom cache policy entry: %s in storage %s for path %s. Expected format: pattern:ttl", configPolicy, storage.GetStorage().MountPath, path) @@ -333,6 +332,9 @@ func MakeDir(ctx context.Context, storage driver.Driver, path string) error { if err != nil { return nil, errors.WithMessagef(err, "failed to get parent dir [%s]", parentPath) } + if !parentDir.IsDir() { + return nil, errs.NotFolder + } if model.ObjHasMask(parentDir, model.NoWrite) { return nil, errors.WithStack(errs.PermissionDenied) } @@ -346,7 +348,7 @@ func MakeDir(ctx context.Context, storage driver.Driver, path string) error { default: return nil, errs.NotImplement } - if err != nil { + if err != nil && !errs.IsObjectAlreadyExists(err) { return nil, errors.WithStack(err) } if storage.Config().NoCache { @@ -643,7 +645,7 @@ func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file mod } } err = MakeDir(ctx, storage, dstDirPath) - if err != nil { + if err != nil && !errs.IsObjectAlreadyExists(err) { return errors.WithMessagef(err, "failed to make dir [%s]", dstDirPath) } parentDir, err := GetUnwrap(ctx, storage, dstDirPath) diff --git a/internal/op/sharing.go b/internal/op/sharing.go index 9db51c59..5be0f8e6 100644 --- a/internal/op/sharing.go +++ b/internal/op/sharing.go @@ -133,6 +133,15 @@ func UpdateSharing(sharing *model.Sharing, skipMarshal ...bool) (err error) { return db.UpdateSharing(sharing.SharingDB) } +func UpdateSharingId(sharing *model.Sharing, newId string) error { + sharingCache.Del(sharing.ID) + if err := db.UpdateSharingId(sharing.ID, newId); err != nil { + return err + } + sharing.ID = newId + return nil +} + func DeleteSharing(sid string) error { sharingCache.Del(sid) return db.DeleteSharingById(sid) diff --git a/pkg/gowebdav/client.go b/pkg/gowebdav/client.go index 7251e084..4341f903 100644 --- a/pkg/gowebdav/client.go +++ b/pkg/gowebdav/client.go @@ -300,9 +300,9 @@ func (c *Client) MkdirAll(path string, _ os.FileMode) (err error) { return nil } if status == 409 { - paths := strings.Split(path, "/") + paths := strings.SplitSeq(path, "/") sub := "/" - for _, e := range paths { + for e := range paths { if e == "" { continue } diff --git a/pkg/gowebdav/digestAuth.go b/pkg/gowebdav/digestAuth.go index 4a5eb62f..6ac0307e 100644 --- a/pkg/gowebdav/digestAuth.go +++ b/pkg/gowebdav/digestAuth.go @@ -45,8 +45,8 @@ func digestParts(resp *http.Response) map[string]string { result := map[string]string{} if len(resp.Header["Www-Authenticate"]) > 0 { wantedHeaders := []string{"nonce", "realm", "qop", "opaque", "algorithm", "entityBody"} - responseHeaders := strings.Split(resp.Header["Www-Authenticate"][0], ",") - for _, r := range responseHeaders { + responseHeaders := strings.SplitSeq(resp.Header["Www-Authenticate"][0], ",") + for r := range responseHeaders { for _, w := range wantedHeaders { if strings.Contains(r, w) { result[w] = strings.Trim( diff --git a/pkg/http_range/range.go b/pkg/http_range/range.go index 5edd210d..74d80e98 100644 --- a/pkg/http_range/range.go +++ b/pkg/http_range/range.go @@ -43,16 +43,16 @@ func ParseRange(s string, size int64) ([]Range, error) { // nolint:gocognit } var ranges []Range noOverlap := false - for _, ra := range strings.Split(s[len(b):], ",") { + for ra := range strings.SplitSeq(s[len(b):], ",") { ra = textproto.TrimString(ra) if ra == "" { continue } - i := strings.Index(ra, "-") - if i < 0 { + before, after, ok := strings.Cut(ra, "-") + if !ok { return nil, ErrInvalid } - start, end := textproto.TrimString(ra[:i]), textproto.TrimString(ra[i+1:]) + start, end := textproto.TrimString(before), textproto.TrimString(after) var r Range if start == "" { // If no start is specified, end specifies the diff --git a/pkg/qbittorrent/client.go b/pkg/qbittorrent/client.go index 102b445d..e4c12db6 100644 --- a/pkg/qbittorrent/client.go +++ b/pkg/qbittorrent/client.go @@ -43,7 +43,7 @@ func New(webuiUrl string) (Client, error) { IdleConnTimeout: 30 * time.Second, DisableKeepAlives: false, // Enable connection reuse } - + var c = &client{ url: u, client: http.Client{ @@ -98,6 +98,15 @@ func (c *client) login() error { } defer resp.Body.Close() + // avoid long waiting time if being upgraded to websocket connections (e.g. 101 responses) + // as per API documentation, qBittorrent returns only 200 on successful login (qBittorrent < 5.2.0) + // qBittorrent 5.2.0 /api/v2/auth/login returns HTTP 204 on success + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + return errors.New("failed to login into qBittorrent webui with status code: " + resp.Status) + } + if resp.StatusCode == http.StatusNoContent { + return nil + } // check result body := make([]byte, 2) _, err = resp.Body.Read(body) @@ -173,6 +182,10 @@ func (c *client) AddFromLink(link string, savePath string, id string) error { return err } defer resp.Body.Close() + // qBittorrent 5.2.0 returns 204 on success. + if resp.StatusCode != http.StatusNoContent { + return nil + } // check result body := make([]byte, 2) diff --git a/pkg/sign/hmac.go b/pkg/sign/hmac.go index 8d7f736b..8ba25bd1 100644 --- a/pkg/sign/hmac.go +++ b/pkg/sign/hmac.go @@ -26,13 +26,13 @@ func (s HMACSign) Sign(data string, expire int64) string { } func (s HMACSign) Verify(data, sign string) error { - signSlice := strings.Split(sign, ":") - // check whether contains expire time - if signSlice[len(signSlice)-1] == "" { + // check whether contains exp time + exp := sign[strings.LastIndex(sign, ":")+1:] + if exp == "" { return ErrExpireMissing } // check whether expire time is expired - expires, err := strconv.ParseInt(signSlice[len(signSlice)-1], 10, 64) + expires, err := strconv.ParseInt(exp, 10, 64) if err != nil { return ErrExpireInvalid } diff --git a/pkg/torrent/bencode.go b/pkg/torrent/bencode.go new file mode 100644 index 00000000..2d4fd782 --- /dev/null +++ b/pkg/torrent/bencode.go @@ -0,0 +1,261 @@ +package torrent + +import ( + "bytes" + "fmt" + "io" + "sort" + "strconv" +) + +// bencode 编码 + +// BencodeEncode 将值编码为 bencode 格式 +func BencodeEncode(v interface{}) ([]byte, error) { + var buf bytes.Buffer + if err := bencodeEncodeValue(&buf, v); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func bencodeEncodeValue(w io.Writer, v interface{}) error { + switch val := v.(type) { + case int: + return bencodeEncodeInt(w, int64(val)) + case int64: + return bencodeEncodeInt(w, val) + case string: + return bencodeEncodeString(w, val) + case []byte: + return bencodeEncodeBytes(w, val) + case []interface{}: + return bencodeEncodeList(w, val) + case map[string]interface{}: + return bencodeEncodeDict(w, val) + case OrderedDict: + return bencodeEncodeOrderedDict(w, val) + default: + return fmt.Errorf("bencode: unsupported type %T", v) + } +} + +func bencodeEncodeInt(w io.Writer, v int64) error { + _, err := fmt.Fprintf(w, "i%de", v) + return err +} + +func bencodeEncodeString(w io.Writer, v string) error { + _, err := fmt.Fprintf(w, "%d:%s", len(v), v) + return err +} + +func bencodeEncodeBytes(w io.Writer, v []byte) error { + _, err := fmt.Fprintf(w, "%d:", len(v)) + if err != nil { + return err + } + _, err = w.Write(v) + return err +} + +func bencodeEncodeList(w io.Writer, v []interface{}) error { + if _, err := w.Write([]byte("l")); err != nil { + return err + } + for _, item := range v { + if err := bencodeEncodeValue(w, item); err != nil { + return err + } + } + _, err := w.Write([]byte("e")) + return err +} + +func bencodeEncodeDict(w io.Writer, v map[string]interface{}) error { + // bencode 字典要求 key 按字典序排列 + keys := make([]string, 0, len(v)) + for k := range v { + keys = append(keys, k) + } + sort.Strings(keys) + + if _, err := w.Write([]byte("d")); err != nil { + return err + } + for _, k := range keys { + if err := bencodeEncodeString(w, k); err != nil { + return err + } + if err := bencodeEncodeValue(w, v[k]); err != nil { + return err + } + } + _, err := w.Write([]byte("e")) + return err +} + +// OrderedDict 有序字典，保持插入顺序 +type OrderedDict struct { + Keys []string + Values map[string]interface{} +} + +func NewOrderedDict() OrderedDict { + return OrderedDict{ + Keys: make([]string, 0), + Values: make(map[string]interface{}), + } +} + +func (d *OrderedDict) Set(key string, value interface{}) { + if _, exists := d.Values[key]; !exists { + d.Keys = append(d.Keys, key) + } + d.Values[key] = value +} + +func (d *OrderedDict) Get(key string) (interface{}, bool) { + v, ok := d.Values[key] + return v, ok +} + +func bencodeEncodeOrderedDict(w io.Writer, d OrderedDict) error { + // 按字典序排列 key（bencode 规范要求） + keys := make([]string, len(d.Keys)) + copy(keys, d.Keys) + sort.Strings(keys) + + if _, err := w.Write([]byte("d")); err != nil { + return err + } + for _, k := range keys { + if err := bencodeEncodeString(w, k); err != nil { + return err + } + if err := bencodeEncodeValue(w, d.Values[k]); err != nil { + return err + } + } + _, err := w.Write([]byte("e")) + return err +} + +// bencode 解码 + +// BencodeDecode 从字节数组解码 bencode 数据 +func BencodeDecode(data []byte) (interface{}, error) { + reader := bytes.NewReader(data) + val, err := bencodeDecodeValue(reader) + if err != nil { + return nil, err + } + return val, nil +} + +func bencodeDecodeValue(r *bytes.Reader) (interface{}, error) { + b, err := r.ReadByte() + if err != nil { + return nil, err + } + + switch { + case b == 'i': + return bencodeDecodeInt(r) + case b == 'l': + return bencodeDecodeList(r) + case b == 'd': + return bencodeDecodeDict(r) + case b >= '0' && b <= '9': + r.UnreadByte() + return bencodeDecodeString(r) + default: + return nil, fmt.Errorf("bencode: unexpected byte '%c' at position %d", b, int64(r.Len())) + } +} + +func bencodeDecodeInt(r *bytes.Reader) (int64, error) { + var buf bytes.Buffer + for { + b, err := r.ReadByte() + if err != nil { + return 0, err + } + if b == 'e' { + break + } + buf.WriteByte(b) + } + return strconv.ParseInt(buf.String(), 10, 64) +} + +func bencodeDecodeString(r *bytes.Reader) ([]byte, error) { + // 读取长度 + var lenBuf bytes.Buffer + for { + b, err := r.ReadByte() + if err != nil { + return nil, err + } + if b == ':' { + break + } + lenBuf.WriteByte(b) + } + length, err := strconv.ParseInt(lenBuf.String(), 10, 64) + if err != nil { + return nil, fmt.Errorf("bencode: invalid string length: %v", err) + } + if length < 0 || length > 100*1024*1024 { + return nil, fmt.Errorf("bencode: string length out of bounds: %d", length) + } + // Safe to convert to int: bounds check above ensures length <= 100MB which fits in int32 + data := make([]byte, int(length)) + _, err = io.ReadFull(r, data) + if err != nil { + return nil, err + } + return data, nil +} + +func bencodeDecodeList(r *bytes.Reader) ([]interface{}, error) { + var list []interface{} + for { + b, err := r.ReadByte() + if err != nil { + return nil, err + } + if b == 'e' { + return list, nil + } + r.UnreadByte() + val, err := bencodeDecodeValue(r) + if err != nil { + return nil, err + } + list = append(list, val) + } +} + +func bencodeDecodeDict(r *bytes.Reader) (map[string]interface{}, error) { + dict := make(map[string]interface{}) + for { + b, err := r.ReadByte() + if err != nil { + return nil, err + } + if b == 'e' { + return dict, nil + } + r.UnreadByte() + keyBytes, err := bencodeDecodeString(r) + if err != nil { + return nil, err + } + val, err := bencodeDecodeValue(r) + if err != nil { + return nil, err + } + dict[string(keyBytes)] = val + } +} diff --git a/pkg/torrent/generate.go b/pkg/torrent/generate.go new file mode 100644 index 00000000..566cad86 --- /dev/null +++ b/pkg/torrent/generate.go @@ -0,0 +1,123 @@ +package torrent + +import ( + "io" + "os" + "strings" +) + +// GenerateFromFile 从文件路径生成通用的 torrent 文件（不含 CAS 扩展） +// 这是一个通用函数，适用于所有驱动 +func GenerateFromFile(filePath string) ([]byte, error) { + f, err := os.Open(filePath) + if err != nil { + return nil, err + } + defer f.Close() + + info, err := f.Stat() + if err != nil { + return nil, err + } + + return GenerateFromReader(f, info.Name(), info.Size(), DefaultPieceSize) +} + +// GenerateFromReader 从 io.Reader 生成通用的 torrent 文件（不含 CAS 扩展） +// 返回 torrent 字节数据 +func GenerateFromReader(reader io.Reader, fileName string, fileSize int64, pieceSize int64) ([]byte, error) { + if pieceSize <= 0 { + pieceSize = DefaultPieceSize + } + + hw := NewHashWriter(pieceSize, pieceSize) + + buf := make([]byte, 32*1024) + for { + n, err := reader.Read(buf) + if n > 0 { + hw.Write(buf[:n]) + } + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + } + hw.Finish() + + fileMD5 := hw.GetFileMD5() + pieceHashes := hw.GetPieceHashes() + + t := NewTorrent(fileName, fileSize, fileMD5) + t.Info.PieceLength = pieceSize + t.SetPieces(pieceHashes) + + return t.Encode() +} + +// GenerateFromReaderWithCAS 从 io.Reader 生成包含 CAS 扩展的 torrent 文件 +// 适用于天翼云等支持秒传的网盘 +func GenerateFromReaderWithCAS(reader io.Reader, fileName string, fileSize int64, pieceSize int64) ([]byte, error) { + if pieceSize <= 0 { + pieceSize = DefaultPieceSize + } + + hw := NewHashWriter(pieceSize, pieceSize) + + buf := make([]byte, 32*1024) + for { + n, err := reader.Read(buf) + if n > 0 { + hw.Write(buf[:n]) + } + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + } + hw.Finish() + + fileMD5 := hw.GetFileMD5() + sliceMD5s := hw.GetSliceMD5s() + pieceHashes := hw.GetPieceHashes() + + // 计算 sliceMD5 + sliceMD5 := fileMD5 + if len(sliceMD5s) > 1 { + joined := strings.Join(sliceMD5s, "\n") + sliceMD5 = strings.ToUpper(GetMD5Str(joined)) + } + + t := NewTorrent(fileName, fileSize, fileMD5) + t.Info.PieceLength = pieceSize + t.SetPieces(pieceHashes) + t.SetCASInfo(&CASInfo{ + FileMD5: fileMD5, + SliceMD5: sliceMD5, + SliceMD5s: sliceMD5s, + SliceSize: pieceSize, + Cloud: "189", + }) + + return t.Encode() +} + +// GenerateFromFileWithCAS 从文件路径生成包含 CAS 扩展的 torrent 文件 +func GenerateFromFileWithCAS(filePath string) ([]byte, error) { + f, err := os.Open(filePath) + if err != nil { + return nil, err + } + defer f.Close() + + info, err := f.Stat() + if err != nil { + return nil, err + } + + return GenerateFromReaderWithCAS(f, info.Name(), info.Size(), DefaultPieceSize) +} diff --git a/pkg/torrent/hash_writer.go b/pkg/torrent/hash_writer.go new file mode 100644 index 00000000..a62f42c6 --- /dev/null +++ b/pkg/torrent/hash_writer.go @@ -0,0 +1,229 @@ +package torrent + +import ( + "crypto/md5" + "crypto/sha1" + "encoding/hex" + "fmt" + "hash" + "io" + "strings" +) + +// HashWriter 同时计算文件的 MD5、分片 MD5 和 SHA-1 piece hash +// 用于在上传过程中一次性计算所有需要的哈希值 +type HashWriter struct { + // 整文件 MD5 + fileMD5 hash.Hash + // 当前分片 MD5 + sliceMD5 hash.Hash + // 当前 piece 的 SHA-1 + pieceSHA1 hash.Hash + + // 分片大小（默认 10MB） + sliceSize int64 + // piece 大小（与 sliceSize 相同，保持对齐） + pieceSize int64 + + // 当前分片已写入字节数 + sliceWritten int64 + // 当前 piece 已写入字节数 + pieceWritten int64 + // 总写入字节数 + totalWritten int64 + + // 每个分片的 MD5（大写十六进制） + sliceMD5Hexs []string + // 所有 piece 的 SHA-1 哈希拼接 + pieceHashes []byte +} + +// NewHashWriter 创建一个新的 HashWriter +// sliceSize: CAS 分片大小（通常 10MB） +// pieceSize: BT piece 大小（设为与 sliceSize 相同以保持对齐） +func NewHashWriter(sliceSize, pieceSize int64) *HashWriter { + if sliceSize <= 0 { + sliceSize = DefaultPieceSize + } + if pieceSize <= 0 { + pieceSize = DefaultPieceSize + } + return &HashWriter{ + fileMD5: md5.New(), + sliceMD5: md5.New(), + pieceSHA1: sha1.New(), + sliceSize: sliceSize, + pieceSize: pieceSize, + } +} + +// NewDefaultHashWriter 创建默认的 HashWriter（10MB 分片） +func NewDefaultHashWriter() *HashWriter { + return NewHashWriter(DefaultPieceSize, DefaultPieceSize) +} + +// Write 实现 io.Writer 接口 +func (hw *HashWriter) Write(p []byte) (n int, err error) { + total := len(p) + offset := 0 + + for offset < total { + // 计算当前可以写入的字节数（取分片和 piece 剩余空间的最小值） + sliceRemain := hw.sliceSize - hw.sliceWritten + pieceRemain := hw.pieceSize - hw.pieceWritten + canWrite := min64(sliceRemain, pieceRemain) + canWrite = min64(canWrite, int64(total-offset)) + + chunk := p[offset : offset+int(canWrite)] + + // 写入整文件 MD5 + hw.fileMD5.Write(chunk) + // 写入当前分片 MD5 + hw.sliceMD5.Write(chunk) + // 写入当前 piece SHA-1 + hw.pieceSHA1.Write(chunk) + + hw.sliceWritten += canWrite + hw.pieceWritten += canWrite + hw.totalWritten += canWrite + offset += int(canWrite) + + // 检查分片是否完成 + if hw.sliceWritten >= hw.sliceSize { + hw.finishSlice() + } + + // 检查 piece 是否完成 + if hw.pieceWritten >= hw.pieceSize { + hw.finishPiece() + } + } + + return total, nil +} + +// finishSlice 完成当前分片的 MD5 计算 +func (hw *HashWriter) finishSlice() { + md5Hex := strings.ToUpper(hex.EncodeToString(hw.sliceMD5.Sum(nil))) + hw.sliceMD5Hexs = append(hw.sliceMD5Hexs, md5Hex) + hw.sliceMD5.Reset() + hw.sliceWritten = 0 +} + +// finishPiece 完成当前 piece 的 SHA-1 计算 +func (hw *HashWriter) finishPiece() { + hw.pieceHashes = append(hw.pieceHashes, hw.pieceSHA1.Sum(nil)...) + hw.pieceSHA1.Reset() + hw.pieceWritten = 0 +} + +// Finish 完成所有哈希计算（处理最后不完整的分片/piece） +func (hw *HashWriter) Finish() { + // 处理最后一个不完整的分片 + if hw.sliceWritten > 0 { + hw.finishSlice() + } + // 处理最后一个不完整的 piece + if hw.pieceWritten > 0 { + hw.finishPiece() + } +} + +// GetFileMD5 获取整文件 MD5（大写十六进制） +func (hw *HashWriter) GetFileMD5() string { + return strings.ToUpper(hex.EncodeToString(hw.fileMD5.Sum(nil))) +} + +// GetSliceMD5s 获取所有分片的 MD5 列表 +func (hw *HashWriter) GetSliceMD5s() []string { + return hw.sliceMD5Hexs +} + +// GetSliceMD5 获取最终的 sliceMD5（用于秒传） +func (hw *HashWriter) GetSliceMD5(fileMD5 string) string { + if len(hw.sliceMD5Hexs) <= 1 { + return fileMD5 + } + joined := strings.Join(hw.sliceMD5Hexs, "\n") + return strings.ToUpper(GetMD5Str(joined)) +} + +// GetPieceHashes 获取所有 piece 的 SHA-1 哈希拼接 +func (hw *HashWriter) GetPieceHashes() []byte { + return hw.pieceHashes +} + +// GetTotalWritten 获取总写入字节数 +func (hw *HashWriter) GetTotalWritten() int64 { + return hw.totalWritten +} + +// BuildTorrent 根据计算结果构建 Torrent 结构 +func (hw *HashWriter) BuildTorrent(fileName string, fileSize int64) *Torrent { + fileMD5 := hw.GetFileMD5() + sliceMD5 := hw.GetSliceMD5(fileMD5) + + t := NewTorrent(fileName, fileSize, fileMD5) + t.SetPieces(hw.GetPieceHashes()) + t.SetCASInfo(&CASInfo{ + FileMD5: fileMD5, + SliceMD5: sliceMD5, + SliceMD5s: hw.GetSliceMD5s(), + SliceSize: hw.sliceSize, + Cloud: "189", + }) + + return t +} + +// BuildTorrentBytes 构建并编码 torrent 文件 +func (hw *HashWriter) BuildTorrentBytes(fileName string, fileSize int64) ([]byte, error) { + t := hw.BuildTorrent(fileName, fileSize) + return t.Encode() +} + +// CopyAndHash 从 reader 读取数据，同时写入 writer 和 HashWriter +func CopyAndHash(dst io.Writer, src io.Reader, hw *HashWriter) (int64, error) { + buf := make([]byte, 32*1024) // 32KB buffer + var written int64 + for { + nr, er := src.Read(buf) + if nr > 0 { + // 写入 HashWriter + hw.Write(buf[:nr]) + // 写入目标 + if dst != nil { + nw, ew := dst.Write(buf[:nr]) + if nw < 0 || nr < nw { + nw = 0 + if ew == nil { + ew = fmt.Errorf("invalid write result") + } + } + written += int64(nw) + if ew != nil { + return written, ew + } + if nr != nw { + return written, io.ErrShortWrite + } + } else { + written += int64(nr) + } + } + if er != nil { + if er == io.EOF { + break + } + return written, er + } + } + return written, nil +} + +func min64(a, b int64) int64 { + if a < b { + return a + } + return b +} diff --git a/pkg/torrent/torrent.go b/pkg/torrent/torrent.go new file mode 100644 index 00000000..8744e636 --- /dev/null +++ b/pkg/torrent/torrent.go @@ -0,0 +1,439 @@ +package torrent + +import ( + "crypto/md5" + "crypto/sha1" + "encoding/hex" + "fmt" + "strings" + "time" +) + +const ( + // DefaultPieceSize 默认分片大小 10MB，与天翼云 CAS 分片大小一致 + DefaultPieceSize int64 = 10 * 1024 * 1024 + + // CASExtensionKey torrent 根字典中的 CAS 扩展 key + CASExtensionKey = "x-cas" + + // CASSliceSizeKey CAS 分片大小 key + CASSliceSizeKey = "slice_size" + // CASSliceMD5sKey 每片 MD5 列表 key + CASSliceMD5sKey = "slice_md5s" + // CASSliceMD5Key 最终 sliceMd5 key + CASSliceMD5Key = "slice_md5" + // CASFileMD5Key 整文件 MD5 key + CASFileMD5Key = "file_md5" + // CASCloudKey 云盘类型 key + CASCloudKey = "cloud" +) + +// CASInfo 天翼云 CAS 秒传所需信息 +type CASInfo struct { + // FileMD5 整文件 MD5（大写十六进制） + FileMD5 string + // SliceMD5 分片 MD5 的摘要（大写十六进制） + SliceMD5 string + // SliceMD5s 每个 10MB 分片的 MD5（大写十六进制） + SliceMD5s []string + // SliceSize 分片大小（字节） + SliceSize int64 + // Cloud 云盘类型标识 + Cloud string +} + +// TorrentFile 表示 torrent 中的单个文件 +type TorrentFile struct { + // Length 文件大小（字节） + Length int64 + // Path 文件路径（多文件模式下的相对路径各段） + Path []string + // MD5Sum 文件的 MD5（可选，BT 标准字段） + MD5Sum string +} + +// TorrentInfo torrent 的 info 字典 +type TorrentInfo struct { + // PieceLength 分片大小 + PieceLength int64 + // Pieces 所有分片的 SHA-1 哈希拼接（每 20 字节一个） + Pieces []byte + // Name 种子名称（单文件模式为文件名，多文件模式为目录名） + Name string + // Length 单文件模式下的文件大小 + Length int64 + // Files 多文件模式下的文件列表 + Files []TorrentFile + // MD5Sum 单文件模式下的文件 MD5（可选） + MD5Sum string +} + +// Torrent 完整的 torrent 文件结构 +type Torrent struct { + // Info info 字典 + Info TorrentInfo + // InfoHash info 字典的 SHA-1 哈希（20 字节） + InfoHash []byte + // Announce tracker URL + Announce string + // AnnounceList tracker 列表 + AnnounceList [][]string + // CreationDate 创建时间 + CreationDate int64 + // Comment 注释 + Comment string + // CreatedBy 创建者 + CreatedBy string + // CAS 天翼云 CAS 扩展信息（存储在 info 字典外部，不影响 info_hash） + CAS *CASInfo +} + +// NewTorrent 创建一个新的 torrent 结构 +func NewTorrent(name string, fileSize int64, fileMD5 string) *Torrent { + return &Torrent{ + Info: TorrentInfo{ + PieceLength: DefaultPieceSize, + Name: name, + Length: fileSize, + MD5Sum: fileMD5, + }, + CreationDate: time.Now().Unix(), + CreatedBy: "OpenList", + Comment: "Generated by OpenList with CAS extension", + } +} + +// SetPieces 设置 SHA-1 分片哈希 +func (t *Torrent) SetPieces(pieces []byte) { + t.Info.Pieces = pieces +} + +// SetCASInfo 设置 CAS 扩展信息 +func (t *Torrent) SetCASInfo(cas *CASInfo) { + t.CAS = cas +} + +// Encode 将 torrent 编码为 bencode 格式的字节数组 +func (t *Torrent) Encode() ([]byte, error) { + // 构建 info 字典 + infoDict := make(map[string]interface{}) + infoDict["piece length"] = int64(t.Info.PieceLength) + infoDict["pieces"] = t.Info.Pieces + infoDict["name"] = t.Info.Name + + if len(t.Info.Files) > 0 { + // 多文件模式 + files := make([]interface{}, 0, len(t.Info.Files)) + for _, f := range t.Info.Files { + fileDict := make(map[string]interface{}) + fileDict["length"] = int64(f.Length) + path := make([]interface{}, 0, len(f.Path)) + for _, p := range f.Path { + path = append(path, p) + } + fileDict["path"] = path + if f.MD5Sum != "" { + fileDict["md5sum"] = f.MD5Sum + } + files = append(files, fileDict) + } + infoDict["files"] = files + } else { + // 单文件模式 + infoDict["length"] = int64(t.Info.Length) + if t.Info.MD5Sum != "" { + infoDict["md5sum"] = t.Info.MD5Sum + } + } + + // 编码 info 字典并计算 info_hash + infoBytes, err := BencodeEncode(infoDict) + if err != nil { + return nil, fmt.Errorf("encode info dict: %w", err) + } + infoHashRaw := sha1.Sum(infoBytes) + t.InfoHash = infoHashRaw[:] + + // 构建根字典 + rootDict := make(map[string]interface{}) + if t.Announce != "" { + rootDict["announce"] = t.Announce + } + if len(t.AnnounceList) > 0 { + announceList := make([]interface{}, 0, len(t.AnnounceList)) + for _, tier := range t.AnnounceList { + tierList := make([]interface{}, 0, len(tier)) + for _, url := range tier { + tierList = append(tierList, url) + } + announceList = append(announceList, tierList) + } + rootDict["announce-list"] = announceList + } + if t.Comment != "" { + rootDict["comment"] = t.Comment + } + if t.CreatedBy != "" { + rootDict["created by"] = t.CreatedBy + } + if t.CreationDate > 0 { + rootDict["creation date"] = t.CreationDate + } + + // info 字典使用原始编码的字节（保证 info_hash 一致） + rootDict["info"] = infoDict + + // CAS 扩展信息（放在 info 外部，不影响 info_hash） + if t.CAS != nil { + casDict := make(map[string]interface{}) + casDict[CASCloudKey] = t.CAS.Cloud + casDict[CASFileMD5Key] = t.CAS.FileMD5 + casDict[CASSliceMD5Key] = t.CAS.SliceMD5 + casDict[CASSliceSizeKey] = t.CAS.SliceSize + + if len(t.CAS.SliceMD5s) > 0 { + md5List := make([]interface{}, 0, len(t.CAS.SliceMD5s)) + for _, md5 := range t.CAS.SliceMD5s { + md5List = append(md5List, md5) + } + casDict[CASSliceMD5sKey] = md5List + } + rootDict[CASExtensionKey] = casDict + } + + return BencodeEncode(rootDict) +} + +// Decode 从 bencode 字节数组解析 torrent +func Decode(data []byte) (*Torrent, error) { + val, err := BencodeDecode(data) + if err != nil { + return nil, fmt.Errorf("bencode decode: %w", err) + } + + rootDict, ok := val.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("torrent: root is not a dict") + } + + t := &Torrent{} + + // 解析 announce + if v, ok := rootDict["announce"]; ok { + if b, ok := v.([]byte); ok { + t.Announce = string(b) + } + } + + // 解析 announce-list + if v, ok := rootDict["announce-list"]; ok { + if list, ok := v.([]interface{}); ok { + for _, tier := range list { + if tierList, ok := tier.([]interface{}); ok { + var urls []string + for _, u := range tierList { + if b, ok := u.([]byte); ok { + urls = append(urls, string(b)) + } + } + if len(urls) > 0 { + t.AnnounceList = append(t.AnnounceList, urls) + } + } + } + } + } + + // 解析 comment + if v, ok := rootDict["comment"]; ok { + if b, ok := v.([]byte); ok { + t.Comment = string(b) + } + } + + // 解析 created by + if v, ok := rootDict["created by"]; ok { + if b, ok := v.([]byte); ok { + t.CreatedBy = string(b) + } + } + + // 解析 creation date + if v, ok := rootDict["creation date"]; ok { + if n, ok := v.(int64); ok { + t.CreationDate = n + } + } + + // 解析 info 字典 + if infoVal, ok := rootDict["info"]; ok { + infoDict, ok := infoVal.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("torrent: info is not a dict") + } + + // 计算 info_hash + infoBytes, err := BencodeEncode(infoDict) + if err != nil { + return nil, fmt.Errorf("encode info for hash: %w", err) + } + infoHashRaw := sha1.Sum(infoBytes) + t.InfoHash = infoHashRaw[:] + + // 解析 info 字段 + if v, ok := infoDict["piece length"]; ok { + if n, ok := v.(int64); ok { + t.Info.PieceLength = n + } + } + if v, ok := infoDict["pieces"]; ok { + if b, ok := v.([]byte); ok { + t.Info.Pieces = b + } + } + if v, ok := infoDict["name"]; ok { + if b, ok := v.([]byte); ok { + t.Info.Name = string(b) + } + } + if v, ok := infoDict["length"]; ok { + if n, ok := v.(int64); ok { + t.Info.Length = n + } + } + if v, ok := infoDict["md5sum"]; ok { + if b, ok := v.([]byte); ok { + t.Info.MD5Sum = string(b) + } + } + + // 解析多文件模式 + if v, ok := infoDict["files"]; ok { + if files, ok := v.([]interface{}); ok { + for _, f := range files { + if fileDict, ok := f.(map[string]interface{}); ok { + tf := TorrentFile{} + if l, ok := fileDict["length"]; ok { + if n, ok := l.(int64); ok { + tf.Length = n + } + } + if p, ok := fileDict["path"]; ok { + if pathList, ok := p.([]interface{}); ok { + for _, pp := range pathList { + if b, ok := pp.([]byte); ok { + tf.Path = append(tf.Path, string(b)) + } + } + } + } + if m, ok := fileDict["md5sum"]; ok { + if b, ok := m.([]byte); ok { + tf.MD5Sum = string(b) + } + } + t.Info.Files = append(t.Info.Files, tf) + } + } + } + } + } + + // 解析 CAS 扩展 + if casVal, ok := rootDict[CASExtensionKey]; ok { + if casDict, ok := casVal.(map[string]interface{}); ok { + cas := &CASInfo{} + if v, ok := casDict[CASCloudKey]; ok { + if b, ok := v.([]byte); ok { + cas.Cloud = string(b) + } + } + if v, ok := casDict[CASFileMD5Key]; ok { + if b, ok := v.([]byte); ok { + cas.FileMD5 = string(b) + } + } + if v, ok := casDict[CASSliceMD5Key]; ok { + if b, ok := v.([]byte); ok { + cas.SliceMD5 = string(b) + } + } + if v, ok := casDict[CASSliceSizeKey]; ok { + if n, ok := v.(int64); ok { + cas.SliceSize = n + } + } + if v, ok := casDict[CASSliceMD5sKey]; ok { + if list, ok := v.([]interface{}); ok { + for _, item := range list { + if b, ok := item.([]byte); ok { + cas.SliceMD5s = append(cas.SliceMD5s, string(b)) + } + } + } + } + t.CAS = cas + } + } + + return t, nil +} + +// GetInfoHashHex 获取 info_hash 的十六进制字符串 +func (t *Torrent) GetInfoHashHex() string { + return hex.EncodeToString(t.InfoHash) +} + +// GetPieceHashes 获取所有分片的 SHA-1 哈希（每个 20 字节） +func (t *Torrent) GetPieceHashes() [][]byte { + if len(t.Info.Pieces) == 0 { + return nil + } + count := len(t.Info.Pieces) / 20 + hashes := make([][]byte, count) + for i := 0; i < count; i++ { + hashes[i] = t.Info.Pieces[i*20 : (i+1)*20] + } + return hashes +} + +// GetTotalSize 获取 torrent 中所有文件的总大小 +func (t *Torrent) GetTotalSize() int64 { + if len(t.Info.Files) > 0 { + var total int64 + for _, f := range t.Info.Files { + total += f.Length + } + return total + } + return t.Info.Length +} + +// HasCASInfo 检查 torrent 是否包含 CAS 扩展信息 +func (t *Torrent) HasCASInfo() bool { + return t.CAS != nil && t.CAS.FileMD5 != "" && t.CAS.SliceMD5 != "" +} + +// BuildCASInfoFromMD5s 从分片 MD5 列表构建 CAS 信息 +func BuildCASInfoFromMD5s(fileMD5 string, sliceMD5s []string, sliceSize int64) *CASInfo { + sliceMD5 := fileMD5 + if len(sliceMD5s) > 1 { + // 所有分片 MD5 用 \n 拼接后再取 MD5 + joined := strings.Join(sliceMD5s, "\n") + sliceMD5 = strings.ToUpper(GetMD5Str(joined)) + } + return &CASInfo{ + FileMD5: fileMD5, + SliceMD5: sliceMD5, + SliceMD5s: sliceMD5s, + SliceSize: sliceSize, + Cloud: "189", + } +} + +// GetMD5Str 计算字符串的 MD5（大写十六进制） +func GetMD5Str(data string) string { + h := md5.New() + h.Write([]byte(data)) + return strings.ToUpper(hex.EncodeToString(h.Sum(nil))) +} diff --git a/pkg/utils/path.go b/pkg/utils/path.go index ec75501e..4ac3dde2 100644 --- a/pkg/utils/path.go +++ b/pkg/utils/path.go @@ -111,14 +111,14 @@ func GetPathHierarchy(path string) []string { hierarchy := []string{"/"} - parts := strings.Split(path, "/") - currentPath := "" - for _, part := range parts { + parts := strings.SplitSeq(path, "/") + var currentPath strings.Builder + for part := range parts { if part == "" { continue } - currentPath += "/" + part - hierarchy = append(hierarchy, currentPath) + currentPath.WriteString("/" + part) + hierarchy = append(hierarchy, currentPath.String()) } return hierarchy diff --git a/server/common/check.go b/server/common/check.go index 27be3103..2c8d3bc8 100644 --- a/server/common/check.go +++ b/server/common/check.go @@ -51,7 +51,7 @@ func CanAccess(user *model.User, meta *model.Meta, reqPath string, password stri // if the reqPath is in hide (only can check the nearest meta) and user can't see hides, can't access if meta != nil && !user.CanSeeHides() && meta.Hide != "" && MetaCoversPath(meta.Path, path.Dir(reqPath), meta.HSub) { // the meta should apply to the parent of current path - for _, hide := range strings.Split(meta.Hide, "\n") { + for hide := range strings.SplitSeq(meta.Hide, "\n") { re := regexp2.MustCompile(hide, regexp2.None) if isMatch, _ := re.MatchString(path.Base(reqPath)); isMatch { return false diff --git a/server/common/common.go b/server/common/common.go index d7805126..a1a9a631 100644 --- a/server/common/common.go +++ b/server/common/common.go @@ -128,13 +128,24 @@ func Pluralize(count int, singular, plural string) string { return plural } -func GinWithValue(c *gin.Context, keyAndValue ...any) { +type requestContext struct { + context.Context +} + +// GinAppendValues 向当前请求上下文追加键值，提供类似 gin.Context Set/Get 的可变语义。 +// 同一请求内，已持有的上下文引用会同步看到后续更新。 +func GinAppendValues(c *gin.Context, keyAndValue ...any) { + ctx := c.Request.Context() + if r, ok := ctx.(*requestContext); ok { + r.Context = ContentWithValues(r.Context, keyAndValue...) + return + } c.Request = c.Request.WithContext( - ContentWithValue(c.Request.Context(), keyAndValue...), + &requestContext{ContentWithValues(ctx, keyAndValue...)}, ) } -func ContentWithValue(ctx context.Context, keyAndValue ...any) context.Context { +func ContentWithValues(ctx context.Context, keyAndValue ...any) context.Context { if len(keyAndValue) < 1 || len(keyAndValue)%2 != 0 { panic("keyAndValue must be an even number of arguments (key, value, ...)") } diff --git a/server/ftp.go b/server/ftp.go index d07a62dd..8999c1f5 100644 --- a/server/ftp.go +++ b/server/ftp.go @@ -226,7 +226,7 @@ func newPortMapper(str string) ftpserver.PasvPortGetter { if str == "" { return nil } - pasvPortMappers := strings.Split(strings.Replace(str, "\n", ",", -1), ",") + pasvPortMappers := strings.Split(strings.ReplaceAll(str, "\n", ","), ",") groups := make([]group, len(pasvPortMappers)) totalLength := 0 convertToPorts := func(str string) (int, int, error) { diff --git a/server/handles/archive.go b/server/handles/archive.go index 96bfd662..12155286 100644 --- a/server/handles/archive.go +++ b/server/handles/archive.go @@ -105,7 +105,7 @@ func FsArchiveMeta(c *gin.Context, req *ArchiveMetaReq, user *model.User) { common.ErrorResp(c, err, 500, true) return } - common.GinWithValue(c, conf.MetaKey, meta) + common.GinAppendValues(c, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return @@ -188,7 +188,7 @@ func FsArchiveList(c *gin.Context, req *ArchiveListReq, user *model.User) { common.ErrorResp(c, err, 500, true) return } - common.GinWithValue(c, conf.MetaKey, meta) + common.GinAppendValues(c, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return diff --git a/server/handles/down.go b/server/handles/down.go index 0c75f4d5..8e12ae47 100644 --- a/server/handles/down.go +++ b/server/handles/down.go @@ -1,11 +1,8 @@ package handles import ( - "bytes" "errors" - "fmt" stdpath "path" - "strconv" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" @@ -17,9 +14,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" - "github.com/microcosm-cc/bluemonday" log "github.com/sirupsen/logrus" - "github.com/yuin/goldmark" ) func Down(c *gin.Context) { @@ -115,33 +110,7 @@ func proxy(c *gin.Context, link *model.Link, file model.Obj, proxyRange bool) { link = common.ProxyRange(c, link, file.GetSize()) } Writer := &common.WrittenResponseWriter{ResponseWriter: c.Writer} - raw, _ := strconv.ParseBool(c.DefaultQuery("raw", "false")) - if utils.Ext(file.GetName()) == "md" && setting.GetBool(conf.FilterReadMeScripts) && !raw { - buf := bytes.NewBuffer(make([]byte, 0, file.GetSize())) - w := &common.InterceptResponseWriter{ResponseWriter: Writer, Writer: buf} - err = common.Proxy(w, c.Request, link, file) - if err == nil && buf.Len() > 0 { - if c.Writer.Status() < 200 || c.Writer.Status() > 300 { - c.Writer.Write(buf.Bytes()) - return - } - - var html bytes.Buffer - if err = goldmark.Convert(buf.Bytes(), &html); err != nil { - err = fmt.Errorf("markdown conversion failed: %w", err) - } else { - buf.Reset() - err = bluemonday.UGCPolicy().SanitizeReaderToWriter(&html, buf) - if err == nil { - Writer.Header().Set("Content-Length", strconv.FormatInt(int64(buf.Len()), 10)) - Writer.Header().Set("Content-Type", "text/html; charset=utf-8") - _, err = utils.CopyWithBuffer(Writer, buf) - } - } - } - } else { - err = common.Proxy(Writer, c.Request, link, file) - } + err = common.Proxy(Writer, c.Request, link, file) if err == nil { return } diff --git a/server/handles/fsbatch.go b/server/handles/fsbatch.go index 28588d66..e672165c 100644 --- a/server/handles/fsbatch.go +++ b/server/handles/fsbatch.go @@ -49,7 +49,7 @@ func FsRecursiveMove(c *gin.Context) { common.ErrorResp(c, errs.PermissionDenied, 403) return } - common.GinWithValue(c, conf.MetaKey, srcMeta) + common.GinAppendValues(c, conf.MetaKey, srcMeta) dstDir, err := user.JoinPath(req.DstDir) if err != nil { @@ -183,7 +183,7 @@ func FsBatchRename(c *gin.Context) { common.ErrorResp(c, errs.PermissionDenied, 403) return } - common.GinWithValue(c, conf.MetaKey, meta) + common.GinAppendValues(c, conf.MetaKey, meta) for _, renameObject := range req.RenameObjects { if renameObject.SrcName == "" || renameObject.NewName == "" { continue @@ -236,7 +236,7 @@ func FsRegexRename(c *gin.Context) { common.ErrorResp(c, errs.PermissionDenied, 403) return } - common.GinWithValue(c, conf.MetaKey, meta) + common.GinAppendValues(c, conf.MetaKey, meta) srcRegexp, err := regexp.Compile(req.SrcNameRegex) if err != nil { diff --git a/server/handles/fsmanage.go b/server/handles/fsmanage.go index bda02f50..9587b358 100644 --- a/server/handles/fsmanage.go +++ b/server/handles/fsmanage.go @@ -111,6 +111,25 @@ func FsMove(c *gin.Context) { return } + if !strings.HasSuffix(srcDir, "/") { + srcDir += "/" + } + // ensure req.Names are not relative/traversal paths + safeNames := make([]string, 0, len(req.Names)) + for _, name := range req.Names { + srcPath := stdpath.Join(srcDir, name) + if !strings.HasPrefix(srcPath+"/", srcDir) { + continue + } + base := stdpath.Base(srcPath) + if base == "." || base == "/" { + common.ErrorStrResp(c, fmt.Sprintf("invalid file name [%s]", name), 400) + return + } + safeNames = append(safeNames, name) + } + req.Names = safeNames + var validNames []string if !req.Overwrite { for _, name := range req.Names { @@ -197,6 +216,25 @@ func FsCopy(c *gin.Context) { return } + if !strings.HasSuffix(srcDir, "/") { + srcDir += "/" + } + // ensure req.Names are not relative/traversal paths + safeNames := make([]string, 0, len(req.Names)) + for _, name := range req.Names { + srcPath := stdpath.Join(srcDir, name) + if !strings.HasPrefix(srcPath+"/", srcDir) { + continue + } + base := stdpath.Base(srcPath) + if base == "." || base == "/" { + common.ErrorStrResp(c, fmt.Sprintf("invalid file name [%s]", name), 400) + return + } + safeNames = append(safeNames, name) + } + req.Names = safeNames + var validNames []string if !req.Overwrite { for _, name := range req.Names { @@ -393,7 +431,7 @@ func FsRemoveEmptyDirectory(c *gin.Context) { common.ErrorResp(c, errs.PermissionDenied, 403) return } - common.GinWithValue(c, conf.MetaKey, meta) + common.GinAppendValues(c, conf.MetaKey, meta) rootFiles, err := fs.List(c.Request.Context(), srcDir, &fs.ListArgs{}) if err != nil { diff --git a/server/handles/fsread.go b/server/handles/fsread.go index 537d986d..ff212e02 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -89,7 +89,7 @@ func FsList(c *gin.Context, req *ListReq, user *model.User) { common.ErrorResp(c, err, 500, true) return } - common.GinWithValue(c, conf.MetaKey, meta) + common.GinAppendValues(c, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return @@ -153,7 +153,7 @@ func FsDirs(c *gin.Context) { common.ErrorResp(c, err, 500, true) return } - common.GinWithValue(c, conf.MetaKey, meta) + common.GinAppendValues(c, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return @@ -293,7 +293,7 @@ func FsGet(c *gin.Context, req *FsGetReq, user *model.User) { common.ErrorResp(c, err, 500, true) return } - common.GinWithValue(c, conf.MetaKey, meta) + common.GinAppendValues(c, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return @@ -417,7 +417,7 @@ func FsOther(c *gin.Context) { common.ErrorResp(c, err, 500) return } - common.GinWithValue(c, conf.MetaKey, meta) + common.GinAppendValues(c, conf.MetaKey, meta) if !common.CanAccess(user, meta, req.Path, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return diff --git a/server/handles/meta.go b/server/handles/meta.go index af4b6527..3ca1cb4c 100644 --- a/server/handles/meta.go +++ b/server/handles/meta.go @@ -69,8 +69,8 @@ func UpdateMeta(c *gin.Context) { } func validHide(hide string) (string, error) { - rs := strings.Split(hide, "\n") - for _, r := range rs { + rs := strings.SplitSeq(hide, "\n") + for r := range rs { _, err := regexp2.Compile(r, regexp2.None) if err != nil { return r, err diff --git a/server/handles/sharing.go b/server/handles/sharing.go index 43f855af..6090fcbd 100644 --- a/server/handles/sharing.go +++ b/server/handles/sharing.go @@ -3,6 +3,7 @@ package handles import ( "fmt" stdpath "path" + "regexp" "strings" "time" @@ -416,6 +417,19 @@ type UpdateSharingReq struct { CreatorName string `json:"creator"` Accessed int `json:"accessed"` ID string `json:"id"` + NewID string `json:"new_id"` +} + +var validSharingID = regexp.MustCompile(`^[\w\p{Han}\-]+$`) + +func validateSharingID(id string) error { + if len([]rune(id)) > 64 { + return errors.New("share id must be at most 64 characters") + } + if !validSharingID.MatchString(id) { + return errors.New("share id can only contain letters, numbers, underscores, hyphens, and CJK characters") + } + return nil } func UpdateSharing(c *gin.Context) { @@ -471,6 +485,20 @@ func UpdateSharing(c *gin.Context) { s.Readme = req.Readme s.Remark = req.Remark s.Creator = user + if req.NewID != "" && req.NewID != req.ID { + if !reqUser.CanCustomizeShareID() { + common.ErrorStrResp(c, "permission denied", 403) + return + } + if err = validateSharingID(req.NewID); err != nil { + common.ErrorResp(c, err, 400) + return + } + if err = op.UpdateSharingId(s, req.NewID); err != nil { + common.ErrorResp(c, err, 500) + return + } + } if err = op.UpdateSharing(s); err != nil { common.ErrorResp(c, err, 500) } else { @@ -493,6 +521,12 @@ func CreateSharing(c *gin.Context) { common.ErrorStrResp(c, "must add at least 1 object", 400) return } + if req.ID != "" { + if err = validateSharingID(req.ID); err != nil { + common.ErrorResp(c, err, 400) + return + } + } var user *model.User reqUser := c.Request.Context().Value(conf.UserKey).(*model.User) if reqUser.IsAdmin() && req.CreatorName != "" { @@ -503,7 +537,7 @@ func CreateSharing(c *gin.Context) { } } else { user = reqUser - if !user.CanShare() || (!user.IsAdmin() && req.ID != "") { + if !user.CanShare() || (!user.CanCustomizeShareID() && req.ID != "") { common.ErrorStrResp(c, "permission denied", 403) return } diff --git a/server/handles/torrent.go b/server/handles/torrent.go new file mode 100644 index 00000000..8b6ee1b6 --- /dev/null +++ b/server/handles/torrent.go @@ -0,0 +1,433 @@ +package handles + +import ( + "encoding/base64" + "fmt" + "io" + "strings" + + _189pc "github.com/OpenListTeam/OpenList/v4/drivers/189pc" + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" + "github.com/OpenListTeam/OpenList/v4/pkg/torrent" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/gin-gonic/gin" + "github.com/pkg/errors" +) + +// maxTorrentBase64Len is the max allowed Base64-encoded torrent size (~10MB decoded) +const maxTorrentBase64Len = 14 * 1024 * 1024 + +// maxTorrentGenFileSize is the max file size allowed for synchronous torrent generation (1GB) +const maxTorrentGenFileSize = 1 * 1024 * 1024 * 1024 + +// validateParsedTorrent checks that basic torrent invariants hold. +func validateParsedTorrent(t *torrent.Torrent) error { + if len(t.Info.Pieces)%20 != 0 { + return fmt.Errorf("torrent pieces 数据无效：长度必须为 20 的整数倍") + } + return nil +} + +// ParseTorrentReq 解析 torrent 文件请求 +type ParseTorrentReq struct { + // TorrentData Base64 编码的 torrent 文件内容 + TorrentData string `json:"torrent_data" binding:"required"` +} + +// ParseTorrentResp 解析 torrent 文件响应 +type ParseTorrentResp struct { + // Name 种子名称 + Name string `json:"name"` + // TotalSize 总大小 + TotalSize int64 `json:"total_size"` + // PieceLength 分片大小 + PieceLength int64 `json:"piece_length"` + // PieceCount 分片数量 + PieceCount int `json:"piece_count"` + // InfoHash info_hash（十六进制） + InfoHash string `json:"info_hash"` + // Files 文件列表（多文件模式） + Files []TorrentFileInfo `json:"files"` + // HasCAS 是否包含 CAS 扩展信息 + HasCAS bool `json:"has_cas"` + // CAS CAS 扩展信息 + CAS *CASInfoResp `json:"cas,omitempty"` +} + +// TorrentFileInfo torrent 中的文件信息 +type TorrentFileInfo struct { + Path string `json:"path"` + Size int64 `json:"size"` +} + +// CASInfoResp CAS 信息响应 +type CASInfoResp struct { + FileMD5 string `json:"file_md5"` + SliceMD5 string `json:"slice_md5"` + SliceSize int64 `json:"slice_size"` + Cloud string `json:"cloud"` +} + +// ParseTorrent 解析 torrent 文件，返回文件列表等信息 +func ParseTorrent(c *gin.Context) { + var req ParseTorrentReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + + // 限制 Base64 输入大小（最大 ~10MB decoded） + if len(req.TorrentData) > maxTorrentBase64Len { + common.ErrorResp(c, fmt.Errorf("torrent 数据过大（最大 10MB）"), 400) + return + } + + // Base64 解码 + torrentData, err := base64.StdEncoding.DecodeString(req.TorrentData) + if err != nil { + common.ErrorResp(c, fmt.Errorf("无效的 Base64 编码: %w", err), 400) + return + } + + // 解析 torrent + t, err := torrent.Decode(torrentData) + if err != nil { + common.ErrorResp(c, fmt.Errorf("解析 torrent 失败: %w", err), 400) + return + } + if err := validateParsedTorrent(t); err != nil { + common.ErrorResp(c, err, 400) + return + } + + resp := ParseTorrentResp{ + Name: t.Info.Name, + TotalSize: t.GetTotalSize(), + PieceLength: t.Info.PieceLength, + PieceCount: len(t.Info.Pieces) / 20, + InfoHash: t.GetInfoHashHex(), + HasCAS: t.HasCASInfo(), + } + + // 文件列表 + if len(t.Info.Files) > 0 { + resp.Files = make([]TorrentFileInfo, 0, len(t.Info.Files)) + for _, f := range t.Info.Files { + resp.Files = append(resp.Files, TorrentFileInfo{ + Path: strings.Join(f.Path, "/"), + Size: f.Length, + }) + } + } else { + // 单文件模式 + resp.Files = []TorrentFileInfo{ + {Path: t.Info.Name, Size: t.Info.Length}, + } + } + + // CAS 信息 + if t.HasCASInfo() { + resp.CAS = &CASInfoResp{ + FileMD5: t.CAS.FileMD5, + SliceMD5: t.CAS.SliceMD5, + SliceSize: t.CAS.SliceSize, + Cloud: t.CAS.Cloud, + } + } + + common.SuccessResp(c, resp) +} + +// TorrentRapidUploadReq 从 torrent 秒传请求 +type TorrentRapidUploadReq struct { + // TorrentData Base64 编码的 torrent 文件内容 + TorrentData string `json:"torrent_data" binding:"required"` + // Path 目标路径 + Path string `json:"path" binding:"required"` +} + +// TorrentRapidUpload 从 torrent 文件中提取 CAS 信息尝试秒传到天翼云 +func TorrentRapidUpload(c *gin.Context) { + user := c.Request.Context().Value(conf.UserKey).(*model.User) + + var req TorrentRapidUploadReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + + reqPath, err := user.JoinPath(req.Path) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + + // 检查权限 + meta, err := op.GetNearestMeta(reqPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if !common.CanWrite(user, meta, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + + // Base64 解码 + torrentData, err := base64.StdEncoding.DecodeString(req.TorrentData) + if err != nil { + common.ErrorResp(c, fmt.Errorf("无效的 Base64 编码: %w", err), 400) + return + } + + // 解析 torrent + t, err := torrent.Decode(torrentData) + if err != nil { + common.ErrorResp(c, fmt.Errorf("解析 torrent 失败: %w", err), 400) + return + } + + if !t.HasCASInfo() { + common.ErrorResp(c, fmt.Errorf("torrent 不包含 CAS 扩展信息，无法秒传"), 400) + return + } + + // 获取目标存储 + storage, dstDirActualPath, err := op.GetStorageAndActualPath(reqPath) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + + // 获取目标目录对象 + dstDir, err := op.Get(c.Request.Context(), storage, dstDirActualPath) + if err != nil { + common.ErrorResp(c, fmt.Errorf("获取目标目录失败: %w", err), 500) + return + } + if !dstDir.IsDir() { + common.ErrorResp(c, errs.NotFolder, 400) + return + } + + // 检查是否是天翼云 PC 驱动 + cloud189PC, ok := storage.(*_189pc.Cloud189PC) + if !ok { + common.ErrorResp(c, fmt.Errorf("目标存储不是天翼云PC驱动，不支持 CAS 秒传"), 400) + return + } + + // 尝试秒传 + obj, err := cloud189PC.RapidUploadFromTorrent(c.Request.Context(), dstDir, torrentData, true) + if err != nil { + common.ErrorResp(c, fmt.Errorf("秒传失败: %w", err), 400) + return + } + + common.SuccessResp(c, gin.H{ + "message": "秒传成功", + "file_name": obj.GetName(), + "file_size": obj.GetSize(), + }) +} + +// UploadTorrentAndParse 通过文件上传方式解析 torrent +func UploadTorrentAndParse(c *gin.Context) { + file, err := c.FormFile("torrent") + if err != nil { + common.ErrorResp(c, fmt.Errorf("获取上传文件失败: %w", err), 400) + return + } + + // 限制文件大小（最大 10MB） + if file.Size > 10*1024*1024 { + common.ErrorResp(c, fmt.Errorf("torrent 文件过大（最大 10MB）"), 400) + return + } + + f, err := file.Open() + if err != nil { + common.ErrorResp(c, fmt.Errorf("打开文件失败: %w", err), 500) + return + } + defer f.Close() + + torrentData, err := io.ReadAll(f) + if err != nil { + common.ErrorResp(c, fmt.Errorf("读取文件失败: %w", err), 500) + return + } + + // 解析 torrent + t, err := torrent.Decode(torrentData) + if err != nil { + common.ErrorResp(c, fmt.Errorf("解析 torrent 失败: %w", err), 400) + return + } + if err := validateParsedTorrent(t); err != nil { + common.ErrorResp(c, err, 400) + return + } + + resp := ParseTorrentResp{ + Name: t.Info.Name, + TotalSize: t.GetTotalSize(), + PieceLength: t.Info.PieceLength, + PieceCount: len(t.Info.Pieces) / 20, + InfoHash: t.GetInfoHashHex(), + HasCAS: t.HasCASInfo(), + } + + // 文件列表 + if len(t.Info.Files) > 0 { + resp.Files = make([]TorrentFileInfo, 0, len(t.Info.Files)) + for _, f := range t.Info.Files { + resp.Files = append(resp.Files, TorrentFileInfo{ + Path: strings.Join(f.Path, "/"), + Size: f.Length, + }) + } + } else { + resp.Files = []TorrentFileInfo{ + {Path: t.Info.Name, Size: t.Info.Length}, + } + } + + // CAS 信息 + if t.HasCASInfo() { + resp.CAS = &CASInfoResp{ + FileMD5: t.CAS.FileMD5, + SliceMD5: t.CAS.SliceMD5, + SliceSize: t.CAS.SliceSize, + Cloud: t.CAS.Cloud, + } + } + + // 同时返回 Base64 编码的 torrent 数据，方便后续使用 + common.SuccessResp(c, gin.H{ + "info": resp, + "torrent_data": base64.StdEncoding.EncodeToString(torrentData), + }) +} + +// GenerateTorrentReq 为指定路径的文件生成 torrent 请求 +type GenerateTorrentReq struct { + // Path 文件在 OpenList 中的路径 + Path string `json:"path" binding:"required"` + // WithCAS 是否注入 CAS 扩展信息（仅天翼云需要） + WithCAS bool `json:"with_cas"` +} + +// GenerateTorrentForPath 为指定路径的文件生成 torrent +// 这是一个通用接口，适用于所有驱动 +// 会获取文件内容计算哈希，然后生成 torrent +func GenerateTorrentForPath(c *gin.Context) { + user := c.Request.Context().Value(conf.UserKey).(*model.User) + + var req GenerateTorrentReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + + reqPath, err := user.JoinPath(req.Path) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + + // 检查读取权限 + meta, err := op.GetNearestMeta(reqPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if !common.CanRead(user, meta, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + + // 获取存储和文件信息 + storage, actualPath, err := op.GetStorageAndActualPath(reqPath) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + + // with_cas 仅支持天翼云PC驱动 + if req.WithCAS { + if _, is189pc := storage.(*_189pc.Cloud189PC); !is189pc { + common.ErrorResp(c, fmt.Errorf("CAS 秒传扩展仅支持天翼云PC驱动"), 400) + return + } + } + + // 获取文件对象 + obj, err := op.Get(c.Request.Context(), storage, actualPath) + if err != nil { + common.ErrorResp(c, fmt.Errorf("获取文件失败: %w", err), 500) + return + } + if obj.IsDir() { + common.ErrorResp(c, fmt.Errorf("不支持为目录生成 torrent"), 400) + return + } + + // 限制可生成 torrent 的文件大小 + if obj.GetSize() > maxTorrentGenFileSize { + common.ErrorResp(c, fmt.Errorf("文件过大，无法生成 torrent（最大 1GB）"), 400) + return + } + + // 获取文件下载链接 + link, _, err := op.Link(c.Request.Context(), storage, actualPath, model.LinkArgs{}) + if err != nil { + common.ErrorResp(c, fmt.Errorf("获取文件链接失败: %w", err), 500) + return + } + defer link.Close() + + // 通过 RangeReader 获取文件内容并计算哈希生成 torrent + if link.RangeReader == nil { + common.ErrorResp(c, fmt.Errorf("该存储不支持流式读取，无法生成 torrent（请先下载文件到本地）"), 400) + return + } + + // 读取整个文件 + rc, err := link.RangeReader.RangeRead(c.Request.Context(), http_range.Range{Length: obj.GetSize()}) + if err != nil { + common.ErrorResp(c, fmt.Errorf("读取文件失败: %w", err), 500) + return + } + defer rc.Close() + + var torrentData []byte + if req.WithCAS { + torrentData, err = torrent.GenerateFromReaderWithCAS(rc, obj.GetName(), obj.GetSize(), torrent.DefaultPieceSize) + } else { + torrentData, err = torrent.GenerateFromReader(rc, obj.GetName(), obj.GetSize(), torrent.DefaultPieceSize) + } + if err != nil { + common.ErrorResp(c, fmt.Errorf("生成 torrent 失败: %w", err), 500) + return + } + + // 解析生成的 torrent 获取 info_hash + t, _ := torrent.Decode(torrentData) + var infoHash string + if t != nil { + infoHash = t.GetInfoHashHex() + } + + common.SuccessResp(c, gin.H{ + "torrent_data": base64.StdEncoding.EncodeToString(torrentData), + "info_hash": infoHash, + "file_name": obj.GetName() + ".torrent", + "size": len(torrentData), + "with_cas": req.WithCAS, + }) +} diff --git a/server/middlewares/auth.go b/server/middlewares/auth.go index 0fc24361..ca67e4a6 100644 --- a/server/middlewares/auth.go +++ b/server/middlewares/auth.go @@ -24,7 +24,7 @@ func Auth(allowDisabledGuest bool) func(c *gin.Context) { c.Abort() return } - common.GinWithValue(c, conf.UserKey, admin) + common.GinAppendValues(c, conf.UserKey, admin) log.Debugf("use admin token: %+v", admin) c.Next() return @@ -41,7 +41,7 @@ func Auth(allowDisabledGuest bool) func(c *gin.Context) { c.Abort() return } - common.GinWithValue(c, conf.UserKey, guest) + common.GinAppendValues(c, conf.UserKey, guest) log.Debugf("use empty token: %+v", guest) c.Next() return @@ -69,7 +69,7 @@ func Auth(allowDisabledGuest bool) func(c *gin.Context) { c.Abort() return } - common.GinWithValue(c, conf.UserKey, user) + common.GinAppendValues(c, conf.UserKey, user) log.Debugf("use login token: %+v", user) c.Next() } @@ -84,7 +84,7 @@ func Authn(c *gin.Context) { c.Abort() return } - common.GinWithValue(c, conf.UserKey, admin) + common.GinAppendValues(c, conf.UserKey, admin) log.Debugf("use admin token: %+v", admin) c.Next() return @@ -96,7 +96,7 @@ func Authn(c *gin.Context) { c.Abort() return } - common.GinWithValue(c, conf.UserKey, guest) + common.GinAppendValues(c, conf.UserKey, guest) log.Debugf("use empty token: %+v", guest) c.Next() return @@ -124,7 +124,7 @@ func Authn(c *gin.Context) { c.Abort() return } - common.GinWithValue(c, conf.UserKey, user) + common.GinAppendValues(c, conf.UserKey, user) log.Debugf("use login token: %+v", user) c.Next() } diff --git a/server/middlewares/check.go b/server/middlewares/check.go index c7203a49..c5e07874 100644 --- a/server/middlewares/check.go +++ b/server/middlewares/check.go @@ -29,7 +29,7 @@ func StoragesLoaded(c *gin.Context) { return } } - common.GinWithValue(c, + common.GinAppendValues(c, conf.ApiUrlKey, common.GetApiUrlFromRequest(c.Request), ) c.Next() diff --git a/server/middlewares/down.go b/server/middlewares/down.go index c1f81b54..d71be00b 100644 --- a/server/middlewares/down.go +++ b/server/middlewares/down.go @@ -17,7 +17,7 @@ import ( func PathParse(c *gin.Context) { rawPath := parsePath(c.Param("path")) - common.GinWithValue(c, conf.PathKey, rawPath) + common.GinAppendValues(c, conf.PathKey, rawPath) c.Next() } @@ -29,7 +29,7 @@ func Down(verifyFunc func(string, string) error) func(c *gin.Context) { common.ErrorPage(c, err, 500, true) return } - common.GinWithValue(c, conf.MetaKey, meta) + common.GinAppendValues(c, conf.MetaKey, meta) // verify sign if needSign(meta, rawPath) { s := c.Query("sign") diff --git a/server/middlewares/sharing.go b/server/middlewares/sharing.go index d7549202..aa0cab0c 100644 --- a/server/middlewares/sharing.go +++ b/server/middlewares/sharing.go @@ -8,11 +8,11 @@ import ( func SharingIdParse(c *gin.Context) { sid := c.Param("sid") - common.GinWithValue(c, conf.SharingIDKey, sid) + common.GinAppendValues(c, conf.SharingIDKey, sid) c.Next() } func EmptyPathParse(c *gin.Context) { - common.GinWithValue(c, conf.PathKey, "/") + common.GinAppendValues(c, conf.PathKey, "/") c.Next() } diff --git a/server/router.go b/server/router.go index fbc6382a..3eb95832 100644 --- a/server/router.go +++ b/server/router.go @@ -234,6 +234,11 @@ func _fs(g *gin.RouterGroup) { // g.POST("/add_transmission", handles.SetTransmission) g.POST("/add_offline_download", handles.AddOfflineDownload) g.POST("/archive/decompress", handles.FsArchiveDecompress) + // Torrent 相关接口 + g.POST("/torrent/parse", handles.ParseTorrent) + g.POST("/torrent/upload_parse", handles.UploadTorrentAndParse) + g.POST("/torrent/rapid_upload", handles.TorrentRapidUpload) + g.POST("/torrent/generate", handles.GenerateTorrentForPath) // Direct upload (client-side upload to storage) g.POST("/get_direct_upload_info", middlewares.FsUp, handles.FsGetDirectUploadInfo) } diff --git a/server/webdav.go b/server/webdav.go index 2e919b3c..008eb26c 100644 --- a/server/webdav.go +++ b/server/webdav.go @@ -109,7 +109,7 @@ func WebDAVAuth(c *gin.Context) { count, cok := model.LoginCache.Get(ip) if cok && count >= model.DefaultMaxAuthRetries { if c.Request.Method == "OPTIONS" { - common.GinWithValue(c, conf.UserKey, guest) + common.GinAppendValues(c, conf.UserKey, guest) c.Next() return } @@ -133,13 +133,13 @@ func WebDAVAuth(c *gin.Context) { c.Abort() return } - common.GinWithValue(c, conf.UserKey, admin) + common.GinAppendValues(c, conf.UserKey, admin) c.Next() return } } if c.Request.Method == "OPTIONS" { - common.GinWithValue(c, conf.UserKey, guest) + common.GinAppendValues(c, conf.UserKey, guest) c.Next() return } @@ -151,7 +151,7 @@ func WebDAVAuth(c *gin.Context) { user, ok := tryLogin(username, password) if !ok { if c.Request.Method == "OPTIONS" { - common.GinWithValue(c, conf.UserKey, guest) + common.GinAppendValues(c, conf.UserKey, guest) c.Next() return } @@ -164,7 +164,7 @@ func WebDAVAuth(c *gin.Context) { model.LoginCache.Del(ip) if user.Disabled || !user.CanWebdavRead() { if c.Request.Method == "OPTIONS" { - common.GinWithValue(c, conf.UserKey, guest) + common.GinAppendValues(c, conf.UserKey, guest) c.Next() return } @@ -197,11 +197,11 @@ func WebDAVAuth(c *gin.Context) { c.Abort() return } - common.GinWithValue(c, conf.UserKey, user) + common.GinAppendValues(c, conf.UserKey, user) if user.IsGuest() { - common.GinWithValue(c, conf.MetaPassKey, password) + common.GinAppendValues(c, conf.MetaPassKey, password) } else { - common.GinWithValue(c, conf.MetaPassKey, "") + common.GinAppendValues(c, conf.MetaPassKey, "") } c.Next() }