From 6f06e698ee29dec77f621b5f1894b3c1e7b6c4a3 Mon Sep 17 00:00:00 2001 From: cofob Date: Sun, 17 Nov 2024 02:22:26 +0400 Subject: [PATCH 1/5] Init work on cloudflare version --- .gitignore | 3 + Cargo.lock | 762 +++++++++--------- Cargo.toml | 8 +- fastside-cloudflare/Cargo.toml | 51 ++ .../node_modules/.cache/wrangler/user-id.json | 3 + .../.cache/wrangler/wrangler-account.json | 6 + fastside-cloudflare/node_modules/.mf/cf.json | 1 + fastside-cloudflare/src/lib.rs | 44 + fastside-cloudflare/wrangler.toml | 6 + fastside-core/Cargo.toml | 35 + {fastside => fastside-core}/src/crawler.rs | 5 +- {fastside => fastside-core}/src/errors.rs | 49 +- {fastside => fastside-core}/src/filters.rs | 0 fastside-core/src/lib.rs | 22 + {fastside => fastside-core}/src/routes/api.rs | 0 fastside-core/src/routes/config.rs | 82 ++ fastside-core/src/routes/index.rs | 57 ++ fastside-core/src/routes/mod.rs | 18 + {fastside => fastside-core}/src/search.rs | 0 fastside-core/src/stub_crawler.rs | 296 +++++++ {fastside => fastside-core}/src/types.rs | 15 +- {fastside => fastside-core}/src/utils/mod.rs | 0 fastside-core/src/utils/user_config.rs | 40 + .../static/favicon.ico | Bin .../templates/base.html | 0 .../templates/cached_redirect.html | 0 .../templates/configure.html | 0 .../templates/error.html | 0 .../templates/fallback_redirect.html | 0 .../templates/history_redirect.html | 0 .../templates/index.html | 0 fastside-shared/Cargo.toml | 25 +- fastside-shared/src/client_builder.rs | 39 +- fastside/Cargo.toml | 9 +- fastside/src/main.rs | 58 +- fastside/src/routes/config.rs | 60 -- fastside/src/routes/index.rs | 77 -- fastside/src/routes/mod.rs | 12 - fastside/src/routes/redirect.rs | 217 ----- fastside/src/utils/user_config.rs | 16 - shell.nix | 8 +- 41 files changed, 1179 insertions(+), 845 deletions(-) create mode 100644 fastside-cloudflare/Cargo.toml create mode 100644 fastside-cloudflare/node_modules/.cache/wrangler/user-id.json create mode 100644 fastside-cloudflare/node_modules/.cache/wrangler/wrangler-account.json create mode 100644 fastside-cloudflare/node_modules/.mf/cf.json create mode 100644 fastside-cloudflare/src/lib.rs create mode 100644 fastside-cloudflare/wrangler.toml create mode 100644 fastside-core/Cargo.toml rename {fastside => fastside-core}/src/crawler.rs (98%) rename {fastside => fastside-core}/src/errors.rs (55%) rename {fastside => fastside-core}/src/filters.rs (100%) create mode 100644 fastside-core/src/lib.rs rename {fastside => fastside-core}/src/routes/api.rs (100%) create mode 100644 fastside-core/src/routes/config.rs create mode 100644 fastside-core/src/routes/index.rs create mode 100644 fastside-core/src/routes/mod.rs rename {fastside => fastside-core}/src/search.rs (100%) create mode 100644 fastside-core/src/stub_crawler.rs rename {fastside => fastside-core}/src/types.rs (51%) rename {fastside => fastside-core}/src/utils/mod.rs (100%) create mode 100644 fastside-core/src/utils/user_config.rs rename {fastside => fastside-core}/static/favicon.ico (100%) rename {fastside => fastside-core}/templates/base.html (100%) rename {fastside => fastside-core}/templates/cached_redirect.html (100%) rename {fastside => fastside-core}/templates/configure.html (100%) rename {fastside => fastside-core}/templates/error.html (100%) rename {fastside => fastside-core}/templates/fallback_redirect.html (100%) rename {fastside => fastside-core}/templates/history_redirect.html (100%) rename {fastside => fastside-core}/templates/index.html (100%) delete mode 100644 fastside/src/routes/config.rs delete mode 100644 fastside/src/routes/index.rs delete mode 100644 fastside/src/routes/mod.rs delete mode 100644 fastside/src/routes/redirect.rs delete mode 100644 fastside/src/utils/user_config.rs diff --git a/.gitignore b/.gitignore index 6e6099bb..6ec83ff9 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,6 @@ /debug/ /target/ /data.json + +/fastside-cloudflare/build/ +/fastside-cloudflare/.wrangler/ diff --git a/Cargo.lock b/Cargo.lock index d91a9be4..c7338a2a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,182 +2,6 @@ # It is not intended for manual editing. version = 3 -[[package]] -name = "actix-codec" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f7b0a21988c1bf877cf4759ef5ddaac04c1c9fe808c9142ecb78ba97d97a28a" -dependencies = [ - "bitflags 2.6.0", - "bytes", - "futures-core", - "futures-sink", - "memchr", - "pin-project-lite", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "actix-http" -version = "3.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ae682f693a9cd7b058f2b0b5d9a6d7728a8555779bedbbc35dd88528611d020" -dependencies = [ - "actix-codec", - "actix-rt", - "actix-service", - "actix-utils", - "ahash", - "base64 0.22.1", - "bitflags 2.6.0", - "bytes", - "bytestring", - "derive_more", - "encoding_rs", - "futures-core", - "http 0.2.12", - "httparse", - "httpdate", - "itoa", - "language-tags", - "local-channel", - "mime", - "percent-encoding", - "pin-project-lite", - "rand", - "sha1", - "smallvec", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "actix-macros" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" -dependencies = [ - "quote", - "syn", -] - -[[package]] -name = "actix-router" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13d324164c51f63867b57e73ba5936ea151b8a41a1d23d1031eeb9f70d0236f8" -dependencies = [ - "bytestring", - "cfg-if", - "http 0.2.12", - "regex-lite", - "serde", - "tracing", -] - -[[package]] -name = "actix-rt" -version = "2.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eda4e2a6e042aa4e55ac438a2ae052d3b5da0ecf83d7411e1a368946925208" -dependencies = [ - "futures-core", - "tokio", -] - -[[package]] -name = "actix-server" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b02303ce8d4e8be5b855af6cf3c3a08f3eff26880faad82bab679c22d3650cb5" -dependencies = [ - "actix-rt", - "actix-service", - "actix-utils", - "futures-core", - "futures-util", - "mio 0.8.11", - "socket2 0.5.7", - "tokio", - "tracing", -] - -[[package]] -name = "actix-service" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b894941f818cfdc7ccc4b9e60fa7e53b5042a2e8567270f9147d5591893373a" -dependencies = [ - "futures-core", - "paste", - "pin-project-lite", -] - -[[package]] -name = "actix-utils" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88a1dcdff1466e3c2488e1cb5c36a71822750ad43839937f85d2f4d9f8b705d8" -dependencies = [ - "local-waker", - "pin-project-lite", -] - -[[package]] -name = "actix-web" -version = "4.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1988c02af8d2b718c05bc4aeb6a66395b7cdf32858c2c71131e5637a8c05a9ff" -dependencies = [ - "actix-codec", - "actix-http", - "actix-macros", - "actix-router", - "actix-rt", - "actix-server", - "actix-service", - "actix-utils", - "actix-web-codegen", - "ahash", - "bytes", - "bytestring", - "cfg-if", - "cookie", - "derive_more", - "encoding_rs", - "futures-core", - "futures-util", - "itoa", - "language-tags", - "log", - "mime", - "once_cell", - "pin-project-lite", - "regex-lite", - "serde", - "serde_json", - "serde_urlencoded", - "smallvec", - "socket2 0.5.7", - "time", - "url", -] - -[[package]] -name = "actix-web-codegen" -version = "4.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f591380e2e68490b5dfaf1dd1aa0ebe78d84ba7067078512b4ea6e4492d622b8" -dependencies = [ - "actix-router", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "addr2line" version = "0.22.0" @@ -193,19 +17,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" -[[package]] -name = "ahash" -version = "0.8.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" -dependencies = [ - "cfg-if", - "getrandom", - "once_cell", - "version_check", - "zerocopy 0.7.35", -] - [[package]] name = "aho-corasick" version = "1.1.3" @@ -545,6 +356,86 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper 1.0.1", + "tokio", + "tower 0.5.1", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper 1.0.1", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-extra" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c794b30c904f0a1c2fb7740f7df7f7972dfaa14ef6f57cb6178dc63e5dca2f04" +dependencies = [ + "axum", + "axum-core", + "bytes", + "cookie", + "fastrand 2.1.0", + "futures-util", + "headers", + "http", + "http-body", + "http-body-util", + "mime", + "multer", + "pin-project-lite", + "serde", + "tower 0.5.1", + "tower-layer", + "tower-service", +] + [[package]] name = "backtrace" version = "0.3.73" @@ -636,15 +527,6 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" -[[package]] -name = "bytestring" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d80203ea6b29df88012294f62733de21cfeab47f17b41af3a38bc30a03ee72" -dependencies = [ - "bytes", -] - [[package]] name = "cc" version = "1.1.7" @@ -733,7 +615,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7328b20597b53c2454f0b1919720c25c7339051c02b72b7e05409e00b14132be" dependencies = [ "async-trait", - "convert_case 0.6.0", + "convert_case", "json5", "lazy_static", "nom", @@ -746,6 +628,16 @@ dependencies = [ "yaml-rust", ] +[[package]] +name = "console_error_panic_hook" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc" +dependencies = [ + "cfg-if", + "wasm-bindgen", +] + [[package]] name = "const-random" version = "0.1.18" @@ -766,12 +658,6 @@ dependencies = [ "tiny-keccak", ] -[[package]] -name = "convert_case" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" - [[package]] name = "convert_case" version = "0.6.0" @@ -783,9 +669,9 @@ dependencies = [ [[package]] name = "cookie" -version = "0.16.2" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb" +checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747" dependencies = [ "percent-encoding", "time", @@ -844,19 +730,6 @@ dependencies = [ "powerfmt", ] -[[package]] -name = "derive_more" -version = "0.99.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f33878137e4dafd7fa914ad4e259e18a4e8e532b9617a2d0150262bf53abfce" -dependencies = [ - "convert_case 0.4.0", - "proc-macro2", - "quote", - "rustc_version", - "syn", -] - [[package]] name = "digest" version = "0.10.7" @@ -884,9 +757,9 @@ checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" [[package]] name = "encoding_rs" -version = "0.8.34" +version = "0.8.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" dependencies = [ "cfg-if", ] @@ -989,12 +862,12 @@ checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" name = "fastside" version = "0.2.0" dependencies = [ - "actix-web", "anyhow", - "askama", + "axum", "base64 0.22.1", "chrono", "clap", + "fastside-core", "fastside-shared", "futures", "log", @@ -1032,6 +905,48 @@ dependencies = [ "url", ] +[[package]] +name = "fastside-cloudflare" +version = "0.2.0" +dependencies = [ + "axum", + "console_error_panic_hook", + "fastside-core", + "fastside-shared", + "getrandom", + "tokio", + "tower-service", + "wasm-bindgen-futures", + "worker", + "worker-macros", +] + +[[package]] +name = "fastside-core" +version = "0.2.0" +dependencies = [ + "anyhow", + "askama", + "axum", + "axum-extra", + "base64 0.22.1", + "chrono", + "fastside-shared", + "futures", + "log", + "rand", + "regex", + "reqwest", + "serde", + "serde_json", + "thiserror", + "time", + "tokio", + "tracing", + "url", + "urlencoding", +] + [[package]] name = "fastside-shared" version = "0.1.0" @@ -1068,9 +983,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", @@ -1083,9 +998,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", "futures-sink", @@ -1093,15 +1008,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", @@ -1110,9 +1025,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-lite" @@ -1144,9 +1059,9 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", @@ -1155,21 +1070,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-util" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-channel", "futures-core", @@ -1200,8 +1115,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -1233,7 +1150,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http 1.1.0", + "http", "indexmap", "slab", "tokio", @@ -1253,6 +1170,30 @@ version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +[[package]] +name = "headers" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "322106e6bd0cba2d5ead589ddb8150a13d7c4217cf80d7c4f682ca994ccc6aa9" +dependencies = [ + "base64 0.21.7", + "bytes", + "headers-core", + "http", + "httpdate", + "mime", + "sha1", +] + +[[package]] +name = "headers-core" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54b4a22553d4242c49fddb9ba998a99962b5cc6f22cb5a3482bec22522403ce4" +dependencies = [ + "http", +] + [[package]] name = "heck" version = "0.4.1" @@ -1331,17 +1272,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "http" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - [[package]] name = "http" version = "1.1.0" @@ -1360,7 +1290,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.1.0", + "http", ] [[package]] @@ -1371,7 +1301,7 @@ checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", "futures-util", - "http 1.1.0", + "http", "http-body", "pin-project-lite", ] @@ -1413,9 +1343,10 @@ dependencies = [ "futures-channel", "futures-util", "h2", - "http 1.1.0", + "http", "http-body", "httparse", + "httpdate", "itoa", "pin-project-lite", "smallvec", @@ -1430,7 +1361,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" dependencies = [ "futures-util", - "http 1.1.0", + "http", "hyper", "hyper-util", "rustls", @@ -1450,13 +1381,13 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.1.0", + "http", "http-body", "hyper", "pin-project-lite", "socket2 0.5.7", "tokio", - "tower", + "tower 0.4.13", "tower-service", "tracing", ] @@ -1577,9 +1508,9 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "js-sys" -version = "0.3.69" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" dependencies = [ "wasm-bindgen", ] @@ -1604,12 +1535,6 @@ dependencies = [ "log", ] -[[package]] -name = "language-tags" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4345964bb142484797b161f473a503a434de77149dd8c7427788c6e13379388" - [[package]] name = "lazy_static" version = "1.5.0" @@ -1624,9 +1549,9 @@ checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" [[package]] name = "libm" -version = "0.2.8" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" [[package]] name = "linked-hash-map" @@ -1646,23 +1571,6 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" -[[package]] -name = "local-channel" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6cbc85e69b8df4b8bb8b89ec634e7189099cea8927a276b7384ce5488e53ec8" -dependencies = [ - "futures-core", - "futures-sink", - "local-waker", -] - -[[package]] -name = "local-waker" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d873d7c67ce09b42110d801813efbc9364414e356be9935700d368351657487" - [[package]] name = "lock_api" version = "0.4.12" @@ -1697,6 +1605,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "memchr" version = "2.7.4" @@ -1736,26 +1650,31 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.11" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +checksum = "4569e456d394deccd22ce1c1913e6ea0e54519f577285001215d33557431afe4" dependencies = [ + "hermit-abi 0.3.9", "libc", - "log", "wasi", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] -name = "mio" -version = "1.0.1" +name = "multer" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4569e456d394deccd22ce1c1913e6ea0e54519f577285001215d33557431afe4" +checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b" dependencies = [ - "hermit-abi 0.3.9", - "libc", - "wasi", - "windows-sys 0.52.0", + "bytes", + "encoding_rs", + "futures-util", + "http", + "httparse", + "memchr", + "mime", + "spin", + "version_check", ] [[package]] @@ -1847,12 +1766,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - [[package]] name = "pathdiff" version = "0.2.1" @@ -1996,7 +1909,7 @@ version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dee4364d9f3b902ef14fab8a1ddffb783a1cb6b4bba3bfc1fa3922732c7de97f" dependencies = [ - "zerocopy 0.6.6", + "zerocopy", ] [[package]] @@ -2141,12 +2054,6 @@ dependencies = [ "regex-syntax", ] -[[package]] -name = "regex-lite" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" - [[package]] name = "regex-syntax" version = "0.8.4" @@ -2164,7 +2071,7 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http 1.1.0", + "http", "http-body", "http-body-util", "hyper", @@ -2184,7 +2091,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "sync_wrapper", + "sync_wrapper 1.0.1", "tokio", "tokio-rustls", "tokio-socks", @@ -2262,15 +2169,6 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" -[[package]] -name = "rustc_version" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" -dependencies = [ - "semver", -] - [[package]] name = "rustix" version = "0.37.27" @@ -2339,6 +2237,12 @@ dependencies = [ "untrusted", ] +[[package]] +name = "rustversion" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" + [[package]] name = "ryu" version = "1.0.18" @@ -2351,12 +2255,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "semver" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" - [[package]] name = "serde" version = "1.0.204" @@ -2366,6 +2264,28 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-wasm-bindgen" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3b143e2833c57ab9ad3ea280d21fd34e285a42837aeb0ee301f4f41890fa00e" +dependencies = [ + "js-sys", + "serde", + "wasm-bindgen", +] + +[[package]] +name = "serde-wasm-bindgen" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8302e169f0eddcc139c70f139d19d6467353af16f9fce27e8c30158036a1e16b" +dependencies = [ + "js-sys", + "serde", + "wasm-bindgen", +] + [[package]] name = "serde_derive" version = "1.0.204" @@ -2389,6 +2309,16 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af99884400da37c88f5e9146b7f1fd0fbcae8f6eec4e9da38b67d05486f814a6" +dependencies = [ + "itoa", + "serde", +] + [[package]] name = "serde_spanned" version = "0.6.7" @@ -2518,6 +2448,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + [[package]] name = "sync_wrapper" version = "1.0.1" @@ -2610,14 +2546,14 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.39.3" +version = "1.41.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9babc99b9923bfa4804bd74722ff02c0381021eafa4db9949217e3be8e84fff5" +checksum = "22cfb5bee7a6a52939ca9224d6ac897bb669134078daa8735560897f69de4d33" dependencies = [ "backtrace", "bytes", "libc", - "mio 1.0.1", + "mio", "parking_lot", "pin-project-lite", "signal-hook-registry", @@ -2722,17 +2658,33 @@ dependencies = [ "tower-service", ] +[[package]] +name = "tower" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2873938d487c3cfb9aed7546dc9f2711d867c9f90c46b889989a2cb84eba6b4f" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper 0.1.2", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tower-layer" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" [[package]] name = "tower-service" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" @@ -2786,12 +2738,9 @@ checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" [[package]] name = "unicase" -version = "2.7.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" -dependencies = [ - "version_check", -] +checksum = "7e51b68083f157f853b6379db119d1c1be0e6e4dec98101079dec41f6f5cf6df" [[package]] name = "unicode-bidi" @@ -2891,19 +2840,20 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" dependencies = [ "cfg-if", + "once_cell", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" dependencies = [ "bumpalo", "log", @@ -2928,9 +2878,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2938,9 +2888,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2", "quote", @@ -2951,15 +2901,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" + +[[package]] +name = "wasm-streams" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e072d4e72f700fb3443d8fe94a39315df013eef1104903cdb0a2abd322bbecd" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] [[package]] name = "web-sys" -version = "0.3.69" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112" dependencies = [ "js-sys", "wasm-bindgen", @@ -3189,49 +3152,104 @@ dependencies = [ ] [[package]] -name = "yaml-rust" -version = "0.4.5" +name = "worker" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +checksum = "d8aca53ec63e508176a89a573c972266f0f98bcc48bd866def7be0d939ef9268" dependencies = [ - "linked-hash-map", + "async-trait", + "axum", + "bytes", + "chrono", + "futures-channel", + "futures-util", + "http", + "http-body", + "js-sys", + "matchit", + "pin-project", + "serde", + "serde-wasm-bindgen 0.6.5", + "serde_json", + "serde_urlencoded", + "tokio", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "worker-kv", + "worker-macros", + "worker-sys", ] [[package]] -name = "zerocopy" -version = "0.6.6" +name = "worker-kv" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854e949ac82d619ee9a14c66a1b674ac730422372ccb759ce0c39cabcf2bf8e6" +checksum = "7f06d4d1416a9f8346ee9123b0d9a11b3cfa38e6cfb5a139698017d1597c4d41" dependencies = [ - "byteorder", - "zerocopy-derive 0.6.6", + "js-sys", + "serde", + "serde-wasm-bindgen 0.5.0", + "serde_json", + "thiserror", + "wasm-bindgen", + "wasm-bindgen-futures", ] [[package]] -name = "zerocopy" -version = "0.7.35" +name = "worker-macros" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +checksum = "1118a0ceb59ddde7fdbaff6c47b6fa6ee47848975ea38b4ae9bb4080f96541cd" dependencies = [ - "zerocopy-derive 0.7.35", + "async-trait", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-bindgen-macro-support", + "worker-sys", ] [[package]] -name = "zerocopy-derive" +name = "worker-sys" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5643a2ba07df61aa50e37212ffcb0944417db7d3960d4331f36aeb2fa5e2fd7" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "yaml-rust" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +dependencies = [ + "linked-hash-map", +] + +[[package]] +name = "zerocopy" version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "125139de3f6b9d625c39e2efdd73d41bdac468ccd556556440e322be0e1bbd91" +checksum = "854e949ac82d619ee9a14c66a1b674ac730422372ccb759ce0c39cabcf2bf8e6" dependencies = [ - "proc-macro2", - "quote", - "syn", + "byteorder", + "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.35" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +checksum = "125139de3f6b9d625c39e2efdd73d41bdac468ccd556556440e322be0e1bbd91" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 095af9d8..12c643c9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,11 @@ [workspace] -members = ["fastside", "fastside-actualizer", "fastside-shared"] +members = [ + "fastside-shared", + "fastside-core", + "fastside", + "fastside-cloudflare", + "fastside-actualizer", +] resolver = "2" [profile.release] diff --git a/fastside-cloudflare/Cargo.toml b/fastside-cloudflare/Cargo.toml new file mode 100644 index 00000000..06133b0a --- /dev/null +++ b/fastside-cloudflare/Cargo.toml @@ -0,0 +1,51 @@ +[package] +name = "fastside-cloudflare" +description = "A smart redirecting gateway for various frontend services. Cloudflare Worker specific version." +version = "0.2.0" +edition = "2021" + +[lib] +crate-type = ["cdylib", "rlib"] + +[dependencies] +fastside-shared = { path = "../fastside-shared" } +fastside-core = { path = "../fastside-core" } + +axum = { version = "0.7.9", default-features = false } # web framework +# askama = "0.12.1" # templating engine +# reqwest = { version = "0.12.4", default-features = false, features = [ +# "rustls-tls", +# "http2", +# "socks", +# ] } # http client + +# clap = { version = "4.5.4", features = ["derive"] } # cli +# serde = { version = "1.0.201", features = ["derive"] } # serialization +# serde_json = "1.0.117" # serialization +# url = { version = "2.5.0", features = ["serde"] } # url +# log = "0.4.21" # logging +# anyhow = "1.0.83" # error +# thiserror = "1.0.60" # error +# tokio = { version = "1.37.0", features = ["full"] } # async +# futures = "0.3.30" # async +# num_cpus = "1.16.0" # get number of cpus +# rand = "0.8.5" # random +# chrono = "0.4.38" # datetime +# time = "0.3.36" # time offsets +# regex = "1.10.5" # regex +# base64 = "0.22.1" # base64 +# urlencoding = "2.1.3" # url encoding + +worker = { version = "0.4.2", features = ["http", "axum"] } +worker-macros = { version = "0.4.2", features = ["http"] } +# axum-extra = { version = "0.9.6", features = ["cookie", "typed-header"] } +# tracing = "0.1.40" +wasm-bindgen-futures = "0.4" +tower-service = "0.3.3" +console_error_panic_hook = "0.1.7" +getrandom = { version = "0.2.15", features = ["js"] } +tokio = { version = "1.41.1", default-features = false, features = ["sync"] } + +# [features] +# default = [] +# worker = ["dep:worker", "worker-macros"] diff --git a/fastside-cloudflare/node_modules/.cache/wrangler/user-id.json b/fastside-cloudflare/node_modules/.cache/wrangler/user-id.json new file mode 100644 index 00000000..6a919303 --- /dev/null +++ b/fastside-cloudflare/node_modules/.cache/wrangler/user-id.json @@ -0,0 +1,3 @@ +{ + "userId": "8632af6789c40dc28769656e59b04edd" +} \ No newline at end of file diff --git a/fastside-cloudflare/node_modules/.cache/wrangler/wrangler-account.json b/fastside-cloudflare/node_modules/.cache/wrangler/wrangler-account.json new file mode 100644 index 00000000..5152560d --- /dev/null +++ b/fastside-cloudflare/node_modules/.cache/wrangler/wrangler-account.json @@ -0,0 +1,6 @@ +{ + "account": { + "id": "025464e503cbb7385dff88bd6d3c9128", + "name": "Cofob@riseup.net's Account" + } +} \ No newline at end of file diff --git a/fastside-cloudflare/node_modules/.mf/cf.json b/fastside-cloudflare/node_modules/.mf/cf.json new file mode 100644 index 00000000..fe38d0ad --- /dev/null +++ b/fastside-cloudflare/node_modules/.mf/cf.json @@ -0,0 +1 @@ +{"clientTcpRtt":8,"longitude":"44.83200","latitude":"41.69590","tlsCipher":"AEAD-AES256-GCM-SHA384","continent":"AS","asn":35805,"clientAcceptEncoding":"br, gzip, deflate","country":"GE","tlsClientAuth":{"certIssuerDNLegacy":"","certIssuerSKI":"","certSubjectDNRFC2253":"","certSubjectDNLegacy":"","certFingerprintSHA256":"","certNotBefore":"","certSKI":"","certSerial":"","certIssuerDN":"","certVerified":"NONE","certNotAfter":"","certSubjectDN":"","certPresented":"0","certRevoked":"0","certIssuerSerial":"","certIssuerDNRFC2253":"","certFingerprintSHA1":""},"verifiedBotCategory":"","tlsExportedAuthenticator":{"clientFinished":"2fbd0eaa083560b4952534f85c144d2bf6cbe3281413a5ab9c801620736342953765a28b594ebd208dba7eef1416a65d","clientHandshake":"6395396631724760823c3f8c064c86bdc6ce0e4523260441c860e7711c7c779e799afe47b29c918274795c6953b25532","serverHandshake":"ea7ffef62e9c8d278bfb00ccd199bfec2591b80132eae9eadbf7d585a8976fd0c80bb1b6b9f6461c3e7a2e8341ba95d2","serverFinished":"94aa5d6c86033bca9802cad73c2e5474b58083867f5d38cc899768a805889c4b2cd8d3f40e410f0d161afea14ecc4ec4"},"tlsVersion":"TLSv1.3","colo":"TBS","timezone":"Asia/Tbilisi","tlsClientHelloLength":"383","edgeRequestKeepAliveStatus":1,"requestPriority":"","tlsClientExtensionsSha1":"1eY97BUYYO8vDaTfHQywB1pcNdM=","region":"Tbilisi","city":"Tbilisi","regionCode":"TB","asOrganization":"JSC Silknet","tlsClientRandom":"pjM/+sdECVpdpI7iVNXDyvcsLWwF9WkAM5mCrxg3L08=","httpProtocol":"HTTP/1.1","botManagement":{"corporateProxy":false,"verifiedBot":false,"jsDetection":{"passed":false},"staticResource":false,"detectionIds":{},"score":99}} \ No newline at end of file diff --git a/fastside-cloudflare/src/lib.rs b/fastside-cloudflare/src/lib.rs new file mode 100644 index 00000000..4c4a92e7 --- /dev/null +++ b/fastside-cloudflare/src/lib.rs @@ -0,0 +1,44 @@ +use std::{collections::HashMap, sync::Arc}; + +use axum::Router; +use fastside_core::{ + crawler::Crawler, + routes::main_router, + types::{AppState, LoadedData}, +}; +use fastside_shared::config::{AppConfig, UserConfig}; +use tokio::sync::RwLock; +use tower_service::Service; +use worker::*; + +fn router() -> Router { + let config = Arc::new(AppConfig::default()); + let loaded_data = Arc::new(RwLock::new(LoadedData { + services: HashMap::new(), + proxies: HashMap::new(), + default_user_config: UserConfig::default(), + })); + let shared_state = Arc::new(AppState { + config: config.clone(), + crawler: Arc::new(Crawler::new( + loaded_data.clone(), + config.clone().crawler.clone(), + )), + loaded_data: loaded_data.clone(), + regexes: HashMap::new(), + }); + Router::new() + .nest("/", main_router()) + .with_state(shared_state) +} + +#[event(fetch)] +async fn fetch( + req: HttpRequest, + _env: Env, + _ctx: Context, +) -> Result> { + console_error_panic_hook::set_once(); + + Ok(router().call(req).await?) +} diff --git a/fastside-cloudflare/wrangler.toml b/fastside-cloudflare/wrangler.toml new file mode 100644 index 00000000..77158b5c --- /dev/null +++ b/fastside-cloudflare/wrangler.toml @@ -0,0 +1,6 @@ +name = "fastside" +main = "build/worker/shim.mjs" +compatibility_date = "2023-03-22" + +[build] +command = "cargo install worker-build --version 0.1.0 && worker-build --release" diff --git a/fastside-core/Cargo.toml b/fastside-core/Cargo.toml new file mode 100644 index 00000000..e268b51a --- /dev/null +++ b/fastside-core/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "fastside-core" +description = "Core component of fastside. Used by standalone and cloudflare worker version." +version = "0.2.0" +edition = "2021" + +[dependencies] +fastside-shared = { path = "../fastside-shared" } +axum = { version = "0.7.9", default-features = false, features = [ + "query", + "json", +] } # web framework +axum-extra = { version = "0.9.6", features = ["cookie", "typed-header"] } +tracing = "0.1.40" +askama = "0.12.1" # templating engine +reqwest = { version = "0.12.4", default-features = false, features = [ + "rustls-tls", + "http2", + "socks", +] } # http client + +serde = { version = "1.0.201", features = ["derive"] } # serialization +serde_json = "1.0.117" # serialization +url = { version = "2.5.0", features = ["serde"] } # url +log = "0.4.21" # logging +anyhow = "1.0.83" # error +thiserror = "1.0.60" # error +tokio = { version = "1.37.0", default-features = false, features = ["sync"] } +futures = "0.3.30" # async +rand = "0.8.5" # random +chrono = "0.4.38" # datetime +time = "0.3.36" # time offsets +regex = "1.10.5" # regex +base64 = "0.22.1" # base64 +urlencoding = "2.1.3" # url encoding diff --git a/fastside/src/crawler.rs b/fastside-core/src/crawler.rs similarity index 98% rename from fastside/src/crawler.rs rename to fastside-core/src/crawler.rs index 5c0511be..c6e3c76a 100644 --- a/fastside/src/crawler.rs +++ b/fastside-core/src/crawler.rs @@ -13,13 +13,15 @@ use tokio::{ }; use url::Url; -use crate::{config::CrawlerConfig, types::LoadedData}; +use fastside_shared::config::CrawlerConfig; use fastside_shared::{ client_builder::build_client, parallel::Parallelise, serde_types::{HttpCodeRanges, Instance, Service}, }; +use crate::types::LoadedData; + #[derive(Error, Debug)] pub enum CrawlerError { #[error("url error: `{0}`")] @@ -191,6 +193,7 @@ impl Crawler { _ if e.is_request() => CrawledInstanceStatus::RequestError, _ if e.is_body() => CrawledInstanceStatus::BodyError, _ if e.is_decode() => CrawledInstanceStatus::DecodeError, + #[cfg(not(target_arch = "wasm32"))] _ if e.is_connect() => CrawledInstanceStatus::ConnectionError, _ => CrawledInstanceStatus::Unknown, }, diff --git a/fastside/src/errors.rs b/fastside-core/src/errors.rs similarity index 55% rename from fastside/src/errors.rs rename to fastside-core/src/errors.rs index 51f9ee47..bca320c8 100644 --- a/fastside/src/errors.rs +++ b/fastside-core/src/errors.rs @@ -1,8 +1,13 @@ -use actix_web::http::StatusCode; +use axum::{ + response::{Html, IntoResponse, Response}, + http::StatusCode, + Json, +}; use askama::Template; use serde::Serialize; use thiserror::Error; +// Error template struct for rendering HTML errors #[derive(Template)] #[template(path = "error.html")] pub struct ErrorTemplate<'a> { @@ -10,19 +15,31 @@ pub struct ErrorTemplate<'a> { pub status_code: StatusCode, } +// Helper macro for implementing IntoResponse for HTML errors macro_rules! impl_template_error { ($err:ty, status => {$($variant:pat => $code:expr),+ $(,)?}) => { - impl actix_web::ResponseError for $err where $err: std::error::Error + 'static { - fn error_response(&self) -> actix_web::HttpResponse { - use askama::Template; - + impl IntoResponse for $err where $err: std::error::Error + 'static { + fn into_response(self) -> Response { + let status_code = self.status_code(); let detail = format!("{}", self); - let error_page = crate::errors::ErrorTemplate { detail: &detail, status_code: self.status_code() }; + let error_page = crate::errors::ErrorTemplate { + detail: &detail, + status_code, + }; + let body = match error_page.render() { + Ok(rendered) => Html(rendered).into_response(), + Err(_) => ( + StatusCode::INTERNAL_SERVER_ERROR, + "Failed to render error page".to_string() + ).into_response(), + }; - actix_web::HttpResponse::build(self.status_code()).body(error_page.render().expect("failed to render error page")) + (status_code, body).into_response() } + } - fn status_code(&self) -> actix_web::http::StatusCode { + impl $err { + pub fn status_code(&self) -> StatusCode { match self { $( $variant => $code, @@ -34,20 +51,26 @@ macro_rules! impl_template_error { } pub(crate) use impl_template_error; +// API error struct for JSON responses #[derive(Serialize)] pub struct ApiError { pub detail: String, } +// Helper macro for implementing IntoResponse for JSON errors macro_rules! impl_api_error { ($err:ty, status => {$($variant:pat => $code:expr),+ $(,)?}) => { - impl actix_web::ResponseError for $err where $err: std::error::Error + 'static { - fn error_response(&self) -> actix_web::HttpResponse { + impl IntoResponse for $err where $err: std::error::Error + 'static { + fn into_response(self) -> Response { + let status_code = self.status_code(); let detail = format!("{}", self); - actix_web::HttpResponse::build(self.status_code()).json(crate::errors::ApiError { detail }) + let error_response = Json(crate::errors::ApiError { detail }); + (status_code, error_response).into_response() } + } - fn status_code(&self) -> actix_web::http::StatusCode { + impl $err { + pub fn status_code(&self) -> StatusCode { match self { $( $variant => $code, @@ -59,6 +82,7 @@ macro_rules! impl_api_error { } pub(crate) use impl_api_error; +// RedirectError definition use crate::search::SearchError; #[derive(Error, Debug)] @@ -82,6 +106,7 @@ impl_template_error!(RedirectError, } ); +// RedirectApiError definition #[derive(Error, Debug)] #[error(transparent)] pub struct RedirectApiError(#[from] pub RedirectError); diff --git a/fastside/src/filters.rs b/fastside-core/src/filters.rs similarity index 100% rename from fastside/src/filters.rs rename to fastside-core/src/filters.rs diff --git a/fastside-core/src/lib.rs b/fastside-core/src/lib.rs new file mode 100644 index 00000000..96c1528e --- /dev/null +++ b/fastside-core/src/lib.rs @@ -0,0 +1,22 @@ +#[cfg(not(target_arch = "wasm32"))] +pub mod crawler; +#[cfg(target_arch = "wasm32")] +pub mod stub_crawler; +#[cfg(target_arch = "wasm32")] +pub use stub_crawler as crawler; +pub mod errors; +pub mod filters; +pub mod routes; +pub mod search; +pub mod types; +pub mod utils; + +#[deny(unused_imports)] +#[deny(unused_variables)] +#[deny(unused_mut)] +#[deny(unsafe_code)] +// Dependencies +#[macro_use] +extern crate log; + +pub const VERSION: &str = env!("CARGO_PKG_VERSION"); diff --git a/fastside/src/routes/api.rs b/fastside-core/src/routes/api.rs similarity index 100% rename from fastside/src/routes/api.rs rename to fastside-core/src/routes/api.rs diff --git a/fastside-core/src/routes/config.rs b/fastside-core/src/routes/config.rs new file mode 100644 index 00000000..0a6f310a --- /dev/null +++ b/fastside-core/src/routes/config.rs @@ -0,0 +1,82 @@ +use askama::Template; +use axum::{ + extract::{Query, State}, + http::StatusCode, + response::{Html, IntoResponse, Redirect}, + routing::get, + Router, +}; +use axum_extra::extract::cookie::{Cookie, CookieJar}; +use serde::Deserialize; +use std::sync::Arc; + +use crate::{types::AppState, utils::user_config::SettingsCookie}; +use fastside_shared::config::UserConfig; + +pub fn router() -> Router> { + Router::new() + .route("/", get(configure_page)) + .route("/save", get(configure_save)) +} + +/// Template struct for rendering the configuration page. +#[derive(Template)] +#[template(path = "configure.html")] +pub struct ConfigureTemplate<'a> { + current_config: &'a str, +} + +/// Query parameters for configuration save route. +#[derive(Deserialize)] +struct QueryParams { + config: Option, +} + +/// Handler for displaying the configuration page. +async fn configure_page( + State(state): State>, + _jar: CookieJar, + SettingsCookie(user_config): SettingsCookie, +) -> impl IntoResponse { + let _loaded_data_guard = state.loaded_data.read().await; + + let template = ConfigureTemplate { + current_config: &user_config + .to_config_string() + .unwrap_or_else(|_| "".to_string()), + }; + + match template.render() { + Ok(rendered) => Html(rendered).into_response(), + Err(_) => ( + StatusCode::INTERNAL_SERVER_ERROR, + "Failed to render the configuration page", + ) + .into_response(), + } +} + +/// Handler for saving configuration via query parameters. +async fn configure_save(Query(params): Query, jar: CookieJar) -> impl IntoResponse { + let query_string = params.config.unwrap_or_default(); + match UserConfig::from_config_string(&query_string) { + Ok(user_config) => { + let cookie_value = user_config + .to_config_string() + .unwrap_or_else(|_| "".to_string()); + + let cookie = Cookie::new("config", cookie_value); + + ( + jar.add(cookie), + Redirect::to("/configure?success").into_response(), + ) + .into_response() + } + Err(_) => ( + StatusCode::BAD_REQUEST, + "Invalid configuration string provided", + ) + .into_response(), + } +} diff --git a/fastside-core/src/routes/index.rs b/fastside-core/src/routes/index.rs new file mode 100644 index 00000000..0c217dec --- /dev/null +++ b/fastside-core/src/routes/index.rs @@ -0,0 +1,57 @@ +use askama::Template; +use axum::{ + extract::State, + http::{header::CONTENT_TYPE, StatusCode}, + response::{Html, IntoResponse}, + routing::get, + Router, +}; +use chrono::{DateTime, Utc}; +use std::{collections::HashMap, sync::Arc}; + +use crate::{crawler::CrawledService, errors::RedirectError, search::SearchError}; +use crate::{filters, types::AppState}; +use fastside_shared::serde_types::ServicesData; + +pub fn router() -> Router> { + Router::new().route("/", get(index)) +} + +/// The `IndexTemplate` structure renders the index page using the Askama template engine. +#[derive(Template)] +#[template(path = "index.html")] +pub struct IndexTemplate<'a> { + pub crawled_services: &'a HashMap, + pub services: &'a ServicesData, + pub time: &'a DateTime, + pub is_reloading: bool, +} + +/// The `index` handler function renders the main page. +pub async fn index(State(state): State>) -> impl IntoResponse { + let data = state.crawler.read().await; + let Some(crawled_services) = data.get_services() else { + return RedirectError::from(SearchError::CrawlerNotFetchedYet).into_response(); + }; + + // Acquire a read lock on the `LoadedData`. + let loaded_data_guard = state.loaded_data.read().await; + + // Render the template with the required data. + let template = IndexTemplate { + services: &loaded_data_guard.services, + crawled_services: &crawled_services.services, + time: &crawled_services.time, + is_reloading: data.is_reloading(), + }; + + match template.render() { + Ok(rendered) => ( + StatusCode::OK, + [(CONTENT_TYPE, "text/html; charset=utf-8")], + Html(rendered), + ) + .into_response(), + Err(_) => StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } +} diff --git a/fastside-core/src/routes/mod.rs b/fastside-core/src/routes/mod.rs new file mode 100644 index 00000000..4a057eaf --- /dev/null +++ b/fastside-core/src/routes/mod.rs @@ -0,0 +1,18 @@ +// mod api; +mod config; +mod index; + +// use actix_web::Scope; + +use std::sync::Arc; + +use axum::Router; + +use crate::types::AppState; + +pub fn main_router() -> Router> { + Router::new() + .nest("/", index::router()) + // .merge(redirect::router()) + .nest("/configure", config::router()) +} diff --git a/fastside/src/search.rs b/fastside-core/src/search.rs similarity index 100% rename from fastside/src/search.rs rename to fastside-core/src/search.rs diff --git a/fastside-core/src/stub_crawler.rs b/fastside-core/src/stub_crawler.rs new file mode 100644 index 00000000..f0916b77 --- /dev/null +++ b/fastside-core/src/stub_crawler.rs @@ -0,0 +1,296 @@ +use std::{ + collections::HashMap, + sync::Arc, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use chrono::{DateTime, Utc}; +use reqwest::StatusCode; +use thiserror::Error; +use tokio::{ + sync::{Mutex, MutexGuard, RwLock}, + time::sleep, +}; +use url::Url; + +use fastside_shared::config::CrawlerConfig; +use fastside_shared::{ + client_builder::build_client, + parallel::Parallelise, + serde_types::{HttpCodeRanges, Instance, Service}, +}; + +use crate::types::LoadedData; + +#[derive(Error, Debug)] +pub enum CrawlerError { + #[error("url error: `{0}`")] + UrlError(#[from] url::ParseError), + #[error("request error: `{0}`")] + RequestError(#[from] reqwest::Error), +} + +#[derive(Clone, Debug)] +pub enum CrawledInstanceStatus { + Ok(Duration), + #[allow(dead_code)] + InvalidStatusCode(StatusCode, Duration), + StringNotFound, + ConnectionError, + RedirectPolicyError, + BuilderError, + RequestError, + BodyError, + DecodeError, + TimedOut, + Unknown, +} + +impl CrawledInstanceStatus { + /// Used for sorting values in index.html template. + pub fn as_isize(&self) -> isize { + match self { + Self::Ok(d) => d.as_millis() as isize, + _ => isize::MAX, + } + } +} + +impl std::fmt::Display for CrawledInstanceStatus { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self) + } +} + +#[derive(Clone, Debug)] +pub struct CrawledInstance { + pub url: Url, + pub status: CrawledInstanceStatus, + pub tags: Vec, +} + +#[derive(Clone, Debug)] +pub struct CrawledService { + pub name: String, + pub instances: Vec, +} + +impl CrawledService { + pub fn get_alive_instances(&self) -> impl Iterator { + self.instances + .iter() + .filter(|s| matches!(&s.status, CrawledInstanceStatus::Ok(_))) + } +} + +#[derive(Clone, Debug)] +pub struct CrawledServices { + pub services: HashMap, + pub time: DateTime, +} + +#[derive(Debug)] +pub enum CrawledData { + CrawledServices(CrawledServices), + InitialLoading, + ReloadingServices(CrawledServices), +} + +impl CrawledData { + pub fn get_services(&self) -> Option<&CrawledServices> { + match self { + Self::CrawledServices(s) => Some(s), + Self::InitialLoading => None, + Self::ReloadingServices(current) => Some(current), + } + } + + pub fn is_reloading(&self) -> bool { + matches!(self, Self::ReloadingServices { .. }) + } + + pub fn replace(&mut self, new: CrawledData) { + *self = new; + } + + pub fn make_reloading(&mut self) { + let current = match self { + Self::CrawledServices(s) => s.clone(), + _ => return, + }; + *self = Self::ReloadingServices(current); + } +} + +impl AsRef for CrawledData { + fn as_ref(&self) -> &CrawledData { + self + } +} + +#[derive(Debug)] +pub struct Crawler { + loaded_data: Arc>, + config: Arc, + data: RwLock, + crawler_lock: Mutex<()>, +} + +impl Crawler { + pub fn new(loaded_data: Arc>, config: CrawlerConfig) -> Self { + Self { + loaded_data, + config: Arc::new(config), + data: RwLock::new(CrawledData::InitialLoading), + crawler_lock: Mutex::new(()), + } + } + + #[inline] + pub async fn read(&self) -> tokio::sync::RwLockReadGuard { + self.data.read().await + } + + async fn crawl_single_instance( + config: Arc, + loaded_data: Arc>, + service: Arc, + instance: Instance, + ) -> Result<(CrawledInstance, String), CrawlerError> { + let client = build_client( + service.as_ref(), + config.as_ref(), + &loaded_data.read().await.proxies, + &instance, + )?; + + let test_url = instance.url.join(&service.test_url)?; + let start = SystemTime::now().duration_since(UNIX_EPOCH).unwrap(); + let response = client.get(test_url).send().await; + let status = match response { + Ok(response) => { + let end = SystemTime::now().duration_since(UNIX_EPOCH).unwrap(); + let status_code = response.status().as_u16(); + if service.allowed_http_codes.is_allowed(status_code) { + if let Some(search_string) = &service.search_string { + let body = response.text().await?; + if !body.contains(search_string) { + CrawledInstanceStatus::StringNotFound + } else { + CrawledInstanceStatus::Ok(end - start) + } + } else { + CrawledInstanceStatus::Ok(end - start) + } + } else { + CrawledInstanceStatus::InvalidStatusCode(response.status(), end - start) + } + } + Err(e) => match e { + _ if e.is_timeout() => CrawledInstanceStatus::TimedOut, + _ if e.is_builder() => CrawledInstanceStatus::BuilderError, + _ if e.is_redirect() => CrawledInstanceStatus::RedirectPolicyError, + _ if e.is_request() => CrawledInstanceStatus::RequestError, + _ if e.is_body() => CrawledInstanceStatus::BodyError, + _ if e.is_decode() => CrawledInstanceStatus::DecodeError, + #[cfg(not(target_arch = "wasm32"))] + _ if e.is_connect() => CrawledInstanceStatus::ConnectionError, + _ => CrawledInstanceStatus::Unknown, + }, + }; + + let ret = ( + CrawledInstance { + url: instance.url.clone(), + tags: instance.tags.clone(), + status, + }, + service.name.clone(), + ); + debug!("Crawled instance: {ret:?}"); + Ok(ret) + } + + async fn crawl<'a>( + &self, + crawler_guard: Option>, + ) -> Result<(), CrawlerError> { + let crawler_guard = match crawler_guard { + Some(guard) => guard, + None => { + let Ok(crawler_guard) = self.crawler_lock.try_lock() else { + warn!("Crawler lock is already acquired, skipping crawl"); + return Ok(()); + }; + crawler_guard + } + }; + + let mut crawled_services: HashMap = self + .loaded_data + .read() + .await + .services + .keys() + .map(|name| { + ( + name.clone(), + CrawledService { + name: name.clone(), + instances: Vec::new(), + }, + ) + }) + .collect(); + + for service in self.loaded_data.read().await.services.values() { + let service = Arc::new(service.clone()); + for instance in &service.instances { + let loaded_data = self.loaded_data.clone(); + let config = self.config.clone(); + let instance = instance.clone(); + } + } + + let mut data = self.data.write().await; + data.replace(CrawledData::CrawledServices(CrawledServices { + services: crawled_services, + time: Utc::now(), + })); + + match data.as_ref() { + CrawledData::ReloadingServices { .. } => { + info!("Finished reloading services"); + } + CrawledData::InitialLoading => { + info!("Finished initial crawl, we are ready to serve requests"); + } + CrawledData::CrawledServices(_) => { + debug!("Finished crawl"); + } + } + + drop(crawler_guard); + Ok(()) + } + + /// Run crawler instantly in update loaded_data mode. + pub async fn update_crawl(&self) -> Result<(), CrawlerError> { + let crawler_guard = self.crawler_lock.lock().await; + let mut data = self.data.write().await; + data.make_reloading(); + drop(data); + self.crawl(Some(crawler_guard)).await + } + + pub async fn crawler_loop(&self) { + loop { + debug!("Starting crawl"); + if let Err(e) = self.crawl(None).await { + error!("Error occured during crawl loop: {e}"); + }; + debug!("Next crawl will start in {:?}", self.config.ping_interval); + sleep(self.config.ping_interval).await; + } + } +} diff --git a/fastside/src/types.rs b/fastside-core/src/types.rs similarity index 51% rename from fastside/src/types.rs rename to fastside-core/src/types.rs index 9f9e6037..1a1a79de 100644 --- a/fastside/src/types.rs +++ b/fastside-core/src/types.rs @@ -1,9 +1,12 @@ -use std::collections::HashMap; +use std::{collections::HashMap, sync::Arc}; use fastside_shared::{ - config::{ProxyData, UserConfig}, + config::{AppConfig, ProxyData, UserConfig}, serde_types::ServicesData, }; +use tokio::sync::RwLock; + +use crate::crawler::Crawler; pub struct CompiledRegexSearch { pub regex: regex::Regex, @@ -18,3 +21,11 @@ pub struct LoadedData { pub proxies: ProxyData, pub default_user_config: UserConfig, } + +// Shared state type +pub struct AppState { + pub config: Arc, + pub crawler: Arc, + pub loaded_data: Arc>, + pub regexes: Regexes, +} diff --git a/fastside/src/utils/mod.rs b/fastside-core/src/utils/mod.rs similarity index 100% rename from fastside/src/utils/mod.rs rename to fastside-core/src/utils/mod.rs diff --git a/fastside-core/src/utils/user_config.rs b/fastside-core/src/utils/user_config.rs new file mode 100644 index 00000000..6503df9a --- /dev/null +++ b/fastside-core/src/utils/user_config.rs @@ -0,0 +1,40 @@ +use axum::{ + async_trait, + extract::FromRequestParts, + http::request::Parts, +}; +use axum_extra::extract::CookieJar; +use fastside_shared::config::UserConfig; +use tracing::debug; + +pub struct SettingsCookie(pub UserConfig); + +#[async_trait] +impl FromRequestParts for SettingsCookie { + type Rejection = (); + + async fn from_request_parts(parts: &mut Parts, state: &S) -> Result { + // Extract the CookieJar from the request parts + let cookie_jar = match CookieJar::from_request_parts(parts, state).await { + Ok(cookies) => cookies, + Err(_) => { + debug!("Failed to extract CookieJar"); + return Err(()); + } + }; + + // Retrieve the "config" cookie + if let Some(config_cookie) = cookie_jar.get("config") { + match UserConfig::from_config_string(config_cookie.value()) { + Ok(user_config) => Ok(SettingsCookie(user_config)), + Err(_) => { + debug!("Invalid config cookie format"); + Err(()) + } + } + } else { + debug!("Config cookie not found"); + Err(()) + } + } +} diff --git a/fastside/static/favicon.ico b/fastside-core/static/favicon.ico similarity index 100% rename from fastside/static/favicon.ico rename to fastside-core/static/favicon.ico diff --git a/fastside/templates/base.html b/fastside-core/templates/base.html similarity index 100% rename from fastside/templates/base.html rename to fastside-core/templates/base.html diff --git a/fastside/templates/cached_redirect.html b/fastside-core/templates/cached_redirect.html similarity index 100% rename from fastside/templates/cached_redirect.html rename to fastside-core/templates/cached_redirect.html diff --git a/fastside/templates/configure.html b/fastside-core/templates/configure.html similarity index 100% rename from fastside/templates/configure.html rename to fastside-core/templates/configure.html diff --git a/fastside/templates/error.html b/fastside-core/templates/error.html similarity index 100% rename from fastside/templates/error.html rename to fastside-core/templates/error.html diff --git a/fastside/templates/fallback_redirect.html b/fastside-core/templates/fallback_redirect.html similarity index 100% rename from fastside/templates/fallback_redirect.html rename to fastside-core/templates/fallback_redirect.html diff --git a/fastside/templates/history_redirect.html b/fastside-core/templates/history_redirect.html similarity index 100% rename from fastside/templates/history_redirect.html rename to fastside-core/templates/history_redirect.html diff --git a/fastside/templates/index.html b/fastside-core/templates/index.html similarity index 100% rename from fastside/templates/index.html rename to fastside-core/templates/index.html diff --git a/fastside-shared/Cargo.toml b/fastside-shared/Cargo.toml index 9b58118a..385d7a68 100644 --- a/fastside-shared/Cargo.toml +++ b/fastside-shared/Cargo.toml @@ -7,17 +7,18 @@ edition = "2021" reqwest = { version = "0.12.4", default-features = false, features = [ "rustls-tls", "http2", + "socks", ] } # http client -serde = { version = "1.0.201", features = ["derive"] } # serialization -serde_json = "1.0.117" # serialization -url = { version = "2.5.0", features = ["serde"] } # url -base64 = "0.22.1" # base64 -thiserror = "1.0.60" # error -config = "0.14.0" # config -anyhow = "1.0.83" # error -log = "0.4.21" # logging -pretty_env_logger = "0.5.0" # logging -tokio = "1.39.3" # async -num_cpus = "1.16.0" # get number of cpus -regex = "1.10.5" # regex +serde = { version = "1.0.201", features = ["derive"] } # serialization +serde_json = "1.0.117" # serialization +url = { version = "2.5.0", features = ["serde"] } # url +base64 = "0.22.1" # base64 +thiserror = "1.0.60" # error +config = "0.14.0" # config +anyhow = "1.0.83" # error +log = "0.4.21" # logging +pretty_env_logger = "0.5.0" # logging +tokio = { version = "1.39.3", default-features = false, features = ["rt", "time"]} # async +num_cpus = "1.16.0" # get number of cpus +regex = "1.10.5" # regex diff --git a/fastside-shared/src/client_builder.rs b/fastside-shared/src/client_builder.rs index cbffad2e..3a912ffd 100644 --- a/fastside-shared/src/client_builder.rs +++ b/fastside-shared/src/client_builder.rs @@ -31,22 +31,27 @@ pub fn build_client( proxies: &ProxyData, instance: &Instance, ) -> Result { - let redirect_policy = if service.follow_redirects { - reqwest::redirect::Policy::default() - } else { - reqwest::redirect::Policy::none() - }; - let timeout = config.get_domain_timeout( - instance - .url - .host_str() - .expect("Failed to get host from instance URL"), - ); - let mut client_builder = Client::builder() - .connect_timeout(timeout) - .read_timeout(timeout) - .default_headers(default_headers()) - .redirect(redirect_policy); + let mut client_builder = Client::builder().default_headers(default_headers()); + + #[cfg(not(target_arch = "wasm32"))] + { + let redirect_policy = if service.follow_redirects { + reqwest::redirect::Policy::default() + } else { + reqwest::redirect::Policy::none() + }; + let timeout = config.get_domain_timeout( + instance + .url + .host_str() + .expect("Failed to get host from instance URL"), + ); + + client_builder = client_builder + .connect_timeout(timeout) + .read_timeout(timeout) + .redirect(redirect_policy); + } let proxy_name: Option = { let mut val: Option = None; @@ -58,6 +63,8 @@ pub fn build_client( } val }; + + #[cfg(not(target_arch = "wasm32"))] if let Some(proxy_name) = proxy_name { let proxy_config = proxies.get(&proxy_name).unwrap(); let proxy = { diff --git a/fastside/Cargo.toml b/fastside/Cargo.toml index f390df00..7fdc173a 100644 --- a/fastside/Cargo.toml +++ b/fastside/Cargo.toml @@ -1,18 +1,15 @@ [package] name = "fastside" -description = "A smart redirecting gateway for various frontend services." +description = "A smart redirecting gateway for various frontend services. Standalone version." version = "0.2.0" edition = "2021" default-run = "fastside" [dependencies] fastside-shared = { path = "../fastside-shared" } +fastside-core = { path = "../fastside-core" } -actix-web = { version = "4.5.1", features = [ - "macros", - "cookies", -], default-features = false } # web framework -askama = "0.12.1" # templating engine +axum = "0.7.9" # web framework reqwest = { version = "0.12.4", default-features = false, features = [ "rustls-tls", "http2", diff --git a/fastside/src/main.rs b/fastside/src/main.rs index 25b31c3f..bbd87f23 100644 --- a/fastside/src/main.rs +++ b/fastside/src/main.rs @@ -1,17 +1,14 @@ //! Fastside API server. -mod crawler; -mod errors; -mod filters; -mod routes; -mod search; -mod types; -mod utils; -use actix_web::{middleware::Logger, web, App, HttpServer}; use anyhow::{Context, Result}; +use axum::Router; use clap::{Parser, Subcommand}; use config::load_config; -use crawler::Crawler; +use fastside_core::{ + crawler::Crawler, + routes::main_router, + types::{AppState, CompiledRegexSearch, LoadedData}, +}; use fastside_shared::{ config::{self, AppConfig}, errors::CliError, @@ -20,7 +17,6 @@ use fastside_shared::{ }; use log_setup::configure_logging; use regex::Regex; -use routes::main_scope; use std::{ collections::HashMap, net::{SocketAddr, SocketAddrV4}, @@ -29,7 +25,6 @@ use std::{ sync::Arc, }; use tokio::sync::RwLock; -use types::{CompiledRegexSearch, LoadedData}; use url::Url; #[deny(unused_imports)] @@ -64,9 +59,6 @@ enum Commands { /// Listen socket address. #[arg(short, long)] listen: Option, - /// Worker count. - #[arg(short, long)] - workers: Option, }, /// Validate services file. Validate { @@ -244,11 +236,7 @@ async fn main() -> Result<()> { configure_logging(&cli.log_level).ok(); match &cli.command { - Some(Commands::Serve { - services, - listen, - workers, - }) => { + Some(Commands::Serve { services, listen }) => { let config = Arc::new(load_config(&cli.config).context("failed to load config")?); let services_source = ServicesSource::from_str( @@ -272,7 +260,6 @@ async fn main() -> Result<()> { let listen: SocketAddr = listen .unwrap_or_else(|| SocketAddr::V4(SocketAddrV4::new([127, 0, 0, 1].into(), 8080))); - let workers: usize = workers.unwrap_or_else(num_cpus::get); let data: Arc> = { let data = load_services(&services_source, &config).await?; @@ -315,26 +302,19 @@ async fn main() -> Result<()> { info!("Listening on {}", listen); - let config_web_data = web::Data::from(config.clone()); - let crawler_web_data = web::Data::from(crawler.clone()); - let data_web_data = web::Data::from(data.clone()); - let regexes_web_data = web::Data::new(regexes); + let shared_state = Arc::new(AppState { + config: config.clone(), + crawler: crawler.clone(), + loaded_data: data.clone(), + regexes, + }); - HttpServer::new(move || { - let logger = Logger::default(); - App::new() - .wrap(logger) - .app_data(config_web_data.clone()) - .app_data(crawler_web_data.clone()) - .app_data(data_web_data.clone()) - .app_data(regexes_web_data.clone()) - .service(main_scope(&config.clone())) - }) - .bind(listen)? - .workers(workers) - .run() - .await - .context("failed to start api server")?; + let router = Router::new() + .nest("/", main_router()) + .with_state(shared_state); + + let listener = tokio::net::TcpListener::bind(listen).await.unwrap(); + axum::serve(listener, router.into_make_service()).await?; reload_services_handle.abort(); crawler_loop_handle.abort(); diff --git a/fastside/src/routes/config.rs b/fastside/src/routes/config.rs deleted file mode 100644 index 443fb4ff..00000000 --- a/fastside/src/routes/config.rs +++ /dev/null @@ -1,60 +0,0 @@ -use actix_web::{cookie::Cookie, get, http::header::LOCATION, web, HttpRequest, Responder, Scope}; -use askama::Template; -use fastside_shared::config::UserConfig; -use tokio::sync::RwLock; - -use crate::{ - config::AppConfig, errors::RedirectError, types::LoadedData, - utils::user_config::load_settings_cookie, -}; - -pub fn scope(_config: &AppConfig) -> Scope { - web::scope("/configure") - .service(configure_page) - .service(configure_save) -} - -#[derive(Template)] -#[template(path = "configure.html")] -pub struct ConfigureTemplate<'a> { - current_config: &'a str, -} - -#[get("")] -async fn configure_page( - req: HttpRequest, - loaded_data: web::Data>, -) -> actix_web::Result { - let loaded_data_guard = loaded_data.read().await; - let user_config = load_settings_cookie(&req, &loaded_data_guard.default_user_config); - - let template = ConfigureTemplate { - current_config: &user_config - .to_config_string() - .map_err(RedirectError::from)?, - }; - - Ok(actix_web::HttpResponse::Ok() - .content_type("text/html; charset=utf-8") - .body(template.render().expect("failed to render error page"))) -} - -#[get("/save")] -async fn configure_save(req: HttpRequest) -> actix_web::Result { - let query_string = req.query_string(); - let user_config = UserConfig::from_config_string(query_string).map_err(RedirectError::from)?; - let cookie = Cookie::build( - "config", - user_config - .to_config_string() - .map_err(RedirectError::from)?, - ) - .path("/") - .expires(time::OffsetDateTime::now_utc() + time::Duration::days(9999)) - .max_age(time::Duration::days(9999)) - .finish(); - Ok(actix_web::HttpResponse::TemporaryRedirect() - .cookie(cookie) - .insert_header((LOCATION, "/configure?success")) - .finish()) -} diff --git a/fastside/src/routes/index.rs b/fastside/src/routes/index.rs deleted file mode 100644 index a4883707..00000000 --- a/fastside/src/routes/index.rs +++ /dev/null @@ -1,77 +0,0 @@ -use std::collections::HashMap; - -use actix_web::{get, web, Responder, Scope}; -use askama::Template; -use chrono::{DateTime, Utc}; -use tokio::sync::RwLock; - -use crate::{ - config::AppConfig, - crawler::{CrawledService, Crawler}, - errors::RedirectError, - filters, - search::SearchError, - types::LoadedData, -}; -use fastside_shared::serde_types::ServicesData; - -use super::{api, config, redirect}; - -pub fn scope(app_config: &AppConfig) -> Scope { - web::scope("") - .service(index) - .service(favicon) - .service(robots_txt) - .service(config::scope(app_config)) - .service(api::scope(app_config)) - .service(redirect::scope(app_config)) -} - -#[derive(Template)] -#[template(path = "index.html")] -pub struct IndexTemplate<'a> { - pub crawled_services: &'a HashMap, - pub services: &'a ServicesData, - pub time: &'a DateTime, - pub is_reloading: bool, -} - -#[get("/")] -async fn index( - crawler: web::Data, - loaded_data: web::Data>, -) -> actix_web::Result { - let data = crawler.read().await; - let Some(crawled_services) = data.get_services() else { - return Err(RedirectError::from(SearchError::CrawlerNotFetchedYet))?; - }; - let loaded_data_guard = loaded_data.read().await; - let template = IndexTemplate { - services: &loaded_data_guard.services, - crawled_services: &crawled_services.services, - time: &crawled_services.time, - is_reloading: data.is_reloading(), - }; - - Ok(actix_web::HttpResponse::Ok() - .content_type("text/html; charset=utf-8") - .body(template.render().expect("failed to render error page"))) -} - -const FAVICON: &[u8] = include_bytes!("../../static/favicon.ico"); - -#[get("/favicon.ico")] -async fn favicon() -> impl Responder { - actix_web::HttpResponse::Ok() - .content_type("image/x-icon") - .body(FAVICON) -} - -const ROBOTS_TXT: &str = "User-agent: *\nDisallow: /\n"; - -#[get("/robots.txt")] -async fn robots_txt() -> impl Responder { - actix_web::HttpResponse::Ok() - .content_type("text/plain") - .body(ROBOTS_TXT) -} diff --git a/fastside/src/routes/mod.rs b/fastside/src/routes/mod.rs deleted file mode 100644 index 35b1e1cd..00000000 --- a/fastside/src/routes/mod.rs +++ /dev/null @@ -1,12 +0,0 @@ -mod api; -mod config; -mod index; -mod redirect; - -use actix_web::Scope; - -use crate::config::AppConfig; - -pub fn main_scope(config: &AppConfig) -> Scope { - index::scope(config) -} diff --git a/fastside/src/routes/redirect.rs b/fastside/src/routes/redirect.rs deleted file mode 100644 index 864808f7..00000000 --- a/fastside/src/routes/redirect.rs +++ /dev/null @@ -1,217 +0,0 @@ -use actix_web::{ - get, - http::{header::LOCATION, Method}, - web, HttpRequest, Responder, Scope, -}; -use askama::Template; -use tokio::sync::RwLock; - -use crate::{ - config::AppConfig, - crawler::{CrawledService, Crawler}, - errors::RedirectError, - search::{ - find_redirect_service_by_name, find_redirect_service_by_url, get_redirect_instance, - get_redirect_instances, SearchError, - }, - types::{LoadedData, Regexes}, - utils::user_config::load_settings_cookie, -}; -use fastside_shared::{ - config::{SelectMethod, UserConfig}, - serde_types::Service, -}; - -pub fn scope(_config: &AppConfig) -> Scope { - web::scope("") - .service(history_redirect) - .service(cached_redirect) - .route("/{path:.*}", web::get().to(base_redirect)) - .route("/{path:.*}", web::post().to(base_redirect)) -} - -#[derive(Template)] -#[template(path = "cached_redirect.html", escape = "none")] -pub struct CachedRedirectTemplate<'a> { - pub urls: Vec<&'a reqwest::Url>, - pub select_method: &'a SelectMethod, -} - -#[get("/@cached/{service_name}/{path:.*}")] -async fn cached_redirect( - req: HttpRequest, - path: web::Path<(String, String)>, - config: web::Data, - crawler: web::Data, - loaded_data: web::Data>, -) -> actix_web::Result { - let (service_name, _) = path.into_inner(); - - let loaded_data_guard = loaded_data.read().await; - let user_config = load_settings_cookie(&req, &loaded_data_guard.default_user_config); - - let guard = crawler.read().await; - let (crawled_service, _) = - find_redirect_service_by_name(&guard, &loaded_data_guard.services, &service_name) - .await - .map_err(RedirectError::from)?; - let mut instances = get_redirect_instances( - crawled_service, - &user_config.required_tags, - &user_config.forbidden_tags, - &user_config.preferred_instances, - ) - .ok_or(RedirectError::from(SearchError::NoInstancesFound))?; - if user_config.select_method == SelectMethod::LowPing { - instances.sort_by(|a, b| a.status.as_isize().cmp(&b.status.as_isize())); - } - debug!("User config: {user_config:?}"); - - let template = CachedRedirectTemplate { - urls: instances.iter().map(|i| &i.url).collect(), - select_method: &user_config.select_method, - }; - - Ok(actix_web::HttpResponse::Ok() - .content_type("text/html; charset=utf-8") - .append_header(( - "cache-control", - format!( - "public, max-age={}, stale-while-revalidate=86400, stale-if-error=86400, immutable", - config.crawler.ping_interval.as_secs() - ), - )) - .body(template.render().expect("failed to render error page"))) -} - -#[derive(Template)] -#[template(path = "history_redirect.html")] -pub struct HistoryRedirectTemplate<'a> { - pub path: &'a str, -} - -#[get("/_/{path:.*}")] -async fn history_redirect( - req: HttpRequest, - path: web::Path, -) -> actix_web::Result { - let mut path = path.into_inner(); - let query = req.query_string(); - if !query.is_empty() { - path.push('?'); - path.push_str(query); - } - - let path = format!("/{path}"); - let template = HistoryRedirectTemplate { path: &path }; - - Ok(actix_web::HttpResponse::Ok() - .content_type("text/html; charset=utf-8") - .append_header(("refresh", format!("1; url={path}"))) - .body(template.render().expect("failed to render error page"))) -} - -#[derive(Template)] -#[template(path = "fallback_redirect.html", escape = "none")] -pub struct FallbackRedirectTemplate<'a> { - pub fallback: &'a str, -} - -pub async fn find_redirect( - crawler: &Crawler, - loaded_data: &LoadedData, - regexes: &Regexes, - user_config: &UserConfig, - path: &str, -) -> Result<(String, bool), RedirectError> { - let is_url_query = if path.starts_with("http://") || path.starts_with("https://") { - true - } else { - path[0..path.find('/').unwrap_or(0)].contains('.') - }; - - let guard = crawler.read().await; - let (redir_path, crawled_service, service): (String, &CrawledService, &Service) = - match is_url_query { - true => { - let (crawled_service, service, redir_path) = - find_redirect_service_by_url(&guard, &loaded_data.services, regexes, path) - .await - .map_err(RedirectError::from)?; - (redir_path, crawled_service, service) - } - false => { - let service_name = path.split('/').next().unwrap(); - let redir_path = path[service_name.len()..].to_string(); - let (crawled_service, service) = - find_redirect_service_by_name(&guard, &loaded_data.services, service_name) - .await - .map_err(RedirectError::from)?; - (redir_path, crawled_service, service) - } - }; - - let (redirect_instance, is_fallback) = - get_redirect_instance(crawled_service, service, user_config) - .map_err(RedirectError::from)?; - - let url = redirect_instance - .url - .clone() - .join(&redir_path) - .map_err(RedirectError::from)? - .to_string(); - - Ok((url, is_fallback)) -} - -async fn base_redirect( - req: HttpRequest, - path: web::Path, - crawler: web::Data, - loaded_data: web::Data>, - regexes: web::Data, -) -> actix_web::Result { - let path = path.into_inner(); - - let loaded_data_guard = loaded_data.read().await; - let user_config = load_settings_cookie(&req, &loaded_data_guard.default_user_config); - - let (mut url, is_fallback) = find_redirect( - crawler.get_ref(), - &loaded_data_guard, - regexes.get_ref(), - &user_config, - &path, - ) - .await?; - - let query = req.query_string(); - if !query.is_empty() { - url.push('?'); - url.push_str(query); - } - - debug!("Redirecting to {url}, is_fallback: {is_fallback}"); - - match ( - is_fallback, - user_config.ignore_fallback_warning, - req.method(), - ) { - (true, false, &Method::GET) => { - let template = FallbackRedirectTemplate { fallback: &url }; - Ok(actix_web::HttpResponse::Ok() - .content_type("text/html; charset=utf-8") - .insert_header(("refresh", format!("15; url={url}"))) - .body( - template - .render() - .expect("failed to render fallback redirect page"), - )) - } - _ => Ok(actix_web::HttpResponse::TemporaryRedirect() - .insert_header((LOCATION, url.to_string())) - .finish()), - } -} diff --git a/fastside/src/utils/user_config.rs b/fastside/src/utils/user_config.rs deleted file mode 100644 index e48148d2..00000000 --- a/fastside/src/utils/user_config.rs +++ /dev/null @@ -1,16 +0,0 @@ -use actix_web::HttpRequest; -use fastside_shared::config::UserConfig; - -pub fn load_settings_cookie(req: &HttpRequest, default: &UserConfig) -> UserConfig { - let cookie = match req.cookie("config") { - Some(cookie) => cookie, - None => { - debug!("Cookie not found"); - return default.clone(); - } - }; - UserConfig::from_config_string(cookie.value()).unwrap_or_else(|_| { - debug!("invalid cookie"); - default.clone() - }) -} diff --git a/shell.nix b/shell.nix index ec119451..5715d2c8 100644 --- a/shell.nix +++ b/shell.nix @@ -9,9 +9,13 @@ in pkgs'.mkShell { nativeBuildInputs = with pkgs'; [ nixfmt-classic # Rust - (if channel == "nightly" then + ((if channel == "nightly" then rust-bin.selectLatestNightlyWith (toolchain: toolchain.${profile}) else - rust-bin.${channel}.latest.${profile}) + rust-bin.${channel}.latest.${profile}).override { + targets = [ "wasm32-unknown-unknown" ]; + }) + wasm-pack + wasm-bindgen-cli ]; } From a376d645a5a8b930275bbfbcd6a50e9f600ed011 Mon Sep 17 00:00:00 2001 From: cofob Date: Sun, 17 Nov 2024 04:11:57 +0400 Subject: [PATCH 2/5] Crawler --- Cargo.lock | 275 ++++++++++++++++++++++++--- fastside-cloudflare/Cargo.toml | 3 + fastside-cloudflare/src/lib.rs | 110 +++++++++-- fastside-cloudflare/wrangler.toml | 11 ++ fastside-core/Cargo.toml | 2 +- fastside-core/src/crawler.rs | 133 +++++++++++--- fastside-core/src/lib.rs | 5 - fastside-core/src/routes/index.rs | 34 +++- fastside-core/src/stub_crawler.rs | 296 ------------------------------ fastside-core/src/types.rs | 3 +- 10 files changed, 512 insertions(+), 360 deletions(-) delete mode 100644 fastside-core/src/stub_crawler.rs diff --git a/Cargo.lock b/Cargo.lock index c7338a2a..9cdb2eba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -549,6 +549,7 @@ dependencies = [ "iana-time-zone", "js-sys", "num-traits", + "serde", "wasm-bindgen", "windows-targets 0.52.6", ] @@ -678,6 +679,16 @@ dependencies = [ "version_check", ] +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.6" @@ -914,6 +925,9 @@ dependencies = [ "fastside-core", "fastside-shared", "getrandom", + "reqwest", + "serde", + "serde_json", "tokio", "tower-service", "wasm-bindgen-futures", @@ -972,6 +986,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -1372,6 +1401,22 @@ dependencies = [ "webpki-roots", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.6" @@ -1474,7 +1519,7 @@ dependencies = [ "socket2 0.5.7", "widestring", "windows-sys 0.48.0", - "winreg 0.50.0", + "winreg", ] [[package]] @@ -1677,6 +1722,23 @@ dependencies = [ "version_check", ] +[[package]] +name = "native-tls" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nom" version = "7.1.3" @@ -1727,6 +1789,50 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "openssl" +version = "0.10.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" +dependencies = [ + "bitflags 2.6.0", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "ordered-multimap" version = "0.6.0" @@ -1866,6 +1972,12 @@ dependencies = [ "futures-io", ] +[[package]] +name = "pkg-config" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" + [[package]] name = "polling" version = "2.8.0" @@ -2062,12 +2174,13 @@ checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" [[package]] name = "reqwest" -version = "0.12.5" +version = "0.12.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7d6d2a27d57148378eb5e111173f4276ad26340ecc5c49a4a2152167a2d6a37" +checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" dependencies = [ "base64 0.22.1", "bytes", + "encoding_rs", "futures-core", "futures-util", "h2", @@ -2076,11 +2189,13 @@ dependencies = [ "http-body-util", "hyper", "hyper-rustls", + "hyper-tls", "hyper-util", "ipnet", "js-sys", "log", "mime", + "native-tls", "once_cell", "percent-encoding", "pin-project-lite", @@ -2092,7 +2207,9 @@ dependencies = [ "serde_json", "serde_urlencoded", "sync_wrapper 1.0.1", + "system-configuration", "tokio", + "tokio-native-tls", "tokio-rustls", "tokio-socks", "tower-service", @@ -2101,7 +2218,7 @@ dependencies = [ "wasm-bindgen-futures", "web-sys", "webpki-roots", - "winreg 0.52.0", + "windows-registry", ] [[package]] @@ -2249,17 +2366,49 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +[[package]] +name = "schannel" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01227be5826fa0690321a2ba6c5cd57a19cf3f6a09e76973b58e61de6ab9d1c1" +dependencies = [ + "windows-sys 0.59.0", +] + [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags 2.6.0", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa39c7303dc58b5543c94d22c1766b0d31f2ee58306363ea622b10bbc075eaa2" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "serde" -version = "1.0.204" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" +checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" dependencies = [ "serde_derive", ] @@ -2288,9 +2437,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.204" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" +checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" dependencies = [ "proc-macro2", "quote", @@ -2299,9 +2448,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.122" +version = "1.0.132" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784b6203951c57ff748476b126ccb5e8e2959a5c19e5c617ab1956be3dbc68da" +checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" dependencies = [ "itoa", "memchr", @@ -2439,9 +2588,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.72" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", @@ -2459,6 +2608,43 @@ name = "sync_wrapper" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +dependencies = [ + "futures-core", +] + +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags 2.6.0", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tempfile" +version = "3.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" +dependencies = [ + "cfg-if", + "fastrand 2.1.0", + "once_cell", + "rustix 0.38.34", + "windows-sys 0.59.0", +] [[package]] name = "termcolor" @@ -2573,6 +2759,16 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.0" @@ -2811,6 +3007,12 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a84c137d37ab0142f0f2ddfe332651fdbf252e7b7dbb4e67b6c1f1b2e925101" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" @@ -2983,6 +3185,36 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-registry" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" +dependencies = [ + "windows-result", + "windows-strings", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-result" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-strings" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" +dependencies = [ + "windows-result", + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -3001,6 +3233,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-targets" version = "0.48.5" @@ -3141,16 +3382,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "winreg" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" -dependencies = [ - "cfg-if", - "windows-sys 0.48.0", -] - [[package]] name = "worker" version = "0.4.2" diff --git a/fastside-cloudflare/Cargo.toml b/fastside-cloudflare/Cargo.toml index 06133b0a..8f95b701 100644 --- a/fastside-cloudflare/Cargo.toml +++ b/fastside-cloudflare/Cargo.toml @@ -45,6 +45,9 @@ tower-service = "0.3.3" console_error_panic_hook = "0.1.7" getrandom = { version = "0.2.15", features = ["js"] } tokio = { version = "1.41.1", default-features = false, features = ["sync"] } +serde = "1.0.215" +serde_json = "1.0.132" +reqwest = "0.12.9" # [features] # default = [] diff --git a/fastside-cloudflare/src/lib.rs b/fastside-cloudflare/src/lib.rs index 4c4a92e7..d90e112d 100644 --- a/fastside-cloudflare/src/lib.rs +++ b/fastside-cloudflare/src/lib.rs @@ -2,27 +2,57 @@ use std::{collections::HashMap, sync::Arc}; use axum::Router; use fastside_core::{ - crawler::Crawler, + crawler::{CrawledData, Crawler}, routes::main_router, types::{AppState, LoadedData}, }; -use fastside_shared::config::{AppConfig, UserConfig}; +use fastside_shared::{ + config::AppConfig, + serde_types::{ServicesData, StoredData}, +}; use tokio::sync::RwLock; use tower_service::Service; use worker::*; -fn router() -> Router { - let config = Arc::new(AppConfig::default()); - let loaded_data = Arc::new(RwLock::new(LoadedData { - services: HashMap::new(), - proxies: HashMap::new(), - default_user_config: UserConfig::default(), - })); +fn load_config(env: &Env) -> AppConfig { + let config_str = env + .var("config") + .expect("config variable is not set") + .to_string(); + let config: AppConfig = serde_json::from_str(&config_str).expect("failed to parse config"); + config +} + +async fn router(env: &Env) -> Router { + let config = Arc::new(load_config(&env)); + let loaded_data: Arc> = Arc::new(RwLock::new( + serde_json::from_str( + &env.kv("fastside") + .expect("failed to get kv") + .get("loaded_data") + .text() + .await + .expect("failed to get loaded_data from kv") + .expect("loaded_data not found"), + ) + .expect("failed to parse loaded_data"), + )); + let crawled_data: CrawledData = serde_json::from_str( + &env.kv("fastside") + .expect("failed to get kv") + .get("crawled_data") + .text() + .await + .expect("failed to get crawled_data from kv") + .expect("crawled_data not found"), + ) + .expect("failed to parse data"); let shared_state = Arc::new(AppState { config: config.clone(), - crawler: Arc::new(Crawler::new( + crawler: Arc::new(Crawler::with_data( loaded_data.clone(), config.clone().crawler.clone(), + crawled_data, )), loaded_data: loaded_data.clone(), regexes: HashMap::new(), @@ -35,10 +65,66 @@ fn router() -> Router { #[event(fetch)] async fn fetch( req: HttpRequest, - _env: Env, + env: Env, _ctx: Context, ) -> Result> { console_error_panic_hook::set_once(); - Ok(router().call(req).await?) + Ok(router(&env).await.call(req).await?) +} + +#[event(scheduled)] +async fn scheduled(_event: ScheduledEvent, env: Env, _ctx: ScheduleContext) { + console_error_panic_hook::set_once(); + + let config = load_config(&env); + let services_url = env + .var("services_url") + .expect("services_url variable is not set") + .to_string(); + + let services_str = reqwest::get(services_url) + .await + .expect("request to services failed") + .text() + .await + .expect("failed to get services text"); + let stored_data: StoredData = + serde_json::from_str(&services_str).expect("failed to parse services"); + + let loaded_data = { + let services_data: ServicesData = stored_data + .services + .into_iter() + .map(|service| (service.name.clone(), service)) + .collect(); + let loaded_data = LoadedData { + services: services_data, + proxies: config.proxies.clone(), + default_user_config: config.default_user_config.clone(), + }; + env.kv("fastside") + .expect("failed to get kv") + .put( + "loaded_data", + serde_json::to_string(&loaded_data).expect("failed to serialize loaded_data"), + ) + .expect("failed to put loaded_data to kv (builder)") + .execute() + .await + .expect("failed to put loaded_data to kv (request)"); + Arc::new(RwLock::new(loaded_data)) + }; + + let crawler = Crawler::new(loaded_data, config.crawler.clone()); + crawler.crawl(None).await.expect("failed to crawl"); + + let data_str = serde_json::to_string(&*crawler.read().await).expect("failed to serialize data"); + env.kv("fastside") + .expect("failed to get kv") + .put("crawled_data", data_str) + .expect("failed to put crawled_data to kv (builder)") + .execute() + .await + .expect("failed to put crawled_data to kv (request)"); } diff --git a/fastside-cloudflare/wrangler.toml b/fastside-cloudflare/wrangler.toml index 77158b5c..bdf6dcf2 100644 --- a/fastside-cloudflare/wrangler.toml +++ b/fastside-cloudflare/wrangler.toml @@ -2,5 +2,16 @@ name = "fastside" main = "build/worker/shim.mjs" compatibility_date = "2023-03-22" +kv_namespaces = [ + { binding = "fastside", id = "989f6480799a4d7c8e2890ac01118fc7" } +] + [build] command = "cargo install worker-build --version 0.1.0 && worker-build --release" + +[vars] +services_url = "https://raw.githubusercontent.com/cofob/fastside/refs/heads/master/services.json" +config = '{"crawler":{"request_timeout":{"secs":1,"nanos":0},"max_concurrent_requests":200},"default_user_config":{"required_tags":["clearnet","https","ipv4"]}}' + +[observability] +enabled = true diff --git a/fastside-core/Cargo.toml b/fastside-core/Cargo.toml index e268b51a..a471cdf8 100644 --- a/fastside-core/Cargo.toml +++ b/fastside-core/Cargo.toml @@ -28,7 +28,7 @@ thiserror = "1.0.60" # tokio = { version = "1.37.0", default-features = false, features = ["sync"] } futures = "0.3.30" # async rand = "0.8.5" # random -chrono = "0.4.38" # datetime +chrono = { version = "0.4.38", features = ["serde"] } time = "0.3.36" # time offsets regex = "1.10.5" # regex base64 = "0.22.1" # base64 diff --git a/fastside-core/src/crawler.rs b/fastside-core/src/crawler.rs index c6e3c76a..e833fbb2 100644 --- a/fastside-core/src/crawler.rs +++ b/fastside-core/src/crawler.rs @@ -5,23 +5,38 @@ use std::{ }; use chrono::{DateTime, Utc}; -use reqwest::StatusCode; +use serde::{Deserialize, Serialize}; use thiserror::Error; use tokio::{ sync::{Mutex, MutexGuard, RwLock}, - time::sleep, + time::{sleep, timeout}, }; use url::Url; use fastside_shared::config::CrawlerConfig; +#[cfg(not(target_arch = "wasm32"))] +use fastside_shared::parallel::Parallelise; use fastside_shared::{ client_builder::build_client, - parallel::Parallelise, serde_types::{HttpCodeRanges, Instance, Service}, }; use crate::types::LoadedData; +fn utc_now() -> DateTime { + #[cfg(not(target_arch = "wasm32"))] + return Utc::now(); + #[cfg(target_arch = "wasm32")] + return DateTime::from_timestamp(0, 0).unwrap(); +} + +fn system_now() -> SystemTime { + #[cfg(not(target_arch = "wasm32"))] + return SystemTime::now(); + #[cfg(target_arch = "wasm32")] + return UNIX_EPOCH; +} + #[derive(Error, Debug)] pub enum CrawlerError { #[error("url error: `{0}`")] @@ -30,11 +45,11 @@ pub enum CrawlerError { RequestError(#[from] reqwest::Error), } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub enum CrawledInstanceStatus { Ok(Duration), #[allow(dead_code)] - InvalidStatusCode(StatusCode, Duration), + InvalidStatusCode(String, Duration), StringNotFound, ConnectionError, RedirectPolicyError, @@ -62,14 +77,14 @@ impl std::fmt::Display for CrawledInstanceStatus { } } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct CrawledInstance { pub url: Url, pub status: CrawledInstanceStatus, pub tags: Vec, } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct CrawledService { pub name: String, pub instances: Vec, @@ -83,13 +98,13 @@ impl CrawledService { } } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct CrawledServices { pub services: HashMap, pub time: DateTime, } -#[derive(Debug)] +#[derive(Debug, Deserialize, Serialize)] pub enum CrawledData { CrawledServices(CrawledServices), InitialLoading, @@ -141,7 +156,23 @@ impl Crawler { Self { loaded_data, config: Arc::new(config), - data: RwLock::new(CrawledData::InitialLoading), + data: RwLock::new(CrawledData::CrawledServices(CrawledServices { + services: HashMap::new(), + time: utc_now(), + })), + crawler_lock: Mutex::new(()), + } + } + + pub fn with_data( + loaded_data: Arc>, + config: CrawlerConfig, + data: CrawledData, + ) -> Self { + Self { + loaded_data, + config: Arc::new(config), + data: RwLock::new(data), crawler_lock: Mutex::new(()), } } @@ -165,11 +196,11 @@ impl Crawler { )?; let test_url = instance.url.join(&service.test_url)?; - let start = SystemTime::now().duration_since(UNIX_EPOCH).unwrap(); + let start = system_now().duration_since(UNIX_EPOCH).unwrap(); let response = client.get(test_url).send().await; let status = match response { Ok(response) => { - let end = SystemTime::now().duration_since(UNIX_EPOCH).unwrap(); + let end = system_now().duration_since(UNIX_EPOCH).unwrap(); let status_code = response.status().as_u16(); if service.allowed_http_codes.is_allowed(status_code) { if let Some(search_string) = &service.search_string { @@ -183,7 +214,10 @@ impl Crawler { CrawledInstanceStatus::Ok(end - start) } } else { - CrawledInstanceStatus::InvalidStatusCode(response.status(), end - start) + CrawledInstanceStatus::InvalidStatusCode( + response.status().to_string(), + end - start, + ) } } Err(e) => match e { @@ -211,7 +245,7 @@ impl Crawler { Ok(ret) } - async fn crawl<'a>( + pub async fn crawl<'a>( &self, crawler_guard: Option>, ) -> Result<(), CrawlerError> { @@ -242,7 +276,63 @@ impl Crawler { ) }) .collect(); - let mut parallelise = Parallelise::with_capacity(self.config.max_concurrent_requests); + + #[cfg(not(target_arch = "wasm32"))] + let mut results = { + let mut parallelise = Parallelise::with_capacity(self.config.max_concurrent_requests); + + for service in self.loaded_data.read().await.services.values() { + let service = Arc::new(service.clone()); + for instance in &service.instances { + let loaded_data = self.loaded_data.clone(); + let config = self.config.clone(); + let instance = instance.clone(); + parallelise + .push(tokio::spawn(Self::crawl_single_instance( + config, + loaded_data.clone(), + service.clone(), + instance, + ))) + .await; + } + } + + parallelise.wait().await + }; + + #[cfg(target_arch = "wasm32")] + let mut results = { + let mut out = Vec::new(); + + for service in self.loaded_data.read().await.services.values() { + let service = Arc::new(service.clone()); + for instance in &service.instances { + let loaded_data = self.loaded_data.clone(); + let config = self.config.clone(); + let instance = instance.clone(); + let inst = match timeout( + Duration::from_secs(1), + Self::crawl_single_instance( + config, + loaded_data.clone(), + service.clone(), + instance, + ), + ) + .await + { + Ok(r) => r, + Err(_) => { + continue; + } + }; + out.push(inst); + } + } + + out + }; for service in self.loaded_data.read().await.services.values() { let service = Arc::new(service.clone()); @@ -250,19 +340,18 @@ impl Crawler { let loaded_data = self.loaded_data.clone(); let config = self.config.clone(); let instance = instance.clone(); - parallelise - .push(tokio::spawn(Self::crawl_single_instance( + results.push( + Self::crawl_single_instance( config, loaded_data.clone(), service.clone(), instance, - ))) - .await; + ) + .await, + ); } } - let results = parallelise.wait().await; - for result in results { let (crawled_instance, name) = match result { Ok(c) => c, @@ -281,7 +370,7 @@ impl Crawler { let mut data = self.data.write().await; data.replace(CrawledData::CrawledServices(CrawledServices { services: crawled_services, - time: Utc::now(), + time: utc_now(), })); match data.as_ref() { diff --git a/fastside-core/src/lib.rs b/fastside-core/src/lib.rs index 96c1528e..008fbfba 100644 --- a/fastside-core/src/lib.rs +++ b/fastside-core/src/lib.rs @@ -1,9 +1,4 @@ -#[cfg(not(target_arch = "wasm32"))] pub mod crawler; -#[cfg(target_arch = "wasm32")] -pub mod stub_crawler; -#[cfg(target_arch = "wasm32")] -pub use stub_crawler as crawler; pub mod errors; pub mod filters; pub mod routes; diff --git a/fastside-core/src/routes/index.rs b/fastside-core/src/routes/index.rs index 0c217dec..18a186a1 100644 --- a/fastside-core/src/routes/index.rs +++ b/fastside-core/src/routes/index.rs @@ -1,5 +1,6 @@ use askama::Template; use axum::{ + body::Bytes, extract::State, http::{header::CONTENT_TYPE, StatusCode}, response::{Html, IntoResponse}, @@ -14,7 +15,10 @@ use crate::{filters, types::AppState}; use fastside_shared::serde_types::ServicesData; pub fn router() -> Router> { - Router::new().route("/", get(index)) + Router::new() + .route("/", get(index)) + .route("/favicon.ico", get(favicon)) + .route("/robots.txt", get(robots_txt)) } /// The `IndexTemplate` structure renders the index page using the Askama template engine. @@ -55,3 +59,31 @@ pub async fn index(State(state): State>) -> impl IntoResponse { Err(_) => StatusCode::INTERNAL_SERVER_ERROR.into_response(), } } + +// Favicon as a static byte slice +const FAVICON: &[u8] = include_bytes!("../../static/favicon.ico"); + +// Handler for /favicon.ico +async fn favicon() -> impl IntoResponse { + ( + [ + ("Content-Type", "image/x-icon"), + ("Cache-Control", "public, max-age=3600"), + ], + Bytes::from_static(FAVICON), + ) +} + +// Robots.txt content as a static string +const ROBOTS_TXT: &str = "User-agent: *\nDisallow: /\n"; + +// Handler for /robots.txt +async fn robots_txt() -> impl IntoResponse { + ( + [ + ("Content-Type", "text/plain"), + ("Cache-Control", "public, max-age=3600"), + ], + ROBOTS_TXT, + ) +} diff --git a/fastside-core/src/stub_crawler.rs b/fastside-core/src/stub_crawler.rs deleted file mode 100644 index f0916b77..00000000 --- a/fastside-core/src/stub_crawler.rs +++ /dev/null @@ -1,296 +0,0 @@ -use std::{ - collections::HashMap, - sync::Arc, - time::{Duration, SystemTime, UNIX_EPOCH}, -}; - -use chrono::{DateTime, Utc}; -use reqwest::StatusCode; -use thiserror::Error; -use tokio::{ - sync::{Mutex, MutexGuard, RwLock}, - time::sleep, -}; -use url::Url; - -use fastside_shared::config::CrawlerConfig; -use fastside_shared::{ - client_builder::build_client, - parallel::Parallelise, - serde_types::{HttpCodeRanges, Instance, Service}, -}; - -use crate::types::LoadedData; - -#[derive(Error, Debug)] -pub enum CrawlerError { - #[error("url error: `{0}`")] - UrlError(#[from] url::ParseError), - #[error("request error: `{0}`")] - RequestError(#[from] reqwest::Error), -} - -#[derive(Clone, Debug)] -pub enum CrawledInstanceStatus { - Ok(Duration), - #[allow(dead_code)] - InvalidStatusCode(StatusCode, Duration), - StringNotFound, - ConnectionError, - RedirectPolicyError, - BuilderError, - RequestError, - BodyError, - DecodeError, - TimedOut, - Unknown, -} - -impl CrawledInstanceStatus { - /// Used for sorting values in index.html template. - pub fn as_isize(&self) -> isize { - match self { - Self::Ok(d) => d.as_millis() as isize, - _ => isize::MAX, - } - } -} - -impl std::fmt::Display for CrawledInstanceStatus { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?}", self) - } -} - -#[derive(Clone, Debug)] -pub struct CrawledInstance { - pub url: Url, - pub status: CrawledInstanceStatus, - pub tags: Vec, -} - -#[derive(Clone, Debug)] -pub struct CrawledService { - pub name: String, - pub instances: Vec, -} - -impl CrawledService { - pub fn get_alive_instances(&self) -> impl Iterator { - self.instances - .iter() - .filter(|s| matches!(&s.status, CrawledInstanceStatus::Ok(_))) - } -} - -#[derive(Clone, Debug)] -pub struct CrawledServices { - pub services: HashMap, - pub time: DateTime, -} - -#[derive(Debug)] -pub enum CrawledData { - CrawledServices(CrawledServices), - InitialLoading, - ReloadingServices(CrawledServices), -} - -impl CrawledData { - pub fn get_services(&self) -> Option<&CrawledServices> { - match self { - Self::CrawledServices(s) => Some(s), - Self::InitialLoading => None, - Self::ReloadingServices(current) => Some(current), - } - } - - pub fn is_reloading(&self) -> bool { - matches!(self, Self::ReloadingServices { .. }) - } - - pub fn replace(&mut self, new: CrawledData) { - *self = new; - } - - pub fn make_reloading(&mut self) { - let current = match self { - Self::CrawledServices(s) => s.clone(), - _ => return, - }; - *self = Self::ReloadingServices(current); - } -} - -impl AsRef for CrawledData { - fn as_ref(&self) -> &CrawledData { - self - } -} - -#[derive(Debug)] -pub struct Crawler { - loaded_data: Arc>, - config: Arc, - data: RwLock, - crawler_lock: Mutex<()>, -} - -impl Crawler { - pub fn new(loaded_data: Arc>, config: CrawlerConfig) -> Self { - Self { - loaded_data, - config: Arc::new(config), - data: RwLock::new(CrawledData::InitialLoading), - crawler_lock: Mutex::new(()), - } - } - - #[inline] - pub async fn read(&self) -> tokio::sync::RwLockReadGuard { - self.data.read().await - } - - async fn crawl_single_instance( - config: Arc, - loaded_data: Arc>, - service: Arc, - instance: Instance, - ) -> Result<(CrawledInstance, String), CrawlerError> { - let client = build_client( - service.as_ref(), - config.as_ref(), - &loaded_data.read().await.proxies, - &instance, - )?; - - let test_url = instance.url.join(&service.test_url)?; - let start = SystemTime::now().duration_since(UNIX_EPOCH).unwrap(); - let response = client.get(test_url).send().await; - let status = match response { - Ok(response) => { - let end = SystemTime::now().duration_since(UNIX_EPOCH).unwrap(); - let status_code = response.status().as_u16(); - if service.allowed_http_codes.is_allowed(status_code) { - if let Some(search_string) = &service.search_string { - let body = response.text().await?; - if !body.contains(search_string) { - CrawledInstanceStatus::StringNotFound - } else { - CrawledInstanceStatus::Ok(end - start) - } - } else { - CrawledInstanceStatus::Ok(end - start) - } - } else { - CrawledInstanceStatus::InvalidStatusCode(response.status(), end - start) - } - } - Err(e) => match e { - _ if e.is_timeout() => CrawledInstanceStatus::TimedOut, - _ if e.is_builder() => CrawledInstanceStatus::BuilderError, - _ if e.is_redirect() => CrawledInstanceStatus::RedirectPolicyError, - _ if e.is_request() => CrawledInstanceStatus::RequestError, - _ if e.is_body() => CrawledInstanceStatus::BodyError, - _ if e.is_decode() => CrawledInstanceStatus::DecodeError, - #[cfg(not(target_arch = "wasm32"))] - _ if e.is_connect() => CrawledInstanceStatus::ConnectionError, - _ => CrawledInstanceStatus::Unknown, - }, - }; - - let ret = ( - CrawledInstance { - url: instance.url.clone(), - tags: instance.tags.clone(), - status, - }, - service.name.clone(), - ); - debug!("Crawled instance: {ret:?}"); - Ok(ret) - } - - async fn crawl<'a>( - &self, - crawler_guard: Option>, - ) -> Result<(), CrawlerError> { - let crawler_guard = match crawler_guard { - Some(guard) => guard, - None => { - let Ok(crawler_guard) = self.crawler_lock.try_lock() else { - warn!("Crawler lock is already acquired, skipping crawl"); - return Ok(()); - }; - crawler_guard - } - }; - - let mut crawled_services: HashMap = self - .loaded_data - .read() - .await - .services - .keys() - .map(|name| { - ( - name.clone(), - CrawledService { - name: name.clone(), - instances: Vec::new(), - }, - ) - }) - .collect(); - - for service in self.loaded_data.read().await.services.values() { - let service = Arc::new(service.clone()); - for instance in &service.instances { - let loaded_data = self.loaded_data.clone(); - let config = self.config.clone(); - let instance = instance.clone(); - } - } - - let mut data = self.data.write().await; - data.replace(CrawledData::CrawledServices(CrawledServices { - services: crawled_services, - time: Utc::now(), - })); - - match data.as_ref() { - CrawledData::ReloadingServices { .. } => { - info!("Finished reloading services"); - } - CrawledData::InitialLoading => { - info!("Finished initial crawl, we are ready to serve requests"); - } - CrawledData::CrawledServices(_) => { - debug!("Finished crawl"); - } - } - - drop(crawler_guard); - Ok(()) - } - - /// Run crawler instantly in update loaded_data mode. - pub async fn update_crawl(&self) -> Result<(), CrawlerError> { - let crawler_guard = self.crawler_lock.lock().await; - let mut data = self.data.write().await; - data.make_reloading(); - drop(data); - self.crawl(Some(crawler_guard)).await - } - - pub async fn crawler_loop(&self) { - loop { - debug!("Starting crawl"); - if let Err(e) = self.crawl(None).await { - error!("Error occured during crawl loop: {e}"); - }; - debug!("Next crawl will start in {:?}", self.config.ping_interval); - sleep(self.config.ping_interval).await; - } - } -} diff --git a/fastside-core/src/types.rs b/fastside-core/src/types.rs index 1a1a79de..054618a2 100644 --- a/fastside-core/src/types.rs +++ b/fastside-core/src/types.rs @@ -4,6 +4,7 @@ use fastside_shared::{ config::{AppConfig, ProxyData, UserConfig}, serde_types::ServicesData, }; +use serde::{Deserialize, Serialize}; use tokio::sync::RwLock; use crate::crawler::Crawler; @@ -15,7 +16,7 @@ pub struct CompiledRegexSearch { pub type Regexes = HashMap>; -#[derive(Debug)] +#[derive(Debug, Deserialize, Serialize)] pub struct LoadedData { pub services: ServicesData, pub proxies: ProxyData, From db3925b8b0522a38eedb6b783d0cfcb94047b8eb Mon Sep 17 00:00:00 2001 From: cofob Date: Mon, 13 Jan 2025 15:04:10 +0400 Subject: [PATCH 3/5] Ignore node_modules --- .gitignore | 1 + .../node_modules/.cache/wrangler/user-id.json | 3 --- .../node_modules/.cache/wrangler/wrangler-account.json | 6 ------ fastside-cloudflare/node_modules/.mf/cf.json | 1 - 4 files changed, 1 insertion(+), 10 deletions(-) delete mode 100644 fastside-cloudflare/node_modules/.cache/wrangler/user-id.json delete mode 100644 fastside-cloudflare/node_modules/.cache/wrangler/wrangler-account.json delete mode 100644 fastside-cloudflare/node_modules/.mf/cf.json diff --git a/.gitignore b/.gitignore index 6ec83ff9..70d9f9e5 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ /fastside-cloudflare/build/ /fastside-cloudflare/.wrangler/ +/fastside-cloudflare/node_modules/ diff --git a/fastside-cloudflare/node_modules/.cache/wrangler/user-id.json b/fastside-cloudflare/node_modules/.cache/wrangler/user-id.json deleted file mode 100644 index 6a919303..00000000 --- a/fastside-cloudflare/node_modules/.cache/wrangler/user-id.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "userId": "8632af6789c40dc28769656e59b04edd" -} \ No newline at end of file diff --git a/fastside-cloudflare/node_modules/.cache/wrangler/wrangler-account.json b/fastside-cloudflare/node_modules/.cache/wrangler/wrangler-account.json deleted file mode 100644 index 5152560d..00000000 --- a/fastside-cloudflare/node_modules/.cache/wrangler/wrangler-account.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "account": { - "id": "025464e503cbb7385dff88bd6d3c9128", - "name": "Cofob@riseup.net's Account" - } -} \ No newline at end of file diff --git a/fastside-cloudflare/node_modules/.mf/cf.json b/fastside-cloudflare/node_modules/.mf/cf.json deleted file mode 100644 index fe38d0ad..00000000 --- a/fastside-cloudflare/node_modules/.mf/cf.json +++ /dev/null @@ -1 +0,0 @@ -{"clientTcpRtt":8,"longitude":"44.83200","latitude":"41.69590","tlsCipher":"AEAD-AES256-GCM-SHA384","continent":"AS","asn":35805,"clientAcceptEncoding":"br, gzip, deflate","country":"GE","tlsClientAuth":{"certIssuerDNLegacy":"","certIssuerSKI":"","certSubjectDNRFC2253":"","certSubjectDNLegacy":"","certFingerprintSHA256":"","certNotBefore":"","certSKI":"","certSerial":"","certIssuerDN":"","certVerified":"NONE","certNotAfter":"","certSubjectDN":"","certPresented":"0","certRevoked":"0","certIssuerSerial":"","certIssuerDNRFC2253":"","certFingerprintSHA1":""},"verifiedBotCategory":"","tlsExportedAuthenticator":{"clientFinished":"2fbd0eaa083560b4952534f85c144d2bf6cbe3281413a5ab9c801620736342953765a28b594ebd208dba7eef1416a65d","clientHandshake":"6395396631724760823c3f8c064c86bdc6ce0e4523260441c860e7711c7c779e799afe47b29c918274795c6953b25532","serverHandshake":"ea7ffef62e9c8d278bfb00ccd199bfec2591b80132eae9eadbf7d585a8976fd0c80bb1b6b9f6461c3e7a2e8341ba95d2","serverFinished":"94aa5d6c86033bca9802cad73c2e5474b58083867f5d38cc899768a805889c4b2cd8d3f40e410f0d161afea14ecc4ec4"},"tlsVersion":"TLSv1.3","colo":"TBS","timezone":"Asia/Tbilisi","tlsClientHelloLength":"383","edgeRequestKeepAliveStatus":1,"requestPriority":"","tlsClientExtensionsSha1":"1eY97BUYYO8vDaTfHQywB1pcNdM=","region":"Tbilisi","city":"Tbilisi","regionCode":"TB","asOrganization":"JSC Silknet","tlsClientRandom":"pjM/+sdECVpdpI7iVNXDyvcsLWwF9WkAM5mCrxg3L08=","httpProtocol":"HTTP/1.1","botManagement":{"corporateProxy":false,"verifiedBot":false,"jsDetection":{"passed":false},"staticResource":false,"detectionIds":{},"score":99}} \ No newline at end of file From 2566ccf944ee81060b13943a442594507b81a81d Mon Sep 17 00:00:00 2001 From: cofob Date: Mon, 13 Jan 2025 15:06:15 +0400 Subject: [PATCH 4/5] Ignore .cache --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 70d9f9e5..0f61c6e4 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ /fastside-cloudflare/build/ /fastside-cloudflare/.wrangler/ +/fastside-cloudflare/.cache/ /fastside-cloudflare/node_modules/ From f3d8ae730bb93889e3def88d77a9c8a0feb683ec Mon Sep 17 00:00:00 2001 From: cofob Date: Sun, 16 Mar 2025 22:55:42 +0400 Subject: [PATCH 5/5] Working redirect --- fastside-cloudflare/package-lock.json | 1440 +++++++++++++++++++++++++ fastside-cloudflare/package.json | 9 + fastside-cloudflare/src/lib.rs | 77 +- fastside-core/src/crawler.rs | 2 +- fastside-core/src/routes/mod.rs | 3 +- fastside-core/src/routes/redirect.rs | 247 +++++ fastside-core/src/types.rs | 2 +- flake.lock | 18 +- shell.nix | 4 +- 9 files changed, 1745 insertions(+), 57 deletions(-) create mode 100644 fastside-cloudflare/package-lock.json create mode 100644 fastside-cloudflare/package.json create mode 100644 fastside-core/src/routes/redirect.rs diff --git a/fastside-cloudflare/package-lock.json b/fastside-cloudflare/package-lock.json new file mode 100644 index 00000000..91f03713 --- /dev/null +++ b/fastside-cloudflare/package-lock.json @@ -0,0 +1,1440 @@ +{ + "name": "fastside-cloudflare", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "fastside-cloudflare", + "dependencies": { + "wrangler": "^4.0.0" + } + }, + "node_modules/@cloudflare/kv-asset-handler": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@cloudflare/kv-asset-handler/-/kv-asset-handler-0.4.0.tgz", + "integrity": "sha512-+tv3z+SPp+gqTIcImN9o0hqE9xyfQjI1XD9pL6NuKjua9B1y7mNYv0S9cP+QEbA4ppVgGZEmKOvHX5G5Ei1CVA==", + "license": "MIT OR Apache-2.0", + "dependencies": { + "mime": "^3.0.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@cloudflare/unenv-preset": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@cloudflare/unenv-preset/-/unenv-preset-2.0.2.tgz", + "integrity": "sha512-nyzYnlZjjV5xT3LizahG1Iu6mnrCaxglJ04rZLpDwlDVDZ7v46lNsfxhV3A/xtfgQuSHmLnc6SVI+KwBpc3Lwg==", + "license": "MIT OR Apache-2.0", + "peerDependencies": { + "unenv": "2.0.0-rc.14", + "workerd": "^1.20250124.0" + }, + "peerDependenciesMeta": { + "workerd": { + "optional": true + } + } + }, + "node_modules/@cloudflare/workerd-darwin-64": { + "version": "1.20250310.0", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20250310.0.tgz", + "integrity": "sha512-LkLJO6F8lRNaCbK5sQCITi66SyCirDpffRuI5/5iILDJWQU4KVvAOKPvHrd4E5h/WDm9FGd22zMJwky7SxaNjg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cloudflare/workerd-darwin-arm64": { + "version": "1.20250310.0", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20250310.0.tgz", + "integrity": "sha512-WythDJQbsU3Ii1hhA7pJZLBQlHezeYWAnaMnv3gS2Exj45oF8G4chFvrO7zCzjlcJXwSeBTtQRJqxw9AiUDhyA==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cloudflare/workerd-linux-64": { + "version": "1.20250310.0", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20250310.0.tgz", + "integrity": "sha512-LbP769tT4/5QBHSj4lCt99QIKTi6cU+wYhLfF7rEtYHBnZS2+nIw9xttAzxeERx/aFrU+mxLcYPFV8fUeVxGng==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cloudflare/workerd-linux-arm64": { + "version": "1.20250310.0", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20250310.0.tgz", + "integrity": "sha512-FzWeKM6id20EMZACaDg0Kkvg1C4lvXZgLBXVI6h6xaXTNFReoyEp4v4eMrRTuja5ec5k+m5iGKjP4/bMWJp9ew==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cloudflare/workerd-windows-64": { + "version": "1.20250310.0", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20250310.0.tgz", + "integrity": "sha512-04OgaDzm8/8nkjF3tovB+WywZLjSdAHCQT2omXKCwH3EDd1kpd8vvzE1pErtdIyKCOf9/sArY4BhPdxRj7ijlg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "0.3.9" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.3.1.tgz", + "integrity": "sha512-kEBmG8KyqtxJZv+ygbEim+KCGtIq1fC22Ms3S4ziXmYKm8uyoLX0MHONVKwp+9opg390VaKRNt4a7A9NwmpNhw==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.24.2.tgz", + "integrity": "sha512-thpVCb/rhxE/BnMLQ7GReQLLN8q9qbHmI55F4489/ByVg2aQaQ6kbcLb6FHkocZzQhxc4gx0sCk0tJkKBFzDhA==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.24.2.tgz", + "integrity": "sha512-tmwl4hJkCfNHwFB3nBa8z1Uy3ypZpxqxfTQOcHX+xRByyYgunVbZ9MzUUfb0RxaHIMnbHagwAxuTL+tnNM+1/Q==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.24.2.tgz", + "integrity": "sha512-cNLgeqCqV8WxfcTIOeL4OAtSmL8JjcN6m09XIgro1Wi7cF4t/THaWEa7eL5CMoMBdjoHOTh/vwTO/o2TRXIyzg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.24.2.tgz", + "integrity": "sha512-B6Q0YQDqMx9D7rvIcsXfmJfvUYLoP722bgfBlO5cGvNVb5V/+Y7nhBE3mHV9OpxBf4eAS2S68KZztiPaWq4XYw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.24.2.tgz", + "integrity": "sha512-kj3AnYWc+CekmZnS5IPu9D+HWtUI49hbnyqk0FLEJDbzCIQt7hg7ucF1SQAilhtYpIujfaHr6O0UHlzzSPdOeA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.24.2.tgz", + "integrity": "sha512-WeSrmwwHaPkNR5H3yYfowhZcbriGqooyu3zI/3GGpF8AyUdsrrP0X6KumITGA9WOyiJavnGZUwPGvxvwfWPHIA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.24.2.tgz", + "integrity": "sha512-UN8HXjtJ0k/Mj6a9+5u6+2eZ2ERD7Edt1Q9IZiB5UZAIdPnVKDoG7mdTVGhHJIeEml60JteamR3qhsr1r8gXvg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.24.2.tgz", + "integrity": "sha512-TvW7wE/89PYW+IevEJXZ5sF6gJRDY/14hyIGFXdIucxCsbRmLUcjseQu1SyTko+2idmCw94TgyaEZi9HUSOe3Q==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.24.2.tgz", + "integrity": "sha512-n0WRM/gWIdU29J57hJyUdIsk0WarGd6To0s+Y+LwvlC55wt+GT/OgkwoXCXvIue1i1sSNWblHEig00GBWiJgfA==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.24.2.tgz", + "integrity": "sha512-7HnAD6074BW43YvvUmE/35Id9/NB7BeX5EoNkK9obndmZBUk8xmJJeU7DwmUeN7tkysslb2eSl6CTrYz6oEMQg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.24.2.tgz", + "integrity": "sha512-sfv0tGPQhcZOgTKO3oBE9xpHuUqguHvSo4jl+wjnKwFpapx+vUDcawbwPNuBIAYdRAvIDBfZVvXprIj3HA+Ugw==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.24.2.tgz", + "integrity": "sha512-CN9AZr8kEndGooS35ntToZLTQLHEjtVB5n7dl8ZcTZMonJ7CCfStrYhrzF97eAecqVbVJ7APOEe18RPI4KLhwQ==", + "cpu": [ + "loong64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.24.2.tgz", + "integrity": "sha512-iMkk7qr/wl3exJATwkISxI7kTcmHKE+BlymIAbHO8xanq/TjHaaVThFF6ipWzPHryoFsesNQJPE/3wFJw4+huw==", + "cpu": [ + "mips64el" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.24.2.tgz", + "integrity": "sha512-shsVrgCZ57Vr2L8mm39kO5PPIb+843FStGt7sGGoqiiWYconSxwTiuswC1VJZLCjNiMLAMh34jg4VSEQb+iEbw==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.24.2.tgz", + "integrity": "sha512-4eSFWnU9Hhd68fW16GD0TINewo1L6dRrB+oLNNbYyMUAeOD2yCK5KXGK1GH4qD/kT+bTEXjsyTCiJGHPZ3eM9Q==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.24.2.tgz", + "integrity": "sha512-S0Bh0A53b0YHL2XEXC20bHLuGMOhFDO6GN4b3YjRLK//Ep3ql3erpNcPlEFed93hsQAjAQDNsvcK+hV90FubSw==", + "cpu": [ + "s390x" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.24.2.tgz", + "integrity": "sha512-8Qi4nQcCTbLnK9WoMjdC9NiTG6/E38RNICU6sUNqK0QFxCYgoARqVqxdFmWkdonVsvGqWhmm7MO0jyTqLqwj0Q==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.24.2.tgz", + "integrity": "sha512-wuLK/VztRRpMt9zyHSazyCVdCXlpHkKm34WUyinD2lzK07FAHTq0KQvZZlXikNWkDGoT6x3TD51jKQ7gMVpopw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.24.2.tgz", + "integrity": "sha512-VefFaQUc4FMmJuAxmIHgUmfNiLXY438XrL4GDNV1Y1H/RW3qow68xTwjZKfj/+Plp9NANmzbH5R40Meudu8mmw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.24.2.tgz", + "integrity": "sha512-YQbi46SBct6iKnszhSvdluqDmxCJA+Pu280Av9WICNwQmMxV7nLRHZfjQzwbPs3jeWnuAhE9Jy0NrnJ12Oz+0A==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.24.2.tgz", + "integrity": "sha512-+iDS6zpNM6EnJyWv0bMGLWSWeXGN/HTaF/LXHXHwejGsVi+ooqDfMCCTerNFxEkM3wYVcExkeGXNqshc9iMaOA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.24.2.tgz", + "integrity": "sha512-hTdsW27jcktEvpwNHJU4ZwWFGkz2zRJUz8pvddmXPtXDzVKTTINmlmga3ZzwcuMpUvLw7JkLy9QLKyGpD2Yxig==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.24.2.tgz", + "integrity": "sha512-LihEQ2BBKVFLOC9ZItT9iFprsE9tqjDjnbulhHoFxYQtQfai7qfluVODIYxt1PgdoyQkz23+01rzwNwYfutxUQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.24.2.tgz", + "integrity": "sha512-q+iGUwfs8tncmFC9pcnD5IvRHAzmbwQ3GPS5/ceCyHdjXubwQWI12MKWSNSMYLJMq23/IUCvJMS76PDqXe1fxA==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.24.2.tgz", + "integrity": "sha512-7VTgWzgMGvup6aSqDPLiW5zHaxYJGTO4OokMjIlrCtf+VpEL+cXKtCvg723iguPYI5oaUNdS+/V7OU2gvXVWEg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@fastify/busboy": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.1.tgz", + "integrity": "sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==", + "license": "MIT", + "engines": { + "node": ">=14" + } + }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.33.5.tgz", + "integrity": "sha512-UT4p+iz/2H4twwAoLCqfA9UH5pI6DggwKEGuaPy7nCVQ8ZsiY5PIcrRvD1DzuY3qYL07NtIQcWnBSY/heikIFQ==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.0.4" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.5.tgz", + "integrity": "sha512-fyHac4jIc1ANYGRDxtiqelIbdWkIuQaI84Mv45KvGRRxSAa7o7d1ZKAOBaYbnepLC1WqxfpimdeWfvqqSGwR2Q==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.0.4" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.0.4.tgz", + "integrity": "sha512-XblONe153h0O2zuFfTAbQYAX2JhYmDHeWikp1LM9Hul9gVPjFY427k6dFEcOL72O01QxQsWi761svJ/ev9xEDg==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.0.4.tgz", + "integrity": "sha512-xnGR8YuZYfJGmWPvmlunFaWJsb9T/AO2ykoP3Fz/0X5XV2aoYBPkX6xqCQvUTKKiLddarLaxpzNe+b1hjeWHAQ==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.0.5.tgz", + "integrity": "sha512-gvcC4ACAOPRNATg/ov8/MnbxFDJqf/pDePbBnuBDcjsI8PssmjoKMAz4LtLaVi+OnSb5FK/yIOamqDwGmXW32g==", + "cpu": [ + "arm" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.0.4.tgz", + "integrity": "sha512-9B+taZ8DlyyqzZQnoeIvDVR/2F4EbMepXMc/NdVbkzsJbzkUjhXv/70GQJ7tdLA4YJgNP25zukcxpX2/SueNrA==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.0.4.tgz", + "integrity": "sha512-u7Wz6ntiSSgGSGcjZ55im6uvTrOxSIS8/dgoVMoiGE9I6JAfU50yH5BoDlYA1tcuGS7g/QNtetJnxA6QEsCVTA==", + "cpu": [ + "s390x" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.0.4.tgz", + "integrity": "sha512-MmWmQ3iPFZr0Iev+BAgVMb3ZyC4KeFc3jFxnNbEPas60e1cIfevbtuyf9nDGIzOaW9PdnDciJm+wFFaTlj5xYw==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.0.4.tgz", + "integrity": "sha512-9Ti+BbTYDcsbp4wfYib8Ctm1ilkugkA/uscUn6UXK1ldpC1JjiXbLfFZtRlBhjPZ5o1NCLiDbg8fhUPKStHoTA==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.0.4.tgz", + "integrity": "sha512-viYN1KX9m+/hGkJtvYYp+CCLgnJXwiQB39damAO7WMdKWlIhmYTfHjwSbQeUK/20vY154mwezd9HflVFM1wVSw==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.5.tgz", + "integrity": "sha512-JTS1eldqZbJxjvKaAkxhZmBqPRGmxgu+qFKSInv8moZ2AmT5Yib3EQ1c6gp493HvrvV8QgdOXdyaIBrhvFhBMQ==", + "cpu": [ + "arm" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.0.5" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.5.tgz", + "integrity": "sha512-JMVv+AMRyGOHtO1RFBiJy/MBsgz0x4AWrT6QoEVVTyh1E39TrCUpTRI7mx9VksGX4awWASxqCYLCV4wBZHAYxA==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.0.4" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.5.tgz", + "integrity": "sha512-y/5PCd+mP4CA/sPDKl2961b+C9d+vPAveS33s6Z3zfASk2j5upL6fXVPZi7ztePZ5CuH+1kW8JtvxgbuXHRa4Q==", + "cpu": [ + "s390x" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.0.4" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.5.tgz", + "integrity": "sha512-opC+Ok5pRNAzuvq1AG0ar+1owsu842/Ab+4qvU879ippJBHvyY5n2mxF1izXqkPYlGuP/M556uh53jRLJmzTWA==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.0.4" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.5.tgz", + "integrity": "sha512-XrHMZwGQGvJg2V/oRSUfSAfjfPxO+4DkiRh6p2AFjLQztWUuY/o8Mq0eMQVIY7HJ1CDQUJlxGGZRw1a5bqmd1g==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.0.4" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.5.tgz", + "integrity": "sha512-WT+d/cgqKkkKySYmqoZ8y3pxx7lx9vVejxW/W4DOFMYVSkErR+w7mf2u8m/y4+xHe7yY9DAXQMWQhpnMuFfScw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.0.4" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.33.5.tgz", + "integrity": "sha512-ykUW4LVGaMcU9lu9thv85CbRMAwfeadCJHRsg2GmeRa/cJxsVY9Rbd57JcMxBkKHag5U/x7TSBpScF4U8ElVzg==", + "cpu": [ + "wasm32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.2.0" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.5.tgz", + "integrity": "sha512-T36PblLaTwuVJ/zw/LaH0PdZkRz5rd3SmMHX8GSmR7vtNSP5Z6bQkExdSK7xGWyxLw4sUknBuugTelgw2faBbQ==", + "cpu": [ + "ia32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.5.tgz", + "integrity": "sha512-MpY/o8/8kj+EcnxwvrP4aTJSWw/aZ7JIGR4aBeZkZw5B7/Jn+tY9/VNwtcoGmdT7GfggGIU4kygOMSbYnOrAbg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz", + "integrity": "sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==", + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", + "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, + "node_modules/acorn": { + "version": "8.14.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.0.tgz", + "integrity": "sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA==", + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-walk": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.2.tgz", + "integrity": "sha512-cjkyv4OtNCIeqhHrfS81QWXoCBPExR/J62oyEqepVw8WaQeSqpW2uhuLPh1m9eWhDuOo/jUXVTlifvesOWp/4A==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/as-table": { + "version": "1.0.55", + "resolved": "https://registry.npmjs.org/as-table/-/as-table-1.0.55.tgz", + "integrity": "sha512-xvsWESUJn0JN421Xb9MQw6AsMHRCUknCe0Wjlxvjud80mU4E6hQf1A6NzQKcYNmYw62MfzEtXc+badstZP3JpQ==", + "license": "MIT", + "dependencies": { + "printable-characters": "^1.0.42" + } + }, + "node_modules/blake3-wasm": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/blake3-wasm/-/blake3-wasm-2.1.5.tgz", + "integrity": "sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==", + "license": "MIT" + }, + "node_modules/color": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz", + "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==", + "license": "MIT", + "optional": true, + "dependencies": { + "color-convert": "^2.0.1", + "color-string": "^1.9.0" + }, + "engines": { + "node": ">=12.5.0" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT", + "optional": true + }, + "node_modules/color-string": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", + "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", + "license": "MIT", + "optional": true, + "dependencies": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } + }, + "node_modules/cookie": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz", + "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/data-uri-to-buffer": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-2.0.2.tgz", + "integrity": "sha512-ND9qDTLc6diwj+Xe5cdAgVTbLVdXbtxTJRXRhli8Mowuaan+0EJOtdqJ0QCHNSSPyoXGx9HX2/VMnKeC34AChA==", + "license": "MIT" + }, + "node_modules/defu": { + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/defu/-/defu-6.1.4.tgz", + "integrity": "sha512-mEQCMmwJu317oSz8CwdIOdwf3xMif1ttiM8LTufzc3g6kR+9Pe236twL8j3IYT1F7GfRgGcW6MWxzZjLIkuHIg==", + "license": "MIT" + }, + "node_modules/detect-libc": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz", + "integrity": "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==", + "license": "Apache-2.0", + "optional": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/esbuild": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.24.2.tgz", + "integrity": "sha512-+9egpBW8I3CD5XPe0n6BfT5fxLzxrlDzqydF3aviG+9ni1lDC/OvMHcxqEFV0+LANZG5R1bFMWfUrjVsdwxJvA==", + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.24.2", + "@esbuild/android-arm": "0.24.2", + "@esbuild/android-arm64": "0.24.2", + "@esbuild/android-x64": "0.24.2", + "@esbuild/darwin-arm64": "0.24.2", + "@esbuild/darwin-x64": "0.24.2", + "@esbuild/freebsd-arm64": "0.24.2", + "@esbuild/freebsd-x64": "0.24.2", + "@esbuild/linux-arm": "0.24.2", + "@esbuild/linux-arm64": "0.24.2", + "@esbuild/linux-ia32": "0.24.2", + "@esbuild/linux-loong64": "0.24.2", + "@esbuild/linux-mips64el": "0.24.2", + "@esbuild/linux-ppc64": "0.24.2", + "@esbuild/linux-riscv64": "0.24.2", + "@esbuild/linux-s390x": "0.24.2", + "@esbuild/linux-x64": "0.24.2", + "@esbuild/netbsd-arm64": "0.24.2", + "@esbuild/netbsd-x64": "0.24.2", + "@esbuild/openbsd-arm64": "0.24.2", + "@esbuild/openbsd-x64": "0.24.2", + "@esbuild/sunos-x64": "0.24.2", + "@esbuild/win32-arm64": "0.24.2", + "@esbuild/win32-ia32": "0.24.2", + "@esbuild/win32-x64": "0.24.2" + } + }, + "node_modules/exit-hook": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/exit-hook/-/exit-hook-2.2.1.tgz", + "integrity": "sha512-eNTPlAD67BmP31LDINZ3U7HSF8l57TxOY2PmBJ1shpCvpnxBF93mWCE8YHBnXs8qiUZJc9WDcWIeC3a2HIAMfw==", + "license": "MIT", + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/exsolve": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/exsolve/-/exsolve-1.0.4.tgz", + "integrity": "sha512-xsZH6PXaER4XoV+NiT7JHp1bJodJVT+cxeSH1G0f0tlT0lJqYuHUP3bUx2HtfTDvOagMINYp8rsqusxud3RXhw==", + "license": "MIT" + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/get-source": { + "version": "2.0.12", + "resolved": "https://registry.npmjs.org/get-source/-/get-source-2.0.12.tgz", + "integrity": "sha512-X5+4+iD+HoSeEED+uwrQ07BOQr0kEDFMVqqpBuI+RaZBpBpHCuXxo70bjar6f0b0u/DQJsJ7ssurpP0V60Az+w==", + "license": "Unlicense", + "dependencies": { + "data-uri-to-buffer": "^2.0.0", + "source-map": "^0.6.1" + } + }, + "node_modules/glob-to-regexp": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz", + "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==", + "license": "BSD-2-Clause" + }, + "node_modules/is-arrayish": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", + "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==", + "license": "MIT", + "optional": true + }, + "node_modules/mime": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz", + "integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/miniflare": { + "version": "4.20250310.0", + "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-4.20250310.0.tgz", + "integrity": "sha512-WL4hKQIfXyTxKyQzxJyyy/v+OYSiF51s3Qe1Q4W4MjHJbtiN8Kg7+oeTdHYgqYehanN2zYoSKJ/xooIy8q5+XA==", + "license": "MIT", + "dependencies": { + "@cspotcode/source-map-support": "0.8.1", + "acorn": "8.14.0", + "acorn-walk": "8.3.2", + "exit-hook": "2.2.1", + "glob-to-regexp": "0.4.1", + "stoppable": "1.1.0", + "undici": "^5.28.5", + "workerd": "1.20250310.0", + "ws": "8.18.0", + "youch": "3.2.3", + "zod": "3.22.3" + }, + "bin": { + "miniflare": "bootstrap.js" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/mustache": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", + "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", + "license": "MIT", + "bin": { + "mustache": "bin/mustache" + } + }, + "node_modules/ohash": { + "version": "2.0.11", + "resolved": "https://registry.npmjs.org/ohash/-/ohash-2.0.11.tgz", + "integrity": "sha512-RdR9FQrFwNBNXAr4GixM8YaRZRJ5PUWbKYbE5eOsrwAjJW0q2REGcf79oYPsLyskQCZG1PLN+S/K1V00joZAoQ==", + "license": "MIT" + }, + "node_modules/path-to-regexp": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz", + "integrity": "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==", + "license": "MIT" + }, + "node_modules/pathe": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", + "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", + "license": "MIT" + }, + "node_modules/printable-characters": { + "version": "1.0.42", + "resolved": "https://registry.npmjs.org/printable-characters/-/printable-characters-1.0.42.tgz", + "integrity": "sha512-dKp+C4iXWK4vVYZmYSd0KBH5F/h1HoZRsbJ82AVKRO3PEo8L4lBS/vLwhVtpwwuYcoIsVY+1JYKR268yn480uQ==", + "license": "Unlicense" + }, + "node_modules/semver": { + "version": "7.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.1.tgz", + "integrity": "sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA==", + "license": "ISC", + "optional": true, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/sharp": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.5.tgz", + "integrity": "sha512-haPVm1EkS9pgvHrQ/F3Xy+hgcuMV0Wm9vfIBSiwZ05k+xgb0PkBQpGsAA/oWdDobNaZTH5ppvHtzCFbnSEwHVw==", + "hasInstallScript": true, + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "color": "^4.2.3", + "detect-libc": "^2.0.3", + "semver": "^7.6.3" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.33.5", + "@img/sharp-darwin-x64": "0.33.5", + "@img/sharp-libvips-darwin-arm64": "1.0.4", + "@img/sharp-libvips-darwin-x64": "1.0.4", + "@img/sharp-libvips-linux-arm": "1.0.5", + "@img/sharp-libvips-linux-arm64": "1.0.4", + "@img/sharp-libvips-linux-s390x": "1.0.4", + "@img/sharp-libvips-linux-x64": "1.0.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.0.4", + "@img/sharp-libvips-linuxmusl-x64": "1.0.4", + "@img/sharp-linux-arm": "0.33.5", + "@img/sharp-linux-arm64": "0.33.5", + "@img/sharp-linux-s390x": "0.33.5", + "@img/sharp-linux-x64": "0.33.5", + "@img/sharp-linuxmusl-arm64": "0.33.5", + "@img/sharp-linuxmusl-x64": "0.33.5", + "@img/sharp-wasm32": "0.33.5", + "@img/sharp-win32-ia32": "0.33.5", + "@img/sharp-win32-x64": "0.33.5" + } + }, + "node_modules/simple-swizzle": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", + "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", + "license": "MIT", + "optional": true, + "dependencies": { + "is-arrayish": "^0.3.1" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stacktracey": { + "version": "2.1.8", + "resolved": "https://registry.npmjs.org/stacktracey/-/stacktracey-2.1.8.tgz", + "integrity": "sha512-Kpij9riA+UNg7TnphqjH7/CzctQ/owJGNbFkfEeve4Z4uxT5+JapVLFXcsurIfN34gnTWZNJ/f7NMG0E8JDzTw==", + "license": "Unlicense", + "dependencies": { + "as-table": "^1.0.36", + "get-source": "^2.0.12" + } + }, + "node_modules/stoppable": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/stoppable/-/stoppable-1.1.0.tgz", + "integrity": "sha512-KXDYZ9dszj6bzvnEMRYvxgeTHU74QBFL54XKtP3nyMuJ81CFYtABZ3bAzL2EdFUaEwJOBOgENyFj3R7oTzDyyw==", + "license": "MIT", + "engines": { + "node": ">=4", + "npm": ">=6" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD", + "optional": true + }, + "node_modules/ufo": { + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/ufo/-/ufo-1.5.4.tgz", + "integrity": "sha512-UsUk3byDzKd04EyoZ7U4DOlxQaD14JUKQl6/P7wiX4FNvUfm3XL246n9W5AmqwW5RSFJ27NAuM0iLscAOYUiGQ==", + "license": "MIT" + }, + "node_modules/undici": { + "version": "5.28.5", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.28.5.tgz", + "integrity": "sha512-zICwjrDrcrUE0pyyJc1I2QzBkLM8FINsgOrt6WjA+BgajVq9Nxu2PbFFXUrAggLfDXlZGZBVZYw7WNV5KiBiBA==", + "license": "MIT", + "dependencies": { + "@fastify/busboy": "^2.0.0" + }, + "engines": { + "node": ">=14.0" + } + }, + "node_modules/unenv": { + "version": "2.0.0-rc.14", + "resolved": "https://registry.npmjs.org/unenv/-/unenv-2.0.0-rc.14.tgz", + "integrity": "sha512-od496pShMen7nOy5VmVJCnq8rptd45vh6Nx/r2iPbrba6pa6p+tS2ywuIHRZ/OBvSbQZB0kWvpO9XBNVFXHD3Q==", + "license": "MIT", + "dependencies": { + "defu": "^6.1.4", + "exsolve": "^1.0.1", + "ohash": "^2.0.10", + "pathe": "^2.0.3", + "ufo": "^1.5.4" + } + }, + "node_modules/workerd": { + "version": "1.20250310.0", + "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20250310.0.tgz", + "integrity": "sha512-bAaZ9Bmts3mArbIrXYAtr+ZRsAJAAUEsCtvwfBavIYXaZ5sgdEOJBEiBbvsHp6CsVObegOM85tIWpYLpbTxQrQ==", + "hasInstallScript": true, + "license": "Apache-2.0", + "bin": { + "workerd": "bin/workerd" + }, + "engines": { + "node": ">=16" + }, + "optionalDependencies": { + "@cloudflare/workerd-darwin-64": "1.20250310.0", + "@cloudflare/workerd-darwin-arm64": "1.20250310.0", + "@cloudflare/workerd-linux-64": "1.20250310.0", + "@cloudflare/workerd-linux-arm64": "1.20250310.0", + "@cloudflare/workerd-windows-64": "1.20250310.0" + } + }, + "node_modules/wrangler": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-4.0.0.tgz", + "integrity": "sha512-9QqqoznS5sfLNqPKPkeEkwPAIe4lPfWLzPxVATmAbMQl4sh3/8iKEYSjZXQxdtcTgiS8iGOUbHq/rdiOFU8H1w==", + "license": "MIT OR Apache-2.0", + "dependencies": { + "@cloudflare/kv-asset-handler": "0.4.0", + "@cloudflare/unenv-preset": "2.0.2", + "blake3-wasm": "2.1.5", + "esbuild": "0.24.2", + "miniflare": "4.20250310.0", + "path-to-regexp": "6.3.0", + "unenv": "2.0.0-rc.14", + "workerd": "1.20250310.0" + }, + "bin": { + "wrangler": "bin/wrangler.js", + "wrangler2": "bin/wrangler.js" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.2", + "sharp": "^0.33.5" + }, + "peerDependencies": { + "@cloudflare/workers-types": "^4.20250310.0" + }, + "peerDependenciesMeta": { + "@cloudflare/workers-types": { + "optional": true + } + } + }, + "node_modules/ws": { + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", + "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/youch": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/youch/-/youch-3.2.3.tgz", + "integrity": "sha512-ZBcWz/uzZaQVdCvfV4uk616Bbpf2ee+F/AvuKDR5EwX/Y4v06xWdtMluqTD7+KlZdM93lLm9gMZYo0sKBS0pgw==", + "license": "MIT", + "dependencies": { + "cookie": "^0.5.0", + "mustache": "^4.2.0", + "stacktracey": "^2.1.8" + } + }, + "node_modules/zod": { + "version": "3.22.3", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.22.3.tgz", + "integrity": "sha512-EjIevzuJRiRPbVH4mGc8nApb/lVLKVpmUhAaR5R5doKGfAnGJ6Gr3CViAVjP+4FWSxCsybeWQdcgCtbX+7oZug==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/fastside-cloudflare/package.json b/fastside-cloudflare/package.json new file mode 100644 index 00000000..b9c6afe6 --- /dev/null +++ b/fastside-cloudflare/package.json @@ -0,0 +1,9 @@ +{ + "name": "fastside-cloudflare", + "dependencies": { + "wrangler": "^4.0.0" + }, + "scripts": { + "deploy": "wrangler deploy" + } +} diff --git a/fastside-cloudflare/src/lib.rs b/fastside-cloudflare/src/lib.rs index d90e112d..10f6c713 100644 --- a/fastside-cloudflare/src/lib.rs +++ b/fastside-cloudflare/src/lib.rs @@ -10,6 +10,7 @@ use fastside_shared::{ config::AppConfig, serde_types::{ServicesData, StoredData}, }; +use serde::{Deserialize, Serialize}; use tokio::sync::RwLock; use tower_service::Service; use worker::*; @@ -23,30 +24,26 @@ fn load_config(env: &Env) -> AppConfig { config } +#[derive(Serialize, Deserialize, Debug)] +struct KvStoredData { + loaded_data: LoadedData, + crawled_data: CrawledData, +} + async fn router(env: &Env) -> Router { let config = Arc::new(load_config(&env)); - let loaded_data: Arc> = Arc::new(RwLock::new( - serde_json::from_str( - &env.kv("fastside") - .expect("failed to get kv") - .get("loaded_data") - .text() - .await - .expect("failed to get loaded_data from kv") - .expect("loaded_data not found"), - ) - .expect("failed to parse loaded_data"), - )); - let crawled_data: CrawledData = serde_json::from_str( + let stored_data: KvStoredData = serde_json::from_str( &env.kv("fastside") .expect("failed to get kv") - .get("crawled_data") + .get("stored_data") .text() .await - .expect("failed to get crawled_data from kv") - .expect("crawled_data not found"), + .expect("failed to get stored_data from kv") + .expect("stored_data not found"), ) - .expect("failed to parse data"); + .expect("failed to parse loaded_data"); + let loaded_data = Arc::new(RwLock::new(stored_data.loaded_data)); + let crawled_data = stored_data.crawled_data; let shared_state = Arc::new(AppState { config: config.clone(), crawler: Arc::new(Crawler::with_data( @@ -92,39 +89,33 @@ async fn scheduled(_event: ScheduledEvent, env: Env, _ctx: ScheduleContext) { let stored_data: StoredData = serde_json::from_str(&services_str).expect("failed to parse services"); - let loaded_data = { - let services_data: ServicesData = stored_data - .services - .into_iter() - .map(|service| (service.name.clone(), service)) - .collect(); - let loaded_data = LoadedData { - services: services_data, - proxies: config.proxies.clone(), - default_user_config: config.default_user_config.clone(), - }; - env.kv("fastside") - .expect("failed to get kv") - .put( - "loaded_data", - serde_json::to_string(&loaded_data).expect("failed to serialize loaded_data"), - ) - .expect("failed to put loaded_data to kv (builder)") - .execute() - .await - .expect("failed to put loaded_data to kv (request)"); - Arc::new(RwLock::new(loaded_data)) + let services_data: ServicesData = stored_data + .services + .into_iter() + .map(|service| (service.name.clone(), service)) + .collect(); + let loaded_data = LoadedData { + services: services_data, + proxies: config.proxies.clone(), + default_user_config: config.default_user_config.clone(), }; + let loaded_data_clone = loaded_data.clone(); + let loaded_data = Arc::new(RwLock::new(loaded_data)); let crawler = Crawler::new(loaded_data, config.crawler.clone()); crawler.crawl(None).await.expect("failed to crawl"); - let data_str = serde_json::to_string(&*crawler.read().await).expect("failed to serialize data"); + let stored_data = KvStoredData { + loaded_data: loaded_data_clone, + crawled_data: crawler.read().await.clone(), + }; + + let data_str = serde_json::to_string(&stored_data).expect("failed to serialize data"); env.kv("fastside") .expect("failed to get kv") - .put("crawled_data", data_str) - .expect("failed to put crawled_data to kv (builder)") + .put("stored_data", data_str) + .expect("failed to put stored_data to kv (builder)") .execute() .await - .expect("failed to put crawled_data to kv (request)"); + .expect("failed to put stored_data to kv (request)"); } diff --git a/fastside-core/src/crawler.rs b/fastside-core/src/crawler.rs index e833fbb2..f2ba682b 100644 --- a/fastside-core/src/crawler.rs +++ b/fastside-core/src/crawler.rs @@ -104,7 +104,7 @@ pub struct CrawledServices { pub time: DateTime, } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub enum CrawledData { CrawledServices(CrawledServices), InitialLoading, diff --git a/fastside-core/src/routes/mod.rs b/fastside-core/src/routes/mod.rs index 4a057eaf..64873aa0 100644 --- a/fastside-core/src/routes/mod.rs +++ b/fastside-core/src/routes/mod.rs @@ -1,6 +1,7 @@ // mod api; mod config; mod index; +mod redirect; // use actix_web::Scope; @@ -13,6 +14,6 @@ use crate::types::AppState; pub fn main_router() -> Router> { Router::new() .nest("/", index::router()) - // .merge(redirect::router()) + .merge(redirect::router()) .nest("/configure", config::router()) } diff --git a/fastside-core/src/routes/redirect.rs b/fastside-core/src/routes/redirect.rs new file mode 100644 index 00000000..71f09aef --- /dev/null +++ b/fastside-core/src/routes/redirect.rs @@ -0,0 +1,247 @@ +use axum::{ + extract::{Path, Query, State}, + http::Method, + response::{Html, IntoResponse, Redirect}, + routing::get, + Router, +}; +use askama::Template; +use std::{collections::HashMap, sync::Arc}; + +use crate::{ + crawler::{Crawler, CrawledService}, + errors::RedirectError, + search::{ + find_redirect_service_by_name, find_redirect_service_by_url, get_redirect_instance, + get_redirect_instances, SearchError, + }, + types::{AppState, LoadedData}, + utils::user_config::SettingsCookie, +}; +use fastside_shared::{ + config::{SelectMethod, UserConfig}, + serde_types::Service, +}; + +pub fn router() -> Router> { + Router::new() + .route("/@cached/:service_name/*path", get(cached_redirect)) + .route("/_/*path", get(history_redirect)) + .route("/*path", get(base_redirect).post(base_redirect)) +} + +#[derive(Template)] +#[template(path = "cached_redirect.html", escape = "none")] +pub struct CachedRedirectTemplate<'a> { + pub urls: Vec<&'a url::Url>, + pub select_method: &'a SelectMethod, +} + +async fn cached_redirect( + State(state): State>, + Path((service_name, _)): Path<(String, String)>, + settings_cookie: Option, +) -> impl IntoResponse { + let loaded_data_guard = state.loaded_data.read().await; + + // Use user config from cookie or fall back to default + let user_config = match settings_cookie { + Some(SettingsCookie(config)) => config, + None => loaded_data_guard.default_user_config.clone(), + }; + + let guard = state.crawler.read().await; + match find_redirect_service_by_name(&guard, &loaded_data_guard.services, &service_name).await { + Ok((crawled_service, _)) => { + match get_redirect_instances( + crawled_service, + &user_config.required_tags, + &user_config.forbidden_tags, + &user_config.preferred_instances, + ) { + Some(mut instances) => { + if user_config.select_method == SelectMethod::LowPing { + instances.sort_by(|a, b| a.status.as_isize().cmp(&b.status.as_isize())); + } + debug!("User config: {user_config:?}"); + + let template = CachedRedirectTemplate { + urls: instances.iter().map(|i| &i.url).collect(), + select_method: &user_config.select_method, + }; + + let cache_control = format!( + "public, max-age={}, stale-while-revalidate=86400, stale-if-error=86400, immutable", + state.config.crawler.ping_interval.as_secs() + ); + + match template.render() { + Ok(rendered) => ( + axum::http::StatusCode::OK, + [ + ("cache-control", cache_control), + ("content-type", "text/html; charset=utf-8".to_string()), + ], + Html(rendered), + ) + .into_response(), + Err(_) => axum::http::StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + } + None => RedirectError::from(SearchError::NoInstancesFound).into_response(), + } + } + Err(err) => RedirectError::from(err).into_response(), + } +} + +#[derive(Template)] +#[template(path = "history_redirect.html")] +pub struct HistoryRedirectTemplate<'a> { + pub path: &'a str, +} + +async fn history_redirect( + Query(query): Query>, + Path(path): Path, +) -> impl IntoResponse { + let mut full_path = format!("/{path}"); + + // Add query parameters if present + if !query.is_empty() { + full_path.push('?'); + let query_string: Vec = query + .iter() + .map(|(k, v)| format!("{}={}", k, v)) + .collect(); + full_path.push_str(&query_string.join("&")); + } + + let template = HistoryRedirectTemplate { path: &full_path }; + + match template.render() { + Ok(rendered) => ( + axum::http::StatusCode::OK, + [ + ("content-type", "text/html; charset=utf-8"), + ("refresh", &format!("1; url={full_path}")), + ], + Html(rendered), + ) + .into_response(), + Err(_) => axum::http::StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } +} + +#[derive(Template)] +#[template(path = "fallback_redirect.html", escape = "none")] +pub struct FallbackRedirectTemplate<'a> { + pub fallback: &'a str, +} + +pub async fn find_redirect( + crawler: &Crawler, + loaded_data: &LoadedData, + regexes: &crate::types::Regexes, + user_config: &UserConfig, + path: &str, +) -> Result<(String, bool), RedirectError> { + let is_url_query = if path.starts_with("http://") || path.starts_with("https://") { + true + } else { + path[0..path.find('/').unwrap_or(path.len())].contains('.') + }; + + let guard = crawler.read().await; + let (redir_path, crawled_service, service): (String, &CrawledService, &Service) = + match is_url_query { + true => { + let (crawled_service, service, redir_path) = + find_redirect_service_by_url(&guard, &loaded_data.services, regexes, path) + .await + .map_err(RedirectError::from)?; + (redir_path, crawled_service, service) + } + false => { + let service_name = path.split('/').next().unwrap_or(""); + let redir_path = path[service_name.len()..].to_string(); + let (crawled_service, service) = + find_redirect_service_by_name(&guard, &loaded_data.services, service_name) + .await + .map_err(RedirectError::from)?; + (redir_path, crawled_service, service) + } + }; + + let (redirect_instance, is_fallback) = + get_redirect_instance(crawled_service, service, user_config) + .map_err(RedirectError::from)?; + + let url = redirect_instance + .url + .clone() + .join(&redir_path) + .map_err(RedirectError::from)? + .to_string(); + + Ok((url, is_fallback)) +} + +async fn base_redirect( + State(state): State>, + method: Method, + Query(query): Query>, + Path(path): Path, + settings_cookie: Option, +) -> impl IntoResponse { + let loaded_data_guard = state.loaded_data.read().await; + + // Use provided user config or fall back to default + let user_config = match settings_cookie { + Some(SettingsCookie(config)) => config, + None => loaded_data_guard.default_user_config.clone(), + }; + + match find_redirect( + &state.crawler, + &loaded_data_guard, + &state.regexes, + &user_config, + &path, + ) + .await { + Ok((mut url, is_fallback)) => { + // Add query parameters if present + if !query.is_empty() { + url.push('?'); + let query_string: Vec = query + .iter() + .map(|(k, v)| format!("{}={}", k, v)) + .collect(); + url.push_str(&query_string.join("&")); + } + + debug!("Redirecting to {url}, is_fallback: {is_fallback}"); + + match (is_fallback, user_config.ignore_fallback_warning, method) { + (true, false, Method::GET) => { + let template = FallbackRedirectTemplate { fallback: &url }; + match template.render() { + Ok(rendered) => ( + axum::http::StatusCode::OK, + [ + ("content-type", "text/html; charset=utf-8"), + ("refresh", &format!("15; url={url}")), + ], + Html(rendered), + ) + .into_response(), + Err(_) => axum::http::StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + } + _ => Redirect::temporary(&url).into_response(), + } + } + Err(e) => e.into_response(), + } +} diff --git a/fastside-core/src/types.rs b/fastside-core/src/types.rs index 054618a2..1bb48454 100644 --- a/fastside-core/src/types.rs +++ b/fastside-core/src/types.rs @@ -16,7 +16,7 @@ pub struct CompiledRegexSearch { pub type Regexes = HashMap>; -#[derive(Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct LoadedData { pub services: ServicesData, pub proxies: ProxyData, diff --git a/flake.lock b/flake.lock index 41d53853..dfac2c5d 100644 --- a/flake.lock +++ b/flake.lock @@ -5,11 +5,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1710146030, - "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", "owner": "numtide", "repo": "flake-utils", - "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", "type": "github" }, "original": { @@ -25,11 +25,11 @@ ] }, "locked": { - "lastModified": 1721727458, - "narHash": "sha256-r/xppY958gmZ4oTfLiHN0ZGuQ+RSTijDblVgVLFi1mw=", + "lastModified": 1739824009, + "narHash": "sha256-fcNrCMUWVLMG3gKC5M9CBqVOAnJtyRvGPxptQFl5mVg=", "owner": "nix-community", "repo": "naersk", - "rev": "3fb418eaf352498f6b6c30592e3beb63df42ef11", + "rev": "e5130d37369bfa600144c2424270c96f0ef0e11d", "type": "github" }, "original": { @@ -40,11 +40,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1724819573, - "narHash": "sha256-GnR7/ibgIH1vhoy8cYdmXE6iyZqKqFxQSVkFgosBh6w=", + "lastModified": 1742069588, + "narHash": "sha256-C7jVfohcGzdZRF6DO+ybyG/sqpo1h6bZi9T56sxLy+k=", "owner": "nixos", "repo": "nixpkgs", - "rev": "71e91c409d1e654808b2621f28a327acfdad8dc2", + "rev": "c80f6a7e10b39afcc1894e02ef785b1ad0b0d7e5", "type": "github" }, "original": { diff --git a/shell.nix b/shell.nix index 5715d2c8..00423293 100644 --- a/shell.nix +++ b/shell.nix @@ -2,8 +2,8 @@ let pkgs' = pkgs.extend (import (builtins.fetchTarball { url = - "https://github.com/oxalica/rust-overlay/archive/87b6cffc276795b46ef544d7ed8d7fed6ad9c8e4.tar.gz"; - sha256 = "01gf3m4a0ljzkxf65lkcvr5kwcjr3mbpjbpppf0djk82mm98qbh4"; + "https://github.com/oxalica/rust-overlay/archive/af76221b285a999ab7d9d77fce8ba1db028f9801.tar.gz"; + sha256 = "03zc2w66zz8dkrxpy39lrh3gqand1ypmnhcakmhibs9ndyi4v3x0"; })); in pkgs'.mkShell { nativeBuildInputs = with pkgs'; [