From b11117ed07f644a3bcd91c2e6a61c81a83a06a40 Mon Sep 17 00:00:00 2001 From: VB2007 Date: Mon, 17 Nov 2025 06:16:19 +0100 Subject: [PATCH 01/17] added packages, fixed 403 error with curl --- package-lock.json | 7 +- package.json | 3 +- src/helpers/darwin/darwinProcess.js | 199 ++++++++++++++++++++++++---- src/sql/darwinCache/table.sql | 2 +- 4 files changed, 179 insertions(+), 32 deletions(-) diff --git a/package-lock.json b/package-lock.json index c995ab14..67dfbf72 100755 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "discordbot", - "version": "4.0.1", + "version": "4.1.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "discordbot", - "version": "4.0.1", + "version": "4.1.1", "license": "AGPL-3.0-only", "dependencies": { "ajv": "^8.17.1", @@ -16,7 +16,8 @@ "fluent-ffmpeg": "^2.1.3", "google-translate-api-x": "^10.7.1", "jest": "^29.7.0", - "mariadb": "^3.3.0" + "mariadb": "^3.3.0", + "undici": "^7.16.0" }, "devDependencies": { "@babel/parser": "^7.28.4", diff --git a/package.json b/package.json index 7feb10b2..7f57c7e9 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,8 @@ "fluent-ffmpeg": "^2.1.3", "google-translate-api-x": "^10.7.1", "jest": "^29.7.0", - "mariadb": "^3.3.0" + "mariadb": "^3.3.0", + "undici": "^7.16.0" }, "devDependencies": { "@babel/parser": "^7.28.4", diff --git a/src/helpers/darwin/darwinProcess.js b/src/helpers/darwin/darwinProcess.js index 58eb7cd1..f7e1ace5 100644 --- a/src/helpers/darwin/darwinProcess.js +++ b/src/helpers/darwin/darwinProcess.js @@ -2,6 +2,8 @@ import { load } from "cheerio"; import fs from "fs"; import path from "path"; import { pipeline } from "stream/promises"; +import { request } from "undici"; +import { spawn } from "child_process"; import { query } from "../db.js"; import { transcodeVideo, getFileSizeMB } from "./darwinTranscode.js"; @@ -10,6 +12,105 @@ import { isInCache, addToCache } from "./darwinCache.js"; import config from "../../../config.json" with { type: "json" }; const darwinConfig = config.darwin; +/** + * Download video using system curl (better TLS fingerprint) + * @param {string} url - Video URL + * @param {string} referer - Referer URL (comments page) + * @param {string} outputPath - Where to save the file + * @returns {Promise} - Whether download was successful + */ +const downloadVideoWithCurl = async (url, referer, outputPath) => { + return new Promise((resolve, reject) => { + const curlArgs = [ + "-L", // Follow redirects + "-A", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36", + "-H", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "-H", + "Accept-Language: en-US,en;q=0.5", + "-H", + "Accept-Encoding: gzip, deflate, br", + "-H", + "Upgrade-Insecure-Requests: 1", + "-H", + "Sec-Fetch-Dest: document", + "-H", + "Sec-Fetch-Mode: navigate", + "-H", + "Sec-Fetch-Site: cross-site", + "-H", + "Sec-Fetch-User: ?1", + "-H", + `Referer: ${referer}`, + "-H", + "Cache-Control: no-cache", + "-H", + "Pragma: no-cache", + "-o", + outputPath, + "--compressed", + "--http2", + url, + ]; + + const curl = spawn("curl", curlArgs); + + let stderr = ""; + + curl.stderr.on("data", (data) => { + stderr += data.toString(); + }); + + curl.on("close", (code) => { + if (code === 0) { + resolve(true); + } else { + reject(new Error(`curl exited with code ${code}: ${stderr}`)); + } + }); + + curl.on("error", (error) => { + reject(new Error(`Failed to spawn curl: ${error.message}`)); + }); + }); +}; + +/** + * Download video using undici with browser-like configuration + * @param {string} url - Video URL + * @param {string} referer - Referer URL (comments page) + * @returns {Promise} - Response object with body stream and headers + */ +const downloadVideo = async (url, referer) => { + const headers = { + "User-Agent": + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36", + Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.5", + "Accept-Encoding": "gzip, deflate, br", + "Upgrade-Insecure-Requests": "1", + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "cross-site", + "Sec-Fetch-User": "?1", + "sec-ch-ua": '"Not/A)Brand";v="8", "Chromium";v="142", "Google Chrome";v="142"', + "sec-ch-ua-mobile": "?0", + "sec-ch-ua-platform": '"Windows"', + Referer: referer, + "Cache-Control": "no-cache", + Pragma: "no-cache", + }; + + const response = await request(url, { + method: "GET", + headers, + maxRedirections: 5, + }); + + return response; +}; + /** * Get the final destination of a URL (follow redirects) * @param {string} url - Initial URL @@ -38,9 +139,9 @@ const getDestination = async (url) => { const getVideoLocation = async (href, markerOne, markerTwo) => { try { const response = await fetch(href, { - //if it isn't set to "darwin" the scraping fails for some reason headers: { "User-Agent": "darwin" }, // headers: { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36 Edg/138.0.0.0" } + // headers: getPageHeaders(), }); const html = await response.text(); const start = html.indexOf(markerOne); @@ -93,38 +194,65 @@ const processVideo = async (video) => { return null; } - const response = await fetch(href); - - if (!response.ok) { - console.error(`${href} ${response.statusText}`); - return null; - } - - const contentLength = parseInt(response.headers.get("Content-Length"), 10); - if (contentLength && contentLength > 50 * 1024 * 1024) { - console.log("Skipping download: File size exceeds limit (50MB)"); - - const fileSize = (contentLength / 1000000).toFixed(0); - // Return info for too large videos - return { - title, - href, - comments, - canBeStreamed: false, - directStreamLink: "", - fileSize, - tooBig: true, - }; - } - const videoId = href.split("/").pop().replace(".mp4", ""); const tempFilePath = path.join(darwinConfig.tempDir, `${videoId}_original.mp4`); const transcodedFilePath = path.join(darwinConfig.tempDir, `${videoId}_transcoded.mp4`); const finalFilePath = path.join(darwinConfig.targetDir, `${videoId}.mp4`); const directStreamLink = `${darwinConfig.cdnUrl}/${videoId}.mp4`; - console.log(`Downloading video to temp location: ${tempFilePath}`); - await pipeline(response.body, fs.createWriteStream(tempFilePath)); + // Maximum download size in bytes from config + const maxSizeBytes = darwinConfig.maxDownloadSize * 1024 * 1024; + + console.log(`Attempting to download video using curl: ${href}`); + + // Try curl first (better TLS fingerprint) + let downloadSuccess = false; + try { + await downloadVideoWithCurl(href, comments, tempFilePath); + console.log("Successfully downloaded using curl"); + downloadSuccess = true; + } catch (curlError) { + console.log(`Curl download failed: ${curlError.message}`); + } + + // Fallback to undici if curl failed + if (!downloadSuccess) { + console.log("Attempting download with undici..."); + try { + const response = await downloadVideo(href, comments); + + if (response.statusCode !== 200) { + console.error(`${href} ${response.statusCode} ${response.statusText || "Error"}`); + return null; + } + + const contentLength = parseInt(response.headers["content-length"], 10); + if (contentLength && contentLength > maxSizeBytes) { + console.log( + `Skipping download: File size exceeds limit (${darwinConfig.maxDownloadSize}MB)` + ); + await response.body.dump(); + + const fileSize = (contentLength / 1000000).toFixed(0); + return { + title, + href, + comments, + canBeStreamed: false, + directStreamLink: "", + fileSize, + tooBig: true, + }; + } + + console.log(`Downloading video to temp location: ${tempFilePath}`); + await pipeline(response.body, fs.createWriteStream(tempFilePath)); + downloadSuccess = true; + } catch (undiciError) { + console.error(`Undici download also failed: ${undiciError.message}`); + return null; + } + } if (!fs.existsSync(tempFilePath)) { console.error(`Error when saving temporary file: ${tempFilePath}`); @@ -134,6 +262,21 @@ const processVideo = async (video) => { const originalSize = getFileSizeMB(tempFilePath); console.log(`Original file size: ${originalSize}MB`); + // Check if downloaded file is too large (for curl downloads which don't check size beforehand) + if (originalSize > darwinConfig.maxDownloadSize) { + console.log(`File too large (${originalSize}MB), removing and skipping`); + fs.unlinkSync(tempFilePath); + return { + title, + href, + comments, + canBeStreamed: false, + directStreamLink: "", + fileSize: originalSize.toFixed(0), + tooBig: true, + }; + } + try { console.log("Starting video transcoding process..."); const transcodingSuccess = await transcodeVideo(tempFilePath, transcodedFilePath); @@ -248,6 +391,8 @@ const fetchVideosFromFeed = async () => { try { const response = await fetch(darwinConfig.feedUrl, { headers: { "User-Agent": "darwin" }, + // headers: { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36" } + // headers: getPageHeaders(), }); const html = await response.text(); diff --git a/src/sql/darwinCache/table.sql b/src/sql/darwinCache/table.sql index 1f1a396d..09fe1355 100644 --- a/src/sql/darwinCache/table.sql +++ b/src/sql/darwinCache/table.sql @@ -7,4 +7,4 @@ CREATE TABLE IF NOT EXISTS `darwinCache` ( `processedAt` TIMESTAMP DEFAULT CURRENT_TIMESTAMP, UNIQUE KEY (`videoUrl`), UNIQUE KEY (`videoId`) -); \ No newline at end of file +); From e695ab4b053d521325270e26984a376db1674ea2 Mon Sep 17 00:00:00 2001 From: VB2007 Date: Mon, 17 Nov 2025 06:20:33 +0100 Subject: [PATCH 02/17] cleaned up codebase after succesful curl approach --- package-lock.json | 3 +- package.json | 3 +- src/helpers/darwin/darwinProcess.js | 87 +---------------------------- 3 files changed, 5 insertions(+), 88 deletions(-) diff --git a/package-lock.json b/package-lock.json index 67dfbf72..9a4cc725 100755 --- a/package-lock.json +++ b/package-lock.json @@ -16,8 +16,7 @@ "fluent-ffmpeg": "^2.1.3", "google-translate-api-x": "^10.7.1", "jest": "^29.7.0", - "mariadb": "^3.3.0", - "undici": "^7.16.0" + "mariadb": "^3.3.0" }, "devDependencies": { "@babel/parser": "^7.28.4", diff --git a/package.json b/package.json index 7f57c7e9..7feb10b2 100644 --- a/package.json +++ b/package.json @@ -38,8 +38,7 @@ "fluent-ffmpeg": "^2.1.3", "google-translate-api-x": "^10.7.1", "jest": "^29.7.0", - "mariadb": "^3.3.0", - "undici": "^7.16.0" + "mariadb": "^3.3.0" }, "devDependencies": { "@babel/parser": "^7.28.4", diff --git a/src/helpers/darwin/darwinProcess.js b/src/helpers/darwin/darwinProcess.js index f7e1ace5..c6eea503 100644 --- a/src/helpers/darwin/darwinProcess.js +++ b/src/helpers/darwin/darwinProcess.js @@ -1,8 +1,6 @@ import { load } from "cheerio"; import fs from "fs"; import path from "path"; -import { pipeline } from "stream/promises"; -import { request } from "undici"; import { spawn } from "child_process"; import { query } from "../db.js"; @@ -76,41 +74,6 @@ const downloadVideoWithCurl = async (url, referer, outputPath) => { }); }; -/** - * Download video using undici with browser-like configuration - * @param {string} url - Video URL - * @param {string} referer - Referer URL (comments page) - * @returns {Promise} - Response object with body stream and headers - */ -const downloadVideo = async (url, referer) => { - const headers = { - "User-Agent": - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36", - Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "Accept-Language": "en-US,en;q=0.5", - "Accept-Encoding": "gzip, deflate, br", - "Upgrade-Insecure-Requests": "1", - "Sec-Fetch-Dest": "document", - "Sec-Fetch-Mode": "navigate", - "Sec-Fetch-Site": "cross-site", - "Sec-Fetch-User": "?1", - "sec-ch-ua": '"Not/A)Brand";v="8", "Chromium";v="142", "Google Chrome";v="142"', - "sec-ch-ua-mobile": "?0", - "sec-ch-ua-platform": '"Windows"', - Referer: referer, - "Cache-Control": "no-cache", - Pragma: "no-cache", - }; - - const response = await request(url, { - method: "GET", - headers, - maxRedirections: 5, - }); - - return response; -}; - /** * Get the final destination of a URL (follow redirects) * @param {string} url - Initial URL @@ -141,7 +104,6 @@ const getVideoLocation = async (href, markerOne, markerTwo) => { const response = await fetch(href, { headers: { "User-Agent": "darwin" }, // headers: { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36 Edg/138.0.0.0" } - // headers: getPageHeaders(), }); const html = await response.text(); const start = html.indexOf(markerOne); @@ -203,55 +165,13 @@ const processVideo = async (video) => { // Maximum download size in bytes from config const maxSizeBytes = darwinConfig.maxDownloadSize * 1024 * 1024; - console.log(`Attempting to download video using curl: ${href}`); - - // Try curl first (better TLS fingerprint) - let downloadSuccess = false; + console.log(`Downloading video using curl: ${href}`); try { await downloadVideoWithCurl(href, comments, tempFilePath); console.log("Successfully downloaded using curl"); - downloadSuccess = true; } catch (curlError) { - console.log(`Curl download failed: ${curlError.message}`); - } - - // Fallback to undici if curl failed - if (!downloadSuccess) { - console.log("Attempting download with undici..."); - try { - const response = await downloadVideo(href, comments); - - if (response.statusCode !== 200) { - console.error(`${href} ${response.statusCode} ${response.statusText || "Error"}`); - return null; - } - - const contentLength = parseInt(response.headers["content-length"], 10); - if (contentLength && contentLength > maxSizeBytes) { - console.log( - `Skipping download: File size exceeds limit (${darwinConfig.maxDownloadSize}MB)` - ); - await response.body.dump(); - - const fileSize = (contentLength / 1000000).toFixed(0); - return { - title, - href, - comments, - canBeStreamed: false, - directStreamLink: "", - fileSize, - tooBig: true, - }; - } - - console.log(`Downloading video to temp location: ${tempFilePath}`); - await pipeline(response.body, fs.createWriteStream(tempFilePath)); - downloadSuccess = true; - } catch (undiciError) { - console.error(`Undici download also failed: ${undiciError.message}`); - return null; - } + console.error(`Curl download failed: ${curlError.message}`); + return null; } if (!fs.existsSync(tempFilePath)) { @@ -392,7 +312,6 @@ const fetchVideosFromFeed = async () => { const response = await fetch(darwinConfig.feedUrl, { headers: { "User-Agent": "darwin" }, // headers: { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36" } - // headers: getPageHeaders(), }); const html = await response.text(); From 370d30abb18d34ed9348a4570825c65c0a36e640 Mon Sep 17 00:00:00 2001 From: VB2007 Date: Tue, 18 Nov 2025 08:47:49 +0100 Subject: [PATCH 03/17] brought back direct stream links for (too) large files, added failsafe check for downloads & distribution --- src/helpers/darwin/darwinProcess.js | 106 +++++++++++++++++++++++----- 1 file changed, 88 insertions(+), 18 deletions(-) diff --git a/src/helpers/darwin/darwinProcess.js b/src/helpers/darwin/darwinProcess.js index c6eea503..fb51dd10 100644 --- a/src/helpers/darwin/darwinProcess.js +++ b/src/helpers/darwin/darwinProcess.js @@ -74,6 +74,60 @@ const downloadVideoWithCurl = async (url, referer, outputPath) => { }); }; +/** + * Get file size using curl HEAD request + * @param {string} url - Video URL + * @param {string} referer - Referer URL (comments page) + * @returns {Promise} - File size in bytes, or null if failed + */ +const getFileSizeWithCurl = async (url, referer) => { + return new Promise((resolve) => { + const curlArgs = [ + "-I", // HEAD request only + "-L", // Follow redirects + "-s", // Silent mode + "-A", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36", + "-H", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "-H", + "Accept-Language: en-US,en;q=0.5", + "-H", + "Sec-Fetch-Dest: document", + "-H", + "Sec-Fetch-Mode: navigate", + "-H", + "Sec-Fetch-Site: cross-site", + "-H", + "Sec-Fetch-User: ?1", + "-H", + `Referer: ${referer}`, + "--http2", + url, + ]; + + const curl = spawn("curl", curlArgs); + let stdout = ""; + + curl.stdout.on("data", (data) => { + stdout += data.toString(); + }); + + curl.on("close", (code) => { + if (code === 0) { + const match = stdout.match(/content-length:\s*(\d+)/i); + resolve(match ? parseInt(match[1], 10) : null); + } else { + resolve(null); + } + }); + + curl.on("error", () => { + resolve(null); + }); + }); +}; + /** * Get the final destination of a URL (follow redirects) * @param {string} url - Initial URL @@ -129,7 +183,7 @@ const getVideoLocation = async (href, markerOne, markerTwo) => { * @param {number} fileSize - File size in MB * @returns {string} - Formatted message */ -const messageGen = (title, href, directStreamLink, comments, canBeStreamed, fileSize) => { +const messageGen = (title, href, comments, canBeStreamed, fileSize, directStreamLink = "") => { return `${canBeStreamed ? `[[ STREAMING & DOWNLOAD ]](${directStreamLink})` : `[[ ORIGINAL MP4 ]](${href})`} - [[ FORUM POST ]](<${comments}>)\n${title}${canBeStreamed ? `\nSize: ${fileSize}MB` : `\nSize (might won't load): ${fileSize}MB`}`; }; @@ -162,8 +216,27 @@ const processVideo = async (video) => { const finalFilePath = path.join(darwinConfig.targetDir, `${videoId}.mp4`); const directStreamLink = `${darwinConfig.cdnUrl}/${videoId}.mp4`; - // Maximum download size in bytes from config - const maxSizeBytes = darwinConfig.maxDownloadSize * 1024 * 1024; + // Check file size before downloading + console.log(`Checking file size for: ${href}`); + const fileSizeBytes = await getFileSizeWithCurl(href, comments); + + if (fileSizeBytes) { + const fileSizeMB = (fileSizeBytes / 1024 / 1024).toFixed(2); + console.log(`File size: ${fileSizeMB}MB`); + + if (fileSizeBytes > darwinConfig.maxDownloadSize * 1024 * 1024) { + console.log(`File exceeds limit (${darwinConfig.maxDownloadSize}MB), using direct link`); + return { + title, + href, + comments, + canBeStreamed: false, + fileSize: fileSizeMB, + }; + } + } else { + console.log("Could not determine file size, will proceed with download"); + } console.log(`Downloading video using curl: ${href}`); try { @@ -182,18 +255,16 @@ const processVideo = async (video) => { const originalSize = getFileSizeMB(tempFilePath); console.log(`Original file size: ${originalSize}MB`); - // Check if downloaded file is too large (for curl downloads which don't check size beforehand) + // Double-check size after download (fallback if HEAD request didn't work) if (originalSize > darwinConfig.maxDownloadSize) { - console.log(`File too large (${originalSize}MB), removing and skipping`); + console.log(`File too large (${originalSize}MB), removing and using direct link`); fs.unlinkSync(tempFilePath); return { title, href, comments, canBeStreamed: false, - directStreamLink: "", - fileSize: originalSize.toFixed(0), - tooBig: true, + fileSize: originalSize.toFixed(2), }; } @@ -222,14 +293,13 @@ const processVideo = async (video) => { } console.log("Temporary files cleaned up"); - // Return success result return { title, href, comments, - directStreamLink, canBeStreamed: true, fileSize: transcodedSize, + directStreamLink, }; } catch (error) { console.error(`Transcoding failed, attempting to use original file: ${error}`); @@ -251,15 +321,15 @@ const processVideo = async (video) => { } const finalSize = getFileSizeMB(finalFilePath); - const canStream = finalSize < 50; // I will run out of storage & network bandwidth quick if that's higher + const canStream = finalSize <= darwinConfig.maxDownloadSize; return { title, href, comments, - directStreamLink, canBeStreamed: canStream, fileSize: finalSize, + directStreamLink, }; } catch (fallbackError) { console.error(`Failed to use original file as fallback: ${fallbackError}`); @@ -280,9 +350,9 @@ const processVideo = async (video) => { * @returns {Promise} */ const distributeVideo = async (client, guildConfigs, processedVideo) => { - const { title, href, comments, directStreamLink, canBeStreamed, fileSize } = processedVideo; + const { title, href, comments, canBeStreamed, fileSize, directStreamLink } = processedVideo; - const message = messageGen(title, href, directStreamLink, comments, canBeStreamed, fileSize); + const message = messageGen(title, href, comments, canBeStreamed, fileSize, directStreamLink); for (const config of guildConfigs) { try { @@ -402,8 +472,8 @@ export const runDarwinProcess = async (client) => { if (result) { processedVideos.push(result); - // If it's a "too big" video, distribute immediately - if (result.tooBig) { + // If video wasn't downloaded, distribute immediately + if (!result.canBeStreamed) { await distributeVideo(client, guildConfigs, result); } } @@ -427,8 +497,8 @@ export const runDarwinProcess = async (client) => { await new Promise((resolve) => setTimeout(resolve, 1000)); } - // Filter out "too big" videos that were already distributed - const videosToDistribute = processedVideos.filter((video) => !video.tooBig); + // Filter out videos that weren't downloaded (already distributed) + const videosToDistribute = processedVideos.filter((video) => video.canBeStreamed); console.log( `Successfully processed ${videosToDistribute.length} videos, distributing to ${guildConfigs.length} channels` ); From 1e9862caf9d576e3b5a7994f228d0a2a764aa3af Mon Sep 17 00:00:00 2001 From: VB2007 Date: Tue, 18 Nov 2025 18:49:38 +0100 Subject: [PATCH 04/17] added curl download & build job to deploy workflow --- .github/workflows/deploy.yml | 102 +++++++++++++++++++++++++++++++++-- 1 file changed, 98 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index c04f556f..382d2746 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -13,8 +13,102 @@ on: type: string jobs: + setup-curl: + runs-on: self-hosted + outputs: + curl-updated: ${{ steps.curl-check.outputs.updated }} + + steps: + - name: Check current curl version + id: curl-check + run: | + echo "Checking current curl version..." + CURRENT_VERSION="" + if command -v /usr/local/bin/curl &> /dev/null; then + CURRENT_VERSION=$(/usr/local/bin/curl --version | head -1 | awk '{print $2}') + echo "Current local curl version: $CURRENT_VERSION" + elif command -v curl &> /dev/null; then + CURRENT_VERSION=$(curl --version | head -1 | awk '{print $2}') + echo "Current system curl version: $CURRENT_VERSION" + else + echo "No curl found" + fi + + LATEST_TAG=$(curl -s https://api.github.com/repos/curl/curl/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/') + LATEST_VERSION=$(echo "$LATEST_TAG" | sed 's/curl-//' | sed 's/_/./g') + echo "Latest available version: $LATEST_VERSION" + + #compare versions + if [ "$CURRENT_VERSION" != "$LATEST_VERSION" ]; then + echo "Curl needs updating: $CURRENT_VERSION -> $LATEST_VERSION" + echo "updated=true" >> $GITHUB_OUTPUT + else + echo "Curl is already up to date: $CURRENT_VERSION" + echo "updated=false" >> $GITHUB_OUTPUT + fi + + - name: Install curl dependencies + if: steps.curl-check.outputs.updated == 'true' + run: | + echo "Installing dependencies for compiling curl..." + sudo apt update + sudo apt install -y \ + build-essential \ + libssl-dev \ + libnghttp2-dev \ + zlib1g-dev \ + libpsl-dev \ + libbrotli-dev \ + libzstd-dev \ + libssh2-1-dev \ + librtmp-dev \ + libldap2-dev \ + libidn2-dev \ + libkrb5-dev \ + autotools-dev + + - name: Build and install latest version of curl + if: steps.curl-check.outputs.updated == 'true' + run: | + echo "Building latest curl..." + cd /tmp + + TAG_NAME=$(curl -s https://api.github.com/repos/curl/curl/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/') + VERSION=$(echo "$TAG_NAME" | sed 's/curl-//' | sed 's/_/./g') + DOWNLOAD_URL="https://curl.se/download/curl-${VERSION}.tar.gz" + + echo "Downloading curl $VERSION..." + wget -v "$DOWNLOAD_URL" -O curl-latest.tar.gz + + echo "Extracting and building..." + tar -xzf curl-latest.tar.gz + cd curl-${VERSION} + + ./configure --prefix=/usr/local \ + --with-openssl \ + --with-nghttp2 \ + --with-libpsl \ + --with-brotli \ + --with-zstd \ + --enable-http2 \ + --enable-ldap \ + --enable-ldaps \ + --disable-static \ + --enable-shared + + make -j$(nproc) + sudo make install + sudo ldconfig + + echo "✅ Curl installation completed" + /usr/local/bin/curl --version + + cd /tmp + rm -rf curl-latest.tar.gz curl-${VERSION} + deploy: runs-on: self-hosted # REQUIRES SOME SUDO EXECUTION PRIVILEGES WITHOUT A PASSWORD PROMPT + needs: setup-curl steps: - name: Stopping systemd service @@ -36,10 +130,10 @@ jobs: # with: # node-version: "24.8.0" - - name: Intalling Node.js dependencies + - name: Installing Node.js dependencies working-directory: /home/vb2007/prod/discordbot env: - PATH: /home/vb2007/.nvm/versions/node/v24.8.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH: /usr/local/bin:/home/vb2007/.nvm/versions/node/v24.8.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin run: | rm -rf node_modules npm ci @@ -48,7 +142,7 @@ jobs: - name: Deploying possible new commands working-directory: /home/vb2007/prod/discordbot env: - PATH: /home/vb2007/.nvm/versions/node/v24.8.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH: /usr/local/bin:/home/vb2007/.nvm/versions/node/v24.8.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin run: | echo "Starting command deployment..." deployOutput=$(npm run deploy 2>&1) @@ -65,7 +159,7 @@ jobs: - name: Updating command data working-directory: /home/vb2007/prod/discordbot env: - PATH: /home/vb2007/.nvm/versions/node/v24.8.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH: /usr/local/bin:/home/vb2007/.nvm/versions/node/v24.8.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin run: | echo "Starting table creation and command data update..." queriesOutput=$(npm run create-tables 2>&1) From 72ab5b803347f3b3872f7d41fa9771f48edcda5d Mon Sep 17 00:00:00 2001 From: VB2007 Date: Tue, 18 Nov 2025 18:50:04 +0100 Subject: [PATCH 05/17] changed branch for testing --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 382d2746..ea5a2469 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -3,7 +3,7 @@ permissions: {} on: push: - branches: [main] + branches: [dev-darwin-curl-fix] workflow_dispatch: inputs: branch: From 87f7dbf9c576b8dcb0f5e3d074c95da09e17d547 Mon Sep 17 00:00:00 2001 From: VB2007 Date: Tue, 18 Nov 2025 19:56:44 +0100 Subject: [PATCH 06/17] created session randomizer for downloads (tls certs, browser sessions, etc.) --- src/helpers/darwin/darwinAntiFingerprint.js | 486 ++++++++++++++++++++ src/helpers/darwin/darwinProcess.js | 156 ++----- 2 files changed, 520 insertions(+), 122 deletions(-) create mode 100644 src/helpers/darwin/darwinAntiFingerprint.js diff --git a/src/helpers/darwin/darwinAntiFingerprint.js b/src/helpers/darwin/darwinAntiFingerprint.js new file mode 100644 index 00000000..a0c638c2 --- /dev/null +++ b/src/helpers/darwin/darwinAntiFingerprint.js @@ -0,0 +1,486 @@ +import { spawn } from "child_process"; +import fs from "fs"; + +/** + * Browser fingerprint configurations for randomization + */ +const BROWSER_CONFIGS = [ + { + name: "Chrome 120 Windows", + userAgent: + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + ciphers: "TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_128_GCM_SHA256", + curves: "X25519:prime256v1:secp384r1", + headers: { + accept: "video/mp4,video/*,application/octet-stream,*/*;q=0.8", + acceptLanguage: "en-US,en;q=0.9", + secFetchSite: "cross-site", + }, + }, + { + name: "Chrome 119 macOS", + userAgent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36", + ciphers: "TLS_CHACHA20_POLY1305_SHA256:TLS_AES_256_GCM_SHA384:TLS_AES_128_GCM_SHA256", + curves: "prime256v1:X25519:secp384r1", + headers: { + accept: "video/webm,video/ogg,video/*;q=0.9,application/ogg;q=0.7,audio/*;q=0.6,*/*;q=0.5", + acceptLanguage: "en-US,en;q=0.9", + secFetchSite: "same-origin", + }, + }, + { + name: "Chrome 118 Linux", + userAgent: + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36", + ciphers: "TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256", + curves: "secp384r1:X25519:prime256v1", + headers: { + accept: "*/*", + acceptLanguage: "en-US,en;q=0.8,de;q=0.6", + secFetchSite: "cross-site", + }, + }, + { + name: "Firefox 120", + userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0", + ciphers: "TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256", + curves: "X25519:secp384r1", + headers: { + accept: "video/*,audio/*,*/*;q=0.8", + acceptLanguage: "en-US,en;q=0.5", + secFetchSite: "same-origin", + }, + }, + { + name: "Safari-like Chrome", + userAgent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15", + ciphers: "TLS_AES_256_GCM_SHA384:TLS_AES_128_GCM_SHA256:TLS_CHACHA20_POLY1305_SHA256", + curves: "prime256v1:secp384r1:X25519", + headers: { + accept: "video/mp4,video/*,*/*;q=0.8", + acceptLanguage: "en-GB,en-US;q=0.9,en;q=0.8", + secFetchSite: "cross-site", + }, + }, +]; + +/** + * Additional header variations for randomization + */ +const HEADER_VARIATIONS = { + acceptEncodings: ["identity", "gzip, deflate, br", "gzip, deflate", "br, gzip, deflate"], + cacheControls: ["no-cache", "no-cache, no-store", "max-age=0", "no-cache, max-age=0"], + connections: ["keep-alive", "close"], + rateLimits: ["5M", "8M", "10M", "12M", "15M"], +}; + +/** + * Generate a randomized request configuration to evade fingerprinting + * @returns {Object} Configuration object with curl args and timing + */ +export const generateRandomRequestConfig = () => { + const config = BROWSER_CONFIGS[Math.floor(Math.random() * BROWSER_CONFIGS.length)]; + + // Add randomized variations + const variations = { + acceptEncoding: + HEADER_VARIATIONS.acceptEncodings[ + Math.floor(Math.random() * HEADER_VARIATIONS.acceptEncodings.length) + ], + cacheControl: + HEADER_VARIATIONS.cacheControls[ + Math.floor(Math.random() * HEADER_VARIATIONS.cacheControls.length) + ], + connection: + HEADER_VARIATIONS.connections[ + Math.floor(Math.random() * HEADER_VARIATIONS.connections.length) + ], + rateLimit: + HEADER_VARIATIONS.rateLimits[Math.floor(Math.random() * HEADER_VARIATIONS.rateLimits.length)], + }; + + // Random timing + const timing = { + preDelay: 1000 + Math.random() * 3000, // 1-4 seconds + keepaliveTime: Math.floor(60 + Math.random() * 60), // 60-120 seconds + connectTimeout: Math.floor(20 + Math.random() * 20), // 20-40 seconds + maxTime: Math.floor(250 + Math.random() * 100), // 250-350 seconds + }; + + // Optional headers (randomly included) + const optionalHeaders = { + includeUpgradeInsecure: Math.random() > 0.5, + includeDNT: Math.random() > 0.6, + includeSecFetchDest: Math.random() > 0.4, + includeSecFetchMode: Math.random() > 0.5, + includePragma: Math.random() > 0.7, + }; + + return { + ...config, + variations, + timing, + optionalHeaders, + sessionId: Math.random().toString(36).substring(2, 15), + }; +}; + +/** + * Download video using randomized curl configuration to evade detection + * @param {string} url - Video URL + * @param {string} referer - Referer URL (comments page) + * @param {string} outputPath - Where to save the file + * @param {number} attempt - Current attempt number (for logging) + * @returns {Promise} - Whether download was successful + */ +export const downloadVideoWithRandomizedCurl = async (url, referer, outputPath, attempt = 1) => { + return new Promise((resolve, reject) => { + const config = generateRandomRequestConfig(); + + const curlArgs = [ + "-L", // Follow redirects + "-s", // Silent mode to reduce log spam + "--max-redirs", + "5", + "--connect-timeout", + config.timing.connectTimeout.toString(), + "--max-time", + config.timing.maxTime.toString(), + + // TLS randomization + "--tls-max", + "1.3", + "--ciphers", + config.ciphers, + "--curves", + config.curves, + "--ssl-reqd", + "--ssl-no-revoke", + + // Browser fingerprint + "-A", + config.userAgent, + "-H", + `Accept: ${config.headers.accept}`, + "-H", + `Accept-Language: ${config.headers.acceptLanguage}`, + "-H", + `Accept-Encoding: ${config.variations.acceptEncoding}`, + "-H", + `Cache-Control: ${config.variations.cacheControl}`, + "-H", + `Connection: ${config.variations.connection}`, + "-H", + `Referer: ${referer}`, + + // Conditional headers for variation + ...(config.optionalHeaders.includeUpgradeInsecure + ? ["-H", "Upgrade-Insecure-Requests: 1"] + : []), + ...(config.optionalHeaders.includeDNT ? ["-H", "DNT: 1"] : []), + ...(config.optionalHeaders.includeSecFetchDest ? ["-H", "Sec-Fetch-Dest: video"] : []), + ...(config.optionalHeaders.includeSecFetchMode ? ["-H", "Sec-Fetch-Mode: no-cors"] : []), + ...(config.optionalHeaders.includePragma ? ["-H", "Pragma: no-cache"] : []), + "-H", + `Sec-Fetch-Site: ${config.headers.secFetchSite}`, + + // Connection management + "--keepalive-time", + config.timing.keepaliveTime.toString(), + "--limit-rate", + config.variations.rateLimit, + + // Output + "-o", + outputPath, + url, + ]; + + console.log( + `[Attempt ${attempt}] Using fingerprint: ${config.name} (Session: ${config.sessionId})` + ); + console.log( + `TLS: ${config.ciphers.split(":")[0]} | Rate: ${config.variations.rateLimit} | Delay: ${Math.round(config.timing.preDelay)}ms` + ); + + // Random pre-request delay + setTimeout(() => { + const curl = spawn("/usr/local/bin/curl", curlArgs); + let stderr = ""; + + curl.stderr.on("data", (data) => { + stderr += data.toString(); + }); + + curl.on("close", (code) => { + // Check for various blocking indicators + const isBlocked = + stderr.includes("403") || + stderr.includes("Forbidden") || + stderr.includes("Access Denied") || + (code === 0 && stderr.includes("text/html")); + + if (isBlocked) { + console.log(`❌ [Attempt ${attempt}] Request blocked with fingerprint: ${config.name}`); + if (stderr.includes("403")) { + console.log("Server returned 403 Forbidden"); + } + reject(new Error(`Request blocked (HTTP 403 or similar)`)); + return; + } + + // Check for successful response codes + const hasSuccessResponse = + stderr.includes("HTTP/2 200") || + stderr.includes("HTTP/1.1 200") || + stderr.includes("200 OK"); + + try { + const stats = fs.statSync(outputPath); + + if (stats.size === 0) { + console.log(`❌ [Attempt ${attempt}] Downloaded empty file`); + reject(new Error(`Downloaded empty file`)); + return; + } + + // Quick file format validation + const fileBuffer = fs.readFileSync(outputPath, { start: 0, end: 100 }); + + // Check for HTML responses (blocked) + const fileStart = fileBuffer.toString("utf8", 0, 50); + if ( + fileStart.includes(" { + console.log(`❌ [Attempt ${attempt}] Curl process error: ${error.message}`); + reject(new Error(`Curl execution failed: ${error.message}`)); + }); + }, config.timing.preDelay); + }); +}; + +/** + * Download with retry logic using different fingerprints + * @param {string} url - Video URL + * @param {string} referer - Referer URL + * @param {string} outputPath - Output file path + * @param {number} maxAttempts - Maximum number of attempts + * @returns {Promise} - Whether download was successful + */ +export const downloadWithRetry = async (url, referer, outputPath, maxAttempts = 3) => { + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + console.log(`Starting download attempt ${attempt}/${maxAttempts}`); + + // Clean up any previous partial download + if (fs.existsSync(outputPath)) { + fs.unlinkSync(outputPath); + } + + // Add progressive delay between attempts + if (attempt > 1) { + const retryDelay = 2000 + attempt * 1000 + Math.random() * 2000; // 3-5s, 4-6s, 5-7s + console.log(`Waiting ${Math.round(retryDelay)}ms before retry with new fingerprint...`); + await new Promise((resolve) => setTimeout(resolve, retryDelay)); + } + + await downloadVideoWithRandomizedCurl(url, referer, outputPath, attempt); + return true; // Success! + } catch (error) { + console.log(`Attempt ${attempt} failed: ${error.message}`); + + if (attempt === maxAttempts) { + console.error(`All ${maxAttempts} download attempts failed`); + throw new Error( + `Download failed after ${maxAttempts} attempts with different fingerprints` + ); + } + + // Clean up failed attempt + if (fs.existsSync(outputPath)) { + fs.unlinkSync(outputPath); + } + } + } +}; + +/** + * Get file size using randomized HEAD request + * @param {string} url - Video URL + * @param {string} referer - Referer URL + * @returns {Promise} - File size in bytes or null if failed + */ +export const getFileSizeWithRandomizedCurl = async (url, referer) => { + return new Promise((resolve) => { + const config = generateRandomRequestConfig(); + + const curlArgs = [ + "-I", // HEAD request only + "-L", // Follow redirects + "-s", // Silent mode + "--connect-timeout", + "20", + "--max-time", + "60", + "--max-redirs", + "3", + + // TLS settings + "--ciphers", + config.ciphers, + "--curves", + config.curves, + + // Headers + "-A", + config.userAgent, + "-H", + `Accept: ${config.headers.accept}`, + "-H", + `Accept-Language: ${config.headers.acceptLanguage}`, + "-H", + `Referer: ${referer}`, + "-H", + `Sec-Fetch-Site: ${config.headers.secFetchSite}`, + + url, + ]; + + const curl = spawn("/usr/local/bin/curl", curlArgs); + let stdout = ""; + + curl.stdout.on("data", (data) => { + stdout += data.toString(); + }); + + curl.on("close", (code) => { + if (code === 0 && !stdout.includes("403") && !stdout.includes("Forbidden")) { + const match = stdout.match(/content-length:\s*(\d+)/i); + const fileSize = match ? parseInt(match[1], 10) : null; + resolve(fileSize); + } else { + resolve(null); + } + }); + + curl.on("error", () => { + resolve(null); + }); + }); +}; + +/** + * Establish a browser session by visiting the referrer page first + * @param {string} referer - The referrer URL to visit + * @returns {Promise} - Whether session establishment was successful + */ +export const establishSession = async (referer) => { + return new Promise((resolve) => { + const config = generateRandomRequestConfig(); + + const curlArgs = [ + "-L", // Follow redirects + "-s", // Silent mode + "--connect-timeout", + "20", + "--max-time", + "60", + "--max-redirs", + "3", + + // TLS settings + "--ciphers", + config.ciphers, + "--curves", + config.curves, + + // Browser headers for page visit + "-A", + config.userAgent, + "-H", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "-H", + `Accept-Language: ${config.headers.acceptLanguage}`, + "-H", + "Accept-Encoding: gzip, deflate, br", + "-H", + "Cache-Control: no-cache", + "-H", + "Upgrade-Insecure-Requests: 1", + "-H", + "Sec-Fetch-Dest: document", + "-H", + "Sec-Fetch-Mode: navigate", + "-H", + "Sec-Fetch-Site: none", + "-H", + "Sec-Fetch-User: ?1", + + // Output to /dev/null since we just want to establish session + "-o", + "/dev/null", + + referer, + ]; + + console.log(`Establishing session by visiting: ${referer.substring(0, 50)}...`); + + const curl = spawn("/usr/local/bin/curl", curlArgs); + + curl.on("close", (code) => { + if (code === 0) { + console.log("✅ Session established successfully"); + resolve(true); + } else { + console.log("⚠️ Session establishment failed, continuing anyway"); + resolve(false); + } + }); + + curl.on("error", () => { + console.log("⚠️ Session establishment error, continuing anyway"); + resolve(false); + }); + }); +}; diff --git a/src/helpers/darwin/darwinProcess.js b/src/helpers/darwin/darwinProcess.js index fb51dd10..e0c048e3 100644 --- a/src/helpers/darwin/darwinProcess.js +++ b/src/helpers/darwin/darwinProcess.js @@ -6,127 +6,20 @@ import { spawn } from "child_process"; import { query } from "../db.js"; import { transcodeVideo, getFileSizeMB } from "./darwinTranscode.js"; import { isInCache, addToCache } from "./darwinCache.js"; +import { + downloadWithRetry, + getFileSizeWithRandomizedCurl, + establishSession, +} from "./darwinAntiFingerprint.js"; import config from "../../../config.json" with { type: "json" }; const darwinConfig = config.darwin; /** - * Download video using system curl (better TLS fingerprint) - * @param {string} url - Video URL - * @param {string} referer - Referer URL (comments page) - * @param {string} outputPath - Where to save the file - * @returns {Promise} - Whether download was successful + * Note: Download functions have been moved to darwinAntiFingerprint.js + * This module now provides advanced TLS fingerprint randomization + * to evade Cloudflare's bot detection systems. */ -const downloadVideoWithCurl = async (url, referer, outputPath) => { - return new Promise((resolve, reject) => { - const curlArgs = [ - "-L", // Follow redirects - "-A", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36", - "-H", - "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "-H", - "Accept-Language: en-US,en;q=0.5", - "-H", - "Accept-Encoding: gzip, deflate, br", - "-H", - "Upgrade-Insecure-Requests: 1", - "-H", - "Sec-Fetch-Dest: document", - "-H", - "Sec-Fetch-Mode: navigate", - "-H", - "Sec-Fetch-Site: cross-site", - "-H", - "Sec-Fetch-User: ?1", - "-H", - `Referer: ${referer}`, - "-H", - "Cache-Control: no-cache", - "-H", - "Pragma: no-cache", - "-o", - outputPath, - "--compressed", - "--http2", - url, - ]; - - const curl = spawn("curl", curlArgs); - - let stderr = ""; - - curl.stderr.on("data", (data) => { - stderr += data.toString(); - }); - - curl.on("close", (code) => { - if (code === 0) { - resolve(true); - } else { - reject(new Error(`curl exited with code ${code}: ${stderr}`)); - } - }); - - curl.on("error", (error) => { - reject(new Error(`Failed to spawn curl: ${error.message}`)); - }); - }); -}; - -/** - * Get file size using curl HEAD request - * @param {string} url - Video URL - * @param {string} referer - Referer URL (comments page) - * @returns {Promise} - File size in bytes, or null if failed - */ -const getFileSizeWithCurl = async (url, referer) => { - return new Promise((resolve) => { - const curlArgs = [ - "-I", // HEAD request only - "-L", // Follow redirects - "-s", // Silent mode - "-A", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36", - "-H", - "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "-H", - "Accept-Language: en-US,en;q=0.5", - "-H", - "Sec-Fetch-Dest: document", - "-H", - "Sec-Fetch-Mode: navigate", - "-H", - "Sec-Fetch-Site: cross-site", - "-H", - "Sec-Fetch-User: ?1", - "-H", - `Referer: ${referer}`, - "--http2", - url, - ]; - - const curl = spawn("curl", curlArgs); - let stdout = ""; - - curl.stdout.on("data", (data) => { - stdout += data.toString(); - }); - - curl.on("close", (code) => { - if (code === 0) { - const match = stdout.match(/content-length:\s*(\d+)/i); - resolve(match ? parseInt(match[1], 10) : null); - } else { - resolve(null); - } - }); - - curl.on("error", () => { - resolve(null); - }); - }); -}; /** * Get the final destination of a URL (follow redirects) @@ -218,7 +111,7 @@ const processVideo = async (video) => { // Check file size before downloading console.log(`Checking file size for: ${href}`); - const fileSizeBytes = await getFileSizeWithCurl(href, comments); + const fileSizeBytes = await getFileSizeWithRandomizedCurl(href, comments); if (fileSizeBytes) { const fileSizeMB = (fileSizeBytes / 1024 / 1024).toFixed(2); @@ -238,12 +131,29 @@ const processVideo = async (video) => { console.log("Could not determine file size, will proceed with download"); } - console.log(`Downloading video using curl: ${href}`); + console.log(`Downloading video using anti-fingerprint curl: ${href}`); + + // Establish session by visiting the referrer page first (like a real browser) + try { + await establishSession(comments); + // Wait a bit after session establishment before downloading video + const sessionDelay = 1000 + Math.random() * 2000; // 1-3 seconds + console.log(`Post-session delay: ${Math.round(sessionDelay)}ms`); + await new Promise((resolve) => setTimeout(resolve, sessionDelay)); + } catch (sessionError) { + console.log(`Session establishment failed, continuing anyway: ${sessionError.message}`); + } + + // Add random delay to avoid appearing automated + const preDownloadDelay = 500 + Math.random() * 1500; // 0.5-2 seconds + console.log(`Pre-download delay: ${Math.round(preDownloadDelay)}ms`); + await new Promise((resolve) => setTimeout(resolve, preDownloadDelay)); + try { - await downloadVideoWithCurl(href, comments, tempFilePath); - console.log("Successfully downloaded using curl"); - } catch (curlError) { - console.error(`Curl download failed: ${curlError.message}`); + await downloadWithRetry(href, comments, tempFilePath, 3); + console.log("Successfully downloaded using randomized fingerprint"); + } catch (downloadError) { + console.error(`All download attempts failed: ${downloadError.message}`); return null; } @@ -484,8 +394,10 @@ export const runDarwinProcess = async (client) => { } } + // Increased delay between video processing to reduce detection if (videoIndex < videosToProcess.length) { - await new Promise((resolve) => setTimeout(resolve, 500)); + const processingDelay = 1000 + Math.random() * 2000; // 1-3 seconds + await new Promise((resolve) => setTimeout(resolve, processingDelay)); } } } From fede69a625c94f373276f9c361f871dff4f7e39c Mon Sep 17 00:00:00 2001 From: VB2007 Date: Tue, 18 Nov 2025 19:56:52 +0100 Subject: [PATCH 07/17] added docs for darwin helper files --- src/helpers/darwin/README.md | 165 +++++++++++++++++++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100644 src/helpers/darwin/README.md diff --git a/src/helpers/darwin/README.md b/src/helpers/darwin/README.md new file mode 100644 index 00000000..5a19da41 --- /dev/null +++ b/src/helpers/darwin/README.md @@ -0,0 +1,165 @@ +# Darwin Video Processing System + +This module handles automated video discovery, downloading, transcoding, and distribution from external sources while evading bot detection systems. + +## Architecture + +### Core Files + +- **`darwinProcess.js`** - Main orchestration logic for video processing +- **`darwinTranscode.js`** - Video transcoding and optimization +- **`darwinCache.js`** - Video caching system to prevent duplicates +- **`darwinAntiFingerprint.js`** - Advanced anti-detection system for bypassing Cloudflare + +## Anti-Fingerprinting System + +### Overview + +The `darwinAntiFingerprint.js` module implements sophisticated techniques to bypass Cloudflare's bot detection by randomizing TLS fingerprints and browser characteristics. + +### Key Features + +#### 1. **TLS Fingerprint Randomization** +- Rotates between different cipher suites and elliptic curves +- Mimics various browser TLS implementations +- Changes SSL/TLS handshake patterns + +#### 2. **Browser Fingerprint Simulation** +- Multiple realistic browser configurations (Chrome, Firefox, Safari) +- Randomized User-Agent strings +- Platform-specific header patterns (Windows, macOS, Linux) + +#### 3. **Request Pattern Obfuscation** +- Random timing delays between requests (1-4 seconds) +- Variable rate limiting (5-15 MB/s) +- Progressive retry delays with new fingerprints + +#### 4. **Session Establishment** +- Visits referrer pages before downloading videos +- Mimics human browsing behavior +- Maintains realistic request flow + +### Browser Configurations + +The system rotates between these browser fingerprints: + +1. **Chrome 120 Windows** - Most common configuration +2. **Chrome 119 macOS** - Alternative Chrome version +3. **Chrome 118 Linux** - Linux-specific Chrome +4. **Firefox 120** - Different engine fingerprint +5. **Safari-like Chrome** - WebKit-based fingerprint + +### Detection Evasion Techniques + +#### TLS Randomization +```javascript +// Example cipher rotation +"TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_128_GCM_SHA256" +"TLS_CHACHA20_POLY1305_SHA256:TLS_AES_256_GCM_SHA384:TLS_AES_128_GCM_SHA256" +"TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256" +``` + +#### Request Headers Variation +- Accept headers that match browser type +- Randomized Accept-Language preferences +- Variable Accept-Encoding support +- Optional security headers (Sec-Fetch-*) + +#### Timing Patterns +- Pre-request delays: 1-4 seconds +- Retry delays: Progressive (3-7 seconds) +- Session establishment: 1-3 seconds +- Processing delays: 1-3 seconds between videos + +### Usage + +#### Download with Anti-Fingerprinting +```javascript +import { downloadWithRetry } from './darwinAntiFingerprint.js'; + +// Attempts download with up to 3 different fingerprints +await downloadWithRetry(videoUrl, refererUrl, outputPath, 3); +``` + +#### File Size Check +```javascript +import { getFileSizeWithRandomizedCurl } from './darwinAntiFingerprint.js'; + +// Uses randomized fingerprint for HEAD request +const fileSize = await getFileSizeWithRandomizedCurl(videoUrl, refererUrl); +``` + +#### Session Establishment +```javascript +import { establishSession } from './darwinAntiFingerprint.js'; + +// Visits page like a real browser before downloading +await establishSession(refererUrl); +``` + +## Configuration + +### Environment Variables +The system uses configuration from `config.json`: + +```json +{ + "darwin": { + "tempDir": "/path/to/temp", + "targetDir": "/path/to/final", + "cdnUrl": "https://your-cdn.com", + "maxDownloadSize": 50, + "feedUrl": "https://source-site.com/feed", + "markerOne": "video_marker_1", + "markerTwo": "https://source-site.com" + } +} +``` + +## Deployment Considerations + +### Curl Requirements +- Requires curl 8.x+ with modern TLS support +- OpenSSL 3.x+ for advanced cipher suites +- HTTP/2 and HTTP/3 support recommended + +### Performance +- Each download attempt includes 1-4 second delays +- Progressive retry delays for failed attempts +- Rate limiting prevents bandwidth saturation + +### Monitoring +The system logs detailed information about: +- Fingerprint selection and success rates +- Blocking detection and retry attempts +- Download success/failure patterns +- TLS configuration effectiveness + +## Troubleshooting + +### Common Issues + +#### 403 Forbidden Errors +- Indicates fingerprint detection +- System automatically retries with different fingerprints +- Check curl version and TLS support + +#### Empty Downloads (0 bytes) +- Usually HTML error pages instead of video +- Triggers automatic retry with new configuration +- May indicate IP-based blocking + +#### TLS Handshake Failures +- Check OpenSSL version compatibility +- Verify cipher suite support +- Update curl to latest version + +### Debug Mode +Enable verbose logging by modifying the curl args to include `-v` for detailed TLS handshake information. + +## Security Notes + +- All requests use legitimate browser fingerprints +- No malicious techniques are employed +- Respects rate limiting and server resources +- Designed to appear as normal browser traffic \ No newline at end of file From 73c094564ec80c7c9abd0494b9a5f4ee8a818bf2 Mon Sep 17 00:00:00 2001 From: VB2007 Date: Tue, 18 Nov 2025 22:05:09 +0100 Subject: [PATCH 08/17] fixed curl parameters, added debug file --- src/helpers/darwin/darwinAntiFingerprint.js | 173 ++++++++--- src/helpers/darwin/darwinProcess.js | 23 ++ src/helpers/darwin/debugCurl.js | 307 ++++++++++++++++++++ 3 files changed, 456 insertions(+), 47 deletions(-) create mode 100644 src/helpers/darwin/debugCurl.js diff --git a/src/helpers/darwin/darwinAntiFingerprint.js b/src/helpers/darwin/darwinAntiFingerprint.js index a0c638c2..81cf567e 100644 --- a/src/helpers/darwin/darwinAntiFingerprint.js +++ b/src/helpers/darwin/darwinAntiFingerprint.js @@ -1,5 +1,6 @@ import { spawn } from "child_process"; import fs from "fs"; +import { execSync } from "child_process"; /** * Browser fingerprint configurations for randomization @@ -9,8 +10,6 @@ const BROWSER_CONFIGS = [ name: "Chrome 120 Windows", userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", - ciphers: "TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_128_GCM_SHA256", - curves: "X25519:prime256v1:secp384r1", headers: { accept: "video/mp4,video/*,application/octet-stream,*/*;q=0.8", acceptLanguage: "en-US,en;q=0.9", @@ -21,8 +20,6 @@ const BROWSER_CONFIGS = [ name: "Chrome 119 macOS", userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36", - ciphers: "TLS_CHACHA20_POLY1305_SHA256:TLS_AES_256_GCM_SHA384:TLS_AES_128_GCM_SHA256", - curves: "prime256v1:X25519:secp384r1", headers: { accept: "video/webm,video/ogg,video/*;q=0.9,application/ogg;q=0.7,audio/*;q=0.6,*/*;q=0.5", acceptLanguage: "en-US,en;q=0.9", @@ -33,8 +30,6 @@ const BROWSER_CONFIGS = [ name: "Chrome 118 Linux", userAgent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36", - ciphers: "TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256", - curves: "secp384r1:X25519:prime256v1", headers: { accept: "*/*", acceptLanguage: "en-US,en;q=0.8,de;q=0.6", @@ -44,8 +39,6 @@ const BROWSER_CONFIGS = [ { name: "Firefox 120", userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0", - ciphers: "TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256", - curves: "X25519:secp384r1", headers: { accept: "video/*,audio/*,*/*;q=0.8", acceptLanguage: "en-US,en;q=0.5", @@ -56,8 +49,6 @@ const BROWSER_CONFIGS = [ name: "Safari-like Chrome", userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15", - ciphers: "TLS_AES_256_GCM_SHA384:TLS_AES_128_GCM_SHA256:TLS_CHACHA20_POLY1305_SHA256", - curves: "prime256v1:secp384r1:X25519", headers: { accept: "video/mp4,video/*,*/*;q=0.8", acceptLanguage: "en-GB,en-US;q=0.9,en;q=0.8", @@ -76,6 +67,119 @@ const HEADER_VARIATIONS = { rateLimits: ["5M", "8M", "10M", "12M", "15M"], }; +/** + * Find the correct curl executable path + * @returns {string} Path to curl executable + */ +const findCurlPath = () => { + const possiblePaths = [ + "/usr/local/bin/curl", + "/usr/bin/curl", + "/opt/homebrew/bin/curl", + "curl", // system PATH + ]; + + for (const path of possiblePaths) { + try { + if (path === "curl") { + // Check if curl is in PATH + execSync("which curl", { stdio: "ignore" }); + return "curl"; + } else { + // Check if file exists and is executable + if (fs.existsSync(path)) { + return path; + } + } + } catch (error) { + // Continue to next path + } + } + + // Default fallback + return "curl"; +}; + +/** + * Verify curl capabilities synchronously + * @param {string} curlPath - Path to curl executable + * @returns {Object} Curl capabilities info + */ +const verifyCurlCapabilities = (curlPath) => { + try { + const result = execSync(`${curlPath} --version`, { encoding: "utf8", timeout: 5000 }); + const versionMatch = result.match(/curl\s+([^\s]+)/); + const version = versionMatch ? versionMatch[1] : "unknown"; + + return { + version, + hasHTTP2: result.includes("HTTP2"), + hasHTTP3: result.includes("HTTP3"), + hasTLS13: result.includes("TLS"), + hasBrotli: result.includes("brotli"), + working: true, + }; + } catch (error) { + return { working: false, error: error.message }; + } +}; + +// Cache the curl path and verify capabilities on module load +const CURL_PATH = findCurlPath(); +const CURL_CAPABILITIES = verifyCurlCapabilities(CURL_PATH); + +if (CURL_CAPABILITIES.working) { + console.log(`✅ Using curl ${CURL_CAPABILITIES.version} at: ${CURL_PATH}`); + console.log( + `Features: HTTP2=${CURL_CAPABILITIES.hasHTTP2}, HTTP3=${CURL_CAPABILITIES.hasHTTP3}, Brotli=${CURL_CAPABILITIES.hasBrotli}` + ); +} else { + console.error(`❌ Curl verification failed: ${CURL_CAPABILITIES.error}`); + console.error(`Please ensure curl is properly installed and accessible`); +} + +/** + * Test curl with a simple request + * @returns {Promise} Whether curl is working + */ +export const testCurl = async () => { + return new Promise((resolve) => { + console.log("🧪 Testing curl functionality..."); + const curl = spawn(CURL_PATH, ["-s", "-I", "https://httpbin.org/get"]); + let stderr = ""; + + curl.stderr.on("data", (data) => { + stderr += data.toString(); + }); + + curl.on("close", (code) => { + if (code === 0) { + console.log("✅ Curl test successful"); + resolve(true); + } else { + console.error(`❌ Curl test failed with code ${code}: ${stderr}`); + resolve(false); + } + }); + + curl.on("error", (error) => { + console.error(`❌ Curl test error: ${error.message}`); + resolve(false); + }); + }); +}; + +/** + * Get curl information for debugging + * @returns {Object} Curl path and capabilities + */ +export const getCurlInfo = () => { + return { + path: CURL_PATH, + capabilities: CURL_CAPABILITIES, + }; +}; + /** * Generate a randomized request configuration to evade fingerprinting * @returns {Object} Configuration object with curl args and timing @@ -149,16 +253,6 @@ export const downloadVideoWithRandomizedCurl = async (url, referer, outputPath, "--max-time", config.timing.maxTime.toString(), - // TLS randomization - "--tls-max", - "1.3", - "--ciphers", - config.ciphers, - "--curves", - config.curves, - "--ssl-reqd", - "--ssl-no-revoke", - // Browser fingerprint "-A", config.userAgent, @@ -167,7 +261,7 @@ export const downloadVideoWithRandomizedCurl = async (url, referer, outputPath, "-H", `Accept-Language: ${config.headers.acceptLanguage}`, "-H", - `Accept-Encoding: ${config.variations.acceptEncoding}`, + `Accept-Encoding: identity`, "-H", `Cache-Control: ${config.variations.cacheControl}`, "-H", @@ -202,12 +296,12 @@ export const downloadVideoWithRandomizedCurl = async (url, referer, outputPath, `[Attempt ${attempt}] Using fingerprint: ${config.name} (Session: ${config.sessionId})` ); console.log( - `TLS: ${config.ciphers.split(":")[0]} | Rate: ${config.variations.rateLimit} | Delay: ${Math.round(config.timing.preDelay)}ms` + `Browser: ${config.userAgent.split(" ")[0]} | Rate: ${config.variations.rateLimit} | Delay: ${Math.round(config.timing.preDelay)}ms` ); // Random pre-request delay setTimeout(() => { - const curl = spawn("/usr/local/bin/curl", curlArgs); + const curl = spawn(CURL_PATH, curlArgs); let stderr = ""; curl.stderr.on("data", (data) => { @@ -231,11 +325,8 @@ export const downloadVideoWithRandomizedCurl = async (url, referer, outputPath, return; } - // Check for successful response codes - const hasSuccessResponse = - stderr.includes("HTTP/2 200") || - stderr.includes("HTTP/1.1 200") || - stderr.includes("200 OK"); + // In silent mode, we rely on exit code and file size for success detection + const hasSuccessResponse = code === 0; try { const stats = fs.statSync(outputPath); @@ -273,16 +364,12 @@ export const downloadVideoWithRandomizedCurl = async (url, referer, outputPath, return; } - if (code === 0 && hasSuccessResponse) { + if (code === 0) { const sizeMB = (stats.size / 1024 / 1024).toFixed(2); console.log( `✅ [Attempt ${attempt}] Successfully downloaded ${sizeMB}MB with ${config.name}` ); resolve(true); - } else if (code === 0) { - // Curl succeeded but no clear success response - might still be valid - console.log(`⚠️ [Attempt ${attempt}] Download completed but unclear response status`); - resolve(true); } else { reject(new Error(`curl exited with code ${code}`)); } @@ -294,6 +381,10 @@ export const downloadVideoWithRandomizedCurl = async (url, referer, outputPath, curl.on("error", (error) => { console.log(`❌ [Attempt ${attempt}] Curl process error: ${error.message}`); + if (error.code === "ENOENT") { + console.log(`Curl not found at: ${CURL_PATH}`); + console.log("Please ensure curl is installed and accessible"); + } reject(new Error(`Curl execution failed: ${error.message}`)); }); }, config.timing.preDelay); @@ -366,12 +457,6 @@ export const getFileSizeWithRandomizedCurl = async (url, referer) => { "--max-redirs", "3", - // TLS settings - "--ciphers", - config.ciphers, - "--curves", - config.curves, - // Headers "-A", config.userAgent, @@ -387,7 +472,7 @@ export const getFileSizeWithRandomizedCurl = async (url, referer) => { url, ]; - const curl = spawn("/usr/local/bin/curl", curlArgs); + const curl = spawn(CURL_PATH, curlArgs); let stdout = ""; curl.stdout.on("data", (data) => { @@ -429,12 +514,6 @@ export const establishSession = async (referer) => { "--max-redirs", "3", - // TLS settings - "--ciphers", - config.ciphers, - "--curves", - config.curves, - // Browser headers for page visit "-A", config.userAgent, @@ -466,7 +545,7 @@ export const establishSession = async (referer) => { console.log(`Establishing session by visiting: ${referer.substring(0, 50)}...`); - const curl = spawn("/usr/local/bin/curl", curlArgs); + const curl = spawn(CURL_PATH, curlArgs); curl.on("close", (code) => { if (code === 0) { diff --git a/src/helpers/darwin/darwinProcess.js b/src/helpers/darwin/darwinProcess.js index e0c048e3..57187de8 100644 --- a/src/helpers/darwin/darwinProcess.js +++ b/src/helpers/darwin/darwinProcess.js @@ -10,6 +10,8 @@ import { downloadWithRetry, getFileSizeWithRandomizedCurl, establishSession, + testCurl, + getCurlInfo, } from "./darwinAntiFingerprint.js"; import config from "../../../config.json" with { type: "json" }; @@ -154,6 +156,16 @@ const processVideo = async (video) => { console.log("Successfully downloaded using randomized fingerprint"); } catch (downloadError) { console.error(`All download attempts failed: ${downloadError.message}`); + + // Test curl again if downloads are failing + if (downloadError.message.includes("ENOENT") || downloadError.message.includes("spawn")) { + console.log("🔧 Re-testing curl due to spawn errors..."); + const curlStillWorking = await testCurl(); + if (!curlStillWorking) { + console.error("❌ Curl is no longer working - stopping Darwin process"); + } + } + return null; } @@ -347,6 +359,17 @@ const fetchVideosFromFeed = async () => { */ export const runDarwinProcess = async (client) => { try { + // Test curl functionality before starting + console.log("🔧 Verifying curl setup..."); + const curlInfo = getCurlInfo(); + console.log(`Curl info:`, curlInfo); + + const curlWorking = await testCurl(); + if (!curlWorking) { + console.error("❌ Curl test failed - cannot proceed with downloads"); + return; + } + // Get all guild configurations const guildConfigs = await query("SELECT * FROM configDarwin"); diff --git a/src/helpers/darwin/debugCurl.js b/src/helpers/darwin/debugCurl.js new file mode 100644 index 00000000..c9eb9722 --- /dev/null +++ b/src/helpers/darwin/debugCurl.js @@ -0,0 +1,307 @@ +#!/usr/bin/env node + +/** + * Debug script to test curl functionality for Darwin anti-fingerprinting system + * Run with: node debugCurl.js + */ + +import { spawn, execSync } from "child_process"; +import fs from "fs"; + +const colors = { + reset: "\x1b[0m", + bright: "\x1b[1m", + red: "\x1b[31m", + green: "\x1b[32m", + yellow: "\x1b[33m", + blue: "\x1b[34m", + magenta: "\x1b[35m", + cyan: "\x1b[36m", +}; + +const log = (color, message) => console.log(`${color}${message}${colors.reset}`); + +/** + * Find curl executable + */ +const findCurl = () => { + const paths = ["/usr/local/bin/curl", "/usr/bin/curl", "/opt/homebrew/bin/curl", "curl"]; + + for (const path of paths) { + try { + if (path === "curl") { + execSync("which curl", { stdio: "ignore" }); + return "curl"; + } else if (fs.existsSync(path)) { + return path; + } + } catch (e) { + // Continue + } + } + return null; +}; + +/** + * Test curl version and features + */ +const testCurlVersion = async (curlPath) => { + return new Promise((resolve) => { + log(colors.blue, `\n🔍 Testing curl version at: ${curlPath}`); + + const curl = spawn(curlPath, ["--version"]); + let stdout = ""; + let stderr = ""; + + curl.stdout.on("data", (data) => (stdout += data.toString())); + curl.stderr.on("data", (data) => (stderr += data.toString())); + + curl.on("close", (code) => { + if (code === 0) { + log(colors.green, "✅ Curl version check successful"); + console.log(stdout); + + const features = { + HTTP2: stdout.includes("HTTP2"), + HTTP3: stdout.includes("HTTP3"), + TLS: stdout.includes("TLS"), + Brotli: stdout.includes("brotli"), + OpenSSL: stdout.includes("OpenSSL"), + }; + + log(colors.cyan, "\n📋 Features:"); + Object.entries(features).forEach(([feature, has]) => { + log(has ? colors.green : colors.red, ` ${feature}: ${has ? "✅" : "❌"}`); + }); + + resolve(true); + } else { + log(colors.red, `❌ Curl version check failed (code ${code})`); + if (stderr) console.error(stderr); + resolve(false); + } + }); + + curl.on("error", (error) => { + log(colors.red, `❌ Curl spawn error: ${error.message}`); + resolve(false); + }); + }); +}; + +/** + * Test basic HTTP request + */ +const testBasicRequest = async (curlPath) => { + return new Promise((resolve) => { + log(colors.blue, "\n🌐 Testing basic HTTP request..."); + + const curl = spawn(curlPath, [ + "-s", + "-I", + "-L", + "--connect-timeout", + "10", + "--max-time", + "30", + "https://httpbin.org/get", + ]); + + let stdout = ""; + let stderr = ""; + + curl.stdout.on("data", (data) => (stdout += data.toString())); + curl.stderr.on("data", (data) => (stderr += data.toString())); + + curl.on("close", (code) => { + if (code === 0 && stdout.includes("200 OK")) { + log(colors.green, "✅ Basic HTTP request successful"); + resolve(true); + } else { + log(colors.red, `❌ Basic HTTP request failed (code ${code})`); + if (stderr) console.error("Error:", stderr); + if (stdout) console.log("Response:", stdout.substring(0, 200)); + resolve(false); + } + }); + + curl.on("error", (error) => { + log(colors.red, `❌ HTTP request error: ${error.message}`); + resolve(false); + }); + }); +}; + +/** + * Test TLS capabilities + */ +const testTLSCapabilities = async (curlPath) => { + return new Promise((resolve) => { + log(colors.blue, "\n🔒 Testing TLS 1.3 capabilities..."); + + const curl = spawn(curlPath, [ + "-s", + "-I", + "-L", + "--tls-max", + "1.3", + "--ciphers", + "TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256", + "--curves", + "X25519:prime256v1", + "--connect-timeout", + "10", + "--max-time", + "30", + "https://tls13.crypto.mozilla.org/", + ]); + + let stdout = ""; + let stderr = ""; + + curl.stdout.on("data", (data) => (stdout += data.toString())); + curl.stderr.on("data", (data) => (stderr += data.toString())); + + curl.on("close", (code) => { + if (code === 0 && (stdout.includes("200 OK") || stdout.includes("HTTP"))) { + log(colors.green, "✅ TLS 1.3 request successful"); + resolve(true); + } else { + log(colors.yellow, `⚠️ TLS 1.3 test inconclusive (code ${code})`); + log(colors.magenta, "This may still work with the target site"); + resolve(true); // Don't fail completely on TLS test + } + }); + + curl.on("error", (error) => { + log(colors.yellow, `⚠️ TLS test error: ${error.message}`); + resolve(true); // Don't fail completely + }); + }); +}; + +/** + * Test actual video site request + */ +const testVideoSiteRequest = async (curlPath) => { + return new Promise((resolve) => { + log(colors.blue, "\n📹 Testing video site request..."); + + const curl = spawn(curlPath, [ + "-s", + "-I", + "-L", + "-A", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "-H", + "Accept: video/mp4,video/*,*/*;q=0.8", + "-H", + "Accept-Language: en-US,en;q=0.9", + "-H", + "Referer: https://theync.com/", + "--connect-timeout", + "15", + "--max-time", + "60", + "https://theync.com/media/videos/6/9/1/c/b/691cb2bb47ee2.mp4", + ]); + + let stdout = ""; + let stderr = ""; + + curl.stdout.on("data", (data) => (stdout += data.toString())); + curl.stderr.on("data", (data) => (stderr += data.toString())); + + curl.on("close", (code) => { + log(colors.cyan, "\n📄 Response headers:"); + console.log(stdout.substring(0, 500)); + + if (code === 0) { + if (stdout.includes("200 OK") && stdout.includes("video/mp4")) { + log(colors.green, "✅ Video site allows access!"); + } else if (stdout.includes("403")) { + log(colors.yellow, "⚠️ Video site returned 403 (blocked)"); + } else if (stdout.includes("404")) { + log(colors.yellow, "⚠️ Video not found (404) - expected for test"); + } else { + log(colors.yellow, `⚠️ Video site returned unexpected response`); + } + resolve(true); + } else { + log(colors.red, `❌ Video site request failed (code ${code})`); + if (stderr) console.error("Error:", stderr); + resolve(false); + } + }); + + curl.on("error", (error) => { + log(colors.red, `❌ Video site request error: ${error.message}`); + resolve(false); + }); + }); +}; + +/** + * Main debug function + */ +const runDebug = async () => { + log(colors.bright + colors.magenta, "🔧 Darwin Curl Debug Tool"); + log(colors.bright + colors.magenta, "============================"); + + // Find curl + const curlPath = findCurl(); + if (!curlPath) { + log(colors.red, "❌ No curl executable found!"); + log(colors.yellow, "Please install curl or ensure it's in your PATH"); + process.exit(1); + } + + log(colors.green, `✅ Found curl at: ${curlPath}`); + + // Run tests + const tests = [ + ["Version Check", () => testCurlVersion(curlPath)], + ["Basic HTTP", () => testBasicRequest(curlPath)], + ["TLS Capabilities", () => testTLSCapabilities(curlPath)], + ["Video Site Access", () => testVideoSiteRequest(curlPath)], + ]; + + const results = []; + for (const [name, testFn] of tests) { + const result = await testFn(); + results.push([name, result]); + } + + // Summary + log(colors.bright + colors.magenta, "\n📊 Test Summary:"); + log(colors.bright + colors.magenta, "================"); + + let allPassed = true; + for (const [name, passed] of results) { + const status = passed ? "✅ PASS" : "❌ FAIL"; + const color = passed ? colors.green : colors.red; + log(color, `${status} ${name}`); + if (!passed) allPassed = false; + } + + if (allPassed) { + log(colors.bright + colors.green, "\n🎉 All tests passed! Curl should work with Darwin."); + } else { + log( + colors.bright + colors.red, + "\n⚠️ Some tests failed. Check curl installation and network connectivity." + ); + } + + log(colors.bright + colors.cyan, "\n💡 Usage tips:"); + log(colors.cyan, "- If TLS test failed, update curl and OpenSSL"); + log(colors.cyan, "- If video site blocks, the anti-fingerprinting system will help"); + log(colors.cyan, "- Run this script periodically to verify connectivity"); +}; + +// Run if called directly +if (import.meta.url === `file://${process.argv[1]}`) { + runDebug().catch(console.error); +} + +export { runDebug as debugCurl }; From 7ae7c99f35943b14b4b38b93a47ab0ebb0a3528b Mon Sep 17 00:00:00 2001 From: VB2007 Date: Tue, 18 Nov 2025 22:05:29 +0100 Subject: [PATCH 09/17] written back deploy workflow trigger --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index ea5a2469..382d2746 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -3,7 +3,7 @@ permissions: {} on: push: - branches: [dev-darwin-curl-fix] + branches: [main] workflow_dispatch: inputs: branch: From 169cf001b86e92080c438c359a524d4988e7e14e Mon Sep 17 00:00:00 2001 From: VB2007 Date: Tue, 18 Nov 2025 23:19:21 +0100 Subject: [PATCH 10/17] tried adaptive fingerprint bypass --- src/helpers/darwin/captureDevPCBehavior.js | 358 ++++++++++++++++++++ src/helpers/darwin/darwinAntiFingerprint.js | 336 +++++++++--------- 2 files changed, 522 insertions(+), 172 deletions(-) create mode 100644 src/helpers/darwin/captureDevPCBehavior.js diff --git a/src/helpers/darwin/captureDevPCBehavior.js b/src/helpers/darwin/captureDevPCBehavior.js new file mode 100644 index 00000000..bf3fdef6 --- /dev/null +++ b/src/helpers/darwin/captureDevPCBehavior.js @@ -0,0 +1,358 @@ +#!/usr/bin/env node + +/** + * Script to capture exact dev PC curl behavior for replication on server + * Run this on your dev PC to capture working curl commands + * Then run on server to test if same commands work + */ + +import { spawn } from "child_process"; +import fs from "fs"; +import { execSync } from "child_process"; + +const colors = { + reset: "\x1b[0m", + bright: "\x1b[1m", + red: "\x1b[31m", + green: "\x1b[32m", + yellow: "\x1b[33m", + blue: "\x1b[34m", + magenta: "\x1b[35m", + cyan: "\x1b[36m", +}; + +const log = (color, message) => console.log(`${color}${message}${colors.reset}`); + +/** + * Capture system information that might affect curl behavior + */ +const captureSystemInfo = () => { + const info = {}; + + try { + info.platform = process.platform; + info.arch = process.arch; + info.nodeVersion = process.version; + + // Get OS info + if (process.platform === 'linux') { + try { + info.osRelease = execSync('cat /etc/os-release', { encoding: 'utf8' }).split('\n')[0]; + } catch (e) { + info.osRelease = 'Unknown Linux'; + } + } + + // Get curl info + try { + info.curlVersion = execSync('curl --version', { encoding: 'utf8' }).split('\n')[0]; + } catch (e) { + info.curlVersion = 'curl not found'; + } + + // Get OpenSSL info + try { + info.opensslVersion = execSync('openssl version', { encoding: 'utf8' }).trim(); + } catch (e) { + info.opensslVersion = 'openssl not found'; + } + + // Get network interface info (might affect routing) + try { + const interfaces = require('os').networkInterfaces(); + info.networkInterfaces = Object.keys(interfaces).length; + } catch (e) { + info.networkInterfaces = 'unknown'; + } + + } catch (error) { + log(colors.red, `Error capturing system info: ${error.message}`); + } + + return info; +}; + +/** + * Test the exact curl command that works on dev PC + */ +const testWorkingCurlCommand = async () => { + return new Promise((resolve) => { + log(colors.blue, "\n🧪 Testing exact working curl command..."); + + const testUrl = "https://theync.com/media/videos/6/9/1/c/b/691cb2bb47ee2.mp4"; + const referer = "https://theync.com/some-page"; + const outputFile = "/tmp/capture-test.mp4"; + + // This is the EXACT command that works on your dev PC + const curlArgs = [ + "-L", + "-v", + "-o", outputFile, + "-A", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36", + "-H", "Accept: video/mp4,video/*,application/octet-stream,*/*;q=0.8", + "-H", "Accept-Language: en-US,en;q=0.5", + "-H", "Accept-Encoding: identity", + "-H", `Referer: ${referer}`, + testUrl + ]; + + log(colors.cyan, `Command: curl ${curlArgs.join(' ')}`); + + const curl = spawn("curl", curlArgs); + let stderr = ""; + + curl.stderr.on("data", (data) => { + stderr += data.toString(); + }); + + curl.on("close", (code) => { + const result = { + exitCode: code, + verboseOutput: stderr, + success: false, + fileSize: 0, + responseCode: null, + contentType: null, + serverHeaders: {}, + tlsInfo: null + }; + + // Parse verbose output for details + if (stderr) { + // Extract HTTP response code + const responseMatch = stderr.match(/HTTP\/[12]\.[01] (\d+)/); + if (responseMatch) { + result.responseCode = parseInt(responseMatch[1]); + } + + // Extract content type + const contentTypeMatch = stderr.match(/content-type:\s*([^\r\n]+)/i); + if (contentTypeMatch) { + result.contentType = contentTypeMatch[1].trim(); + } + + // Extract TLS connection info + const tlsMatch = stderr.match(/SSL connection using\s+([^\r\n]+)/); + if (tlsMatch) { + result.tlsInfo = tlsMatch[1].trim(); + } + + // Extract server + const serverMatch = stderr.match(/server:\s*([^\r\n]+)/i); + if (serverMatch) { + result.serverHeaders.server = serverMatch[1].trim(); + } + } + + // Check file + try { + if (fs.existsSync(outputFile)) { + const stats = fs.statSync(outputFile); + result.fileSize = stats.size; + + if (stats.size > 0) { + const buffer = fs.readFileSync(outputFile, { start: 0, end: 100 }); + const isVideo = buffer.includes(Buffer.from('ftyp')) && buffer.includes(Buffer.from('isom')); + const isHtml = buffer.toString('utf8', 0, 50).includes(' { + resolve({ + success: false, + error: error.message + }); + }); + }); +}; + +/** + * Test various curl configurations to identify working patterns + */ +const testCurlVariations = async () => { + const variations = [ + { + name: "Basic", + args: ["-L", "-v", "-o", "/tmp/var-basic.mp4", "https://theync.com/media/videos/6/9/1/c/b/691cb2bb47ee2.mp4"] + }, + { + name: "With User-Agent", + args: ["-L", "-v", "-A", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", "-o", "/tmp/var-ua.mp4", "https://theync.com/media/videos/6/9/1/c/b/691cb2bb47ee2.mp4"] + }, + { + name: "With Headers", + args: ["-L", "-v", "-A", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", "-H", "Accept: video/mp4,*/*", "-H", "Referer: https://theync.com/", "-o", "/tmp/var-headers.mp4", "https://theync.com/media/videos/6/9/1/c/b/691cb2bb47ee2.mp4"] + }, + { + name: "With HTTP2", + args: ["-L", "-v", "--http2", "-A", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", "-H", "Accept: video/mp4,*/*", "-o", "/tmp/var-http2.mp4", "https://theync.com/media/videos/6/9/1/c/b/691cb2bb47ee2.mp4"] + }, + { + name: "With Compression", + args: ["-L", "-v", "--compressed", "-A", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", "-o", "/tmp/var-compressed.mp4", "https://theync.com/media/videos/6/9/1/c/b/691cb2bb47ee2.mp4"] + } + ]; + + const results = []; + + for (const variation of variations) { + log(colors.blue, `\n🔄 Testing variation: ${variation.name}`); + + const result = await new Promise((resolve) => { + const curl = spawn("curl", variation.args); + let stderr = ""; + + curl.stderr.on("data", (data) => { + stderr += data.toString(); + }); + + curl.on("close", (code) => { + const outputFile = variation.args.find((arg, i) => variation.args[i-1] === '-o'); + let success = false; + let fileSize = 0; + let responseCode = null; + + // Parse response code + const responseMatch = stderr.match(/HTTP\/[12]\.[01] (\d+)/); + if (responseMatch) { + responseCode = parseInt(responseMatch[1]); + } + + // Check file + try { + if (fs.existsSync(outputFile)) { + const stats = fs.statSync(outputFile); + fileSize = stats.size; + + if (fileSize > 0) { + const buffer = fs.readFileSync(outputFile, { start: 0, end: 50 }); + const isVideo = buffer.includes(Buffer.from('ftyp')); + const isHtml = buffer.toString('utf8').includes(' { + resolve({ + name: variation.name, + error: error.message, + success: false + }); + }); + }); + + results.push(result); + + const status = result.success ? colors.green + "✅ SUCCESS" : colors.red + "❌ FAILED"; + log(status, `${variation.name}: ${result.responseCode || 'No response'} - ${result.fileSize} bytes`); + } + + return results; +}; + +/** + * Generate a comprehensive report + */ +const generateReport = async () => { + log(colors.bright + colors.magenta, "🔍 Dev PC Curl Behavior Capture"); + log(colors.bright + colors.magenta, "=================================="); + + // System info + log(colors.cyan, "\n📋 System Information:"); + const systemInfo = captureSystemInfo(); + Object.entries(systemInfo).forEach(([key, value]) => { + log(colors.cyan, ` ${key}: ${value}`); + }); + + // Test exact working command + log(colors.yellow, "\n🎯 Testing exact working command..."); + const mainResult = await testWorkingCurlCommand(); + + if (mainResult.success) { + log(colors.green, "✅ Main command successful!"); + log(colors.green, ` Response: ${mainResult.responseCode}`); + log(colors.green, ` Content-Type: ${mainResult.contentType}`); + log(colors.green, ` File Size: ${mainResult.fileSize} bytes`); + log(colors.green, ` TLS: ${mainResult.tlsInfo}`); + } else { + log(colors.red, "❌ Main command failed!"); + if (mainResult.error) { + log(colors.red, ` Error: ${mainResult.error}`); + } else { + log(colors.red, ` Response: ${mainResult.responseCode}`); + log(colors.red, ` File Size: ${mainResult.fileSize} bytes`); + } + } + + // Test variations + log(colors.yellow, "\n🔬 Testing curl variations..."); + const variations = await testCurlVariations(); + + // Summary + log(colors.bright + colors.magenta, "\n📊 Summary:"); + const successful = variations.filter(v => v.success); + log(colors.cyan, `Working variations: ${successful.length}/${variations.length}`); + + if (successful.length > 0) { + log(colors.green, "✅ Successful configurations:"); + successful.forEach(v => { + log(colors.green, ` - ${v.name}: ${v.responseCode} (${v.fileSize} bytes)`); + }); + } + + const failed = variations.filter(v => !v.success); + if (failed.length > 0) { + log(colors.red, "\n❌ Failed configurations:"); + failed.forEach(v => { + log(colors.red, ` - ${v.name}: ${v.responseCode || 'Error'} (${v.fileSize} bytes)`); + }); + } + + // Export data for server testing + const exportData = { + systemInfo, + mainResult, + variations, + timestamp: new Date().toISOString() + }; + + const exportFile = '/tmp/curl-behavior-capture.json'; + fs.writeFileSync(exportFile, JSON.stringify(exportData, null, 2)); + log(colors.bright + colors.cyan, `\n💾 Data exported to: ${exportFile}`); + log(colors.cyan, "Copy this file to your server and run the same script there for comparison"); + + return exportData; +}; + +// Run if called directly +if (import.meta.url === `file://${process.argv[1]}`) { + generateReport().catch(console.error); +} + +export { generateReport as captureDevPCBehavior }; diff --git a/src/helpers/darwin/darwinAntiFingerprint.js b/src/helpers/darwin/darwinAntiFingerprint.js index 81cf567e..afaebf27 100644 --- a/src/helpers/darwin/darwinAntiFingerprint.js +++ b/src/helpers/darwin/darwinAntiFingerprint.js @@ -232,162 +232,137 @@ export const generateRandomRequestConfig = () => { }; /** - * Download video using randomized curl configuration to evade detection + * Create adaptive curl configuration based on system capabilities + * @param {string} url - Video URL + * @param {string} referer - Referer URL + * @param {string} outputPath - Output file path + * @returns {Array} Curl arguments array + */ +const createAdaptiveCurlArgs = (url, referer, outputPath) => { + // Base arguments that work across systems + const baseArgs = [ + "-L", // Follow redirects + "-s", // Silent mode to reduce noise + "--max-redirs", + "5", + "--connect-timeout", + "30", + "--max-time", + "300", + + // Core headers that match successful dev PC behavior + "-A", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36", + "-H", + "Accept: video/mp4,video/*,application/octet-stream,*/*;q=0.8", + "-H", + "Accept-Language: en-US,en;q=0.5", + "-H", + "Accept-Encoding: identity", + "-H", + `Referer: ${referer}`, + + "-o", + outputPath, + url, + ]; + + // Try enhanced options that work on newer systems + if (CURL_CAPABILITIES.hasHTTP2) { + baseArgs.splice(-3, 0, "--http2"); + } + + if (CURL_CAPABILITIES.hasBrotli) { + baseArgs.splice(-3, 0, "--compressed"); + } + + return baseArgs; +}; + +/** + * Download video with system-adaptive approach * @param {string} url - Video URL * @param {string} referer - Referer URL (comments page) * @param {string} outputPath - Where to save the file * @param {number} attempt - Current attempt number (for logging) * @returns {Promise} - Whether download was successful */ -export const downloadVideoWithRandomizedCurl = async (url, referer, outputPath, attempt = 1) => { +const downloadVideoWithRandomizedCurl = async (url, referer, outputPath, attempt = 1) => { return new Promise((resolve, reject) => { - const config = generateRandomRequestConfig(); + const curlArgs = createAdaptiveCurlArgs(url, referer, outputPath); - const curlArgs = [ - "-L", // Follow redirects - "-s", // Silent mode to reduce log spam - "--max-redirs", - "5", - "--connect-timeout", - config.timing.connectTimeout.toString(), - "--max-time", - config.timing.maxTime.toString(), + console.log( + `[Attempt ${attempt}] Adaptive download (${CURL_CAPABILITIES.hasHTTP2 ? "HTTP2" : "HTTP1"}, ${CURL_CAPABILITIES.version})` + ); - // Browser fingerprint - "-A", - config.userAgent, - "-H", - `Accept: ${config.headers.accept}`, - "-H", - `Accept-Language: ${config.headers.acceptLanguage}`, - "-H", - `Accept-Encoding: identity`, - "-H", - `Cache-Control: ${config.variations.cacheControl}`, - "-H", - `Connection: ${config.variations.connection}`, - "-H", - `Referer: ${referer}`, + const curl = spawn(CURL_PATH, curlArgs); + let stderr = ""; - // Conditional headers for variation - ...(config.optionalHeaders.includeUpgradeInsecure - ? ["-H", "Upgrade-Insecure-Requests: 1"] - : []), - ...(config.optionalHeaders.includeDNT ? ["-H", "DNT: 1"] : []), - ...(config.optionalHeaders.includeSecFetchDest ? ["-H", "Sec-Fetch-Dest: video"] : []), - ...(config.optionalHeaders.includeSecFetchMode ? ["-H", "Sec-Fetch-Mode: no-cors"] : []), - ...(config.optionalHeaders.includePragma ? ["-H", "Pragma: no-cache"] : []), - "-H", - `Sec-Fetch-Site: ${config.headers.secFetchSite}`, + curl.stderr.on("data", (data) => { + stderr += data.toString(); + }); - // Connection management - "--keepalive-time", - config.timing.keepaliveTime.toString(), - "--limit-rate", - config.variations.rateLimit, + curl.on("close", (code) => { + // Log any curl errors for debugging + if (code !== 0) { + console.log(`Curl exit code: ${code}`); + if (stderr) { + console.log(`Curl error: ${stderr.substring(0, 200)}...`); + } + } - // Output - "-o", - outputPath, - url, - ]; + try { + const stats = fs.statSync(outputPath); - console.log( - `[Attempt ${attempt}] Using fingerprint: ${config.name} (Session: ${config.sessionId})` - ); - console.log( - `Browser: ${config.userAgent.split(" ")[0]} | Rate: ${config.variations.rateLimit} | Delay: ${Math.round(config.timing.preDelay)}ms` - ); + if (stats.size === 0) { + console.log(`❌ [Attempt ${attempt}] Downloaded empty file`); + reject(new Error(`Downloaded empty file`)); + return; + } - // Random pre-request delay - setTimeout(() => { - const curl = spawn(CURL_PATH, curlArgs); - let stderr = ""; - - curl.stderr.on("data", (data) => { - stderr += data.toString(); - }); - - curl.on("close", (code) => { - // Check for various blocking indicators - const isBlocked = - stderr.includes("403") || - stderr.includes("Forbidden") || - stderr.includes("Access Denied") || - (code === 0 && stderr.includes("text/html")); - - if (isBlocked) { - console.log(`❌ [Attempt ${attempt}] Request blocked with fingerprint: ${config.name}`); - if (stderr.includes("403")) { - console.log("Server returned 403 Forbidden"); - } - reject(new Error(`Request blocked (HTTP 403 or similar)`)); + // Validate file content + const fileBuffer = fs.readFileSync(outputPath, { start: 0, end: 100 }); + + // Check for HTML error pages (Cloudflare blocking) + const fileStart = fileBuffer.toString("utf8", 0, 50); + if ( + fileStart.includes(" { - console.log(`❌ [Attempt ${attempt}] Curl process error: ${error.message}`); - if (error.code === "ENOENT") { - console.log(`Curl not found at: ${CURL_PATH}`); - console.log("Please ensure curl is installed and accessible"); + if (code === 0) { + const sizeMB = (stats.size / 1024 / 1024).toFixed(2); + console.log(`✅ [Attempt ${attempt}] Successfully downloaded ${sizeMB}MB video`); + resolve(true); + } else { + reject(new Error(`curl exited with code ${code}: ${stderr}`)); } - reject(new Error(`Curl execution failed: ${error.message}`)); - }); - }, config.timing.preDelay); + } catch (statError) { + console.log(`❌ [Attempt ${attempt}] File validation failed: ${statError.message}`); + reject(new Error(`File validation failed: ${statError.message}`)); + } + }); + + curl.on("error", (error) => { + console.log(`❌ [Attempt ${attempt}] Curl process error: ${error.message}`); + reject(new Error(`Curl execution failed: ${error.message}`)); + }); }); }; @@ -400,32 +375,50 @@ export const downloadVideoWithRandomizedCurl = async (url, referer, outputPath, * @returns {Promise} - Whether download was successful */ export const downloadWithRetry = async (url, referer, outputPath, maxAttempts = 3) => { + // Try different strategies on each attempt + const strategies = [ + { name: "Direct", delay: 0 }, + { name: "With session delay", delay: 2000 }, + { name: "Fallback mode", delay: 5000 }, + ]; + for (let attempt = 1; attempt <= maxAttempts; attempt++) { try { - console.log(`Starting download attempt ${attempt}/${maxAttempts}`); + const strategy = strategies[Math.min(attempt - 1, strategies.length - 1)]; + console.log(`Download attempt ${attempt}/${maxAttempts} - Strategy: ${strategy.name}`); // Clean up any previous partial download if (fs.existsSync(outputPath)) { fs.unlinkSync(outputPath); } - // Add progressive delay between attempts - if (attempt > 1) { - const retryDelay = 2000 + attempt * 1000 + Math.random() * 2000; // 3-5s, 4-6s, 5-7s - console.log(`Waiting ${Math.round(retryDelay)}ms before retry with new fingerprint...`); - await new Promise((resolve) => setTimeout(resolve, retryDelay)); + // Apply strategy-specific delay + if (strategy.delay > 0) { + console.log(`Applying ${strategy.delay}ms delay for strategy: ${strategy.name}`); + await new Promise((resolve) => setTimeout(resolve, strategy.delay)); } await downloadVideoWithRandomizedCurl(url, referer, outputPath, attempt); + console.log(`✅ Success with strategy: ${strategy.name}`); return true; // Success! } catch (error) { - console.log(`Attempt ${attempt} failed: ${error.message}`); + console.log(`❌ Attempt ${attempt} failed: ${error.message}`); if (attempt === maxAttempts) { console.error(`All ${maxAttempts} download attempts failed`); - throw new Error( - `Download failed after ${maxAttempts} attempts with different fingerprints` - ); + + // Enhanced error reporting + if (error.message.includes("HTML error page")) { + throw new Error( + `Server consistently blocking requests - received HTML instead of video after ${maxAttempts} attempts` + ); + } else if (error.message.includes("empty file")) { + throw new Error( + `Server returning empty responses - possible network or authentication issue` + ); + } else { + throw new Error(`Download failed after ${maxAttempts} attempts: ${error.message}`); + } } // Clean up failed attempt @@ -444,8 +437,7 @@ export const downloadWithRetry = async (url, referer, outputPath, maxAttempts = */ export const getFileSizeWithRandomizedCurl = async (url, referer) => { return new Promise((resolve) => { - const config = generateRandomRequestConfig(); - + // Create adaptive HEAD request const curlArgs = [ "-I", // HEAD request only "-L", // Follow redirects @@ -457,21 +449,24 @@ export const getFileSizeWithRandomizedCurl = async (url, referer) => { "--max-redirs", "3", - // Headers + // Core headers "-A", - config.userAgent, + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36", "-H", - `Accept: ${config.headers.accept}`, + "Accept: video/mp4,video/*,application/octet-stream,*/*;q=0.8", "-H", - `Accept-Language: ${config.headers.acceptLanguage}`, + "Accept-Language: en-US,en;q=0.5", "-H", `Referer: ${referer}`, - "-H", - `Sec-Fetch-Site: ${config.headers.secFetchSite}`, url, ]; + // Add HTTP/2 if supported + if (CURL_CAPABILITIES.hasHTTP2) { + curlArgs.splice(-1, 0, "--http2"); + } + const curl = spawn(CURL_PATH, curlArgs); let stdout = ""; @@ -485,6 +480,7 @@ export const getFileSizeWithRandomizedCurl = async (url, referer) => { const fileSize = match ? parseInt(match[1], 10) : null; resolve(fileSize); } else { + // Don't log failures for HEAD requests - they often fail but downloads still work resolve(null); } }); @@ -502,8 +498,7 @@ export const getFileSizeWithRandomizedCurl = async (url, referer) => { */ export const establishSession = async (referer) => { return new Promise((resolve) => { - const config = generateRandomRequestConfig(); - + // Create adaptive session establishment const curlArgs = [ "-L", // Follow redirects "-s", // Silent mode @@ -514,35 +509,32 @@ export const establishSession = async (referer) => { "--max-redirs", "3", - // Browser headers for page visit + // Browser headers for page visits "-A", - config.userAgent, - "-H", - "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36", "-H", - `Accept-Language: ${config.headers.acceptLanguage}`, + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "-H", - "Accept-Encoding: gzip, deflate, br", + "Accept-Language: en-US,en;q=0.5", "-H", "Cache-Control: no-cache", "-H", "Upgrade-Insecure-Requests: 1", - "-H", - "Sec-Fetch-Dest: document", - "-H", - "Sec-Fetch-Mode: navigate", - "-H", - "Sec-Fetch-Site: none", - "-H", - "Sec-Fetch-User: ?1", - // Output to /dev/null since we just want to establish session "-o", "/dev/null", - referer, ]; + // Add enhanced features if available + if (CURL_CAPABILITIES.hasHTTP2) { + curlArgs.splice(-2, 0, "--http2"); + } + if (CURL_CAPABILITIES.hasBrotli) { + curlArgs.splice(-2, 0, "--compressed"); + curlArgs.splice(-2, 0, "-H", "Accept-Encoding: gzip, deflate, br"); + } + console.log(`Establishing session by visiting: ${referer.substring(0, 50)}...`); const curl = spawn(CURL_PATH, curlArgs); From 115b6ac6400f85726598b793a21c1f0bf1853ac6 Mon Sep 17 00:00:00 2001 From: VB2007 Date: Wed, 19 Nov 2025 00:00:58 +0100 Subject: [PATCH 11/17] removed download, transcode, tls spoof, etc. logic from darwin, used direct links --- src/helpers/darwin/README.md | 165 ------ src/helpers/darwin/captureDevPCBehavior.js | 358 ------------- src/helpers/darwin/darwinAntiFingerprint.js | 557 -------------------- src/helpers/darwin/darwinProcess.js | 395 +++----------- src/helpers/darwin/darwinTranscode.js | 225 -------- src/helpers/darwin/debugCurl.js | 307 ----------- 6 files changed, 71 insertions(+), 1936 deletions(-) delete mode 100644 src/helpers/darwin/README.md delete mode 100644 src/helpers/darwin/captureDevPCBehavior.js delete mode 100644 src/helpers/darwin/darwinAntiFingerprint.js delete mode 100644 src/helpers/darwin/darwinTranscode.js delete mode 100644 src/helpers/darwin/debugCurl.js diff --git a/src/helpers/darwin/README.md b/src/helpers/darwin/README.md deleted file mode 100644 index 5a19da41..00000000 --- a/src/helpers/darwin/README.md +++ /dev/null @@ -1,165 +0,0 @@ -# Darwin Video Processing System - -This module handles automated video discovery, downloading, transcoding, and distribution from external sources while evading bot detection systems. - -## Architecture - -### Core Files - -- **`darwinProcess.js`** - Main orchestration logic for video processing -- **`darwinTranscode.js`** - Video transcoding and optimization -- **`darwinCache.js`** - Video caching system to prevent duplicates -- **`darwinAntiFingerprint.js`** - Advanced anti-detection system for bypassing Cloudflare - -## Anti-Fingerprinting System - -### Overview - -The `darwinAntiFingerprint.js` module implements sophisticated techniques to bypass Cloudflare's bot detection by randomizing TLS fingerprints and browser characteristics. - -### Key Features - -#### 1. **TLS Fingerprint Randomization** -- Rotates between different cipher suites and elliptic curves -- Mimics various browser TLS implementations -- Changes SSL/TLS handshake patterns - -#### 2. **Browser Fingerprint Simulation** -- Multiple realistic browser configurations (Chrome, Firefox, Safari) -- Randomized User-Agent strings -- Platform-specific header patterns (Windows, macOS, Linux) - -#### 3. **Request Pattern Obfuscation** -- Random timing delays between requests (1-4 seconds) -- Variable rate limiting (5-15 MB/s) -- Progressive retry delays with new fingerprints - -#### 4. **Session Establishment** -- Visits referrer pages before downloading videos -- Mimics human browsing behavior -- Maintains realistic request flow - -### Browser Configurations - -The system rotates between these browser fingerprints: - -1. **Chrome 120 Windows** - Most common configuration -2. **Chrome 119 macOS** - Alternative Chrome version -3. **Chrome 118 Linux** - Linux-specific Chrome -4. **Firefox 120** - Different engine fingerprint -5. **Safari-like Chrome** - WebKit-based fingerprint - -### Detection Evasion Techniques - -#### TLS Randomization -```javascript -// Example cipher rotation -"TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_128_GCM_SHA256" -"TLS_CHACHA20_POLY1305_SHA256:TLS_AES_256_GCM_SHA384:TLS_AES_128_GCM_SHA256" -"TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256" -``` - -#### Request Headers Variation -- Accept headers that match browser type -- Randomized Accept-Language preferences -- Variable Accept-Encoding support -- Optional security headers (Sec-Fetch-*) - -#### Timing Patterns -- Pre-request delays: 1-4 seconds -- Retry delays: Progressive (3-7 seconds) -- Session establishment: 1-3 seconds -- Processing delays: 1-3 seconds between videos - -### Usage - -#### Download with Anti-Fingerprinting -```javascript -import { downloadWithRetry } from './darwinAntiFingerprint.js'; - -// Attempts download with up to 3 different fingerprints -await downloadWithRetry(videoUrl, refererUrl, outputPath, 3); -``` - -#### File Size Check -```javascript -import { getFileSizeWithRandomizedCurl } from './darwinAntiFingerprint.js'; - -// Uses randomized fingerprint for HEAD request -const fileSize = await getFileSizeWithRandomizedCurl(videoUrl, refererUrl); -``` - -#### Session Establishment -```javascript -import { establishSession } from './darwinAntiFingerprint.js'; - -// Visits page like a real browser before downloading -await establishSession(refererUrl); -``` - -## Configuration - -### Environment Variables -The system uses configuration from `config.json`: - -```json -{ - "darwin": { - "tempDir": "/path/to/temp", - "targetDir": "/path/to/final", - "cdnUrl": "https://your-cdn.com", - "maxDownloadSize": 50, - "feedUrl": "https://source-site.com/feed", - "markerOne": "video_marker_1", - "markerTwo": "https://source-site.com" - } -} -``` - -## Deployment Considerations - -### Curl Requirements -- Requires curl 8.x+ with modern TLS support -- OpenSSL 3.x+ for advanced cipher suites -- HTTP/2 and HTTP/3 support recommended - -### Performance -- Each download attempt includes 1-4 second delays -- Progressive retry delays for failed attempts -- Rate limiting prevents bandwidth saturation - -### Monitoring -The system logs detailed information about: -- Fingerprint selection and success rates -- Blocking detection and retry attempts -- Download success/failure patterns -- TLS configuration effectiveness - -## Troubleshooting - -### Common Issues - -#### 403 Forbidden Errors -- Indicates fingerprint detection -- System automatically retries with different fingerprints -- Check curl version and TLS support - -#### Empty Downloads (0 bytes) -- Usually HTML error pages instead of video -- Triggers automatic retry with new configuration -- May indicate IP-based blocking - -#### TLS Handshake Failures -- Check OpenSSL version compatibility -- Verify cipher suite support -- Update curl to latest version - -### Debug Mode -Enable verbose logging by modifying the curl args to include `-v` for detailed TLS handshake information. - -## Security Notes - -- All requests use legitimate browser fingerprints -- No malicious techniques are employed -- Respects rate limiting and server resources -- Designed to appear as normal browser traffic \ No newline at end of file diff --git a/src/helpers/darwin/captureDevPCBehavior.js b/src/helpers/darwin/captureDevPCBehavior.js deleted file mode 100644 index bf3fdef6..00000000 --- a/src/helpers/darwin/captureDevPCBehavior.js +++ /dev/null @@ -1,358 +0,0 @@ -#!/usr/bin/env node - -/** - * Script to capture exact dev PC curl behavior for replication on server - * Run this on your dev PC to capture working curl commands - * Then run on server to test if same commands work - */ - -import { spawn } from "child_process"; -import fs from "fs"; -import { execSync } from "child_process"; - -const colors = { - reset: "\x1b[0m", - bright: "\x1b[1m", - red: "\x1b[31m", - green: "\x1b[32m", - yellow: "\x1b[33m", - blue: "\x1b[34m", - magenta: "\x1b[35m", - cyan: "\x1b[36m", -}; - -const log = (color, message) => console.log(`${color}${message}${colors.reset}`); - -/** - * Capture system information that might affect curl behavior - */ -const captureSystemInfo = () => { - const info = {}; - - try { - info.platform = process.platform; - info.arch = process.arch; - info.nodeVersion = process.version; - - // Get OS info - if (process.platform === 'linux') { - try { - info.osRelease = execSync('cat /etc/os-release', { encoding: 'utf8' }).split('\n')[0]; - } catch (e) { - info.osRelease = 'Unknown Linux'; - } - } - - // Get curl info - try { - info.curlVersion = execSync('curl --version', { encoding: 'utf8' }).split('\n')[0]; - } catch (e) { - info.curlVersion = 'curl not found'; - } - - // Get OpenSSL info - try { - info.opensslVersion = execSync('openssl version', { encoding: 'utf8' }).trim(); - } catch (e) { - info.opensslVersion = 'openssl not found'; - } - - // Get network interface info (might affect routing) - try { - const interfaces = require('os').networkInterfaces(); - info.networkInterfaces = Object.keys(interfaces).length; - } catch (e) { - info.networkInterfaces = 'unknown'; - } - - } catch (error) { - log(colors.red, `Error capturing system info: ${error.message}`); - } - - return info; -}; - -/** - * Test the exact curl command that works on dev PC - */ -const testWorkingCurlCommand = async () => { - return new Promise((resolve) => { - log(colors.blue, "\n🧪 Testing exact working curl command..."); - - const testUrl = "https://theync.com/media/videos/6/9/1/c/b/691cb2bb47ee2.mp4"; - const referer = "https://theync.com/some-page"; - const outputFile = "/tmp/capture-test.mp4"; - - // This is the EXACT command that works on your dev PC - const curlArgs = [ - "-L", - "-v", - "-o", outputFile, - "-A", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36", - "-H", "Accept: video/mp4,video/*,application/octet-stream,*/*;q=0.8", - "-H", "Accept-Language: en-US,en;q=0.5", - "-H", "Accept-Encoding: identity", - "-H", `Referer: ${referer}`, - testUrl - ]; - - log(colors.cyan, `Command: curl ${curlArgs.join(' ')}`); - - const curl = spawn("curl", curlArgs); - let stderr = ""; - - curl.stderr.on("data", (data) => { - stderr += data.toString(); - }); - - curl.on("close", (code) => { - const result = { - exitCode: code, - verboseOutput: stderr, - success: false, - fileSize: 0, - responseCode: null, - contentType: null, - serverHeaders: {}, - tlsInfo: null - }; - - // Parse verbose output for details - if (stderr) { - // Extract HTTP response code - const responseMatch = stderr.match(/HTTP\/[12]\.[01] (\d+)/); - if (responseMatch) { - result.responseCode = parseInt(responseMatch[1]); - } - - // Extract content type - const contentTypeMatch = stderr.match(/content-type:\s*([^\r\n]+)/i); - if (contentTypeMatch) { - result.contentType = contentTypeMatch[1].trim(); - } - - // Extract TLS connection info - const tlsMatch = stderr.match(/SSL connection using\s+([^\r\n]+)/); - if (tlsMatch) { - result.tlsInfo = tlsMatch[1].trim(); - } - - // Extract server - const serverMatch = stderr.match(/server:\s*([^\r\n]+)/i); - if (serverMatch) { - result.serverHeaders.server = serverMatch[1].trim(); - } - } - - // Check file - try { - if (fs.existsSync(outputFile)) { - const stats = fs.statSync(outputFile); - result.fileSize = stats.size; - - if (stats.size > 0) { - const buffer = fs.readFileSync(outputFile, { start: 0, end: 100 }); - const isVideo = buffer.includes(Buffer.from('ftyp')) && buffer.includes(Buffer.from('isom')); - const isHtml = buffer.toString('utf8', 0, 50).includes(' { - resolve({ - success: false, - error: error.message - }); - }); - }); -}; - -/** - * Test various curl configurations to identify working patterns - */ -const testCurlVariations = async () => { - const variations = [ - { - name: "Basic", - args: ["-L", "-v", "-o", "/tmp/var-basic.mp4", "https://theync.com/media/videos/6/9/1/c/b/691cb2bb47ee2.mp4"] - }, - { - name: "With User-Agent", - args: ["-L", "-v", "-A", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", "-o", "/tmp/var-ua.mp4", "https://theync.com/media/videos/6/9/1/c/b/691cb2bb47ee2.mp4"] - }, - { - name: "With Headers", - args: ["-L", "-v", "-A", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", "-H", "Accept: video/mp4,*/*", "-H", "Referer: https://theync.com/", "-o", "/tmp/var-headers.mp4", "https://theync.com/media/videos/6/9/1/c/b/691cb2bb47ee2.mp4"] - }, - { - name: "With HTTP2", - args: ["-L", "-v", "--http2", "-A", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", "-H", "Accept: video/mp4,*/*", "-o", "/tmp/var-http2.mp4", "https://theync.com/media/videos/6/9/1/c/b/691cb2bb47ee2.mp4"] - }, - { - name: "With Compression", - args: ["-L", "-v", "--compressed", "-A", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", "-o", "/tmp/var-compressed.mp4", "https://theync.com/media/videos/6/9/1/c/b/691cb2bb47ee2.mp4"] - } - ]; - - const results = []; - - for (const variation of variations) { - log(colors.blue, `\n🔄 Testing variation: ${variation.name}`); - - const result = await new Promise((resolve) => { - const curl = spawn("curl", variation.args); - let stderr = ""; - - curl.stderr.on("data", (data) => { - stderr += data.toString(); - }); - - curl.on("close", (code) => { - const outputFile = variation.args.find((arg, i) => variation.args[i-1] === '-o'); - let success = false; - let fileSize = 0; - let responseCode = null; - - // Parse response code - const responseMatch = stderr.match(/HTTP\/[12]\.[01] (\d+)/); - if (responseMatch) { - responseCode = parseInt(responseMatch[1]); - } - - // Check file - try { - if (fs.existsSync(outputFile)) { - const stats = fs.statSync(outputFile); - fileSize = stats.size; - - if (fileSize > 0) { - const buffer = fs.readFileSync(outputFile, { start: 0, end: 50 }); - const isVideo = buffer.includes(Buffer.from('ftyp')); - const isHtml = buffer.toString('utf8').includes(' { - resolve({ - name: variation.name, - error: error.message, - success: false - }); - }); - }); - - results.push(result); - - const status = result.success ? colors.green + "✅ SUCCESS" : colors.red + "❌ FAILED"; - log(status, `${variation.name}: ${result.responseCode || 'No response'} - ${result.fileSize} bytes`); - } - - return results; -}; - -/** - * Generate a comprehensive report - */ -const generateReport = async () => { - log(colors.bright + colors.magenta, "🔍 Dev PC Curl Behavior Capture"); - log(colors.bright + colors.magenta, "=================================="); - - // System info - log(colors.cyan, "\n📋 System Information:"); - const systemInfo = captureSystemInfo(); - Object.entries(systemInfo).forEach(([key, value]) => { - log(colors.cyan, ` ${key}: ${value}`); - }); - - // Test exact working command - log(colors.yellow, "\n🎯 Testing exact working command..."); - const mainResult = await testWorkingCurlCommand(); - - if (mainResult.success) { - log(colors.green, "✅ Main command successful!"); - log(colors.green, ` Response: ${mainResult.responseCode}`); - log(colors.green, ` Content-Type: ${mainResult.contentType}`); - log(colors.green, ` File Size: ${mainResult.fileSize} bytes`); - log(colors.green, ` TLS: ${mainResult.tlsInfo}`); - } else { - log(colors.red, "❌ Main command failed!"); - if (mainResult.error) { - log(colors.red, ` Error: ${mainResult.error}`); - } else { - log(colors.red, ` Response: ${mainResult.responseCode}`); - log(colors.red, ` File Size: ${mainResult.fileSize} bytes`); - } - } - - // Test variations - log(colors.yellow, "\n🔬 Testing curl variations..."); - const variations = await testCurlVariations(); - - // Summary - log(colors.bright + colors.magenta, "\n📊 Summary:"); - const successful = variations.filter(v => v.success); - log(colors.cyan, `Working variations: ${successful.length}/${variations.length}`); - - if (successful.length > 0) { - log(colors.green, "✅ Successful configurations:"); - successful.forEach(v => { - log(colors.green, ` - ${v.name}: ${v.responseCode} (${v.fileSize} bytes)`); - }); - } - - const failed = variations.filter(v => !v.success); - if (failed.length > 0) { - log(colors.red, "\n❌ Failed configurations:"); - failed.forEach(v => { - log(colors.red, ` - ${v.name}: ${v.responseCode || 'Error'} (${v.fileSize} bytes)`); - }); - } - - // Export data for server testing - const exportData = { - systemInfo, - mainResult, - variations, - timestamp: new Date().toISOString() - }; - - const exportFile = '/tmp/curl-behavior-capture.json'; - fs.writeFileSync(exportFile, JSON.stringify(exportData, null, 2)); - log(colors.bright + colors.cyan, `\n💾 Data exported to: ${exportFile}`); - log(colors.cyan, "Copy this file to your server and run the same script there for comparison"); - - return exportData; -}; - -// Run if called directly -if (import.meta.url === `file://${process.argv[1]}`) { - generateReport().catch(console.error); -} - -export { generateReport as captureDevPCBehavior }; diff --git a/src/helpers/darwin/darwinAntiFingerprint.js b/src/helpers/darwin/darwinAntiFingerprint.js deleted file mode 100644 index afaebf27..00000000 --- a/src/helpers/darwin/darwinAntiFingerprint.js +++ /dev/null @@ -1,557 +0,0 @@ -import { spawn } from "child_process"; -import fs from "fs"; -import { execSync } from "child_process"; - -/** - * Browser fingerprint configurations for randomization - */ -const BROWSER_CONFIGS = [ - { - name: "Chrome 120 Windows", - userAgent: - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", - headers: { - accept: "video/mp4,video/*,application/octet-stream,*/*;q=0.8", - acceptLanguage: "en-US,en;q=0.9", - secFetchSite: "cross-site", - }, - }, - { - name: "Chrome 119 macOS", - userAgent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36", - headers: { - accept: "video/webm,video/ogg,video/*;q=0.9,application/ogg;q=0.7,audio/*;q=0.6,*/*;q=0.5", - acceptLanguage: "en-US,en;q=0.9", - secFetchSite: "same-origin", - }, - }, - { - name: "Chrome 118 Linux", - userAgent: - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36", - headers: { - accept: "*/*", - acceptLanguage: "en-US,en;q=0.8,de;q=0.6", - secFetchSite: "cross-site", - }, - }, - { - name: "Firefox 120", - userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0", - headers: { - accept: "video/*,audio/*,*/*;q=0.8", - acceptLanguage: "en-US,en;q=0.5", - secFetchSite: "same-origin", - }, - }, - { - name: "Safari-like Chrome", - userAgent: - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15", - headers: { - accept: "video/mp4,video/*,*/*;q=0.8", - acceptLanguage: "en-GB,en-US;q=0.9,en;q=0.8", - secFetchSite: "cross-site", - }, - }, -]; - -/** - * Additional header variations for randomization - */ -const HEADER_VARIATIONS = { - acceptEncodings: ["identity", "gzip, deflate, br", "gzip, deflate", "br, gzip, deflate"], - cacheControls: ["no-cache", "no-cache, no-store", "max-age=0", "no-cache, max-age=0"], - connections: ["keep-alive", "close"], - rateLimits: ["5M", "8M", "10M", "12M", "15M"], -}; - -/** - * Find the correct curl executable path - * @returns {string} Path to curl executable - */ -const findCurlPath = () => { - const possiblePaths = [ - "/usr/local/bin/curl", - "/usr/bin/curl", - "/opt/homebrew/bin/curl", - "curl", // system PATH - ]; - - for (const path of possiblePaths) { - try { - if (path === "curl") { - // Check if curl is in PATH - execSync("which curl", { stdio: "ignore" }); - return "curl"; - } else { - // Check if file exists and is executable - if (fs.existsSync(path)) { - return path; - } - } - } catch (error) { - // Continue to next path - } - } - - // Default fallback - return "curl"; -}; - -/** - * Verify curl capabilities synchronously - * @param {string} curlPath - Path to curl executable - * @returns {Object} Curl capabilities info - */ -const verifyCurlCapabilities = (curlPath) => { - try { - const result = execSync(`${curlPath} --version`, { encoding: "utf8", timeout: 5000 }); - const versionMatch = result.match(/curl\s+([^\s]+)/); - const version = versionMatch ? versionMatch[1] : "unknown"; - - return { - version, - hasHTTP2: result.includes("HTTP2"), - hasHTTP3: result.includes("HTTP3"), - hasTLS13: result.includes("TLS"), - hasBrotli: result.includes("brotli"), - working: true, - }; - } catch (error) { - return { working: false, error: error.message }; - } -}; - -// Cache the curl path and verify capabilities on module load -const CURL_PATH = findCurlPath(); -const CURL_CAPABILITIES = verifyCurlCapabilities(CURL_PATH); - -if (CURL_CAPABILITIES.working) { - console.log(`✅ Using curl ${CURL_CAPABILITIES.version} at: ${CURL_PATH}`); - console.log( - `Features: HTTP2=${CURL_CAPABILITIES.hasHTTP2}, HTTP3=${CURL_CAPABILITIES.hasHTTP3}, Brotli=${CURL_CAPABILITIES.hasBrotli}` - ); -} else { - console.error(`❌ Curl verification failed: ${CURL_CAPABILITIES.error}`); - console.error(`Please ensure curl is properly installed and accessible`); -} - -/** - * Test curl with a simple request - * @returns {Promise} Whether curl is working - */ -export const testCurl = async () => { - return new Promise((resolve) => { - console.log("🧪 Testing curl functionality..."); - const curl = spawn(CURL_PATH, ["-s", "-I", "https://httpbin.org/get"]); - let stderr = ""; - - curl.stderr.on("data", (data) => { - stderr += data.toString(); - }); - - curl.on("close", (code) => { - if (code === 0) { - console.log("✅ Curl test successful"); - resolve(true); - } else { - console.error(`❌ Curl test failed with code ${code}: ${stderr}`); - resolve(false); - } - }); - - curl.on("error", (error) => { - console.error(`❌ Curl test error: ${error.message}`); - resolve(false); - }); - }); -}; - -/** - * Get curl information for debugging - * @returns {Object} Curl path and capabilities - */ -export const getCurlInfo = () => { - return { - path: CURL_PATH, - capabilities: CURL_CAPABILITIES, - }; -}; - -/** - * Generate a randomized request configuration to evade fingerprinting - * @returns {Object} Configuration object with curl args and timing - */ -export const generateRandomRequestConfig = () => { - const config = BROWSER_CONFIGS[Math.floor(Math.random() * BROWSER_CONFIGS.length)]; - - // Add randomized variations - const variations = { - acceptEncoding: - HEADER_VARIATIONS.acceptEncodings[ - Math.floor(Math.random() * HEADER_VARIATIONS.acceptEncodings.length) - ], - cacheControl: - HEADER_VARIATIONS.cacheControls[ - Math.floor(Math.random() * HEADER_VARIATIONS.cacheControls.length) - ], - connection: - HEADER_VARIATIONS.connections[ - Math.floor(Math.random() * HEADER_VARIATIONS.connections.length) - ], - rateLimit: - HEADER_VARIATIONS.rateLimits[Math.floor(Math.random() * HEADER_VARIATIONS.rateLimits.length)], - }; - - // Random timing - const timing = { - preDelay: 1000 + Math.random() * 3000, // 1-4 seconds - keepaliveTime: Math.floor(60 + Math.random() * 60), // 60-120 seconds - connectTimeout: Math.floor(20 + Math.random() * 20), // 20-40 seconds - maxTime: Math.floor(250 + Math.random() * 100), // 250-350 seconds - }; - - // Optional headers (randomly included) - const optionalHeaders = { - includeUpgradeInsecure: Math.random() > 0.5, - includeDNT: Math.random() > 0.6, - includeSecFetchDest: Math.random() > 0.4, - includeSecFetchMode: Math.random() > 0.5, - includePragma: Math.random() > 0.7, - }; - - return { - ...config, - variations, - timing, - optionalHeaders, - sessionId: Math.random().toString(36).substring(2, 15), - }; -}; - -/** - * Create adaptive curl configuration based on system capabilities - * @param {string} url - Video URL - * @param {string} referer - Referer URL - * @param {string} outputPath - Output file path - * @returns {Array} Curl arguments array - */ -const createAdaptiveCurlArgs = (url, referer, outputPath) => { - // Base arguments that work across systems - const baseArgs = [ - "-L", // Follow redirects - "-s", // Silent mode to reduce noise - "--max-redirs", - "5", - "--connect-timeout", - "30", - "--max-time", - "300", - - // Core headers that match successful dev PC behavior - "-A", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36", - "-H", - "Accept: video/mp4,video/*,application/octet-stream,*/*;q=0.8", - "-H", - "Accept-Language: en-US,en;q=0.5", - "-H", - "Accept-Encoding: identity", - "-H", - `Referer: ${referer}`, - - "-o", - outputPath, - url, - ]; - - // Try enhanced options that work on newer systems - if (CURL_CAPABILITIES.hasHTTP2) { - baseArgs.splice(-3, 0, "--http2"); - } - - if (CURL_CAPABILITIES.hasBrotli) { - baseArgs.splice(-3, 0, "--compressed"); - } - - return baseArgs; -}; - -/** - * Download video with system-adaptive approach - * @param {string} url - Video URL - * @param {string} referer - Referer URL (comments page) - * @param {string} outputPath - Where to save the file - * @param {number} attempt - Current attempt number (for logging) - * @returns {Promise} - Whether download was successful - */ -const downloadVideoWithRandomizedCurl = async (url, referer, outputPath, attempt = 1) => { - return new Promise((resolve, reject) => { - const curlArgs = createAdaptiveCurlArgs(url, referer, outputPath); - - console.log( - `[Attempt ${attempt}] Adaptive download (${CURL_CAPABILITIES.hasHTTP2 ? "HTTP2" : "HTTP1"}, ${CURL_CAPABILITIES.version})` - ); - - const curl = spawn(CURL_PATH, curlArgs); - let stderr = ""; - - curl.stderr.on("data", (data) => { - stderr += data.toString(); - }); - - curl.on("close", (code) => { - // Log any curl errors for debugging - if (code !== 0) { - console.log(`Curl exit code: ${code}`); - if (stderr) { - console.log(`Curl error: ${stderr.substring(0, 200)}...`); - } - } - - try { - const stats = fs.statSync(outputPath); - - if (stats.size === 0) { - console.log(`❌ [Attempt ${attempt}] Downloaded empty file`); - reject(new Error(`Downloaded empty file`)); - return; - } - - // Validate file content - const fileBuffer = fs.readFileSync(outputPath, { start: 0, end: 100 }); - - // Check for HTML error pages (Cloudflare blocking) - const fileStart = fileBuffer.toString("utf8", 0, 50); - if ( - fileStart.includes(" { - console.log(`❌ [Attempt ${attempt}] Curl process error: ${error.message}`); - reject(new Error(`Curl execution failed: ${error.message}`)); - }); - }); -}; - -/** - * Download with retry logic using different fingerprints - * @param {string} url - Video URL - * @param {string} referer - Referer URL - * @param {string} outputPath - Output file path - * @param {number} maxAttempts - Maximum number of attempts - * @returns {Promise} - Whether download was successful - */ -export const downloadWithRetry = async (url, referer, outputPath, maxAttempts = 3) => { - // Try different strategies on each attempt - const strategies = [ - { name: "Direct", delay: 0 }, - { name: "With session delay", delay: 2000 }, - { name: "Fallback mode", delay: 5000 }, - ]; - - for (let attempt = 1; attempt <= maxAttempts; attempt++) { - try { - const strategy = strategies[Math.min(attempt - 1, strategies.length - 1)]; - console.log(`Download attempt ${attempt}/${maxAttempts} - Strategy: ${strategy.name}`); - - // Clean up any previous partial download - if (fs.existsSync(outputPath)) { - fs.unlinkSync(outputPath); - } - - // Apply strategy-specific delay - if (strategy.delay > 0) { - console.log(`Applying ${strategy.delay}ms delay for strategy: ${strategy.name}`); - await new Promise((resolve) => setTimeout(resolve, strategy.delay)); - } - - await downloadVideoWithRandomizedCurl(url, referer, outputPath, attempt); - console.log(`✅ Success with strategy: ${strategy.name}`); - return true; // Success! - } catch (error) { - console.log(`❌ Attempt ${attempt} failed: ${error.message}`); - - if (attempt === maxAttempts) { - console.error(`All ${maxAttempts} download attempts failed`); - - // Enhanced error reporting - if (error.message.includes("HTML error page")) { - throw new Error( - `Server consistently blocking requests - received HTML instead of video after ${maxAttempts} attempts` - ); - } else if (error.message.includes("empty file")) { - throw new Error( - `Server returning empty responses - possible network or authentication issue` - ); - } else { - throw new Error(`Download failed after ${maxAttempts} attempts: ${error.message}`); - } - } - - // Clean up failed attempt - if (fs.existsSync(outputPath)) { - fs.unlinkSync(outputPath); - } - } - } -}; - -/** - * Get file size using randomized HEAD request - * @param {string} url - Video URL - * @param {string} referer - Referer URL - * @returns {Promise} - File size in bytes or null if failed - */ -export const getFileSizeWithRandomizedCurl = async (url, referer) => { - return new Promise((resolve) => { - // Create adaptive HEAD request - const curlArgs = [ - "-I", // HEAD request only - "-L", // Follow redirects - "-s", // Silent mode - "--connect-timeout", - "20", - "--max-time", - "60", - "--max-redirs", - "3", - - // Core headers - "-A", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36", - "-H", - "Accept: video/mp4,video/*,application/octet-stream,*/*;q=0.8", - "-H", - "Accept-Language: en-US,en;q=0.5", - "-H", - `Referer: ${referer}`, - - url, - ]; - - // Add HTTP/2 if supported - if (CURL_CAPABILITIES.hasHTTP2) { - curlArgs.splice(-1, 0, "--http2"); - } - - const curl = spawn(CURL_PATH, curlArgs); - let stdout = ""; - - curl.stdout.on("data", (data) => { - stdout += data.toString(); - }); - - curl.on("close", (code) => { - if (code === 0 && !stdout.includes("403") && !stdout.includes("Forbidden")) { - const match = stdout.match(/content-length:\s*(\d+)/i); - const fileSize = match ? parseInt(match[1], 10) : null; - resolve(fileSize); - } else { - // Don't log failures for HEAD requests - they often fail but downloads still work - resolve(null); - } - }); - - curl.on("error", () => { - resolve(null); - }); - }); -}; - -/** - * Establish a browser session by visiting the referrer page first - * @param {string} referer - The referrer URL to visit - * @returns {Promise} - Whether session establishment was successful - */ -export const establishSession = async (referer) => { - return new Promise((resolve) => { - // Create adaptive session establishment - const curlArgs = [ - "-L", // Follow redirects - "-s", // Silent mode - "--connect-timeout", - "20", - "--max-time", - "60", - "--max-redirs", - "3", - - // Browser headers for page visits - "-A", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36", - "-H", - "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "-H", - "Accept-Language: en-US,en;q=0.5", - "-H", - "Cache-Control: no-cache", - "-H", - "Upgrade-Insecure-Requests: 1", - - "-o", - "/dev/null", - referer, - ]; - - // Add enhanced features if available - if (CURL_CAPABILITIES.hasHTTP2) { - curlArgs.splice(-2, 0, "--http2"); - } - if (CURL_CAPABILITIES.hasBrotli) { - curlArgs.splice(-2, 0, "--compressed"); - curlArgs.splice(-2, 0, "-H", "Accept-Encoding: gzip, deflate, br"); - } - - console.log(`Establishing session by visiting: ${referer.substring(0, 50)}...`); - - const curl = spawn(CURL_PATH, curlArgs); - - curl.on("close", (code) => { - if (code === 0) { - console.log("✅ Session established successfully"); - resolve(true); - } else { - console.log("⚠️ Session establishment failed, continuing anyway"); - resolve(false); - } - }); - - curl.on("error", () => { - console.log("⚠️ Session establishment error, continuing anyway"); - resolve(false); - }); - }); -}; diff --git a/src/helpers/darwin/darwinProcess.js b/src/helpers/darwin/darwinProcess.js index 57187de8..c0902320 100644 --- a/src/helpers/darwin/darwinProcess.js +++ b/src/helpers/darwin/darwinProcess.js @@ -1,60 +1,61 @@ import { load } from "cheerio"; -import fs from "fs"; -import path from "path"; -import { spawn } from "child_process"; - +import https from "https"; import { query } from "../db.js"; -import { transcodeVideo, getFileSizeMB } from "./darwinTranscode.js"; import { isInCache, addToCache } from "./darwinCache.js"; -import { - downloadWithRetry, - getFileSizeWithRandomizedCurl, - establishSession, - testCurl, - getCurlInfo, -} from "./darwinAntiFingerprint.js"; - import config from "../../../config.json" with { type: "json" }; + const darwinConfig = config.darwin; /** - * Note: Download functions have been moved to darwinAntiFingerprint.js - * This module now provides advanced TLS fingerprint randomization - * to evade Cloudflare's bot detection systems. + * Simple HTTPS GET request + * @param {string} url - URL to fetch + * @param {Object} headers - Request headers + * @returns {Promise} - Response body */ +const httpsGet = (url, headers = {}) => { + return new Promise((resolve, reject) => { + const options = { + headers: { + "User-Agent": "darwin", + ...headers, + }, + }; + + https + .get(url, options, (res) => { + let data = ""; + res.on("data", (chunk) => (data += chunk)); + res.on("end", () => resolve(data)); + }) + .on("error", reject); + }); +}; /** - * Get the final destination of a URL (follow redirects) + * Get final URL after redirects * @param {string} url - Initial URL - * @returns {Promise} - Final URL after redirects + * @returns {Promise} - Final URL */ const getDestination = async (url) => { try { - const response = await fetch(url, { - method: "HEAD", - redirect: "follow", - }); + const response = await fetch(url, { method: "HEAD", redirect: "follow" }); return response.url; } catch (error) { - console.error(`Failed to resolve URL destination: ${error}`); + console.error(`Failed to resolve URL: ${error}`); return url; } }; /** - * Extract video location from a page + * Extract video location from page * @param {string} href - Page URL - * @param {string} markerOne - First marker to identify video - * @param {string} markerTwo - Second marker to identify video + * @param {string} markerOne - First marker + * @param {string} markerTwo - Second marker * @returns {Promise} - Direct video URL */ const getVideoLocation = async (href, markerOne, markerTwo) => { try { - const response = await fetch(href, { - headers: { "User-Agent": "darwin" }, - // headers: { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36 Edg/138.0.0.0" } - }); - const html = await response.text(); + const html = await httpsGet(href); const start = html.indexOf(markerOne); if (start === -1) return ""; @@ -69,230 +70,14 @@ const getVideoLocation = async (href, markerOne, markerTwo) => { }; /** - * Generate message for Discord channel + * Generate Discord message * @param {string} title - Video title - * @param {string} href - Source URL - * @param {string} directStreamLink - Direct streaming link - * @param {string} comments - Comments/forum URL - * @param {boolean} canBeStreamed - Whether video can be streamed - * @param {number} fileSize - File size in MB + * @param {string} href - Video URL + * @param {string} comments - Forum post URL * @returns {string} - Formatted message */ -const messageGen = (title, href, comments, canBeStreamed, fileSize, directStreamLink = "") => { - return `${canBeStreamed ? `[[ STREAMING & DOWNLOAD ]](${directStreamLink})` : `[[ ORIGINAL MP4 ]](${href})`} - [[ FORUM POST ]](<${comments}>)\n${title}${canBeStreamed ? `\nSize: ${fileSize}MB` : `\nSize (might won't load): ${fileSize}MB`}`; -}; - -/** - * Process a single video - * @param {Object} video - Video metadata - * @returns {Promise} - Processed video result or null if failed - */ -const processVideo = async (video) => { - const { title, href, comments } = video; - console.log(`Processing ${title} ${href}`); - - try { - const isVideoInCache = await isInCache(href); - if (isVideoInCache) { - console.log(`Video already in cache, skipping: ${href}`); - return null; - } - - // Add to cache BEFORE downloading to prevent multiple uploads - const addedToCache = await addToCache(href); - if (!addedToCache) { - console.log(`Failed to add to cache, skipping to prevent duplicates: ${href}`); - return null; - } - - const videoId = href.split("/").pop().replace(".mp4", ""); - const tempFilePath = path.join(darwinConfig.tempDir, `${videoId}_original.mp4`); - const transcodedFilePath = path.join(darwinConfig.tempDir, `${videoId}_transcoded.mp4`); - const finalFilePath = path.join(darwinConfig.targetDir, `${videoId}.mp4`); - const directStreamLink = `${darwinConfig.cdnUrl}/${videoId}.mp4`; - - // Check file size before downloading - console.log(`Checking file size for: ${href}`); - const fileSizeBytes = await getFileSizeWithRandomizedCurl(href, comments); - - if (fileSizeBytes) { - const fileSizeMB = (fileSizeBytes / 1024 / 1024).toFixed(2); - console.log(`File size: ${fileSizeMB}MB`); - - if (fileSizeBytes > darwinConfig.maxDownloadSize * 1024 * 1024) { - console.log(`File exceeds limit (${darwinConfig.maxDownloadSize}MB), using direct link`); - return { - title, - href, - comments, - canBeStreamed: false, - fileSize: fileSizeMB, - }; - } - } else { - console.log("Could not determine file size, will proceed with download"); - } - - console.log(`Downloading video using anti-fingerprint curl: ${href}`); - - // Establish session by visiting the referrer page first (like a real browser) - try { - await establishSession(comments); - // Wait a bit after session establishment before downloading video - const sessionDelay = 1000 + Math.random() * 2000; // 1-3 seconds - console.log(`Post-session delay: ${Math.round(sessionDelay)}ms`); - await new Promise((resolve) => setTimeout(resolve, sessionDelay)); - } catch (sessionError) { - console.log(`Session establishment failed, continuing anyway: ${sessionError.message}`); - } - - // Add random delay to avoid appearing automated - const preDownloadDelay = 500 + Math.random() * 1500; // 0.5-2 seconds - console.log(`Pre-download delay: ${Math.round(preDownloadDelay)}ms`); - await new Promise((resolve) => setTimeout(resolve, preDownloadDelay)); - - try { - await downloadWithRetry(href, comments, tempFilePath, 3); - console.log("Successfully downloaded using randomized fingerprint"); - } catch (downloadError) { - console.error(`All download attempts failed: ${downloadError.message}`); - - // Test curl again if downloads are failing - if (downloadError.message.includes("ENOENT") || downloadError.message.includes("spawn")) { - console.log("🔧 Re-testing curl due to spawn errors..."); - const curlStillWorking = await testCurl(); - if (!curlStillWorking) { - console.error("❌ Curl is no longer working - stopping Darwin process"); - } - } - - return null; - } - - if (!fs.existsSync(tempFilePath)) { - console.error(`Error when saving temporary file: ${tempFilePath}`); - return null; - } - - const originalSize = getFileSizeMB(tempFilePath); - console.log(`Original file size: ${originalSize}MB`); - - // Double-check size after download (fallback if HEAD request didn't work) - if (originalSize > darwinConfig.maxDownloadSize) { - console.log(`File too large (${originalSize}MB), removing and using direct link`); - fs.unlinkSync(tempFilePath); - return { - title, - href, - comments, - canBeStreamed: false, - fileSize: originalSize.toFixed(2), - }; - } - - try { - console.log("Starting video transcoding process..."); - const transcodingSuccess = await transcodeVideo(tempFilePath, transcodedFilePath); - - const transcodedSize = getFileSizeMB(transcodedFilePath); - if (transcodingSuccess) { - console.log( - `Transcoded file size: ${transcodedSize}MB (${Math.round((transcodedSize / originalSize) * 100)}% of original)` - ); - - fs.copyFileSync(transcodedFilePath, finalFilePath); - console.log(`File moved to final destination: ${finalFilePath}`); - } else { - throw new Error("Transcoding failed to produce valid output"); - } - - // Clean up temp files - if (fs.existsSync(tempFilePath)) { - fs.unlinkSync(tempFilePath); - } - if (fs.existsSync(transcodedFilePath)) { - fs.unlinkSync(transcodedFilePath); - } - console.log("Temporary files cleaned up"); - - return { - title, - href, - comments, - canBeStreamed: true, - fileSize: transcodedSize, - directStreamLink, - }; - } catch (error) { - console.error(`Transcoding failed, attempting to use original file: ${error}`); - - try { - // Use original file as fallback - fs.copyFileSync(tempFilePath, finalFilePath); - console.log(`Original file moved to final destination: ${finalFilePath}`); - - // Clean up temporary files - if (fs.existsSync(tempFilePath)) { - fs.unlinkSync(tempFilePath); - console.log("Original temporary file cleaned up"); - } - - if (fs.existsSync(transcodedFilePath)) { - fs.unlinkSync(transcodedFilePath); - console.log("Partial transcoded file cleaned up"); - } - - const finalSize = getFileSizeMB(finalFilePath); - const canStream = finalSize <= darwinConfig.maxDownloadSize; - - return { - title, - href, - comments, - canBeStreamed: canStream, - fileSize: finalSize, - directStreamLink, - }; - } catch (fallbackError) { - console.error(`Failed to use original file as fallback: ${fallbackError}`); - return null; - } - } - } catch (error) { - console.error(`Error processing video ${title}: ${error}`); - return null; - } -}; - -/** - * Distribute a processed video to all configured channels - * @param {Object} client - Discord client - * @param {Array} guildConfigs - Array of guild configurations - * @param {Object} processedVideo - Processed video result - * @returns {Promise} - */ -const distributeVideo = async (client, guildConfigs, processedVideo) => { - const { title, href, comments, canBeStreamed, fileSize, directStreamLink } = processedVideo; - - const message = messageGen(title, href, comments, canBeStreamed, fileSize, directStreamLink); - - for (const config of guildConfigs) { - try { - console.log(`Sending video to guild ${config.guildId}, channel ${config.channelId}`); - const channel = await client.channels.fetch(config.channelId); - - if (channel) { - await channel.send(message); - console.log( - `Successfully sent video to channel ${config.channelName} (${config.channelId})` - ); - } else { - console.error(`Failed to fetch channel ${config.channelId}`); - } - } catch (error) { - console.error(`Error sending to channel ${config.channelId}: ${error}`); - } - } +const messageGen = (title, href, comments) => { + return `[[ STREAMING & DOWNLOAD ]](${href}) - [[ FORUM POST ]](<${comments}>)\n${title}`; }; /** @@ -301,16 +86,9 @@ const distributeVideo = async (client, guildConfigs, processedVideo) => { */ const fetchVideosFromFeed = async () => { try { - const response = await fetch(darwinConfig.feedUrl, { - headers: { "User-Agent": "darwin" }, - // headers: { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36" } - }); - - const html = await response.text(); + const html = await httpsGet(darwinConfig.feedUrl); const $ = load(html); - const contentBlock = $(".content-block > div"); - const videosToProcess = []; for (const node of contentBlock) { @@ -334,7 +112,7 @@ const fetchVideosFromFeed = async () => { const isVideoInCache = await isInCache(videoLocation); if (isVideoInCache) { - console.log(`Skipping cached video: ${title.trim()} (${videoLocation})`); + console.log(`Skipping cached video: ${title.trim()}`); continue; } @@ -353,24 +131,38 @@ const fetchVideosFromFeed = async () => { }; /** - * Run the Darwin process for all configured guilds + * Distribute video to all configured channels * @param {Object} client - Discord client - * @returns {Promise} + * @param {Array} guildConfigs - Guild configurations + * @param {Object} video - Video object */ -export const runDarwinProcess = async (client) => { - try { - // Test curl functionality before starting - console.log("🔧 Verifying curl setup..."); - const curlInfo = getCurlInfo(); - console.log(`Curl info:`, curlInfo); +const distributeVideo = async (client, guildConfigs, video) => { + const { title, href, comments } = video; + const message = messageGen(title, href, comments); - const curlWorking = await testCurl(); - if (!curlWorking) { - console.error("❌ Curl test failed - cannot proceed with downloads"); - return; + for (const config of guildConfigs) { + try { + console.log(`Sending video to guild ${config.guildId}, channel ${config.channelId}`); + const channel = await client.channels.fetch(config.channelId); + + if (channel) { + await channel.send(message); + console.log(`Successfully sent to ${config.channelName} (${config.channelId})`); + } else { + console.error(`Failed to fetch channel ${config.channelId}`); + } + } catch (error) { + console.error(`Error sending to channel ${config.channelId}: ${error}`); } + } +}; - // Get all guild configurations +/** + * Run the Darwin process + * @param {Object} client - Discord client + */ +export const runDarwinProcess = async (client) => { + try { const guildConfigs = await query("SELECT * FROM configDarwin"); if (guildConfigs.length === 0) { @@ -380,7 +172,6 @@ export const runDarwinProcess = async (client) => { console.log(`Found ${guildConfigs.length} Darwin configurations`); - // Fetch videos from feed once const videosToProcess = await fetchVideosFromFeed(); console.log(`Discovered ${videosToProcess.length} new videos to process`); @@ -389,57 +180,13 @@ export const runDarwinProcess = async (client) => { return; } - // Process videos one at a time, with concurrency limit - const concurrencyLimit = 2; - let activeTasks = 0; - let videoIndex = 0; - const processedVideos = []; - - async function processNextVideos() { - while (activeTasks < concurrencyLimit && videoIndex < videosToProcess.length) { - const video = videosToProcess[videoIndex++]; - activeTasks++; - - try { - const result = await processVideo(video); - if (result) { - processedVideos.push(result); - - // If video wasn't downloaded, distribute immediately - if (!result.canBeStreamed) { - await distributeVideo(client, guildConfigs, result); - } - } - } finally { - activeTasks--; - if (videoIndex < videosToProcess.length) { - processNextVideos(); - } - } - - // Increased delay between video processing to reduce detection - if (videoIndex < videosToProcess.length) { - const processingDelay = 1000 + Math.random() * 2000; // 1-3 seconds - await new Promise((resolve) => setTimeout(resolve, processingDelay)); - } + for (const video of videosToProcess) { + const addedToCache = await addToCache(video.href); + if (!addedToCache) { + console.log(`Failed to add to cache, skipping: ${video.href}`); + continue; } - } - - await processNextVideos(); - - // Wait for all processing to complete - while (activeTasks > 0) { - await new Promise((resolve) => setTimeout(resolve, 1000)); - } - - // Filter out videos that weren't downloaded (already distributed) - const videosToDistribute = processedVideos.filter((video) => video.canBeStreamed); - console.log( - `Successfully processed ${videosToDistribute.length} videos, distributing to ${guildConfigs.length} channels` - ); - // Distribute each processed video to all configured channels - for (const video of videosToDistribute) { await distributeVideo(client, guildConfigs, video); } diff --git a/src/helpers/darwin/darwinTranscode.js b/src/helpers/darwin/darwinTranscode.js deleted file mode 100644 index 41da8920..00000000 --- a/src/helpers/darwin/darwinTranscode.js +++ /dev/null @@ -1,225 +0,0 @@ -import ffmpeg from "fluent-ffmpeg"; -import fs from "fs"; - -/** - * Get video information including dimensions and duration - * @param {string} inputPath - Path to the video file - * @returns {Promise} - Video metadata object - */ -export const getVideoInfo = async (inputPath) => { - return new Promise((resolve, reject) => { - ffmpeg.ffprobe(inputPath, (err, metadata) => { - if (err) { - return reject(err); - } - - const videoStream = metadata.streams.find((stream) => stream.codec_type === "video"); - if (!videoStream) { - return reject(new Error("No video stream found")); - } - - resolve({ - width: videoStream.width, - height: videoStream.height, - duration: videoStream.duration, - bitrate: videoStream.bit_rate, - rotation: - videoStream.tags && videoStream.tags.rotate ? parseInt(videoStream.tags.rotate) : 0, - codec_name: videoStream.codec_name, - }); - }); - }); -}; - -/** - * Get video file size in MB - * @param {string} filePath - Path to the video file - * @returns {number} - File size in MB - */ -export const getFileSizeMB = (filePath) => { - try { - const stats = fs.statSync(filePath); - return (stats.size / (1024 * 1024)).toFixed(2); - } catch (error) { - console.error(`Failed to get file size: ${error}`); - return 0; - } -}; - -/** - * Calculate optimal CRF value based on video resolution - * @param {number} totalPixels - Total pixels in the video - * @returns {number} - Optimal CRF value - */ -const calculateOptimalCRF = (totalPixels) => { - // Higher resolution can use higher CRF (more compression) while still looking good - if (totalPixels > 2073600) { - // > 1080p - return 28; - } else if (totalPixels > 921600) { - // > 720p - return 26; - } else if (totalPixels > 409920) { - // > 480p - return 24; - } else { - // Lower resolutions need less compression to look good - return 23; - } -}; - -/** - * Calculate optimal dimensions for transcoding while preserving aspect ratio - * @param {Object} videoInfo - Original video information - * @returns {Object} - Calculated dimensions and scaling information - */ -const calculateOptimalDimensions = (videoInfo) => { - const { width, height, rotation } = videoInfo; - - const effectiveWidth = rotation === 90 || rotation === 270 ? height : width; - const effectiveHeight = rotation === 90 || rotation === 270 ? width : height; - - const aspectRatio = effectiveWidth / effectiveHeight; - const totalPixels = effectiveWidth * effectiveHeight; - - // Base limits - only apply if video exceeds these dimensions - const MAX_WIDTH = 1920; - const MAX_HEIGHT = 1280; - const MAX_PIXELS = 2073600; // 1080p equivalent (1920×1080) - - // Initialize with original dimensions - don't change unless needed - let targetWidth = effectiveWidth; - let targetHeight = effectiveHeight; - let needsScaling = false; - - // Check if we need to scale down - if (totalPixels > MAX_PIXELS || effectiveWidth > MAX_WIDTH || effectiveHeight > MAX_HEIGHT) { - needsScaling = true; - - // Scale by the most constraining dimension - if (effectiveWidth / MAX_WIDTH > effectiveHeight / MAX_HEIGHT) { - // Width is the limiting factor - targetWidth = MAX_WIDTH; - targetHeight = Math.round(MAX_WIDTH / aspectRatio); - } else { - // Height is the limiting factor - targetHeight = MAX_HEIGHT; - targetWidth = Math.round(MAX_HEIGHT * aspectRatio); - } - - // Ensure we don't exceed the pixel count limit - if (targetWidth * targetHeight > MAX_PIXELS) { - const scale = Math.sqrt(MAX_PIXELS / (targetWidth * targetHeight)); - targetWidth = Math.round(targetWidth * scale); - targetHeight = Math.round(targetHeight * scale); - } - } - - console.log( - `Original dimensions: ${effectiveWidth}x${effectiveHeight}, aspect ratio: ${aspectRatio.toFixed(2)}` - ); - if (needsScaling) { - console.log(`Scaling to: ${targetWidth}x${targetHeight}`); - } else { - console.log(`Keeping original dimensions: ${targetWidth}x${targetHeight}`); - } - - return { - width: targetWidth, - height: targetHeight, - aspectRatio, - needsScaling, - isLandscape: aspectRatio > 1, - isPortrait: aspectRatio < 1, - isSquare: Math.abs(aspectRatio - 1) < 0.1, - totalPixels, - }; -}; - -/** - * Transcode video to reduce file size while maintaining Discord compatibility - * @param {string} inputPath - Path to the input video file - * @param {string} outputPath - Path for the transcoded output file - * @returns {Promise} - Whether transcoding was successful - */ -export const transcodeVideo = async (inputPath, outputPath) => { - try { - // Get original video dimensions - const videoInfo = await getVideoInfo(inputPath); - console.log( - `Original video: ${videoInfo.width}x${videoInfo.height}, rotation: ${videoInfo.rotation}°, codec: ${videoInfo.codec_name}` - ); - - // Calculate optimal dimensions based on actual video properties - const dimensions = calculateOptimalDimensions(videoInfo); - - return await performTranscode(inputPath, outputPath, dimensions, videoInfo); - } catch (error) { - console.error(`Error in transcodeVideo: ${error.message}`); - // Last resort fallback with conservative settings - try { - console.log("Using conservative fallback settings"); - return await performTranscode(inputPath, outputPath, null, null); - } catch (finalError) { - console.error(`Final transcoding attempt failed: ${finalError.message}`); - throw finalError; - } - } -}; - -/** - * Transcode video using software encoding - * @param {string} inputPath - Path to the input video file - * @param {string} outputPath - Path for the transcoded output file - * @param {Object} dimensions - Calculated video dimensions or null for fallback - * @param {Object} videoInfo - Original video metadata or null for fallback - * @returns {Promise} - Whether transcoding was successful - */ -const performTranscode = async (inputPath, outputPath, dimensions, videoInfo) => { - return new Promise((resolve, reject) => { - console.log(`Starting software transcoding: ${inputPath} -> ${outputPath}`); - - // Use conservative defaults if dimensions aren't available - const crf = dimensions ? calculateOptimalCRF(dimensions.totalPixels) : 26; - console.log(`Using CRF value: ${crf} for quality/size optimization`); - - const command = ffmpeg(inputPath) - .videoCodec("libx264") - .audioCodec("aac") - .outputOptions([ - `-crf ${crf}`, - "-preset fast", - "-profile:v main", - "-level 3.1", - "-movflags +faststart", - "-b:a 128k", - "-map_metadata -1", - ]); - - // Apply scaling only if we have dimensions and scaling is needed - if (dimensions && dimensions.needsScaling) { - command.size(`${dimensions.width}x${dimensions.height}`); - } else if (!dimensions) { - // Fallback scaling that ensures video isn't too large - command.outputOptions([ - "-vf scale='min(1280,iw):min(720,ih):force_original_aspect_ratio=decrease'", - ]); - } - - command - // .on('progress', (progress) => { - // if (progress.percent) { - // console.log(`Software transcoding progress: ${Math.round(progress.percent)}%`); - // } - // }) - .on("end", () => { - console.log(`Software transcoding completed: ${outputPath}`); - resolve(true); - }) - .on("error", (err) => { - console.error(`Software transcoding failed: ${err.message}`); - reject(err); - }) - .save(outputPath); - }); -}; diff --git a/src/helpers/darwin/debugCurl.js b/src/helpers/darwin/debugCurl.js deleted file mode 100644 index c9eb9722..00000000 --- a/src/helpers/darwin/debugCurl.js +++ /dev/null @@ -1,307 +0,0 @@ -#!/usr/bin/env node - -/** - * Debug script to test curl functionality for Darwin anti-fingerprinting system - * Run with: node debugCurl.js - */ - -import { spawn, execSync } from "child_process"; -import fs from "fs"; - -const colors = { - reset: "\x1b[0m", - bright: "\x1b[1m", - red: "\x1b[31m", - green: "\x1b[32m", - yellow: "\x1b[33m", - blue: "\x1b[34m", - magenta: "\x1b[35m", - cyan: "\x1b[36m", -}; - -const log = (color, message) => console.log(`${color}${message}${colors.reset}`); - -/** - * Find curl executable - */ -const findCurl = () => { - const paths = ["/usr/local/bin/curl", "/usr/bin/curl", "/opt/homebrew/bin/curl", "curl"]; - - for (const path of paths) { - try { - if (path === "curl") { - execSync("which curl", { stdio: "ignore" }); - return "curl"; - } else if (fs.existsSync(path)) { - return path; - } - } catch (e) { - // Continue - } - } - return null; -}; - -/** - * Test curl version and features - */ -const testCurlVersion = async (curlPath) => { - return new Promise((resolve) => { - log(colors.blue, `\n🔍 Testing curl version at: ${curlPath}`); - - const curl = spawn(curlPath, ["--version"]); - let stdout = ""; - let stderr = ""; - - curl.stdout.on("data", (data) => (stdout += data.toString())); - curl.stderr.on("data", (data) => (stderr += data.toString())); - - curl.on("close", (code) => { - if (code === 0) { - log(colors.green, "✅ Curl version check successful"); - console.log(stdout); - - const features = { - HTTP2: stdout.includes("HTTP2"), - HTTP3: stdout.includes("HTTP3"), - TLS: stdout.includes("TLS"), - Brotli: stdout.includes("brotli"), - OpenSSL: stdout.includes("OpenSSL"), - }; - - log(colors.cyan, "\n📋 Features:"); - Object.entries(features).forEach(([feature, has]) => { - log(has ? colors.green : colors.red, ` ${feature}: ${has ? "✅" : "❌"}`); - }); - - resolve(true); - } else { - log(colors.red, `❌ Curl version check failed (code ${code})`); - if (stderr) console.error(stderr); - resolve(false); - } - }); - - curl.on("error", (error) => { - log(colors.red, `❌ Curl spawn error: ${error.message}`); - resolve(false); - }); - }); -}; - -/** - * Test basic HTTP request - */ -const testBasicRequest = async (curlPath) => { - return new Promise((resolve) => { - log(colors.blue, "\n🌐 Testing basic HTTP request..."); - - const curl = spawn(curlPath, [ - "-s", - "-I", - "-L", - "--connect-timeout", - "10", - "--max-time", - "30", - "https://httpbin.org/get", - ]); - - let stdout = ""; - let stderr = ""; - - curl.stdout.on("data", (data) => (stdout += data.toString())); - curl.stderr.on("data", (data) => (stderr += data.toString())); - - curl.on("close", (code) => { - if (code === 0 && stdout.includes("200 OK")) { - log(colors.green, "✅ Basic HTTP request successful"); - resolve(true); - } else { - log(colors.red, `❌ Basic HTTP request failed (code ${code})`); - if (stderr) console.error("Error:", stderr); - if (stdout) console.log("Response:", stdout.substring(0, 200)); - resolve(false); - } - }); - - curl.on("error", (error) => { - log(colors.red, `❌ HTTP request error: ${error.message}`); - resolve(false); - }); - }); -}; - -/** - * Test TLS capabilities - */ -const testTLSCapabilities = async (curlPath) => { - return new Promise((resolve) => { - log(colors.blue, "\n🔒 Testing TLS 1.3 capabilities..."); - - const curl = spawn(curlPath, [ - "-s", - "-I", - "-L", - "--tls-max", - "1.3", - "--ciphers", - "TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256", - "--curves", - "X25519:prime256v1", - "--connect-timeout", - "10", - "--max-time", - "30", - "https://tls13.crypto.mozilla.org/", - ]); - - let stdout = ""; - let stderr = ""; - - curl.stdout.on("data", (data) => (stdout += data.toString())); - curl.stderr.on("data", (data) => (stderr += data.toString())); - - curl.on("close", (code) => { - if (code === 0 && (stdout.includes("200 OK") || stdout.includes("HTTP"))) { - log(colors.green, "✅ TLS 1.3 request successful"); - resolve(true); - } else { - log(colors.yellow, `⚠️ TLS 1.3 test inconclusive (code ${code})`); - log(colors.magenta, "This may still work with the target site"); - resolve(true); // Don't fail completely on TLS test - } - }); - - curl.on("error", (error) => { - log(colors.yellow, `⚠️ TLS test error: ${error.message}`); - resolve(true); // Don't fail completely - }); - }); -}; - -/** - * Test actual video site request - */ -const testVideoSiteRequest = async (curlPath) => { - return new Promise((resolve) => { - log(colors.blue, "\n📹 Testing video site request..."); - - const curl = spawn(curlPath, [ - "-s", - "-I", - "-L", - "-A", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", - "-H", - "Accept: video/mp4,video/*,*/*;q=0.8", - "-H", - "Accept-Language: en-US,en;q=0.9", - "-H", - "Referer: https://theync.com/", - "--connect-timeout", - "15", - "--max-time", - "60", - "https://theync.com/media/videos/6/9/1/c/b/691cb2bb47ee2.mp4", - ]); - - let stdout = ""; - let stderr = ""; - - curl.stdout.on("data", (data) => (stdout += data.toString())); - curl.stderr.on("data", (data) => (stderr += data.toString())); - - curl.on("close", (code) => { - log(colors.cyan, "\n📄 Response headers:"); - console.log(stdout.substring(0, 500)); - - if (code === 0) { - if (stdout.includes("200 OK") && stdout.includes("video/mp4")) { - log(colors.green, "✅ Video site allows access!"); - } else if (stdout.includes("403")) { - log(colors.yellow, "⚠️ Video site returned 403 (blocked)"); - } else if (stdout.includes("404")) { - log(colors.yellow, "⚠️ Video not found (404) - expected for test"); - } else { - log(colors.yellow, `⚠️ Video site returned unexpected response`); - } - resolve(true); - } else { - log(colors.red, `❌ Video site request failed (code ${code})`); - if (stderr) console.error("Error:", stderr); - resolve(false); - } - }); - - curl.on("error", (error) => { - log(colors.red, `❌ Video site request error: ${error.message}`); - resolve(false); - }); - }); -}; - -/** - * Main debug function - */ -const runDebug = async () => { - log(colors.bright + colors.magenta, "🔧 Darwin Curl Debug Tool"); - log(colors.bright + colors.magenta, "============================"); - - // Find curl - const curlPath = findCurl(); - if (!curlPath) { - log(colors.red, "❌ No curl executable found!"); - log(colors.yellow, "Please install curl or ensure it's in your PATH"); - process.exit(1); - } - - log(colors.green, `✅ Found curl at: ${curlPath}`); - - // Run tests - const tests = [ - ["Version Check", () => testCurlVersion(curlPath)], - ["Basic HTTP", () => testBasicRequest(curlPath)], - ["TLS Capabilities", () => testTLSCapabilities(curlPath)], - ["Video Site Access", () => testVideoSiteRequest(curlPath)], - ]; - - const results = []; - for (const [name, testFn] of tests) { - const result = await testFn(); - results.push([name, result]); - } - - // Summary - log(colors.bright + colors.magenta, "\n📊 Test Summary:"); - log(colors.bright + colors.magenta, "================"); - - let allPassed = true; - for (const [name, passed] of results) { - const status = passed ? "✅ PASS" : "❌ FAIL"; - const color = passed ? colors.green : colors.red; - log(color, `${status} ${name}`); - if (!passed) allPassed = false; - } - - if (allPassed) { - log(colors.bright + colors.green, "\n🎉 All tests passed! Curl should work with Darwin."); - } else { - log( - colors.bright + colors.red, - "\n⚠️ Some tests failed. Check curl installation and network connectivity." - ); - } - - log(colors.bright + colors.cyan, "\n💡 Usage tips:"); - log(colors.cyan, "- If TLS test failed, update curl and OpenSSL"); - log(colors.cyan, "- If video site blocks, the anti-fingerprinting system will help"); - log(colors.cyan, "- Run this script periodically to verify connectivity"); -}; - -// Run if called directly -if (import.meta.url === `file://${process.argv[1]}`) { - runDebug().catch(console.error); -} - -export { runDebug as debugCurl }; From b0ba7fe1c02df4f887071a76c20f3698c3c2001c Mon Sep 17 00:00:00 2001 From: VB2007 Date: Wed, 19 Nov 2025 00:10:01 +0100 Subject: [PATCH 12/17] updated darwinCache table schema, added migration script --- src/sql/darwinCache/migrate-to-new-schema.sql | 14 ++++++++++++++ src/sql/darwinCache/table.sql | 6 ++++-- 2 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 src/sql/darwinCache/migrate-to-new-schema.sql diff --git a/src/sql/darwinCache/migrate-to-new-schema.sql b/src/sql/darwinCache/migrate-to-new-schema.sql new file mode 100644 index 00000000..22c9bb2a --- /dev/null +++ b/src/sql/darwinCache/migrate-to-new-schema.sql @@ -0,0 +1,14 @@ +ALTER TABLE `darwinCache` +CHANGE COLUMN `videoUrl` `directVideoUrl` VARCHAR(255) NOT NULL; + +ALTER TABLE `darwinCache` +ADD COLUMN `forumPostUrl` VARCHAR(255) NOT NULL DEFAULT '' AFTER `directVideoUrl`, +ADD COLUMN `videoTitle` VARCHAR(180) NOT NULL DEFAULT '' AFTER `videoId`; + +ALTER TABLE `darwinCache` +DROP INDEX `videoUrl`; + +ALTER TABLE `darwinCache` +ADD UNIQUE KEY `directVideoUrl` (`directVideoUrl`); + +DESCRIBE `darwinCache`; diff --git a/src/sql/darwinCache/table.sql b/src/sql/darwinCache/table.sql index 09fe1355..25e4bf05 100644 --- a/src/sql/darwinCache/table.sql +++ b/src/sql/darwinCache/table.sql @@ -2,9 +2,11 @@ CREATE TABLE IF NOT EXISTS `darwinCache` ( `id` INT AUTO_INCREMENT PRIMARY KEY, - `videoUrl` VARCHAR(255) NOT NULL, + `directVideoUrl` VARCHAR(255) NOT NULL, + `forumPostUrl` VARCHAR(255) NOT NULL, `videoId` VARCHAR(100) NOT NULL, + `videoTitle` VARCHAR(180) NOT NULL, `processedAt` TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE KEY (`videoUrl`), + UNIQUE KEY (`directVideoUrl`), UNIQUE KEY (`videoId`) ); From 29e0efa47d710a7ca213f5c98ca7b4c75df44b49 Mon Sep 17 00:00:00 2001 From: VB2007 Date: Wed, 19 Nov 2025 00:28:46 +0100 Subject: [PATCH 13/17] updated distribution logic, used new table column names in helpers --- src/helpers/darwin/darwinCache.js | 25 +++++++++------- src/helpers/darwin/darwinProcess.js | 44 ++++++++++++++--------------- 2 files changed, 36 insertions(+), 33 deletions(-) diff --git a/src/helpers/darwin/darwinCache.js b/src/helpers/darwin/darwinCache.js index 63ba320d..ed5df7ee 100644 --- a/src/helpers/darwin/darwinCache.js +++ b/src/helpers/darwin/darwinCache.js @@ -22,23 +22,28 @@ const extractVideoId = (url) => { }; /** - * Add a URL to Darwin's cache - * @param {string} url - The video URL to cache + * Add a video to Darwin's cache + * @param {string} directVideoUrl - The direct video URL + * @param {string} forumPostUrl - The forum post URL + * @param {string} videoTitle - The video title * @returns {Promise} - Whether the operation was successful */ -export const addToCache = async (url) => { +export const addToCache = async (directVideoUrl, forumPostUrl, videoTitle) => { try { - const exists = await isInCache(url); + const exists = await isInCache(directVideoUrl); if (exists) { - console.log(`URL already in cache: ${url}`); + console.log(`URL already in cache: ${directVideoUrl}`); return true; } - const videoId = extractVideoId(url); - console.log(`Adding "${url}" (ID: ${videoId}) to Darwin's cache`); + const videoId = extractVideoId(directVideoUrl); + console.log(`Adding "${videoTitle}" (ID: ${videoId}) to Darwin's cache`); try { - await query("INSERT INTO darwinCache (videoUrl, videoId) VALUES (?, ?)", [url, videoId]); + await query( + "INSERT INTO darwinCache (directVideoUrl, forumPostUrl, videoId, videoTitle) VALUES (?, ?, ?, ?)", + [directVideoUrl, forumPostUrl, videoId, videoTitle] + ); return true; } catch (sqlError) { //handle duplicate key error (MySQL error code 1062) (edge case, if source reuses the same id) @@ -49,7 +54,7 @@ export const addToCache = async (url) => { if (idExists.length > 0) { console.log(`Another URL already uses videoId ${videoId}, treating as cached`); } else { - console.log(`URL ${url} is a duplicate entry`); + console.log(`URL ${directVideoUrl} is a duplicate entry`); } return true; @@ -74,7 +79,7 @@ export const isInCache = async (url) => { console.log(`Checking "${url}" (ID: ${videoId}) in Darwin's cache`); const result = await query( - "SELECT videoUrl FROM darwinCache WHERE videoUrl = ? OR videoId = ?", + "SELECT directVideoUrl FROM darwinCache WHERE directVideoUrl = ? OR videoId = ?", [url, videoId] ); return result.length > 0; diff --git a/src/helpers/darwin/darwinProcess.js b/src/helpers/darwin/darwinProcess.js index c0902320..a27cd257 100644 --- a/src/helpers/darwin/darwinProcess.js +++ b/src/helpers/darwin/darwinProcess.js @@ -84,12 +84,12 @@ const messageGen = (title, href, comments) => { * Fetch videos from feed * @returns {Promise} - Array of video objects */ -const fetchVideosFromFeed = async () => { +const fetchAndDistributeVideos = async (client, guildConfigs) => { try { const html = await httpsGet(darwinConfig.feedUrl); const $ = load(html); const contentBlock = $(".content-block > div"); - const videosToProcess = []; + let distributedCount = 0; for (const node of contentBlock) { try { @@ -116,17 +116,27 @@ const fetchVideosFromFeed = async () => { continue; } - console.log(`Discovered "${title.trim()}" at "${videoLocation}"`); - videosToProcess.push({ title, href: videoLocation, comments: href }); + console.log(`Discovered new video: "${title.trim()}" at "${videoLocation}"`); + + //immediately distribute to all channels + const video = { title: title.trim(), href: videoLocation, comments: href }; + await distributeVideo(client, guildConfigs, video); + + const addedToCache = await addToCache(videoLocation, href, title.trim()); + if (addedToCache) { + distributedCount++; + } else { + console.log(`Warning: Failed to add to cache: ${videoLocation}`); + } } catch (error) { console.error(`Error processing item: ${error}`); } } - return videosToProcess; + return distributedCount; } catch (error) { console.error(`Error fetching videos from feed: ${error}`); - return []; + return 0; } }; @@ -172,25 +182,13 @@ export const runDarwinProcess = async (client) => { console.log(`Found ${guildConfigs.length} Darwin configurations`); - const videosToProcess = await fetchVideosFromFeed(); - console.log(`Discovered ${videosToProcess.length} new videos to process`); + const distributedCount = await fetchAndDistributeVideos(client, guildConfigs); - if (videosToProcess.length === 0) { - console.log("No new videos to process"); - return; + if (distributedCount === 0) { + console.log("No new videos to distribute"); + } else { + console.log(`Darwin process completed: ${distributedCount} video(s) distributed`); } - - for (const video of videosToProcess) { - const addedToCache = await addToCache(video.href); - if (!addedToCache) { - console.log(`Failed to add to cache, skipping: ${video.href}`); - continue; - } - - await distributeVideo(client, guildConfigs, video); - } - - console.log("Darwin process completed successfully"); } catch (error) { console.error(`Error running Darwin process: ${error}`); } From d5220e8a036d941e3085806629c019dc60485824 Mon Sep 17 00:00:00 2001 From: VB2007 Date: Wed, 19 Nov 2025 00:30:47 +0100 Subject: [PATCH 14/17] rolled back deploy workflow (removed curl) --- .github/workflows/deploy.yml | 100 ++--------------------------------- 1 file changed, 3 insertions(+), 97 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 382d2746..28f80b80 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -13,102 +13,8 @@ on: type: string jobs: - setup-curl: - runs-on: self-hosted - outputs: - curl-updated: ${{ steps.curl-check.outputs.updated }} - - steps: - - name: Check current curl version - id: curl-check - run: | - echo "Checking current curl version..." - CURRENT_VERSION="" - if command -v /usr/local/bin/curl &> /dev/null; then - CURRENT_VERSION=$(/usr/local/bin/curl --version | head -1 | awk '{print $2}') - echo "Current local curl version: $CURRENT_VERSION" - elif command -v curl &> /dev/null; then - CURRENT_VERSION=$(curl --version | head -1 | awk '{print $2}') - echo "Current system curl version: $CURRENT_VERSION" - else - echo "No curl found" - fi - - LATEST_TAG=$(curl -s https://api.github.com/repos/curl/curl/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/') - LATEST_VERSION=$(echo "$LATEST_TAG" | sed 's/curl-//' | sed 's/_/./g') - echo "Latest available version: $LATEST_VERSION" - - #compare versions - if [ "$CURRENT_VERSION" != "$LATEST_VERSION" ]; then - echo "Curl needs updating: $CURRENT_VERSION -> $LATEST_VERSION" - echo "updated=true" >> $GITHUB_OUTPUT - else - echo "Curl is already up to date: $CURRENT_VERSION" - echo "updated=false" >> $GITHUB_OUTPUT - fi - - - name: Install curl dependencies - if: steps.curl-check.outputs.updated == 'true' - run: | - echo "Installing dependencies for compiling curl..." - sudo apt update - sudo apt install -y \ - build-essential \ - libssl-dev \ - libnghttp2-dev \ - zlib1g-dev \ - libpsl-dev \ - libbrotli-dev \ - libzstd-dev \ - libssh2-1-dev \ - librtmp-dev \ - libldap2-dev \ - libidn2-dev \ - libkrb5-dev \ - autotools-dev - - - name: Build and install latest version of curl - if: steps.curl-check.outputs.updated == 'true' - run: | - echo "Building latest curl..." - cd /tmp - - TAG_NAME=$(curl -s https://api.github.com/repos/curl/curl/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/') - VERSION=$(echo "$TAG_NAME" | sed 's/curl-//' | sed 's/_/./g') - DOWNLOAD_URL="https://curl.se/download/curl-${VERSION}.tar.gz" - - echo "Downloading curl $VERSION..." - wget -v "$DOWNLOAD_URL" -O curl-latest.tar.gz - - echo "Extracting and building..." - tar -xzf curl-latest.tar.gz - cd curl-${VERSION} - - ./configure --prefix=/usr/local \ - --with-openssl \ - --with-nghttp2 \ - --with-libpsl \ - --with-brotli \ - --with-zstd \ - --enable-http2 \ - --enable-ldap \ - --enable-ldaps \ - --disable-static \ - --enable-shared - - make -j$(nproc) - sudo make install - sudo ldconfig - - echo "✅ Curl installation completed" - /usr/local/bin/curl --version - - cd /tmp - rm -rf curl-latest.tar.gz curl-${VERSION} - deploy: runs-on: self-hosted # REQUIRES SOME SUDO EXECUTION PRIVILEGES WITHOUT A PASSWORD PROMPT - needs: setup-curl steps: - name: Stopping systemd service @@ -133,7 +39,7 @@ jobs: - name: Installing Node.js dependencies working-directory: /home/vb2007/prod/discordbot env: - PATH: /usr/local/bin:/home/vb2007/.nvm/versions/node/v24.8.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH: /home/vb2007/.nvm/versions/node/v24.8.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin run: | rm -rf node_modules npm ci @@ -142,7 +48,7 @@ jobs: - name: Deploying possible new commands working-directory: /home/vb2007/prod/discordbot env: - PATH: /usr/local/bin:/home/vb2007/.nvm/versions/node/v24.8.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH: /home/vb2007/.nvm/versions/node/v24.8.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin run: | echo "Starting command deployment..." deployOutput=$(npm run deploy 2>&1) @@ -159,7 +65,7 @@ jobs: - name: Updating command data working-directory: /home/vb2007/prod/discordbot env: - PATH: /usr/local/bin:/home/vb2007/.nvm/versions/node/v24.8.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH: /home/vb2007/.nvm/versions/node/v24.8.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin run: | echo "Starting table creation and command data update..." queriesOutput=$(npm run create-tables 2>&1) From e65ee1204c67955aadd646197e582674ff5efedb Mon Sep 17 00:00:00 2001 From: VB2007 Date: Wed, 19 Nov 2025 00:31:31 +0100 Subject: [PATCH 15/17] updated packages --- package-lock.json | 91 ++++++++++++++++++++++++----------------------- 1 file changed, 47 insertions(+), 44 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9a4cc725..34701662 100755 --- a/package-lock.json +++ b/package-lock.json @@ -553,12 +553,12 @@ } }, "node_modules/@discordjs/formatters": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/@discordjs/formatters/-/formatters-0.6.1.tgz", - "integrity": "sha512-5cnX+tASiPCqCWtFcFslxBVUaCetB0thvM/JyavhbXInP1HJIEU+Qv/zMrnuwSsX3yWH2lVXNJZeDK3EiP4HHg==", + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/@discordjs/formatters/-/formatters-0.6.2.tgz", + "integrity": "sha512-y4UPwWhH6vChKRkGdMB4odasUbHOUwy7KL+OVwF86PvT6QVOwElx+TiI1/6kcmcEe+g5YRXJFiXSXUdabqZOvQ==", "license": "Apache-2.0", "dependencies": { - "discord-api-types": "^0.38.1" + "discord-api-types": "^0.38.33" }, "engines": { "node": ">=16.11.0" @@ -612,10 +612,13 @@ } }, "node_modules/@discordjs/util": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@discordjs/util/-/util-1.1.1.tgz", - "integrity": "sha512-eddz6UnOBEB1oITPinyrB2Pttej49M9FZQY8NxgEvc3tq6ZICZ19m70RsmzRdDHk80O9NoYN/25AqJl8vPVf/g==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@discordjs/util/-/util-1.2.0.tgz", + "integrity": "sha512-3LKP7F2+atl9vJFhaBjn4nOaSWahZ/yWjOvA4e5pnXkt2qyXRCHLxoBQy81GFtLGCq7K9lPm9R517M1U+/90Qg==", "license": "Apache-2.0", + "dependencies": { + "discord-api-types": "^0.38.33" + }, "engines": { "node": ">=18" }, @@ -1176,9 +1179,9 @@ } }, "node_modules/@types/node": { - "version": "24.9.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-24.9.1.tgz", - "integrity": "sha512-QoiaXANRkSXK6p0Duvt56W208du4P9Uye9hWLWgGMDTEoKPhuenzNcC4vGUmrNkiOKTlIrBoyNQYNpSwfEZXSg==", + "version": "24.10.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.1.tgz", + "integrity": "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==", "license": "MIT", "dependencies": { "undici-types": "~7.16.0" @@ -1201,9 +1204,9 @@ } }, "node_modules/@types/yargs": { - "version": "17.0.34", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.34.tgz", - "integrity": "sha512-KExbHVa92aJpw9WDQvzBaGVE2/Pz+pLZQloT2hjL8IqsZnV62rlPOYvNnLmf/L2dyllfVUOVBj64M0z/46eR2A==", + "version": "17.0.35", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.35.tgz", + "integrity": "sha512-qUHkeCyQFxMXg79wQfTtfndEC+N9ZZg76HJftDJp+qH2tV7Gj4OJi7l+PiWwJ+pWtW8GwSmqsDj/oymhrTWXjg==", "dev": true, "license": "MIT", "dependencies": { @@ -1438,9 +1441,9 @@ "license": "MIT" }, "node_modules/baseline-browser-mapping": { - "version": "2.8.20", - "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.8.20.tgz", - "integrity": "sha512-JMWsdF+O8Orq3EMukbUN1QfbLK9mX2CkUmQBcW2T0s8OmdAUL5LLM/6wFwSrqXzlXB13yhyK9gTKS1rIizOduQ==", + "version": "2.8.29", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.8.29.tgz", + "integrity": "sha512-sXdt2elaVnhpDNRDz+1BDx1JQoJRuNk7oVlAlbGiFkLikHCAQiccexF/9e91zVi6RCgqspl04aP+6Cnl9zRLrA==", "dev": true, "license": "Apache-2.0", "bin": { @@ -1478,9 +1481,9 @@ } }, "node_modules/browserslist": { - "version": "4.27.0", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.27.0.tgz", - "integrity": "sha512-AXVQwdhot1eqLihwasPElhX2tAZiBjWdJ9i/Zcj2S6QYIjkx62OKSfnobkriB81C3l4w0rVy3Nt4jaTBltYEpw==", + "version": "4.28.0", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.0.tgz", + "integrity": "sha512-tbydkR/CxfMwelN0vwdP/pLkDwyAASZ+VfWm4EOwlB6SWhx1sYnWLqo8N5j0rAzPfzfRaxt0mM/4wPU/Su84RQ==", "dev": true, "funding": [ { @@ -1499,10 +1502,10 @@ "license": "MIT", "peer": true, "dependencies": { - "baseline-browser-mapping": "^2.8.19", - "caniuse-lite": "^1.0.30001751", - "electron-to-chromium": "^1.5.238", - "node-releases": "^2.0.26", + "baseline-browser-mapping": "^2.8.25", + "caniuse-lite": "^1.0.30001754", + "electron-to-chromium": "^1.5.249", + "node-releases": "^2.0.27", "update-browserslist-db": "^1.1.4" }, "bin": { @@ -1550,9 +1553,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001751", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001751.tgz", - "integrity": "sha512-A0QJhug0Ly64Ii3eIqHu5X51ebln3k4yTUkY1j8drqpWHVreg/VLijN48cZ1bYPiqOQuqpkIKnzr/Ul8V+p6Cw==", + "version": "1.0.30001755", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001755.tgz", + "integrity": "sha512-44V+Jm6ctPj7R52Na4TLi3Zri4dWUljJd+RDm+j8LtNCc/ihLCT+X1TzoOAkRETEWqjuLnh9581Tl80FvK7jVA==", "dev": true, "funding": [ { @@ -1883,28 +1886,28 @@ } }, "node_modules/discord-api-types": { - "version": "0.38.31", - "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.38.31.tgz", - "integrity": "sha512-kC94ANsk8ackj8ENTuO8joTNEL0KtymVhHy9dyEC/s4QAZ7GCx40dYEzQaadyo8w+oP0X8QydE/nzAWRylTGtQ==", + "version": "0.38.34", + "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.38.34.tgz", + "integrity": "sha512-muq7xKGznj5MSFCzuIm/2TO7DpttuomUTemVM82fRqgnMl70YRzEyY24jlbiV6R9tzOTq6A6UnZ0bsfZeKD38Q==", "license": "MIT", "workspaces": [ "scripts/actions/documentation" ] }, "node_modules/discord.js": { - "version": "14.24.0", - "resolved": "https://registry.npmjs.org/discord.js/-/discord.js-14.24.0.tgz", - "integrity": "sha512-KNq/ekT8bsmT3ZAfVre8cPbl+DfVYSdlLnDmGZPoz7Cw21LYeWHllRA9MivqNq5b1GPGAxGvyUN1vxbTb/PQWw==", + "version": "14.25.0", + "resolved": "https://registry.npmjs.org/discord.js/-/discord.js-14.25.0.tgz", + "integrity": "sha512-9rnJWTAkfauKTieYJ3oI4oncrSzpbJPWN1/XU+2H6wsHQPtqbOrXvgM0nFhSVnIbTo7nfUF7fzcYRevvRGjpzA==", "license": "Apache-2.0", "dependencies": { "@discordjs/builders": "^1.13.0", "@discordjs/collection": "1.5.3", - "@discordjs/formatters": "^0.6.1", + "@discordjs/formatters": "^0.6.2", "@discordjs/rest": "^2.6.0", - "@discordjs/util": "^1.1.1", + "@discordjs/util": "^1.2.0", "@discordjs/ws": "^1.2.3", "@sapphire/snowflake": "3.5.3", - "discord-api-types": "^0.38.31", + "discord-api-types": "^0.38.33", "fast-deep-equal": "3.1.3", "lodash.snakecase": "4.1.1", "magic-bytes.js": "^1.10.0", @@ -1995,9 +1998,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.5.240", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.240.tgz", - "integrity": "sha512-OBwbZjWgrCOH+g6uJsA2/7Twpas2OlepS9uvByJjR2datRDuKGYeD+nP8lBBks2qnB7bGJNHDUx7c/YLaT3QMQ==", + "version": "1.5.256", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.256.tgz", + "integrity": "sha512-uqYq1IQhpXXLX+HgiXdyOZml7spy4xfy42yPxcCCRjswp0fYM2X+JwCON07lqnpLEGVCj739B7Yr+FngmHBMEQ==", "dev": true, "license": "ISC" }, @@ -3229,9 +3232,9 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", - "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "version": "3.14.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz", + "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==", "dev": true, "license": "MIT", "dependencies": { @@ -3476,9 +3479,9 @@ "license": "MIT" }, "node_modules/node-releases": { - "version": "2.0.26", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.26.tgz", - "integrity": "sha512-S2M9YimhSjBSvYnlr5/+umAnPHE++ODwt5e2Ij6FoX45HA/s4vHdkDx1eax2pAPeAOqu4s9b7ppahsyEFdVqQA==", + "version": "2.0.27", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.27.tgz", + "integrity": "sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA==", "dev": true, "license": "MIT" }, From 4c475191202ea167b85a0ab4b4151436ee594bf4 Mon Sep 17 00:00:00 2001 From: VB2007 Date: Wed, 19 Nov 2025 00:33:43 +0100 Subject: [PATCH 16/17] commented out non-fatal "spammy" logs in darwin helpers --- src/helpers/darwin/darwinCache.js | 2 +- src/helpers/darwin/darwinProcess.js | 16 ++++++++-------- src/index.js | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/helpers/darwin/darwinCache.js b/src/helpers/darwin/darwinCache.js index ed5df7ee..d3b8ca10 100644 --- a/src/helpers/darwin/darwinCache.js +++ b/src/helpers/darwin/darwinCache.js @@ -37,7 +37,7 @@ export const addToCache = async (directVideoUrl, forumPostUrl, videoTitle) => { } const videoId = extractVideoId(directVideoUrl); - console.log(`Adding "${videoTitle}" (ID: ${videoId}) to Darwin's cache`); + // console.log(`Adding "${videoTitle}" (ID: ${videoId}) to Darwin's cache`); try { await query( diff --git a/src/helpers/darwin/darwinProcess.js b/src/helpers/darwin/darwinProcess.js index a27cd257..977b2167 100644 --- a/src/helpers/darwin/darwinProcess.js +++ b/src/helpers/darwin/darwinProcess.js @@ -112,11 +112,11 @@ const fetchAndDistributeVideos = async (client, guildConfigs) => { const isVideoInCache = await isInCache(videoLocation); if (isVideoInCache) { - console.log(`Skipping cached video: ${title.trim()}`); + // console.log(`Skipping cached video: ${title.trim()}`); continue; } - console.log(`Discovered new video: "${title.trim()}" at "${videoLocation}"`); + // console.log(`Discovered new video: "${title.trim()}" at "${videoLocation}"`); //immediately distribute to all channels const video = { title: title.trim(), href: videoLocation, comments: href }; @@ -152,12 +152,12 @@ const distributeVideo = async (client, guildConfigs, video) => { for (const config of guildConfigs) { try { - console.log(`Sending video to guild ${config.guildId}, channel ${config.channelId}`); + // console.log(`Sending video to guild ${config.guildId}, channel ${config.channelId}`); const channel = await client.channels.fetch(config.channelId); if (channel) { await channel.send(message); - console.log(`Successfully sent to ${config.channelName} (${config.channelId})`); + // console.log(`Successfully sent to ${config.channelName} (${config.channelId})`); } else { console.error(`Failed to fetch channel ${config.channelId}`); } @@ -176,18 +176,18 @@ export const runDarwinProcess = async (client) => { const guildConfigs = await query("SELECT * FROM configDarwin"); if (guildConfigs.length === 0) { - console.log("No Darwin configurations found"); + // console.log("No Darwin configurations found"); return; } - console.log(`Found ${guildConfigs.length} Darwin configurations`); + // console.log(`Found ${guildConfigs.length} Darwin configurations`); const distributedCount = await fetchAndDistributeVideos(client, guildConfigs); if (distributedCount === 0) { - console.log("No new videos to distribute"); + // console.log("No new videos to distribute"); } else { - console.log(`Darwin process completed: ${distributedCount} video(s) distributed`); + // console.log(`Darwin process completed: ${distributedCount} video(s) distributed`); } } catch (error) { console.error(`Error running Darwin process: ${error}`); diff --git a/src/index.js b/src/index.js index dd9e290f..b50936d9 100644 --- a/src/index.js +++ b/src/index.js @@ -75,7 +75,7 @@ let darwinProcessRunning = false; // Notify hoster on console if the app is ready & about the Darwin process's status client.once(Events.ClientReady, (readyClient) => { console.log(`Bot is ready! Logged in as ${readyClient.user.tag}`); - console.log("Initializing Darwin video processing system..."); + // console.log("Initializing Darwin video processing system..."); setInterval(() => { if (darwinProcessRunning) { From ec23ce3a69c267cce1bc2da2fbb4d2e20743770d Mon Sep 17 00:00:00 2001 From: VB2007 Date: Wed, 19 Nov 2025 00:39:59 +0100 Subject: [PATCH 17/17] updated version in package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 7feb10b2..2fa15ea1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "discordbot", - "version": "4.1.1", + "version": "4.2.0", "description": "A simple discord bot of mine developed with discord.js (Node.js).", "type": "module", "main": "index.js",