From b11117ed07f644a3bcd91c2e6a61c81a83a06a40 Mon Sep 17 00:00:00 2001 From: VB2007 Date: Mon, 17 Nov 2025 06:16:19 +0100 Subject: [PATCH 1/3] added packages, fixed 403 error with curl --- package-lock.json | 7 +- package.json | 3 +- src/helpers/darwin/darwinProcess.js | 199 ++++++++++++++++++++++++---- src/sql/darwinCache/table.sql | 2 +- 4 files changed, 179 insertions(+), 32 deletions(-) diff --git a/package-lock.json b/package-lock.json index c995ab14..67dfbf72 100755 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "discordbot", - "version": "4.0.1", + "version": "4.1.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "discordbot", - "version": "4.0.1", + "version": "4.1.1", "license": "AGPL-3.0-only", "dependencies": { "ajv": "^8.17.1", @@ -16,7 +16,8 @@ "fluent-ffmpeg": "^2.1.3", "google-translate-api-x": "^10.7.1", "jest": "^29.7.0", - "mariadb": "^3.3.0" + "mariadb": "^3.3.0", + "undici": "^7.16.0" }, "devDependencies": { "@babel/parser": "^7.28.4", diff --git a/package.json b/package.json index 7feb10b2..7f57c7e9 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,8 @@ "fluent-ffmpeg": "^2.1.3", "google-translate-api-x": "^10.7.1", "jest": "^29.7.0", - "mariadb": "^3.3.0" + "mariadb": "^3.3.0", + "undici": "^7.16.0" }, "devDependencies": { "@babel/parser": "^7.28.4", diff --git a/src/helpers/darwin/darwinProcess.js b/src/helpers/darwin/darwinProcess.js index 58eb7cd1..f7e1ace5 100644 --- a/src/helpers/darwin/darwinProcess.js +++ b/src/helpers/darwin/darwinProcess.js @@ -2,6 +2,8 @@ import { load } from "cheerio"; import fs from "fs"; import path from "path"; import { pipeline } from "stream/promises"; +import { request } from "undici"; +import { spawn } from "child_process"; import { query } from "../db.js"; import { transcodeVideo, getFileSizeMB } from "./darwinTranscode.js"; @@ -10,6 +12,105 @@ import { isInCache, addToCache } from "./darwinCache.js"; import config from "../../../config.json" with { type: "json" }; const darwinConfig = config.darwin; +/** + * Download video using system curl (better TLS fingerprint) + * @param {string} url - Video URL + * @param {string} referer - Referer URL (comments page) + * @param {string} outputPath - Where to save the file + * @returns {Promise} - Whether download was successful + */ +const downloadVideoWithCurl = async (url, referer, outputPath) => { + return new Promise((resolve, reject) => { + const curlArgs = [ + "-L", // Follow redirects + "-A", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36", + "-H", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "-H", + "Accept-Language: en-US,en;q=0.5", + "-H", + "Accept-Encoding: gzip, deflate, br", + "-H", + "Upgrade-Insecure-Requests: 1", + "-H", + "Sec-Fetch-Dest: document", + "-H", + "Sec-Fetch-Mode: navigate", + "-H", + "Sec-Fetch-Site: cross-site", + "-H", + "Sec-Fetch-User: ?1", + "-H", + `Referer: ${referer}`, + "-H", + "Cache-Control: no-cache", + "-H", + "Pragma: no-cache", + "-o", + outputPath, + "--compressed", + "--http2", + url, + ]; + + const curl = spawn("curl", curlArgs); + + let stderr = ""; + + curl.stderr.on("data", (data) => { + stderr += data.toString(); + }); + + curl.on("close", (code) => { + if (code === 0) { + resolve(true); + } else { + reject(new Error(`curl exited with code ${code}: ${stderr}`)); + } + }); + + curl.on("error", (error) => { + reject(new Error(`Failed to spawn curl: ${error.message}`)); + }); + }); +}; + +/** + * Download video using undici with browser-like configuration + * @param {string} url - Video URL + * @param {string} referer - Referer URL (comments page) + * @returns {Promise} - Response object with body stream and headers + */ +const downloadVideo = async (url, referer) => { + const headers = { + "User-Agent": + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36", + Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.5", + "Accept-Encoding": "gzip, deflate, br", + "Upgrade-Insecure-Requests": "1", + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "cross-site", + "Sec-Fetch-User": "?1", + "sec-ch-ua": '"Not/A)Brand";v="8", "Chromium";v="142", "Google Chrome";v="142"', + "sec-ch-ua-mobile": "?0", + "sec-ch-ua-platform": '"Windows"', + Referer: referer, + "Cache-Control": "no-cache", + Pragma: "no-cache", + }; + + const response = await request(url, { + method: "GET", + headers, + maxRedirections: 5, + }); + + return response; +}; + /** * Get the final destination of a URL (follow redirects) * @param {string} url - Initial URL @@ -38,9 +139,9 @@ const getDestination = async (url) => { const getVideoLocation = async (href, markerOne, markerTwo) => { try { const response = await fetch(href, { - //if it isn't set to "darwin" the scraping fails for some reason headers: { "User-Agent": "darwin" }, // headers: { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36 Edg/138.0.0.0" } + // headers: getPageHeaders(), }); const html = await response.text(); const start = html.indexOf(markerOne); @@ -93,38 +194,65 @@ const processVideo = async (video) => { return null; } - const response = await fetch(href); - - if (!response.ok) { - console.error(`${href} ${response.statusText}`); - return null; - } - - const contentLength = parseInt(response.headers.get("Content-Length"), 10); - if (contentLength && contentLength > 50 * 1024 * 1024) { - console.log("Skipping download: File size exceeds limit (50MB)"); - - const fileSize = (contentLength / 1000000).toFixed(0); - // Return info for too large videos - return { - title, - href, - comments, - canBeStreamed: false, - directStreamLink: "", - fileSize, - tooBig: true, - }; - } - const videoId = href.split("/").pop().replace(".mp4", ""); const tempFilePath = path.join(darwinConfig.tempDir, `${videoId}_original.mp4`); const transcodedFilePath = path.join(darwinConfig.tempDir, `${videoId}_transcoded.mp4`); const finalFilePath = path.join(darwinConfig.targetDir, `${videoId}.mp4`); const directStreamLink = `${darwinConfig.cdnUrl}/${videoId}.mp4`; - console.log(`Downloading video to temp location: ${tempFilePath}`); - await pipeline(response.body, fs.createWriteStream(tempFilePath)); + // Maximum download size in bytes from config + const maxSizeBytes = darwinConfig.maxDownloadSize * 1024 * 1024; + + console.log(`Attempting to download video using curl: ${href}`); + + // Try curl first (better TLS fingerprint) + let downloadSuccess = false; + try { + await downloadVideoWithCurl(href, comments, tempFilePath); + console.log("Successfully downloaded using curl"); + downloadSuccess = true; + } catch (curlError) { + console.log(`Curl download failed: ${curlError.message}`); + } + + // Fallback to undici if curl failed + if (!downloadSuccess) { + console.log("Attempting download with undici..."); + try { + const response = await downloadVideo(href, comments); + + if (response.statusCode !== 200) { + console.error(`${href} ${response.statusCode} ${response.statusText || "Error"}`); + return null; + } + + const contentLength = parseInt(response.headers["content-length"], 10); + if (contentLength && contentLength > maxSizeBytes) { + console.log( + `Skipping download: File size exceeds limit (${darwinConfig.maxDownloadSize}MB)` + ); + await response.body.dump(); + + const fileSize = (contentLength / 1000000).toFixed(0); + return { + title, + href, + comments, + canBeStreamed: false, + directStreamLink: "", + fileSize, + tooBig: true, + }; + } + + console.log(`Downloading video to temp location: ${tempFilePath}`); + await pipeline(response.body, fs.createWriteStream(tempFilePath)); + downloadSuccess = true; + } catch (undiciError) { + console.error(`Undici download also failed: ${undiciError.message}`); + return null; + } + } if (!fs.existsSync(tempFilePath)) { console.error(`Error when saving temporary file: ${tempFilePath}`); @@ -134,6 +262,21 @@ const processVideo = async (video) => { const originalSize = getFileSizeMB(tempFilePath); console.log(`Original file size: ${originalSize}MB`); + // Check if downloaded file is too large (for curl downloads which don't check size beforehand) + if (originalSize > darwinConfig.maxDownloadSize) { + console.log(`File too large (${originalSize}MB), removing and skipping`); + fs.unlinkSync(tempFilePath); + return { + title, + href, + comments, + canBeStreamed: false, + directStreamLink: "", + fileSize: originalSize.toFixed(0), + tooBig: true, + }; + } + try { console.log("Starting video transcoding process..."); const transcodingSuccess = await transcodeVideo(tempFilePath, transcodedFilePath); @@ -248,6 +391,8 @@ const fetchVideosFromFeed = async () => { try { const response = await fetch(darwinConfig.feedUrl, { headers: { "User-Agent": "darwin" }, + // headers: { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36" } + // headers: getPageHeaders(), }); const html = await response.text(); diff --git a/src/sql/darwinCache/table.sql b/src/sql/darwinCache/table.sql index 1f1a396d..09fe1355 100644 --- a/src/sql/darwinCache/table.sql +++ b/src/sql/darwinCache/table.sql @@ -7,4 +7,4 @@ CREATE TABLE IF NOT EXISTS `darwinCache` ( `processedAt` TIMESTAMP DEFAULT CURRENT_TIMESTAMP, UNIQUE KEY (`videoUrl`), UNIQUE KEY (`videoId`) -); \ No newline at end of file +); From e695ab4b053d521325270e26984a376db1674ea2 Mon Sep 17 00:00:00 2001 From: VB2007 Date: Mon, 17 Nov 2025 06:20:33 +0100 Subject: [PATCH 2/3] cleaned up codebase after succesful curl approach --- package-lock.json | 3 +- package.json | 3 +- src/helpers/darwin/darwinProcess.js | 87 +---------------------------- 3 files changed, 5 insertions(+), 88 deletions(-) diff --git a/package-lock.json b/package-lock.json index 67dfbf72..9a4cc725 100755 --- a/package-lock.json +++ b/package-lock.json @@ -16,8 +16,7 @@ "fluent-ffmpeg": "^2.1.3", "google-translate-api-x": "^10.7.1", "jest": "^29.7.0", - "mariadb": "^3.3.0", - "undici": "^7.16.0" + "mariadb": "^3.3.0" }, "devDependencies": { "@babel/parser": "^7.28.4", diff --git a/package.json b/package.json index 7f57c7e9..7feb10b2 100644 --- a/package.json +++ b/package.json @@ -38,8 +38,7 @@ "fluent-ffmpeg": "^2.1.3", "google-translate-api-x": "^10.7.1", "jest": "^29.7.0", - "mariadb": "^3.3.0", - "undici": "^7.16.0" + "mariadb": "^3.3.0" }, "devDependencies": { "@babel/parser": "^7.28.4", diff --git a/src/helpers/darwin/darwinProcess.js b/src/helpers/darwin/darwinProcess.js index f7e1ace5..c6eea503 100644 --- a/src/helpers/darwin/darwinProcess.js +++ b/src/helpers/darwin/darwinProcess.js @@ -1,8 +1,6 @@ import { load } from "cheerio"; import fs from "fs"; import path from "path"; -import { pipeline } from "stream/promises"; -import { request } from "undici"; import { spawn } from "child_process"; import { query } from "../db.js"; @@ -76,41 +74,6 @@ const downloadVideoWithCurl = async (url, referer, outputPath) => { }); }; -/** - * Download video using undici with browser-like configuration - * @param {string} url - Video URL - * @param {string} referer - Referer URL (comments page) - * @returns {Promise} - Response object with body stream and headers - */ -const downloadVideo = async (url, referer) => { - const headers = { - "User-Agent": - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36", - Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "Accept-Language": "en-US,en;q=0.5", - "Accept-Encoding": "gzip, deflate, br", - "Upgrade-Insecure-Requests": "1", - "Sec-Fetch-Dest": "document", - "Sec-Fetch-Mode": "navigate", - "Sec-Fetch-Site": "cross-site", - "Sec-Fetch-User": "?1", - "sec-ch-ua": '"Not/A)Brand";v="8", "Chromium";v="142", "Google Chrome";v="142"', - "sec-ch-ua-mobile": "?0", - "sec-ch-ua-platform": '"Windows"', - Referer: referer, - "Cache-Control": "no-cache", - Pragma: "no-cache", - }; - - const response = await request(url, { - method: "GET", - headers, - maxRedirections: 5, - }); - - return response; -}; - /** * Get the final destination of a URL (follow redirects) * @param {string} url - Initial URL @@ -141,7 +104,6 @@ const getVideoLocation = async (href, markerOne, markerTwo) => { const response = await fetch(href, { headers: { "User-Agent": "darwin" }, // headers: { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36 Edg/138.0.0.0" } - // headers: getPageHeaders(), }); const html = await response.text(); const start = html.indexOf(markerOne); @@ -203,55 +165,13 @@ const processVideo = async (video) => { // Maximum download size in bytes from config const maxSizeBytes = darwinConfig.maxDownloadSize * 1024 * 1024; - console.log(`Attempting to download video using curl: ${href}`); - - // Try curl first (better TLS fingerprint) - let downloadSuccess = false; + console.log(`Downloading video using curl: ${href}`); try { await downloadVideoWithCurl(href, comments, tempFilePath); console.log("Successfully downloaded using curl"); - downloadSuccess = true; } catch (curlError) { - console.log(`Curl download failed: ${curlError.message}`); - } - - // Fallback to undici if curl failed - if (!downloadSuccess) { - console.log("Attempting download with undici..."); - try { - const response = await downloadVideo(href, comments); - - if (response.statusCode !== 200) { - console.error(`${href} ${response.statusCode} ${response.statusText || "Error"}`); - return null; - } - - const contentLength = parseInt(response.headers["content-length"], 10); - if (contentLength && contentLength > maxSizeBytes) { - console.log( - `Skipping download: File size exceeds limit (${darwinConfig.maxDownloadSize}MB)` - ); - await response.body.dump(); - - const fileSize = (contentLength / 1000000).toFixed(0); - return { - title, - href, - comments, - canBeStreamed: false, - directStreamLink: "", - fileSize, - tooBig: true, - }; - } - - console.log(`Downloading video to temp location: ${tempFilePath}`); - await pipeline(response.body, fs.createWriteStream(tempFilePath)); - downloadSuccess = true; - } catch (undiciError) { - console.error(`Undici download also failed: ${undiciError.message}`); - return null; - } + console.error(`Curl download failed: ${curlError.message}`); + return null; } if (!fs.existsSync(tempFilePath)) { @@ -392,7 +312,6 @@ const fetchVideosFromFeed = async () => { const response = await fetch(darwinConfig.feedUrl, { headers: { "User-Agent": "darwin" }, // headers: { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36" } - // headers: getPageHeaders(), }); const html = await response.text(); From 370d30abb18d34ed9348a4570825c65c0a36e640 Mon Sep 17 00:00:00 2001 From: VB2007 Date: Tue, 18 Nov 2025 08:47:49 +0100 Subject: [PATCH 3/3] brought back direct stream links for (too) large files, added failsafe check for downloads & distribution --- src/helpers/darwin/darwinProcess.js | 106 +++++++++++++++++++++++----- 1 file changed, 88 insertions(+), 18 deletions(-) diff --git a/src/helpers/darwin/darwinProcess.js b/src/helpers/darwin/darwinProcess.js index c6eea503..fb51dd10 100644 --- a/src/helpers/darwin/darwinProcess.js +++ b/src/helpers/darwin/darwinProcess.js @@ -74,6 +74,60 @@ const downloadVideoWithCurl = async (url, referer, outputPath) => { }); }; +/** + * Get file size using curl HEAD request + * @param {string} url - Video URL + * @param {string} referer - Referer URL (comments page) + * @returns {Promise} - File size in bytes, or null if failed + */ +const getFileSizeWithCurl = async (url, referer) => { + return new Promise((resolve) => { + const curlArgs = [ + "-I", // HEAD request only + "-L", // Follow redirects + "-s", // Silent mode + "-A", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36", + "-H", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "-H", + "Accept-Language: en-US,en;q=0.5", + "-H", + "Sec-Fetch-Dest: document", + "-H", + "Sec-Fetch-Mode: navigate", + "-H", + "Sec-Fetch-Site: cross-site", + "-H", + "Sec-Fetch-User: ?1", + "-H", + `Referer: ${referer}`, + "--http2", + url, + ]; + + const curl = spawn("curl", curlArgs); + let stdout = ""; + + curl.stdout.on("data", (data) => { + stdout += data.toString(); + }); + + curl.on("close", (code) => { + if (code === 0) { + const match = stdout.match(/content-length:\s*(\d+)/i); + resolve(match ? parseInt(match[1], 10) : null); + } else { + resolve(null); + } + }); + + curl.on("error", () => { + resolve(null); + }); + }); +}; + /** * Get the final destination of a URL (follow redirects) * @param {string} url - Initial URL @@ -129,7 +183,7 @@ const getVideoLocation = async (href, markerOne, markerTwo) => { * @param {number} fileSize - File size in MB * @returns {string} - Formatted message */ -const messageGen = (title, href, directStreamLink, comments, canBeStreamed, fileSize) => { +const messageGen = (title, href, comments, canBeStreamed, fileSize, directStreamLink = "") => { return `${canBeStreamed ? `[[ STREAMING & DOWNLOAD ]](${directStreamLink})` : `[[ ORIGINAL MP4 ]](${href})`} - [[ FORUM POST ]](<${comments}>)\n${title}${canBeStreamed ? `\nSize: ${fileSize}MB` : `\nSize (might won't load): ${fileSize}MB`}`; }; @@ -162,8 +216,27 @@ const processVideo = async (video) => { const finalFilePath = path.join(darwinConfig.targetDir, `${videoId}.mp4`); const directStreamLink = `${darwinConfig.cdnUrl}/${videoId}.mp4`; - // Maximum download size in bytes from config - const maxSizeBytes = darwinConfig.maxDownloadSize * 1024 * 1024; + // Check file size before downloading + console.log(`Checking file size for: ${href}`); + const fileSizeBytes = await getFileSizeWithCurl(href, comments); + + if (fileSizeBytes) { + const fileSizeMB = (fileSizeBytes / 1024 / 1024).toFixed(2); + console.log(`File size: ${fileSizeMB}MB`); + + if (fileSizeBytes > darwinConfig.maxDownloadSize * 1024 * 1024) { + console.log(`File exceeds limit (${darwinConfig.maxDownloadSize}MB), using direct link`); + return { + title, + href, + comments, + canBeStreamed: false, + fileSize: fileSizeMB, + }; + } + } else { + console.log("Could not determine file size, will proceed with download"); + } console.log(`Downloading video using curl: ${href}`); try { @@ -182,18 +255,16 @@ const processVideo = async (video) => { const originalSize = getFileSizeMB(tempFilePath); console.log(`Original file size: ${originalSize}MB`); - // Check if downloaded file is too large (for curl downloads which don't check size beforehand) + // Double-check size after download (fallback if HEAD request didn't work) if (originalSize > darwinConfig.maxDownloadSize) { - console.log(`File too large (${originalSize}MB), removing and skipping`); + console.log(`File too large (${originalSize}MB), removing and using direct link`); fs.unlinkSync(tempFilePath); return { title, href, comments, canBeStreamed: false, - directStreamLink: "", - fileSize: originalSize.toFixed(0), - tooBig: true, + fileSize: originalSize.toFixed(2), }; } @@ -222,14 +293,13 @@ const processVideo = async (video) => { } console.log("Temporary files cleaned up"); - // Return success result return { title, href, comments, - directStreamLink, canBeStreamed: true, fileSize: transcodedSize, + directStreamLink, }; } catch (error) { console.error(`Transcoding failed, attempting to use original file: ${error}`); @@ -251,15 +321,15 @@ const processVideo = async (video) => { } const finalSize = getFileSizeMB(finalFilePath); - const canStream = finalSize < 50; // I will run out of storage & network bandwidth quick if that's higher + const canStream = finalSize <= darwinConfig.maxDownloadSize; return { title, href, comments, - directStreamLink, canBeStreamed: canStream, fileSize: finalSize, + directStreamLink, }; } catch (fallbackError) { console.error(`Failed to use original file as fallback: ${fallbackError}`); @@ -280,9 +350,9 @@ const processVideo = async (video) => { * @returns {Promise} */ const distributeVideo = async (client, guildConfigs, processedVideo) => { - const { title, href, comments, directStreamLink, canBeStreamed, fileSize } = processedVideo; + const { title, href, comments, canBeStreamed, fileSize, directStreamLink } = processedVideo; - const message = messageGen(title, href, directStreamLink, comments, canBeStreamed, fileSize); + const message = messageGen(title, href, comments, canBeStreamed, fileSize, directStreamLink); for (const config of guildConfigs) { try { @@ -402,8 +472,8 @@ export const runDarwinProcess = async (client) => { if (result) { processedVideos.push(result); - // If it's a "too big" video, distribute immediately - if (result.tooBig) { + // If video wasn't downloaded, distribute immediately + if (!result.canBeStreamed) { await distributeVideo(client, guildConfigs, result); } } @@ -427,8 +497,8 @@ export const runDarwinProcess = async (client) => { await new Promise((resolve) => setTimeout(resolve, 1000)); } - // Filter out "too big" videos that were already distributed - const videosToDistribute = processedVideos.filter((video) => !video.tooBig); + // Filter out videos that weren't downloaded (already distributed) + const videosToDistribute = processedVideos.filter((video) => video.canBeStreamed); console.log( `Successfully processed ${videosToDistribute.length} videos, distributing to ${guildConfigs.length} channels` );