diff --git a/package-lock.json b/package-lock.json index c995ab14..9a4cc725 100755 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "discordbot", - "version": "4.0.1", + "version": "4.1.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "discordbot", - "version": "4.0.1", + "version": "4.1.1", "license": "AGPL-3.0-only", "dependencies": { "ajv": "^8.17.1", diff --git a/src/helpers/darwin/darwinProcess.js b/src/helpers/darwin/darwinProcess.js index 58eb7cd1..fb51dd10 100644 --- a/src/helpers/darwin/darwinProcess.js +++ b/src/helpers/darwin/darwinProcess.js @@ -1,7 +1,7 @@ import { load } from "cheerio"; import fs from "fs"; import path from "path"; -import { pipeline } from "stream/promises"; +import { spawn } from "child_process"; import { query } from "../db.js"; import { transcodeVideo, getFileSizeMB } from "./darwinTranscode.js"; @@ -10,6 +10,124 @@ import { isInCache, addToCache } from "./darwinCache.js"; import config from "../../../config.json" with { type: "json" }; const darwinConfig = config.darwin; +/** + * Download video using system curl (better TLS fingerprint) + * @param {string} url - Video URL + * @param {string} referer - Referer URL (comments page) + * @param {string} outputPath - Where to save the file + * @returns {Promise} - Whether download was successful + */ +const downloadVideoWithCurl = async (url, referer, outputPath) => { + return new Promise((resolve, reject) => { + const curlArgs = [ + "-L", // Follow redirects + "-A", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36", + "-H", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "-H", + "Accept-Language: en-US,en;q=0.5", + "-H", + "Accept-Encoding: gzip, deflate, br", + "-H", + "Upgrade-Insecure-Requests: 1", + "-H", + "Sec-Fetch-Dest: document", + "-H", + "Sec-Fetch-Mode: navigate", + "-H", + "Sec-Fetch-Site: cross-site", + "-H", + "Sec-Fetch-User: ?1", + "-H", + `Referer: ${referer}`, + "-H", + "Cache-Control: no-cache", + "-H", + "Pragma: no-cache", + "-o", + outputPath, + "--compressed", + "--http2", + url, + ]; + + const curl = spawn("curl", curlArgs); + + let stderr = ""; + + curl.stderr.on("data", (data) => { + stderr += data.toString(); + }); + + curl.on("close", (code) => { + if (code === 0) { + resolve(true); + } else { + reject(new Error(`curl exited with code ${code}: ${stderr}`)); + } + }); + + curl.on("error", (error) => { + reject(new Error(`Failed to spawn curl: ${error.message}`)); + }); + }); +}; + +/** + * Get file size using curl HEAD request + * @param {string} url - Video URL + * @param {string} referer - Referer URL (comments page) + * @returns {Promise} - File size in bytes, or null if failed + */ +const getFileSizeWithCurl = async (url, referer) => { + return new Promise((resolve) => { + const curlArgs = [ + "-I", // HEAD request only + "-L", // Follow redirects + "-s", // Silent mode + "-A", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36", + "-H", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "-H", + "Accept-Language: en-US,en;q=0.5", + "-H", + "Sec-Fetch-Dest: document", + "-H", + "Sec-Fetch-Mode: navigate", + "-H", + "Sec-Fetch-Site: cross-site", + "-H", + "Sec-Fetch-User: ?1", + "-H", + `Referer: ${referer}`, + "--http2", + url, + ]; + + const curl = spawn("curl", curlArgs); + let stdout = ""; + + curl.stdout.on("data", (data) => { + stdout += data.toString(); + }); + + curl.on("close", (code) => { + if (code === 0) { + const match = stdout.match(/content-length:\s*(\d+)/i); + resolve(match ? parseInt(match[1], 10) : null); + } else { + resolve(null); + } + }); + + curl.on("error", () => { + resolve(null); + }); + }); +}; + /** * Get the final destination of a URL (follow redirects) * @param {string} url - Initial URL @@ -38,7 +156,6 @@ const getDestination = async (url) => { const getVideoLocation = async (href, markerOne, markerTwo) => { try { const response = await fetch(href, { - //if it isn't set to "darwin" the scraping fails for some reason headers: { "User-Agent": "darwin" }, // headers: { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36 Edg/138.0.0.0" } }); @@ -66,7 +183,7 @@ const getVideoLocation = async (href, markerOne, markerTwo) => { * @param {number} fileSize - File size in MB * @returns {string} - Formatted message */ -const messageGen = (title, href, directStreamLink, comments, canBeStreamed, fileSize) => { +const messageGen = (title, href, comments, canBeStreamed, fileSize, directStreamLink = "") => { return `${canBeStreamed ? `[[ STREAMING & DOWNLOAD ]](${directStreamLink})` : `[[ ORIGINAL MP4 ]](${href})`} - [[ FORUM POST ]](<${comments}>)\n${title}${canBeStreamed ? `\nSize: ${fileSize}MB` : `\nSize (might won't load): ${fileSize}MB`}`; }; @@ -93,38 +210,42 @@ const processVideo = async (video) => { return null; } - const response = await fetch(href); - - if (!response.ok) { - console.error(`${href} ${response.statusText}`); - return null; - } - - const contentLength = parseInt(response.headers.get("Content-Length"), 10); - if (contentLength && contentLength > 50 * 1024 * 1024) { - console.log("Skipping download: File size exceeds limit (50MB)"); - - const fileSize = (contentLength / 1000000).toFixed(0); - // Return info for too large videos - return { - title, - href, - comments, - canBeStreamed: false, - directStreamLink: "", - fileSize, - tooBig: true, - }; - } - const videoId = href.split("/").pop().replace(".mp4", ""); const tempFilePath = path.join(darwinConfig.tempDir, `${videoId}_original.mp4`); const transcodedFilePath = path.join(darwinConfig.tempDir, `${videoId}_transcoded.mp4`); const finalFilePath = path.join(darwinConfig.targetDir, `${videoId}.mp4`); const directStreamLink = `${darwinConfig.cdnUrl}/${videoId}.mp4`; - console.log(`Downloading video to temp location: ${tempFilePath}`); - await pipeline(response.body, fs.createWriteStream(tempFilePath)); + // Check file size before downloading + console.log(`Checking file size for: ${href}`); + const fileSizeBytes = await getFileSizeWithCurl(href, comments); + + if (fileSizeBytes) { + const fileSizeMB = (fileSizeBytes / 1024 / 1024).toFixed(2); + console.log(`File size: ${fileSizeMB}MB`); + + if (fileSizeBytes > darwinConfig.maxDownloadSize * 1024 * 1024) { + console.log(`File exceeds limit (${darwinConfig.maxDownloadSize}MB), using direct link`); + return { + title, + href, + comments, + canBeStreamed: false, + fileSize: fileSizeMB, + }; + } + } else { + console.log("Could not determine file size, will proceed with download"); + } + + console.log(`Downloading video using curl: ${href}`); + try { + await downloadVideoWithCurl(href, comments, tempFilePath); + console.log("Successfully downloaded using curl"); + } catch (curlError) { + console.error(`Curl download failed: ${curlError.message}`); + return null; + } if (!fs.existsSync(tempFilePath)) { console.error(`Error when saving temporary file: ${tempFilePath}`); @@ -134,6 +255,19 @@ const processVideo = async (video) => { const originalSize = getFileSizeMB(tempFilePath); console.log(`Original file size: ${originalSize}MB`); + // Double-check size after download (fallback if HEAD request didn't work) + if (originalSize > darwinConfig.maxDownloadSize) { + console.log(`File too large (${originalSize}MB), removing and using direct link`); + fs.unlinkSync(tempFilePath); + return { + title, + href, + comments, + canBeStreamed: false, + fileSize: originalSize.toFixed(2), + }; + } + try { console.log("Starting video transcoding process..."); const transcodingSuccess = await transcodeVideo(tempFilePath, transcodedFilePath); @@ -159,14 +293,13 @@ const processVideo = async (video) => { } console.log("Temporary files cleaned up"); - // Return success result return { title, href, comments, - directStreamLink, canBeStreamed: true, fileSize: transcodedSize, + directStreamLink, }; } catch (error) { console.error(`Transcoding failed, attempting to use original file: ${error}`); @@ -188,15 +321,15 @@ const processVideo = async (video) => { } const finalSize = getFileSizeMB(finalFilePath); - const canStream = finalSize < 50; // I will run out of storage & network bandwidth quick if that's higher + const canStream = finalSize <= darwinConfig.maxDownloadSize; return { title, href, comments, - directStreamLink, canBeStreamed: canStream, fileSize: finalSize, + directStreamLink, }; } catch (fallbackError) { console.error(`Failed to use original file as fallback: ${fallbackError}`); @@ -217,9 +350,9 @@ const processVideo = async (video) => { * @returns {Promise} */ const distributeVideo = async (client, guildConfigs, processedVideo) => { - const { title, href, comments, directStreamLink, canBeStreamed, fileSize } = processedVideo; + const { title, href, comments, canBeStreamed, fileSize, directStreamLink } = processedVideo; - const message = messageGen(title, href, directStreamLink, comments, canBeStreamed, fileSize); + const message = messageGen(title, href, comments, canBeStreamed, fileSize, directStreamLink); for (const config of guildConfigs) { try { @@ -248,6 +381,7 @@ const fetchVideosFromFeed = async () => { try { const response = await fetch(darwinConfig.feedUrl, { headers: { "User-Agent": "darwin" }, + // headers: { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36" } }); const html = await response.text(); @@ -338,8 +472,8 @@ export const runDarwinProcess = async (client) => { if (result) { processedVideos.push(result); - // If it's a "too big" video, distribute immediately - if (result.tooBig) { + // If video wasn't downloaded, distribute immediately + if (!result.canBeStreamed) { await distributeVideo(client, guildConfigs, result); } } @@ -363,8 +497,8 @@ export const runDarwinProcess = async (client) => { await new Promise((resolve) => setTimeout(resolve, 1000)); } - // Filter out "too big" videos that were already distributed - const videosToDistribute = processedVideos.filter((video) => !video.tooBig); + // Filter out videos that weren't downloaded (already distributed) + const videosToDistribute = processedVideos.filter((video) => video.canBeStreamed); console.log( `Successfully processed ${videosToDistribute.length} videos, distributing to ${guildConfigs.length} channels` ); diff --git a/src/sql/darwinCache/table.sql b/src/sql/darwinCache/table.sql index 1f1a396d..09fe1355 100644 --- a/src/sql/darwinCache/table.sql +++ b/src/sql/darwinCache/table.sql @@ -7,4 +7,4 @@ CREATE TABLE IF NOT EXISTS `darwinCache` ( `processedAt` TIMESTAMP DEFAULT CURRENT_TIMESTAMP, UNIQUE KEY (`videoUrl`), UNIQUE KEY (`videoId`) -); \ No newline at end of file +);