diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index c04f556f..28f80b80 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -36,7 +36,7 @@ jobs: # with: # node-version: "24.8.0" - - name: Intalling Node.js dependencies + - name: Installing Node.js dependencies working-directory: /home/vb2007/prod/discordbot env: PATH: /home/vb2007/.nvm/versions/node/v24.8.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin diff --git a/package-lock.json b/package-lock.json index c995ab14..34701662 100755 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "discordbot", - "version": "4.0.1", + "version": "4.1.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "discordbot", - "version": "4.0.1", + "version": "4.1.1", "license": "AGPL-3.0-only", "dependencies": { "ajv": "^8.17.1", @@ -553,12 +553,12 @@ } }, "node_modules/@discordjs/formatters": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/@discordjs/formatters/-/formatters-0.6.1.tgz", - "integrity": "sha512-5cnX+tASiPCqCWtFcFslxBVUaCetB0thvM/JyavhbXInP1HJIEU+Qv/zMrnuwSsX3yWH2lVXNJZeDK3EiP4HHg==", + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/@discordjs/formatters/-/formatters-0.6.2.tgz", + "integrity": "sha512-y4UPwWhH6vChKRkGdMB4odasUbHOUwy7KL+OVwF86PvT6QVOwElx+TiI1/6kcmcEe+g5YRXJFiXSXUdabqZOvQ==", "license": "Apache-2.0", "dependencies": { - "discord-api-types": "^0.38.1" + "discord-api-types": "^0.38.33" }, "engines": { "node": ">=16.11.0" @@ -612,10 +612,13 @@ } }, "node_modules/@discordjs/util": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@discordjs/util/-/util-1.1.1.tgz", - "integrity": "sha512-eddz6UnOBEB1oITPinyrB2Pttej49M9FZQY8NxgEvc3tq6ZICZ19m70RsmzRdDHk80O9NoYN/25AqJl8vPVf/g==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@discordjs/util/-/util-1.2.0.tgz", + "integrity": "sha512-3LKP7F2+atl9vJFhaBjn4nOaSWahZ/yWjOvA4e5pnXkt2qyXRCHLxoBQy81GFtLGCq7K9lPm9R517M1U+/90Qg==", "license": "Apache-2.0", + "dependencies": { + "discord-api-types": "^0.38.33" + }, "engines": { "node": ">=18" }, @@ -1176,9 +1179,9 @@ } }, "node_modules/@types/node": { - "version": "24.9.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-24.9.1.tgz", - "integrity": "sha512-QoiaXANRkSXK6p0Duvt56W208du4P9Uye9hWLWgGMDTEoKPhuenzNcC4vGUmrNkiOKTlIrBoyNQYNpSwfEZXSg==", + "version": "24.10.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.1.tgz", + "integrity": "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==", "license": "MIT", "dependencies": { "undici-types": "~7.16.0" @@ -1201,9 +1204,9 @@ } }, "node_modules/@types/yargs": { - "version": "17.0.34", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.34.tgz", - "integrity": "sha512-KExbHVa92aJpw9WDQvzBaGVE2/Pz+pLZQloT2hjL8IqsZnV62rlPOYvNnLmf/L2dyllfVUOVBj64M0z/46eR2A==", + "version": "17.0.35", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.35.tgz", + "integrity": "sha512-qUHkeCyQFxMXg79wQfTtfndEC+N9ZZg76HJftDJp+qH2tV7Gj4OJi7l+PiWwJ+pWtW8GwSmqsDj/oymhrTWXjg==", "dev": true, "license": "MIT", "dependencies": { @@ -1438,9 +1441,9 @@ "license": "MIT" }, "node_modules/baseline-browser-mapping": { - "version": "2.8.20", - "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.8.20.tgz", - "integrity": "sha512-JMWsdF+O8Orq3EMukbUN1QfbLK9mX2CkUmQBcW2T0s8OmdAUL5LLM/6wFwSrqXzlXB13yhyK9gTKS1rIizOduQ==", + "version": "2.8.29", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.8.29.tgz", + "integrity": "sha512-sXdt2elaVnhpDNRDz+1BDx1JQoJRuNk7oVlAlbGiFkLikHCAQiccexF/9e91zVi6RCgqspl04aP+6Cnl9zRLrA==", "dev": true, "license": "Apache-2.0", "bin": { @@ -1478,9 +1481,9 @@ } }, "node_modules/browserslist": { - "version": "4.27.0", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.27.0.tgz", - "integrity": "sha512-AXVQwdhot1eqLihwasPElhX2tAZiBjWdJ9i/Zcj2S6QYIjkx62OKSfnobkriB81C3l4w0rVy3Nt4jaTBltYEpw==", + "version": "4.28.0", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.0.tgz", + "integrity": "sha512-tbydkR/CxfMwelN0vwdP/pLkDwyAASZ+VfWm4EOwlB6SWhx1sYnWLqo8N5j0rAzPfzfRaxt0mM/4wPU/Su84RQ==", "dev": true, "funding": [ { @@ -1499,10 +1502,10 @@ "license": "MIT", "peer": true, "dependencies": { - "baseline-browser-mapping": "^2.8.19", - "caniuse-lite": "^1.0.30001751", - "electron-to-chromium": "^1.5.238", - "node-releases": "^2.0.26", + "baseline-browser-mapping": "^2.8.25", + "caniuse-lite": "^1.0.30001754", + "electron-to-chromium": "^1.5.249", + "node-releases": "^2.0.27", "update-browserslist-db": "^1.1.4" }, "bin": { @@ -1550,9 +1553,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001751", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001751.tgz", - "integrity": "sha512-A0QJhug0Ly64Ii3eIqHu5X51ebln3k4yTUkY1j8drqpWHVreg/VLijN48cZ1bYPiqOQuqpkIKnzr/Ul8V+p6Cw==", + "version": "1.0.30001755", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001755.tgz", + "integrity": "sha512-44V+Jm6ctPj7R52Na4TLi3Zri4dWUljJd+RDm+j8LtNCc/ihLCT+X1TzoOAkRETEWqjuLnh9581Tl80FvK7jVA==", "dev": true, "funding": [ { @@ -1883,28 +1886,28 @@ } }, "node_modules/discord-api-types": { - "version": "0.38.31", - "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.38.31.tgz", - "integrity": "sha512-kC94ANsk8ackj8ENTuO8joTNEL0KtymVhHy9dyEC/s4QAZ7GCx40dYEzQaadyo8w+oP0X8QydE/nzAWRylTGtQ==", + "version": "0.38.34", + "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.38.34.tgz", + "integrity": "sha512-muq7xKGznj5MSFCzuIm/2TO7DpttuomUTemVM82fRqgnMl70YRzEyY24jlbiV6R9tzOTq6A6UnZ0bsfZeKD38Q==", "license": "MIT", "workspaces": [ "scripts/actions/documentation" ] }, "node_modules/discord.js": { - "version": "14.24.0", - "resolved": "https://registry.npmjs.org/discord.js/-/discord.js-14.24.0.tgz", - "integrity": "sha512-KNq/ekT8bsmT3ZAfVre8cPbl+DfVYSdlLnDmGZPoz7Cw21LYeWHllRA9MivqNq5b1GPGAxGvyUN1vxbTb/PQWw==", + "version": "14.25.0", + "resolved": "https://registry.npmjs.org/discord.js/-/discord.js-14.25.0.tgz", + "integrity": "sha512-9rnJWTAkfauKTieYJ3oI4oncrSzpbJPWN1/XU+2H6wsHQPtqbOrXvgM0nFhSVnIbTo7nfUF7fzcYRevvRGjpzA==", "license": "Apache-2.0", "dependencies": { "@discordjs/builders": "^1.13.0", "@discordjs/collection": "1.5.3", - "@discordjs/formatters": "^0.6.1", + "@discordjs/formatters": "^0.6.2", "@discordjs/rest": "^2.6.0", - "@discordjs/util": "^1.1.1", + "@discordjs/util": "^1.2.0", "@discordjs/ws": "^1.2.3", "@sapphire/snowflake": "3.5.3", - "discord-api-types": "^0.38.31", + "discord-api-types": "^0.38.33", "fast-deep-equal": "3.1.3", "lodash.snakecase": "4.1.1", "magic-bytes.js": "^1.10.0", @@ -1995,9 +1998,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.5.240", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.240.tgz", - "integrity": "sha512-OBwbZjWgrCOH+g6uJsA2/7Twpas2OlepS9uvByJjR2datRDuKGYeD+nP8lBBks2qnB7bGJNHDUx7c/YLaT3QMQ==", + "version": "1.5.256", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.256.tgz", + "integrity": "sha512-uqYq1IQhpXXLX+HgiXdyOZml7spy4xfy42yPxcCCRjswp0fYM2X+JwCON07lqnpLEGVCj739B7Yr+FngmHBMEQ==", "dev": true, "license": "ISC" }, @@ -3229,9 +3232,9 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", - "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "version": "3.14.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz", + "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==", "dev": true, "license": "MIT", "dependencies": { @@ -3476,9 +3479,9 @@ "license": "MIT" }, "node_modules/node-releases": { - "version": "2.0.26", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.26.tgz", - "integrity": "sha512-S2M9YimhSjBSvYnlr5/+umAnPHE++ODwt5e2Ij6FoX45HA/s4vHdkDx1eax2pAPeAOqu4s9b7ppahsyEFdVqQA==", + "version": "2.0.27", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.27.tgz", + "integrity": "sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA==", "dev": true, "license": "MIT" }, diff --git a/package.json b/package.json index 7feb10b2..2fa15ea1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "discordbot", - "version": "4.1.1", + "version": "4.2.0", "description": "A simple discord bot of mine developed with discord.js (Node.js).", "type": "module", "main": "index.js", diff --git a/src/helpers/darwin/darwinCache.js b/src/helpers/darwin/darwinCache.js index 63ba320d..d3b8ca10 100644 --- a/src/helpers/darwin/darwinCache.js +++ b/src/helpers/darwin/darwinCache.js @@ -22,23 +22,28 @@ const extractVideoId = (url) => { }; /** - * Add a URL to Darwin's cache - * @param {string} url - The video URL to cache + * Add a video to Darwin's cache + * @param {string} directVideoUrl - The direct video URL + * @param {string} forumPostUrl - The forum post URL + * @param {string} videoTitle - The video title * @returns {Promise} - Whether the operation was successful */ -export const addToCache = async (url) => { +export const addToCache = async (directVideoUrl, forumPostUrl, videoTitle) => { try { - const exists = await isInCache(url); + const exists = await isInCache(directVideoUrl); if (exists) { - console.log(`URL already in cache: ${url}`); + console.log(`URL already in cache: ${directVideoUrl}`); return true; } - const videoId = extractVideoId(url); - console.log(`Adding "${url}" (ID: ${videoId}) to Darwin's cache`); + const videoId = extractVideoId(directVideoUrl); + // console.log(`Adding "${videoTitle}" (ID: ${videoId}) to Darwin's cache`); try { - await query("INSERT INTO darwinCache (videoUrl, videoId) VALUES (?, ?)", [url, videoId]); + await query( + "INSERT INTO darwinCache (directVideoUrl, forumPostUrl, videoId, videoTitle) VALUES (?, ?, ?, ?)", + [directVideoUrl, forumPostUrl, videoId, videoTitle] + ); return true; } catch (sqlError) { //handle duplicate key error (MySQL error code 1062) (edge case, if source reuses the same id) @@ -49,7 +54,7 @@ export const addToCache = async (url) => { if (idExists.length > 0) { console.log(`Another URL already uses videoId ${videoId}, treating as cached`); } else { - console.log(`URL ${url} is a duplicate entry`); + console.log(`URL ${directVideoUrl} is a duplicate entry`); } return true; @@ -74,7 +79,7 @@ export const isInCache = async (url) => { console.log(`Checking "${url}" (ID: ${videoId}) in Darwin's cache`); const result = await query( - "SELECT videoUrl FROM darwinCache WHERE videoUrl = ? OR videoId = ?", + "SELECT directVideoUrl FROM darwinCache WHERE directVideoUrl = ? OR videoId = ?", [url, videoId] ); return result.length > 0; diff --git a/src/helpers/darwin/darwinProcess.js b/src/helpers/darwin/darwinProcess.js index 58eb7cd1..977b2167 100644 --- a/src/helpers/darwin/darwinProcess.js +++ b/src/helpers/darwin/darwinProcess.js @@ -1,48 +1,61 @@ import { load } from "cheerio"; -import fs from "fs"; -import path from "path"; -import { pipeline } from "stream/promises"; - +import https from "https"; import { query } from "../db.js"; -import { transcodeVideo, getFileSizeMB } from "./darwinTranscode.js"; import { isInCache, addToCache } from "./darwinCache.js"; - import config from "../../../config.json" with { type: "json" }; + const darwinConfig = config.darwin; /** - * Get the final destination of a URL (follow redirects) + * Simple HTTPS GET request + * @param {string} url - URL to fetch + * @param {Object} headers - Request headers + * @returns {Promise} - Response body + */ +const httpsGet = (url, headers = {}) => { + return new Promise((resolve, reject) => { + const options = { + headers: { + "User-Agent": "darwin", + ...headers, + }, + }; + + https + .get(url, options, (res) => { + let data = ""; + res.on("data", (chunk) => (data += chunk)); + res.on("end", () => resolve(data)); + }) + .on("error", reject); + }); +}; + +/** + * Get final URL after redirects * @param {string} url - Initial URL - * @returns {Promise} - Final URL after redirects + * @returns {Promise} - Final URL */ const getDestination = async (url) => { try { - const response = await fetch(url, { - method: "HEAD", - redirect: "follow", - }); + const response = await fetch(url, { method: "HEAD", redirect: "follow" }); return response.url; } catch (error) { - console.error(`Failed to resolve URL destination: ${error}`); + console.error(`Failed to resolve URL: ${error}`); return url; } }; /** - * Extract video location from a page + * Extract video location from page * @param {string} href - Page URL - * @param {string} markerOne - First marker to identify video - * @param {string} markerTwo - Second marker to identify video + * @param {string} markerOne - First marker + * @param {string} markerTwo - Second marker * @returns {Promise} - Direct video URL */ const getVideoLocation = async (href, markerOne, markerTwo) => { try { - const response = await fetch(href, { - //if it isn't set to "darwin" the scraping fails for some reason - headers: { "User-Agent": "darwin" }, - // headers: { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36 Edg/138.0.0.0" } - }); - const html = await response.text(); + const html = await httpsGet(href); const start = html.indexOf(markerOne); if (start === -1) return ""; @@ -57,205 +70,26 @@ const getVideoLocation = async (href, markerOne, markerTwo) => { }; /** - * Generate message for Discord channel + * Generate Discord message * @param {string} title - Video title - * @param {string} href - Source URL - * @param {string} directStreamLink - Direct streaming link - * @param {string} comments - Comments/forum URL - * @param {boolean} canBeStreamed - Whether video can be streamed - * @param {number} fileSize - File size in MB + * @param {string} href - Video URL + * @param {string} comments - Forum post URL * @returns {string} - Formatted message */ -const messageGen = (title, href, directStreamLink, comments, canBeStreamed, fileSize) => { - return `${canBeStreamed ? `[[ STREAMING & DOWNLOAD ]](${directStreamLink})` : `[[ ORIGINAL MP4 ]](${href})`} - [[ FORUM POST ]](<${comments}>)\n${title}${canBeStreamed ? `\nSize: ${fileSize}MB` : `\nSize (might won't load): ${fileSize}MB`}`; -}; - -/** - * Process a single video - * @param {Object} video - Video metadata - * @returns {Promise} - Processed video result or null if failed - */ -const processVideo = async (video) => { - const { title, href, comments } = video; - console.log(`Processing ${title} ${href}`); - - try { - const isVideoInCache = await isInCache(href); - if (isVideoInCache) { - console.log(`Video already in cache, skipping: ${href}`); - return null; - } - - // Add to cache BEFORE downloading to prevent multiple uploads - const addedToCache = await addToCache(href); - if (!addedToCache) { - console.log(`Failed to add to cache, skipping to prevent duplicates: ${href}`); - return null; - } - - const response = await fetch(href); - - if (!response.ok) { - console.error(`${href} ${response.statusText}`); - return null; - } - - const contentLength = parseInt(response.headers.get("Content-Length"), 10); - if (contentLength && contentLength > 50 * 1024 * 1024) { - console.log("Skipping download: File size exceeds limit (50MB)"); - - const fileSize = (contentLength / 1000000).toFixed(0); - // Return info for too large videos - return { - title, - href, - comments, - canBeStreamed: false, - directStreamLink: "", - fileSize, - tooBig: true, - }; - } - - const videoId = href.split("/").pop().replace(".mp4", ""); - const tempFilePath = path.join(darwinConfig.tempDir, `${videoId}_original.mp4`); - const transcodedFilePath = path.join(darwinConfig.tempDir, `${videoId}_transcoded.mp4`); - const finalFilePath = path.join(darwinConfig.targetDir, `${videoId}.mp4`); - const directStreamLink = `${darwinConfig.cdnUrl}/${videoId}.mp4`; - - console.log(`Downloading video to temp location: ${tempFilePath}`); - await pipeline(response.body, fs.createWriteStream(tempFilePath)); - - if (!fs.existsSync(tempFilePath)) { - console.error(`Error when saving temporary file: ${tempFilePath}`); - return null; - } - - const originalSize = getFileSizeMB(tempFilePath); - console.log(`Original file size: ${originalSize}MB`); - - try { - console.log("Starting video transcoding process..."); - const transcodingSuccess = await transcodeVideo(tempFilePath, transcodedFilePath); - - const transcodedSize = getFileSizeMB(transcodedFilePath); - if (transcodingSuccess) { - console.log( - `Transcoded file size: ${transcodedSize}MB (${Math.round((transcodedSize / originalSize) * 100)}% of original)` - ); - - fs.copyFileSync(transcodedFilePath, finalFilePath); - console.log(`File moved to final destination: ${finalFilePath}`); - } else { - throw new Error("Transcoding failed to produce valid output"); - } - - // Clean up temp files - if (fs.existsSync(tempFilePath)) { - fs.unlinkSync(tempFilePath); - } - if (fs.existsSync(transcodedFilePath)) { - fs.unlinkSync(transcodedFilePath); - } - console.log("Temporary files cleaned up"); - - // Return success result - return { - title, - href, - comments, - directStreamLink, - canBeStreamed: true, - fileSize: transcodedSize, - }; - } catch (error) { - console.error(`Transcoding failed, attempting to use original file: ${error}`); - - try { - // Use original file as fallback - fs.copyFileSync(tempFilePath, finalFilePath); - console.log(`Original file moved to final destination: ${finalFilePath}`); - - // Clean up temporary files - if (fs.existsSync(tempFilePath)) { - fs.unlinkSync(tempFilePath); - console.log("Original temporary file cleaned up"); - } - - if (fs.existsSync(transcodedFilePath)) { - fs.unlinkSync(transcodedFilePath); - console.log("Partial transcoded file cleaned up"); - } - - const finalSize = getFileSizeMB(finalFilePath); - const canStream = finalSize < 50; // I will run out of storage & network bandwidth quick if that's higher - - return { - title, - href, - comments, - directStreamLink, - canBeStreamed: canStream, - fileSize: finalSize, - }; - } catch (fallbackError) { - console.error(`Failed to use original file as fallback: ${fallbackError}`); - return null; - } - } - } catch (error) { - console.error(`Error processing video ${title}: ${error}`); - return null; - } -}; - -/** - * Distribute a processed video to all configured channels - * @param {Object} client - Discord client - * @param {Array} guildConfigs - Array of guild configurations - * @param {Object} processedVideo - Processed video result - * @returns {Promise} - */ -const distributeVideo = async (client, guildConfigs, processedVideo) => { - const { title, href, comments, directStreamLink, canBeStreamed, fileSize } = processedVideo; - - const message = messageGen(title, href, directStreamLink, comments, canBeStreamed, fileSize); - - for (const config of guildConfigs) { - try { - console.log(`Sending video to guild ${config.guildId}, channel ${config.channelId}`); - const channel = await client.channels.fetch(config.channelId); - - if (channel) { - await channel.send(message); - console.log( - `Successfully sent video to channel ${config.channelName} (${config.channelId})` - ); - } else { - console.error(`Failed to fetch channel ${config.channelId}`); - } - } catch (error) { - console.error(`Error sending to channel ${config.channelId}: ${error}`); - } - } +const messageGen = (title, href, comments) => { + return `[[ STREAMING & DOWNLOAD ]](${href}) - [[ FORUM POST ]](<${comments}>)\n${title}`; }; /** * Fetch videos from feed * @returns {Promise} - Array of video objects */ -const fetchVideosFromFeed = async () => { +const fetchAndDistributeVideos = async (client, guildConfigs) => { try { - const response = await fetch(darwinConfig.feedUrl, { - headers: { "User-Agent": "darwin" }, - }); - - const html = await response.text(); + const html = await httpsGet(darwinConfig.feedUrl); const $ = load(html); - const contentBlock = $(".content-block > div"); - - const videosToProcess = []; + let distributedCount = 0; for (const node of contentBlock) { try { @@ -278,103 +112,83 @@ const fetchVideosFromFeed = async () => { const isVideoInCache = await isInCache(videoLocation); if (isVideoInCache) { - console.log(`Skipping cached video: ${title.trim()} (${videoLocation})`); + // console.log(`Skipping cached video: ${title.trim()}`); continue; } - console.log(`Discovered "${title.trim()}" at "${videoLocation}"`); - videosToProcess.push({ title, href: videoLocation, comments: href }); + // console.log(`Discovered new video: "${title.trim()}" at "${videoLocation}"`); + + //immediately distribute to all channels + const video = { title: title.trim(), href: videoLocation, comments: href }; + await distributeVideo(client, guildConfigs, video); + + const addedToCache = await addToCache(videoLocation, href, title.trim()); + if (addedToCache) { + distributedCount++; + } else { + console.log(`Warning: Failed to add to cache: ${videoLocation}`); + } } catch (error) { console.error(`Error processing item: ${error}`); } } - return videosToProcess; + return distributedCount; } catch (error) { console.error(`Error fetching videos from feed: ${error}`); - return []; + return 0; + } +}; + +/** + * Distribute video to all configured channels + * @param {Object} client - Discord client + * @param {Array} guildConfigs - Guild configurations + * @param {Object} video - Video object + */ +const distributeVideo = async (client, guildConfigs, video) => { + const { title, href, comments } = video; + const message = messageGen(title, href, comments); + + for (const config of guildConfigs) { + try { + // console.log(`Sending video to guild ${config.guildId}, channel ${config.channelId}`); + const channel = await client.channels.fetch(config.channelId); + + if (channel) { + await channel.send(message); + // console.log(`Successfully sent to ${config.channelName} (${config.channelId})`); + } else { + console.error(`Failed to fetch channel ${config.channelId}`); + } + } catch (error) { + console.error(`Error sending to channel ${config.channelId}: ${error}`); + } } }; /** - * Run the Darwin process for all configured guilds + * Run the Darwin process * @param {Object} client - Discord client - * @returns {Promise} */ export const runDarwinProcess = async (client) => { try { - // Get all guild configurations const guildConfigs = await query("SELECT * FROM configDarwin"); if (guildConfigs.length === 0) { - console.log("No Darwin configurations found"); + // console.log("No Darwin configurations found"); return; } - console.log(`Found ${guildConfigs.length} Darwin configurations`); + // console.log(`Found ${guildConfigs.length} Darwin configurations`); - // Fetch videos from feed once - const videosToProcess = await fetchVideosFromFeed(); - console.log(`Discovered ${videosToProcess.length} new videos to process`); + const distributedCount = await fetchAndDistributeVideos(client, guildConfigs); - if (videosToProcess.length === 0) { - console.log("No new videos to process"); - return; + if (distributedCount === 0) { + // console.log("No new videos to distribute"); + } else { + // console.log(`Darwin process completed: ${distributedCount} video(s) distributed`); } - - // Process videos one at a time, with concurrency limit - const concurrencyLimit = 2; - let activeTasks = 0; - let videoIndex = 0; - const processedVideos = []; - - async function processNextVideos() { - while (activeTasks < concurrencyLimit && videoIndex < videosToProcess.length) { - const video = videosToProcess[videoIndex++]; - activeTasks++; - - try { - const result = await processVideo(video); - if (result) { - processedVideos.push(result); - - // If it's a "too big" video, distribute immediately - if (result.tooBig) { - await distributeVideo(client, guildConfigs, result); - } - } - } finally { - activeTasks--; - if (videoIndex < videosToProcess.length) { - processNextVideos(); - } - } - - if (videoIndex < videosToProcess.length) { - await new Promise((resolve) => setTimeout(resolve, 500)); - } - } - } - - await processNextVideos(); - - // Wait for all processing to complete - while (activeTasks > 0) { - await new Promise((resolve) => setTimeout(resolve, 1000)); - } - - // Filter out "too big" videos that were already distributed - const videosToDistribute = processedVideos.filter((video) => !video.tooBig); - console.log( - `Successfully processed ${videosToDistribute.length} videos, distributing to ${guildConfigs.length} channels` - ); - - // Distribute each processed video to all configured channels - for (const video of videosToDistribute) { - await distributeVideo(client, guildConfigs, video); - } - - console.log("Darwin process completed successfully"); } catch (error) { console.error(`Error running Darwin process: ${error}`); } diff --git a/src/helpers/darwin/darwinTranscode.js b/src/helpers/darwin/darwinTranscode.js deleted file mode 100644 index 41da8920..00000000 --- a/src/helpers/darwin/darwinTranscode.js +++ /dev/null @@ -1,225 +0,0 @@ -import ffmpeg from "fluent-ffmpeg"; -import fs from "fs"; - -/** - * Get video information including dimensions and duration - * @param {string} inputPath - Path to the video file - * @returns {Promise} - Video metadata object - */ -export const getVideoInfo = async (inputPath) => { - return new Promise((resolve, reject) => { - ffmpeg.ffprobe(inputPath, (err, metadata) => { - if (err) { - return reject(err); - } - - const videoStream = metadata.streams.find((stream) => stream.codec_type === "video"); - if (!videoStream) { - return reject(new Error("No video stream found")); - } - - resolve({ - width: videoStream.width, - height: videoStream.height, - duration: videoStream.duration, - bitrate: videoStream.bit_rate, - rotation: - videoStream.tags && videoStream.tags.rotate ? parseInt(videoStream.tags.rotate) : 0, - codec_name: videoStream.codec_name, - }); - }); - }); -}; - -/** - * Get video file size in MB - * @param {string} filePath - Path to the video file - * @returns {number} - File size in MB - */ -export const getFileSizeMB = (filePath) => { - try { - const stats = fs.statSync(filePath); - return (stats.size / (1024 * 1024)).toFixed(2); - } catch (error) { - console.error(`Failed to get file size: ${error}`); - return 0; - } -}; - -/** - * Calculate optimal CRF value based on video resolution - * @param {number} totalPixels - Total pixels in the video - * @returns {number} - Optimal CRF value - */ -const calculateOptimalCRF = (totalPixels) => { - // Higher resolution can use higher CRF (more compression) while still looking good - if (totalPixels > 2073600) { - // > 1080p - return 28; - } else if (totalPixels > 921600) { - // > 720p - return 26; - } else if (totalPixels > 409920) { - // > 480p - return 24; - } else { - // Lower resolutions need less compression to look good - return 23; - } -}; - -/** - * Calculate optimal dimensions for transcoding while preserving aspect ratio - * @param {Object} videoInfo - Original video information - * @returns {Object} - Calculated dimensions and scaling information - */ -const calculateOptimalDimensions = (videoInfo) => { - const { width, height, rotation } = videoInfo; - - const effectiveWidth = rotation === 90 || rotation === 270 ? height : width; - const effectiveHeight = rotation === 90 || rotation === 270 ? width : height; - - const aspectRatio = effectiveWidth / effectiveHeight; - const totalPixels = effectiveWidth * effectiveHeight; - - // Base limits - only apply if video exceeds these dimensions - const MAX_WIDTH = 1920; - const MAX_HEIGHT = 1280; - const MAX_PIXELS = 2073600; // 1080p equivalent (1920×1080) - - // Initialize with original dimensions - don't change unless needed - let targetWidth = effectiveWidth; - let targetHeight = effectiveHeight; - let needsScaling = false; - - // Check if we need to scale down - if (totalPixels > MAX_PIXELS || effectiveWidth > MAX_WIDTH || effectiveHeight > MAX_HEIGHT) { - needsScaling = true; - - // Scale by the most constraining dimension - if (effectiveWidth / MAX_WIDTH > effectiveHeight / MAX_HEIGHT) { - // Width is the limiting factor - targetWidth = MAX_WIDTH; - targetHeight = Math.round(MAX_WIDTH / aspectRatio); - } else { - // Height is the limiting factor - targetHeight = MAX_HEIGHT; - targetWidth = Math.round(MAX_HEIGHT * aspectRatio); - } - - // Ensure we don't exceed the pixel count limit - if (targetWidth * targetHeight > MAX_PIXELS) { - const scale = Math.sqrt(MAX_PIXELS / (targetWidth * targetHeight)); - targetWidth = Math.round(targetWidth * scale); - targetHeight = Math.round(targetHeight * scale); - } - } - - console.log( - `Original dimensions: ${effectiveWidth}x${effectiveHeight}, aspect ratio: ${aspectRatio.toFixed(2)}` - ); - if (needsScaling) { - console.log(`Scaling to: ${targetWidth}x${targetHeight}`); - } else { - console.log(`Keeping original dimensions: ${targetWidth}x${targetHeight}`); - } - - return { - width: targetWidth, - height: targetHeight, - aspectRatio, - needsScaling, - isLandscape: aspectRatio > 1, - isPortrait: aspectRatio < 1, - isSquare: Math.abs(aspectRatio - 1) < 0.1, - totalPixels, - }; -}; - -/** - * Transcode video to reduce file size while maintaining Discord compatibility - * @param {string} inputPath - Path to the input video file - * @param {string} outputPath - Path for the transcoded output file - * @returns {Promise} - Whether transcoding was successful - */ -export const transcodeVideo = async (inputPath, outputPath) => { - try { - // Get original video dimensions - const videoInfo = await getVideoInfo(inputPath); - console.log( - `Original video: ${videoInfo.width}x${videoInfo.height}, rotation: ${videoInfo.rotation}°, codec: ${videoInfo.codec_name}` - ); - - // Calculate optimal dimensions based on actual video properties - const dimensions = calculateOptimalDimensions(videoInfo); - - return await performTranscode(inputPath, outputPath, dimensions, videoInfo); - } catch (error) { - console.error(`Error in transcodeVideo: ${error.message}`); - // Last resort fallback with conservative settings - try { - console.log("Using conservative fallback settings"); - return await performTranscode(inputPath, outputPath, null, null); - } catch (finalError) { - console.error(`Final transcoding attempt failed: ${finalError.message}`); - throw finalError; - } - } -}; - -/** - * Transcode video using software encoding - * @param {string} inputPath - Path to the input video file - * @param {string} outputPath - Path for the transcoded output file - * @param {Object} dimensions - Calculated video dimensions or null for fallback - * @param {Object} videoInfo - Original video metadata or null for fallback - * @returns {Promise} - Whether transcoding was successful - */ -const performTranscode = async (inputPath, outputPath, dimensions, videoInfo) => { - return new Promise((resolve, reject) => { - console.log(`Starting software transcoding: ${inputPath} -> ${outputPath}`); - - // Use conservative defaults if dimensions aren't available - const crf = dimensions ? calculateOptimalCRF(dimensions.totalPixels) : 26; - console.log(`Using CRF value: ${crf} for quality/size optimization`); - - const command = ffmpeg(inputPath) - .videoCodec("libx264") - .audioCodec("aac") - .outputOptions([ - `-crf ${crf}`, - "-preset fast", - "-profile:v main", - "-level 3.1", - "-movflags +faststart", - "-b:a 128k", - "-map_metadata -1", - ]); - - // Apply scaling only if we have dimensions and scaling is needed - if (dimensions && dimensions.needsScaling) { - command.size(`${dimensions.width}x${dimensions.height}`); - } else if (!dimensions) { - // Fallback scaling that ensures video isn't too large - command.outputOptions([ - "-vf scale='min(1280,iw):min(720,ih):force_original_aspect_ratio=decrease'", - ]); - } - - command - // .on('progress', (progress) => { - // if (progress.percent) { - // console.log(`Software transcoding progress: ${Math.round(progress.percent)}%`); - // } - // }) - .on("end", () => { - console.log(`Software transcoding completed: ${outputPath}`); - resolve(true); - }) - .on("error", (err) => { - console.error(`Software transcoding failed: ${err.message}`); - reject(err); - }) - .save(outputPath); - }); -}; diff --git a/src/index.js b/src/index.js index dd9e290f..b50936d9 100644 --- a/src/index.js +++ b/src/index.js @@ -75,7 +75,7 @@ let darwinProcessRunning = false; // Notify hoster on console if the app is ready & about the Darwin process's status client.once(Events.ClientReady, (readyClient) => { console.log(`Bot is ready! Logged in as ${readyClient.user.tag}`); - console.log("Initializing Darwin video processing system..."); + // console.log("Initializing Darwin video processing system..."); setInterval(() => { if (darwinProcessRunning) { diff --git a/src/sql/darwinCache/migrate-to-new-schema.sql b/src/sql/darwinCache/migrate-to-new-schema.sql new file mode 100644 index 00000000..22c9bb2a --- /dev/null +++ b/src/sql/darwinCache/migrate-to-new-schema.sql @@ -0,0 +1,14 @@ +ALTER TABLE `darwinCache` +CHANGE COLUMN `videoUrl` `directVideoUrl` VARCHAR(255) NOT NULL; + +ALTER TABLE `darwinCache` +ADD COLUMN `forumPostUrl` VARCHAR(255) NOT NULL DEFAULT '' AFTER `directVideoUrl`, +ADD COLUMN `videoTitle` VARCHAR(180) NOT NULL DEFAULT '' AFTER `videoId`; + +ALTER TABLE `darwinCache` +DROP INDEX `videoUrl`; + +ALTER TABLE `darwinCache` +ADD UNIQUE KEY `directVideoUrl` (`directVideoUrl`); + +DESCRIBE `darwinCache`; diff --git a/src/sql/darwinCache/table.sql b/src/sql/darwinCache/table.sql index 1f1a396d..25e4bf05 100644 --- a/src/sql/darwinCache/table.sql +++ b/src/sql/darwinCache/table.sql @@ -2,9 +2,11 @@ CREATE TABLE IF NOT EXISTS `darwinCache` ( `id` INT AUTO_INCREMENT PRIMARY KEY, - `videoUrl` VARCHAR(255) NOT NULL, + `directVideoUrl` VARCHAR(255) NOT NULL, + `forumPostUrl` VARCHAR(255) NOT NULL, `videoId` VARCHAR(100) NOT NULL, + `videoTitle` VARCHAR(180) NOT NULL, `processedAt` TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE KEY (`videoUrl`), + UNIQUE KEY (`directVideoUrl`), UNIQUE KEY (`videoId`) -); \ No newline at end of file +);