From 8b4242f004738d15df6810be37ff18865181aa44 Mon Sep 17 00:00:00 2001 From: Saurabh Kumar Bajpai <157192462+saurabhhhcodes@users.noreply.github.com> Date: Mon, 1 Jun 2026 00:45:40 +0530 Subject: [PATCH] fix: require structured challenge test results --- backend/controllers/challenge.controller.js | 191 +++++++++++++++++++- backend/models/ChallengeRoom.js | 19 ++ backend/tests/challenge.test.js | 52 +++++- 3 files changed, 253 insertions(+), 9 deletions(-) diff --git a/backend/controllers/challenge.controller.js b/backend/controllers/challenge.controller.js index c84fc19..287ae43 100644 --- a/backend/controllers/challenge.controller.js +++ b/backend/controllers/challenge.controller.js @@ -3,6 +3,7 @@ import Problem from "../models/Problem.js" import User from "../models/User.js" const PISTON_URL = "https://emkc.org/api/v2/piston/execute" +const JUDGE_RESULT_PREFIX = "__NPMCHAT_JUDGE_RESULT__" const LANGUAGE_VERSIONS = { javascript: "18.15.0", @@ -12,6 +13,175 @@ const LANGUAGE_VERSIONS = { go: "1.16.2", } +const splitTopLevelAssignments = (input) => { + const assignments = [] + let current = "" + let depth = 0 + let quote = null + + for (const char of input) { + if (quote) { + current += char + if (char === quote) quote = null + continue + } + + if (char === '"' || char === "'") { + quote = char + current += char + continue + } + + if (char === "[" || char === "{" || char === "(") depth += 1 + if (char === "]" || char === "}" || char === ")") depth -= 1 + + if (char === "," && depth === 0) { + assignments.push(current.trim()) + current = "" + } else { + current += char + } + } + + if (current.trim()) assignments.push(current.trim()) + return assignments +} + +const parseExpectedOutput = (expectedOutput) => { + try { + return JSON.parse(expectedOutput) + } catch { + return expectedOutput + } +} + +const buildJavascriptAssignments = (input) => + splitTopLevelAssignments(input) + .map((assignment) => { + const separatorIndex = assignment.indexOf("=") + if (separatorIndex === -1) return "" + + const name = assignment.slice(0, separatorIndex).trim() + const value = assignment.slice(separatorIndex + 1).trim() + + if (!/^[A-Za-z_$][\w$]*$/.test(name)) return "" + return `const ${name} = ${value};` + }) + .filter(Boolean) + .join("\n") + +const detectJavascriptFunctionName = (code) => { + const declarationMatch = code.match(/function\s+([A-Za-z_$][\w$]*)\s*\(/) + if (declarationMatch) return declarationMatch[1] + + const assignmentMatch = code.match( + /(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*(?:async\s*)?\(?/, + ) + return assignmentMatch?.[1] || null +} + +const buildJavascriptTestHarness = (code, testCases = []) => { + if (testCases.length === 0) return code + + const functionName = detectJavascriptFunctionName(code) + if (!functionName) return code + + const caseRunners = testCases + .map((testCase, index) => { + const assignments = buildJavascriptAssignments(testCase.input) + const args = splitTopLevelAssignments(testCase.input) + .map((assignment) => assignment.split("=")[0]?.trim()) + .filter((name) => /^[A-Za-z_$][\w$]*$/.test(name)) + .join(", ") + const expected = JSON.stringify(parseExpectedOutput(testCase.expectedOutput)) + + return ` +(() => { + try { +${assignments + .split("\n") + .filter(Boolean) + .map((line) => ` ${line}`) + .join("\n")} + const actual = ${functionName}(${args}); + const expected = ${expected}; + const passed = JSON.stringify(actual) === JSON.stringify(expected); + console.log("${JUDGE_RESULT_PREFIX}" + JSON.stringify({ + index: ${index}, + input: ${JSON.stringify(testCase.input)}, + expected, + actual, + passed + })); + } catch (error) { + console.log("${JUDGE_RESULT_PREFIX}" + JSON.stringify({ + index: ${index}, + input: ${JSON.stringify(testCase.input)}, + expected: ${expected}, + passed: false, + error: error.message + })); + } +})();` + }) + .join("\n") + + return `${code}\n\n// NPMChat structured challenge judge\n${caseRunners}` +} + +const buildExecutableCode = (code, language, problem) => { + if (language === "javascript") { + return buildJavascriptTestHarness(code, problem.testCases || []) + } + + return code +} + +const parseStructuredJudgeResults = (stdout) => + stdout + .split("\n") + .filter((line) => line.startsWith(JUDGE_RESULT_PREFIX)) + .map((line) => { + try { + return JSON.parse(line.slice(JUDGE_RESULT_PREFIX.length)) + } catch { + return { passed: false, error: "Malformed judge result" } + } + }) + +const evaluateSubmissionResult = (result, testCases = []) => { + const stdout = result.run?.stdout || "" + const stderr = result.run?.stderr || "" + const exitCode = Number(result.run?.code ?? 0) + const testResults = parseStructuredJudgeResults(stdout) + + if (testCases.length > 0) { + const hasAllResults = testResults.length === testCases.length + const allPassed = testResults.every((testResult) => testResult.passed) + const failedTestCases = testResults + .filter((testResult) => !testResult.passed) + .map((testResult) => ({ + index: testResult.index, + input: testResult.input, + expected: testResult.expected, + actual: testResult.actual, + error: testResult.error, + })) + + return { + isCorrect: exitCode === 0 && !stderr && hasAllResults && allPassed, + testResults, + failedTestCases, + } + } + + return { + isCorrect: exitCode === 0 && !stderr, + testResults: [], + failedTestCases: [], + } +} + export const createChallenge = async (req, res) => { try { const { title, problemId, timeLimit, allowedLanguages, isPublic } = req.body @@ -87,9 +257,7 @@ export const submitSolution = async (req, res) => { const problem = challenge.problemId - // Simple verification (assuming problem has a check/testcases setup) - // In a real scenario, we inject the test cases into the code - const testCode = `${code}\n\n// Run Test Cases\nconsole.log("Passed");` // Placeholder + const testCode = buildExecutableCode(code, language, problem) const response = await fetch(PISTON_URL, { method: "POST", @@ -104,9 +272,9 @@ export const submitSolution = async (req, res) => { const result = await response.json() const output = result.run?.stdout || "" const errorOutput = result.run?.stderr || "" - - const isCorrect = !errorOutput && output.includes("Passed") // Simplified evaluation - const executionTimeMs = parseFloat(result.run?.code || 0) // Placeholder for execution time metric + const { isCorrect, testResults, failedTestCases } = + evaluateSubmissionResult(result, problem.testCases || []) + const executionTimeMs = Number.parseFloat(result.run?.time || 0) const submission = { userId, @@ -116,6 +284,8 @@ export const submitSolution = async (req, res) => { isCorrect, executionTimeMs, timeComplexity: "O(n)", // AI placeholder + testResults, + failedTestCases, submittedAt: Date.now(), } @@ -169,7 +339,14 @@ export const submitSolution = async (req, res) => { } } - res.status(200).json({ submission, isCorrect, output, errorOutput }) + res.status(200).json({ + submission, + isCorrect, + output, + errorOutput, + testResults, + failedTestCases, + }) } catch (error) { res.status(500).json({ error: error.message }) } diff --git a/backend/models/ChallengeRoom.js b/backend/models/ChallengeRoom.js index ce4a0e1..eb33ca1 100644 --- a/backend/models/ChallengeRoom.js +++ b/backend/models/ChallengeRoom.js @@ -12,6 +12,25 @@ const submissionSchema = new mongoose.Schema({ isCorrect: Boolean, executionTimeMs: Number, timeComplexity: String, + testResults: [ + { + index: Number, + input: String, + expected: mongoose.Schema.Types.Mixed, + actual: mongoose.Schema.Types.Mixed, + passed: Boolean, + error: String, + }, + ], + failedTestCases: [ + { + index: Number, + input: String, + expected: mongoose.Schema.Types.Mixed, + actual: mongoose.Schema.Types.Mixed, + error: String, + }, + ], submittedAt: { type: Date, default: Date.now }, }) diff --git a/backend/tests/challenge.test.js b/backend/tests/challenge.test.js index c5d295f..b704837 100644 --- a/backend/tests/challenge.test.js +++ b/backend/tests/challenge.test.js @@ -10,6 +10,8 @@ import jwt from "jsonwebtoken" // Mock global fetch for Piston API calls global.fetch = vi.fn() +const JUDGE_RESULT_PREFIX = "__NPMCHAT_JUDGE_RESULT__" + describe("ChallengeRoom API Endpoints", () => { let user, problem, challenge, token @@ -35,6 +37,12 @@ describe("ChallengeRoom API Endpoints", () => { difficulty: "Easy", category: "Array", solutionTemplate: "function twoSum() {}", + testCases: [ + { + input: "nums = [2,7,11,15], target = 9", + expectedOutput: "[0,1]", + }, + ], }) challenge = await ChallengeRoom.create({ @@ -84,9 +92,18 @@ describe("ChallengeRoom API Endpoints", () => { fetch.mockResolvedValueOnce({ json: async () => ({ run: { - stdout: "Passed", + stdout: + `${JUDGE_RESULT_PREFIX}` + + JSON.stringify({ + index: 0, + input: "nums = [2,7,11,15], target = 9", + expected: [0, 1], + actual: [0, 1], + passed: true, + }), stderr: "", - code: 15, + code: 0, + time: "15", }, }), }) @@ -102,6 +119,8 @@ describe("ChallengeRoom API Endpoints", () => { expect(res.status).toBe(200) expect(res.body.isCorrect).toBe(true) expect(res.body.submission.language).toBe("javascript") + expect(res.body.testResults).toHaveLength(1) + expect(res.body.failedTestCases).toHaveLength(0) // Verify badge assignment in DB const updatedUser = await User.findById(user._id) @@ -172,4 +191,33 @@ describe("ChallengeRoom API Endpoints", () => { expect(res.body.isCorrect).toBe(false) expect(res.body.errorOutput).toContain("SyntaxError") }) + + it("POST /api/v1/challenges/:id/submit - should not trust Passed text in stdout", async () => { + fetch.mockResolvedValueOnce({ + json: async () => ({ + run: { + stdout: "Passed all tests according to user code", + stderr: "", + code: 1, + time: "7", + }, + }), + }) + + const res = await request(app) + .post(`/api/v1/challenges/${challenge._id}/submit`) + .set("Authorization", `Bearer ${token}`) + .send({ + code: "function twoSum() { console.log('Passed'); return [] }", + language: "javascript", + }) + + expect(res.status).toBe(200) + expect(res.body.isCorrect).toBe(false) + expect(res.body.testResults).toEqual([]) + expect(res.body.submission.isCorrect).toBe(false) + + const updatedChallenge = await ChallengeRoom.findById(challenge._id) + expect(updatedChallenge.submissions[0].isCorrect).toBe(false) + }) })