fix: require structured challenge test results

backend/controllers/challenge.controller.js

@@ -3,6 +3,7 @@ import Problem from "../models/Problem.js"
 import User from "../models/User.js"
 
 const PISTON_URL = "https://emkc.org/api/v2/piston/execute"
+const JUDGE_RESULT_PREFIX = "__NPMCHAT_JUDGE_RESULT__"
 
 const LANGUAGE_VERSIONS = {
   javascript: "18.15.0",
@@ -12,6 +13,175 @@ const LANGUAGE_VERSIONS = {
   go: "1.16.2",
 }
 
+const splitTopLevelAssignments = (input) => {
+  const assignments = []
+  let current = ""
+  let depth = 0
+  let quote = null
+
+  for (const char of input) {
+    if (quote) {
+      current += char
+      if (char === quote) quote = null
+      continue
+    }
+
+    if (char === '"' || char === "'") {
+      quote = char
+      current += char
+      continue
+    }
+
+    if (char === "[" || char === "{" || char === "(") depth += 1
+    if (char === "]" || char === "}" || char === ")") depth -= 1
+
+    if (char === "," && depth === 0) {
+      assignments.push(current.trim())
+      current = ""
+    } else {
+      current += char
+    }
+  }
+
+  if (current.trim()) assignments.push(current.trim())
+  return assignments
+}
+
+const parseExpectedOutput = (expectedOutput) => {
+  try {
+    return JSON.parse(expectedOutput)
+  } catch {
+    return expectedOutput
+  }
+}
+
+const buildJavascriptAssignments = (input) =>
+  splitTopLevelAssignments(input)
+    .map((assignment) => {
+      const separatorIndex = assignment.indexOf("=")
+      if (separatorIndex === -1) return ""
+
+      const name = assignment.slice(0, separatorIndex).trim()
+      const value = assignment.slice(separatorIndex + 1).trim()
+
+      if (!/^[A-Za-z_$][\w$]*$/.test(name)) return ""
+      return `const ${name} = ${value};`
+    })
+    .filter(Boolean)
+    .join("\n")
+
+const detectJavascriptFunctionName = (code) => {
+  const declarationMatch = code.match(/function\s+([A-Za-z_$][\w$]*)\s*\(/)
+  if (declarationMatch) return declarationMatch[1]
+
+  const assignmentMatch = code.match(
+    /(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*(?:async\s*)?\(?/,
+  )
+  return assignmentMatch?.[1] || null
+}
+
+const buildJavascriptTestHarness = (code, testCases = []) => {
+  if (testCases.length === 0) return code
+
+  const functionName = detectJavascriptFunctionName(code)
+  if (!functionName) return code
+
+  const caseRunners = testCases
+    .map((testCase, index) => {
+      const assignments = buildJavascriptAssignments(testCase.input)
+      const args = splitTopLevelAssignments(testCase.input)
+        .map((assignment) => assignment.split("=")[0]?.trim())
+        .filter((name) => /^[A-Za-z_$][\w$]*$/.test(name))
+        .join(", ")
+      const expected = JSON.stringify(parseExpectedOutput(testCase.expectedOutput))
+
+      return `
+(() => {
+  try {
+${assignments
+  .split("\n")
+  .filter(Boolean)
+  .map((line) => `    ${line}`)
+  .join("\n")}
+    const actual = ${functionName}(${args});
+    const expected = ${expected};
+    const passed = JSON.stringify(actual) === JSON.stringify(expected);
+    console.log("${JUDGE_RESULT_PREFIX}" + JSON.stringify({
+      index: ${index},
+      input: ${JSON.stringify(testCase.input)},
+      expected,
+      actual,
+      passed
+    }));
+  } catch (error) {
+    console.log("${JUDGE_RESULT_PREFIX}" + JSON.stringify({
+      index: ${index},
+      input: ${JSON.stringify(testCase.input)},
+      expected: ${expected},
+      passed: false,
+      error: error.message
+    }));
+  }
+})();`
+    })
+    .join("\n")
+
+  return `${code}\n\n// NPMChat structured challenge judge\n${caseRunners}`
+}
+
+const buildExecutableCode = (code, language, problem) => {
+  if (language === "javascript") {
+    return buildJavascriptTestHarness(code, problem.testCases || [])
+  }
+
+  return code
+}
+
+const parseStructuredJudgeResults = (stdout) =>
+  stdout
+    .split("\n")
+    .filter((line) => line.startsWith(JUDGE_RESULT_PREFIX))
+    .map((line) => {
+      try {
+        return JSON.parse(line.slice(JUDGE_RESULT_PREFIX.length))
+      } catch {
+        return { passed: false, error: "Malformed judge result" }
+      }
+    })
+
+const evaluateSubmissionResult = (result, testCases = []) => {
+  const stdout = result.run?.stdout || ""
+  const stderr = result.run?.stderr || ""
+  const exitCode = Number(result.run?.code ?? 0)
+  const testResults = parseStructuredJudgeResults(stdout)
+
+  if (testCases.length > 0) {
+    const hasAllResults = testResults.length === testCases.length
+    const allPassed = testResults.every((testResult) => testResult.passed)
+    const failedTestCases = testResults
+      .filter((testResult) => !testResult.passed)
+      .map((testResult) => ({
+        index: testResult.index,
+        input: testResult.input,
+        expected: testResult.expected,
+        actual: testResult.actual,
+        error: testResult.error,
+      }))
+
+    return {
+      isCorrect: exitCode === 0 && !stderr && hasAllResults && allPassed,
+      testResults,
+      failedTestCases,
+    }
+  }
+
+  return {
+    isCorrect: exitCode === 0 && !stderr,
+    testResults: [],
+    failedTestCases: [],
+  }
+}
+
 export const createChallenge = async (req, res) => {
   try {
     const { title, problemId, timeLimit, allowedLanguages, isPublic } = req.body
@@ -87,9 +257,7 @@ export const submitSolution = async (req, res) => {
 
     const problem = challenge.problemId
 
-    // Simple verification (assuming problem has a check/testcases setup)
-    // In a real scenario, we inject the test cases into the code
-    const testCode = `${code}\n\n// Run Test Cases\nconsole.log("Passed");` // Placeholder
+    const testCode = buildExecutableCode(code, language, problem)
 
     const response = await fetch(PISTON_URL, {
       method: "POST",
@@ -104,9 +272,9 @@ export const submitSolution = async (req, res) => {
     const result = await response.json()
     const output = result.run?.stdout || ""
     const errorOutput = result.run?.stderr || ""
-
-    const isCorrect = !errorOutput && output.includes("Passed") // Simplified evaluation
-    const executionTimeMs = parseFloat(result.run?.code || 0) // Placeholder for execution time metric
+    const { isCorrect, testResults, failedTestCases } =
+      evaluateSubmissionResult(result, problem.testCases || [])
+    const executionTimeMs = Number.parseFloat(result.run?.time || 0)
 
     const submission = {
       userId,
@@ -116,6 +284,8 @@ export const submitSolution = async (req, res) => {
       isCorrect,
       executionTimeMs,
       timeComplexity: "O(n)", // AI placeholder
+      testResults,
+      failedTestCases,
       submittedAt: Date.now(),
     }
 
@@ -169,7 +339,14 @@ export const submitSolution = async (req, res) => {
       }
     }
 
-    res.status(200).json({ submission, isCorrect, output, errorOutput })
+    res.status(200).json({
+      submission,
+      isCorrect,
+      output,
+      errorOutput,
+      testResults,
+      failedTestCases,
+    })
   } catch (error) {
     res.status(500).json({ error: error.message })
   }

backend/models/ChallengeRoom.js

@@ -12,6 +12,25 @@ const submissionSchema = new mongoose.Schema({
   isCorrect: Boolean,
   executionTimeMs: Number,
   timeComplexity: String,
+  testResults: [
+    {
+      index: Number,
+      input: String,
+      expected: mongoose.Schema.Types.Mixed,
+      actual: mongoose.Schema.Types.Mixed,
+      passed: Boolean,
+      error: String,
+    },
+  ],
+  failedTestCases: [
+    {
+      index: Number,
+      input: String,
+      expected: mongoose.Schema.Types.Mixed,
+      actual: mongoose.Schema.Types.Mixed,
+      error: String,
+    },
+  ],
   submittedAt: { type: Date, default: Date.now },
 })
 

backend/tests/challenge.test.js

@@ -10,6 +10,8 @@ import jwt from "jsonwebtoken"
 // Mock global fetch for Piston API calls
 global.fetch = vi.fn()
 
+const JUDGE_RESULT_PREFIX = "__NPMCHAT_JUDGE_RESULT__"
+
 describe("ChallengeRoom API Endpoints", () => {
   let user, problem, challenge, token
 
@@ -35,6 +37,12 @@ describe("ChallengeRoom API Endpoints", () => {
       difficulty: "Easy",
       category: "Array",
       solutionTemplate: "function twoSum() {}",
+      testCases: [
+        {
+          input: "nums = [2,7,11,15], target = 9",
+          expectedOutput: "[0,1]",
+        },
+      ],
     })
 
     challenge = await ChallengeRoom.create({
@@ -84,9 +92,18 @@ describe("ChallengeRoom API Endpoints", () => {
     fetch.mockResolvedValueOnce({
       json: async () => ({
         run: {
-          stdout: "Passed",
+          stdout:
+            `${JUDGE_RESULT_PREFIX}` +
+            JSON.stringify({
+              index: 0,
+              input: "nums = [2,7,11,15], target = 9",
+              expected: [0, 1],
+              actual: [0, 1],
+              passed: true,
+            }),
           stderr: "",
-          code: 15,
+          code: 0,
+          time: "15",
         },
       }),
     })
@@ -102,6 +119,8 @@ describe("ChallengeRoom API Endpoints", () => {
     expect(res.status).toBe(200)
     expect(res.body.isCorrect).toBe(true)
     expect(res.body.submission.language).toBe("javascript")
+    expect(res.body.testResults).toHaveLength(1)
+    expect(res.body.failedTestCases).toHaveLength(0)
 
     // Verify badge assignment in DB
     const updatedUser = await User.findById(user._id)
@@ -172,4 +191,33 @@ describe("ChallengeRoom API Endpoints", () => {
     expect(res.body.isCorrect).toBe(false)
     expect(res.body.errorOutput).toContain("SyntaxError")
   })
+
+  it("POST /api/v1/challenges/:id/submit - should not trust Passed text in stdout", async () => {
+    fetch.mockResolvedValueOnce({
+      json: async () => ({
+        run: {
+          stdout: "Passed all tests according to user code",
+          stderr: "",
+          code: 1,
+          time: "7",
+        },
+      }),
+    })
+
+    const res = await request(app)
+      .post(`/api/v1/challenges/${challenge._id}/submit`)
+      .set("Authorization", `Bearer ${token}`)
+      .send({
+        code: "function twoSum() { console.log('Passed'); return [] }",
+        language: "javascript",
+      })
+
+    expect(res.status).toBe(200)
+    expect(res.body.isCorrect).toBe(false)
+    expect(res.body.testResults).toEqual([])
+    expect(res.body.submission.isCorrect).toBe(false)
+
+    const updatedChallenge = await ChallengeRoom.findById(challenge._id)
+    expect(updatedChallenge.submissions[0].isCorrect).toBe(false)
+  })
 })