diff --git a/app/abTests.js b/app/abTests.js new file mode 100644 index 0000000..3b774ad --- /dev/null +++ b/app/abTests.js @@ -0,0 +1,46 @@ +const $ = require('jquery'); +const lookup = require('./lookup'); +const abConfig = require('../shared/ab/config'); + +// don't cache always fetch +const getRunningTests = (randomisationTypeIds, callback) => + $.getJSON(`/ab/running?randomisationTypeId=${randomisationTypeIds.join('&randomisationTypeId=')}`, tests => callback(tests)); + +module.exports = { + + clearPageGroups: () => { + if (lookup.pageGroups) { + lookup.pageGroups.forEach((test) => { + delete lookup.tests[test]; + }); + } + }, + + process: (randomisationTypeIds, callback) => { + if (!lookup.tests) lookup.tests = {}; + if (!lookup.pageGroups) lookup.pageGroups = []; + getRunningTests(randomisationTypeIds, (tests) => { + if (tests) { + tests.forEach((test) => { + // assign the user to a group (might be already defined) + test.group = abConfig.assign(test); + // add to the state object + lookup.tests[test.name] = test.group; + + // keep track of perPage and perPatientView + if (test.randomisationTypeId === abConfig.randomisationTypes.perPage.id || + test.randomisationTypeId === abConfig.randomisationTypes.perPatientViewed.id) { + lookup.pageGroups.push(test.name); + } + + // if the user is in the feature group execute + abConfig.init($, test); + console.log(`User ${lookup.userName} assigned to the '${test.group}' group for test '${test.name}'`); + }); + } + if (callback) callback(); + }); + }, + +}; + diff --git a/app/log.js b/app/log.js index 0007441..c4506c9 100644 --- a/app/log.js +++ b/app/log.js @@ -20,6 +20,7 @@ const log = { event(type, url, dataProp, xpath) { const dataToSend = { event: { type, url, pageId, data: dataProp } }; + if (lookup.tests && Object.keys(lookup.tests).length > 0) dataToSend.event.tests = lookup.tests; if (xpath && xpath.length > 0) dataToSend.event.xpath = xpath; $.ajax({ type: 'POST', @@ -95,10 +96,12 @@ const log = { // rwhere recordIndividualPlan(text, practiceId, patientId, indicatorList, done) { + const dataToSend = { actionText: text, indicatorList, pageId }; + if (lookup.tests && Object.keys(lookup.tests).length > 0) dataToSend.tests = lookup.tests; $.ajax({ type: 'POST', url: `/api/action/addIndividual/${practiceId}/${patientId}`, - data: JSON.stringify({ actionText: text, indicatorList, pageId }), + data: JSON.stringify(dataToSend), success(action) { notify.showSaved(); data.addOrUpdatePatientAction(patientId, action, () => done(null, action)); @@ -124,10 +127,12 @@ const log = { // rwhere updateIndividualAction(practiceId, patientId, updatedAction, callback) { + const dataToSend = { action: updatedAction, url: window.location.href, pageId }; + if (lookup.tests && Object.keys(lookup.tests).length > 0) dataToSend.tests = lookup.tests; $.ajax({ type: 'POST', url: `/api/action/update/individual/${practiceId}/${patientId}`, - data: JSON.stringify({ action: updatedAction, url: window.location.href, pageId }), + data: JSON.stringify(dataToSend), success(action) { notify.showSaved(); if (action.agree === true) { @@ -143,10 +148,12 @@ const log = { }, updateTeamAction(practiceId, indicatorId, dataProp, done) { + const dataToSend = { action: dataProp, url: window.location.href, pageId }; + if (lookup.tests && Object.keys(lookup.tests).length > 0) dataToSend.tests = lookup.tests; $.ajax({ type: 'POST', url: `/api/action/update/team/${practiceId}/${indicatorId}`, - data: JSON.stringify({ action: dataProp, url: window.location.href, pageId }), + data: JSON.stringify(dataToSend), success(d) { if (!done) return notify.showSaved(); return done(null, d); @@ -157,10 +164,12 @@ const log = { }, updateUserDefinedPatientAction(patientId, actionTextId, dataProp, callback) { + const dataToSend = { action: dataProp }; + if (lookup.tests && Object.keys(lookup.tests).length > 0) dataToSend.tests = lookup.tests; $.ajax({ type: 'POST', url: `/api/action/update/userdefinedpatient/${patientId}/${actionTextId}`, - data: JSON.stringify({ action: dataProp }), + data: JSON.stringify(dataToSend), success(action) { notify.showSaved(); action.oldActionTextId = actionTextId; @@ -185,10 +194,12 @@ const log = { }, updateUserDefinedTeamAction(actionTextId, dataProp, done) { + const dataToSend = { action: dataProp }; + if (lookup.tests && Object.keys(lookup.tests).length > 0) dataToSend.tests = lookup.tests; $.ajax({ type: 'POST', url: `/api/action/update/userdefinedteam/${actionTextId}`, - data: JSON.stringify({ action: dataProp }), + data: JSON.stringify(dataToSend), success(d) { if (!done) return notify.showSaved(); return done(null, d); @@ -200,10 +211,12 @@ const log = { recordTeamPlan(practiceId, text, indicatorId, done) { const url = `/api/action/addTeam/${practiceId}/${indicatorId || ''}`; + const dataToSend = { actionText: text, pageId }; + if (lookup.tests && Object.keys(lookup.tests).length > 0) dataToSend.tests = lookup.tests; $.ajax({ type: 'POST', url, - data: JSON.stringify({ actionText: text, pageId }), + data: JSON.stringify(dataToSend), success(d) { notify.showSaved(); return done(null, d); diff --git a/app/script.js b/app/script.js index 6950fa8..8c89a05 100644 --- a/app/script.js +++ b/app/script.js @@ -8,6 +8,8 @@ require('datatables.net-buttons-bs')(window, $); require('datatables.net-buttons/js/buttons.colVis.js')(window, $); require('datatables.net-buttons/js/buttons.html5.js')(window, $); +const { randomisationTypes } = require('../shared/ab/config'); + /* * For each disease area there will be 4 stages: Diagnosis, Monitoring, Treatment and Exclusions. * Each stage gets a single panel on the front screen. @@ -21,6 +23,7 @@ const events = require('./events'); const state = require('./state'); const layout = require('./layout'); const tutorial = require('./tutorial'); +const abTests = require('./abTests'); // TODO not sure why i did this - was in local variable // maybe a separate module @@ -110,6 +113,8 @@ const App = { getServerParameters(); + abTests.process([randomisationTypes.perUser.id, randomisationTypes.perSession.id]); + main.getInitialData(() => { gotInitialData = true; if (gotInitialData && pageIsReady) { diff --git a/app/template.js b/app/template.js index 8f01be8..3e407a1 100644 --- a/app/template.js +++ b/app/template.js @@ -1,16 +1,21 @@ const log = require('./log'); const lookup = require('./lookup'); const base = require('./base'); +const abTests = require('./abTests'); const layout = require('./layout'); const overview = require('./views/overview'); const indicatorView = require('./views/indicator'); const patientView = require('./views/patient'); const actionPlan = require('./views/actions'); +const { randomisationTypes } = require('../shared/ab/config'); const template = { loadContent(hash, isPoppingState, callback) { base.hideTooltips(); + // Must do this before we log the navigate page event + abTests.clearPageGroups(); + log.navigatePage(hash, []); let patientId; @@ -27,6 +32,7 @@ const template = { $('html').removeClass('scroll-bar'); shouldCallCallback = true; } else { + abTests.process([randomisationTypes.perPage.id]); base.hideFooter(); // $('html').addClass('scroll-bar'); const params = {}; @@ -66,6 +72,7 @@ const template = { shouldCallCallback = true; } else if (urlBits[0] === '#patient') { // create(pathwayId, pathwayStage, standard, patientId) + abTests.process([randomisationTypes.perPatientViewed]); patientView.create( urlBits[2], urlBits[3], diff --git a/brunch-server.js b/brunch-server.js index 23e1500..a5b75c8 100644 --- a/brunch-server.js +++ b/brunch-server.js @@ -20,7 +20,7 @@ const routeIndex = require('./server/routes/index'); const debug = require('debug')('passport-mongo:server'); const http = require('http'); -const DEBUG = false; +const DEBUG = true; module.exports = (PORT, PATH, CALLBACK) => { mongoose.set('debug', DEBUG); diff --git a/docs/AB_TESTING.md b/docs/AB_TESTING.md new file mode 100644 index 0000000..13ee53b --- /dev/null +++ b/docs/AB_TESTING.md @@ -0,0 +1,7 @@ +Create tests with UI + +Each test then gets an entry in shared/ab/tests.js. Can be autogenerated with `node shared/ab/generate.js` + +Names of tests must be unique + +Once code for init and pull down are set and ready to go set the `readyToDeploy` flag to true diff --git a/server/controllers/ab/outcomes.js b/server/controllers/ab/outcomes.js new file mode 100644 index 0000000..a28a946 --- /dev/null +++ b/server/controllers/ab/outcomes.js @@ -0,0 +1,199 @@ +const Event = require('../../models/event'); +const { outcomes, trials } = require('../../../shared/ab/config'); + +const getNDaysAgo = (n) => { + const today = new Date(); + const nDaysAgo = new Date(); + nDaysAgo.setDate(today.getDate() - n); + return nDaysAgo; +}; + +const getMatchForPageViewThumbClicks = (daysOrDate) => { + const $match = { $match: { pageId: { $exists: true } } }; + if (daysOrDate) { + const nDaysAgo = daysOrDate.getMinutes ? daysOrDate : getNDaysAgo(daysOrDate); + $match.$match.date = { $gt: nDaysAgo }; + } + return $match; +}; + +const getMatchForPatientPageViewThumbClicks = (daysOrDate, eventTypes) => { + const $match = { $match: { pageId: { $exists: true }, $or: [{ url: /patients?\/\d/ }, { type: { $in: eventTypes } }] } }; + if (daysOrDate) { + const nDaysAgo = daysOrDate.getMinutes ? daysOrDate : getNDaysAgo(daysOrDate); + $match.$match.date = { $gt: nDaysAgo }; + } + return $match; +}; + +const getProjectForPageViewThumbClicks = eventTypes => ({ $project: { pageId: 1, hasEvent: { $cond: { if: { $in: ['$type', eventTypes] }, then: 1, else: 0 } } } }); + +const getProjectForPatientPageViewThumbClicks = eventTypes => ({ $project: { pageId: 1, url: 1, hasEvent: { $cond: { if: { $in: ['$type', eventTypes] }, then: 1, else: 0 } } } }); + +const getQueryForPageViewThumbClicks = (days, eventTypes) => [ + // Only things with a page id within the last n days + getMatchForPageViewThumbClicks(days), + + // Determine if there is a thumb click event + getProjectForPageViewThumbClicks(eventTypes), + + // Determine if there is a thumb click event for each pageId + { $group: { _id: '$pageId', clicks: { $max: '$hasEvent' } } }, + + // Average to get the proportion of page views with a thumb click + { $group: { _id: null, value: { $avg: '$clicks' } } }, + + // Convert proportion into a percentage + { $project: { result: { $multiply: [100, '$value'] } } }, +]; + +const getQueryForPatientPageViewThumbClicks = (days, eventTypes) => [ + // Only things with a page id within the last n days that are either one of the event types + // or a patient page url (agree/disagree don't have a url so aren't matched otherwise) + getMatchForPatientPageViewThumbClicks(days, eventTypes), + + // Determine if there is a thumb click event + getProjectForPatientPageViewThumbClicks(eventTypes), + + // Determine if there is a thumb click event for each pageId + { $group: { _id: '$pageId', clicks: { $max: '$hasEvent' }, url: { $max: '$url' } } }, + + // If url is null then it is a thumb click on a non-patient page + { $match: { url: { $ne: null } } }, + + // Average to get the proportion of page views with a thumb click + { $group: { _id: null, value: { $avg: '$clicks' } } }, + + // Convert proportion into a percentage + { $project: { result: { $multiply: [100, '$value'] } } }, +]; + +const getHitQueryForPageViewThumbClicks = (test, eventTypes) => { + const $match = getMatchForPageViewThumbClicks(test.startDate); + $match.$match[`tests.${test.name}`] = { $exists: true }; + const $project = getProjectForPageViewThumbClicks(eventTypes); + $project.$project.groop = `$tests.${test.name}`; + return [ + // Only things with a page id within the last n days + $match, + + // Determine if there is a thumb click event + $project, + + { $group: { _id: '$pageId', groop: { $max: '$groop' }, hits: { $max: '$hasEvent' } } }, + + // Determine if there is a thumb click event for each pageId + { $group: { _id: '$groop', total: { $sum: 1 }, hits: { $sum: '$hits' } } }, + ]; +}; + +const getHitQueryForPatientPageViewThumbClicks = (test, eventTypes) => { + const $match = getMatchForPatientPageViewThumbClicks(test.startDate, eventTypes); + $match.$match[`tests.${test.name}`] = { $exists: true }; + const $project = getProjectForPatientPageViewThumbClicks(eventTypes); + $project.$project.groop = `$tests.${test.name}`; + return [ + // Only things with a page id within the last n days that are either one of the event types + // or a patient page url (agree/disagree don't have a url so aren't matched otherwise) + $match, + + // Determine if there is a thumb click event + $project, + + // Determine if there is a thumb click event for each pageId + { $group: { _id: '$pageId', url: { $max: '$url' }, groop: { $max: '$groop' }, hits: { $max: '$hasEvent' } } }, + + // If url is null then it is a thumb click on a non-patient page + { $match: { url: { $ne: null } } }, + + // Determine if there is a thumb click event for each pageId + { $group: { _id: '$groop', total: { $sum: 1 }, hits: { $sum: '$hits' } } }, + ]; +}; + +const getConversionQuery = (trialId, outcomeId, days) => { + switch (trialId) { + case trials.pageView.id: + switch (outcomeId) { + case outcomes.thumbClicks.id: + return getQueryForPageViewThumbClicks(days, ['agree', 'disagree']); + case outcomes.thumbAgreeClicks.id: + return getQueryForPageViewThumbClicks(days, ['agree']); + case outcomes.thumbDisagreeClicks.id: + return getQueryForPageViewThumbClicks(days, ['disagree']); + default: + return false; + } + case trials.patientView.id: + switch (outcomeId) { + case outcomes.thumbClicks.id: + return getQueryForPatientPageViewThumbClicks(days, ['agree', 'disagree']); + case outcomes.thumbAgreeClicks.id: + return getQueryForPatientPageViewThumbClicks(days, ['agree']); + case outcomes.thumbDisagreeClicks.id: + return getQueryForPatientPageViewThumbClicks(days, ['disagree']); + default: + return false; + } + default: + return false; + } +}; + +const getTrialHitQuery = (test) => { + switch (test.trialId) { + case trials.pageView.id: + switch (test.outcomeId) { + case outcomes.thumbClicks.id: + return getHitQueryForPageViewThumbClicks(test, ['agree', 'disagree']); + case outcomes.thumbAgreeClicks.id: + return getHitQueryForPageViewThumbClicks(test, ['agree']); + case outcomes.thumbDisagreeClicks.id: + return getHitQueryForPageViewThumbClicks(test, ['disagree']); + default: + return false; + } + case trials.patientView.id: + switch (test.outcomeId) { + case outcomes.thumbClicks.id: + return getHitQueryForPatientPageViewThumbClicks(test, ['agree', 'disagree']); + case outcomes.thumbAgreeClicks.id: + return getHitQueryForPatientPageViewThumbClicks(test, ['agree']); + case outcomes.thumbDisagreeClicks.id: + return getHitQueryForPatientPageViewThumbClicks(test, ['disagree']); + default: + return false; + } + default: + return false; + } +}; + +module.exports = { + + conversion: (req, res) => { + const { trialId, outcomeId, days } = req.params; + const query = getConversionQuery(trialId, outcomeId, days); + Event.aggregate(query, (err, output) => { + if (output && output.length > 0) { + res.send({ error: err, value: output[0].result }); + } else { + res.send({ error: err, value: 0 }); + } + }); + }, + + trialHits: (test, callback) => { + const query = getTrialHitQuery(test); + Event.aggregate(query, (err, output) => { + let baseline = { total: 0, hits: 0 }; + let feature = { total: 0, hits: 0 }; + output.forEach((o) => { + if (o._id === 'baseline') baseline = { total: o.total, hits: o.hits }; + else feature = { total: o.total, hits: o.hits }; + }); + return callback(err, { baseline, feature }); + }); + }, + +}; diff --git a/server/controllers/ab/significanceCalculator.js b/server/controllers/ab/significanceCalculator.js new file mode 100644 index 0000000..0b397b7 --- /dev/null +++ b/server/controllers/ab/significanceCalculator.js @@ -0,0 +1,94 @@ +const lg = Math.log; +const ep = Math.exp; +const ab = Math.abs; + +const LogGamma = (Z) => { + const S = 1 + 76.18009173 / Z - 86.50532033 / (Z + 1) + 24.01409822 / (Z + 2) - 1.231739516 / (Z + 3) + 0.00120858003 / (Z + 4) - 0.00000536382 / (Z + 5); + const LG = (Z - 0.5) * lg(Z + 4.5) - (Z + 4.5) + lg(S * 2.50662827465); + return LG; +}; + +const Gcf = (X, A) => { // Good for X>A+=> 1 + let A0 = 0; + let B0 = 1; + let A1 = 1; + let B1 = X; + let AOLD = 0; + let N = 0; + while (ab((A1 - AOLD) / A1) > 0.00001) { + AOLD = A1; + N += 1; + A0 = A1 + (N - A) * A0; + B0 = B1 + (N - A) * B0; + A1 = X * A0 + N * A1; + B1 = X * B0 + N * B1; + A0 /= B1; + B0 /= B1; + A1 /= B1; + B1 = 1; + } + const Prob = ep(A * lg(X) - X - LogGamma(A)) * A1; + + return 1 - Prob; +}; + +const Gser = (X, A) => { // Good for X . + let T9 = 1 / A; + let G = T9; + let I = 1; + while (T9 > G * 0.00001) { + T9 = T9 * X / (A + I); + G += T9; + I += 1; + } + G *= ep(A * lg(X) - X - LogGamma(A)); + + return G; +}; + +const normalcdf = (X) => { // HASTINGS. MAX ERROR = .00000=> 1 + const T = 1 / (1 + 0.2316419 * ab(X)); + const D = 0.3989423 * ep(-X * X / 2); + let Prob = D * T * (0.3193815 + T * (-0.3565638 + T * (1.781478 + T * (-1.821256 + T * 1.330274)))); + if (X > 0) { + Prob = 1 - Prob; + } + return Prob; +}; + +const Gammacdf = (x, a) => { + let GI; + if (x <= 0) { + GI = 0; + } else if (a > 200) { + z = (x - a) / Math.sqrt(a); + y = normalcdf(z); + b1 = 2 / Math.sqrt(a); + phiz = 0.39894228 * ep(-z * z / 2); + w = y - b1 * (z * z - 1) * phiz / 6; // Edgeworth1 + b2 = 6 / a; + u = 3 * b2 * (z * z - 3) + b1 * b1 * (z ^ 4 - 10 * z * z + 15); + GI = w - phiz * z * u / 72; // Edgeworth2 + } else if (x < a + 1) { + GI = Gser(x, a); + } else { + GI = Gcf(x, a); + } + return GI; +}; + +const compute = (X, A, B) => Math.round(Gammacdf(X * B, A) * 100000) / 100000; + +const chi_squared_term = (e, o) => (e - o) * (e - o) / e; + +const chi_squared = (s1, t1, s2, t2) => { + let test_stat = 0.0; + const mean_p = (s1 + s2) / (t1 + t2); + test_stat += chi_squared_term(mean_p * t1, s1); + test_stat += chi_squared_term((1 - mean_p) * t1, t1 - s1); + test_stat += chi_squared_term(mean_p * t2, s2); + test_stat += chi_squared_term((1 - mean_p) * t2, t2 - s2); + return test_stat; +}; + +exports.pValue = (baseSuccesses, baseTrials, featureSuccesses, featureTrials) => 1.0 - compute(chi_squared(baseSuccesses, baseTrials, featureSuccesses, featureTrials), 0.5, 0.5); diff --git a/server/controllers/ab/tests.js b/server/controllers/ab/tests.js new file mode 100644 index 0000000..401be1a --- /dev/null +++ b/server/controllers/ab/tests.js @@ -0,0 +1,194 @@ +const Test = require('../../models/ab/test'); +const { statuses, randomisationTypeArray, trialArray, outcomeArray } = require('../../../shared/ab/config'); +const outcomeCtl = require('./outcomes'); +const significance = require('./significanceCalculator'); + +const isTestFullyConfigured = test => test.name && + test.researchQuestion && + test.randomisationTypeId && + test.trialId && + test.outcomeId; + +module.exports = { + + index: (req, res) => { + const query = req.query.showArchived ? {} : { statusId: { $ne: statuses.archived.id } }; + Test.find(query, (err, tests) => { + res.render('pages/ab/index.jade', { tests, showArchived: req.query.showArchived }); + }); + }, + + running: (req, res) => { + const query = { statusId: statuses.running.id }; + if (req.query.randomisationTypeId) { + if (typeof req.query.randomisationType === 'string') { + query.randomisationTypeId = req.query.randomisationTypeId; + } else { + query.randomisationTypeId = { $in: req.query.randomisationTypeId }; + } + } + Test.find(query, (err, tests) => { + res.send(tests); + }); + }, + + create: { + get: (req, res) => { + res.render('pages/ab/create.jade', { test: {}, outcomeArray, randomisationTypeArray, trialArray }); + }, + post: (req, res) => { + const newTest = new Test(req.body); + if (isTestFullyConfigured(newTest)) newTest.statusId = statuses.configured.id; + else newTest.statusId = statuses.new.id; + newTest.save((err) => { + if (err) res.render('pages/ab/create.jade', { test: newTest, message: { error: err }, outcomeArray, randomisationTypeArray, trialArray }); + else res.redirect('/ab'); + }); + }, + }, + + configure: { + get: (req, res) => { + Test.findById(req.params.testId, (err, test) => { + res.render('pages/ab/configure.jade', { test, outcomeArray, randomisationTypeArray, trialArray }); + }); + }, + post: (req, res) => { + Test.findById(req.params.testId, (err, test) => { + test.name = req.body.name; + test.description = req.body.description; + test.researchQuestion = req.body.researchQuestion; + test.days = req.body.days; + test.randomisationTypeId = req.body.randomisationTypeId; + test.outcomeId = req.body.outcomeId; + test.outcomeBaselineValue = req.body.outcomeBaselineValue; + test.conversionMinDetectableEffect = req.body.conversionMinDetectableEffect; + test.conversionPower = req.body.conversionPower; + test.conversionAlpha = req.body.conversionAlpha; + test.conversionSampleSize = req.body.conversionSampleSize; + test.trialId = req.body.trialId; + if (req.body.startDate) test.startDate = new Date(req.body.startDate); + if (isTestFullyConfigured(test)) test.statusId = statuses.configured.id; + else test.statusId = statuses.new.id; + test.save(() => { + res.redirect('/ab'); + }); + }); + }, + }, + + remove: { + get: (req, res) => { + Test.findById(req.params.testId, (err, test) => { + res.render('pages/ab/remove.jade', { test }); + }); + }, + post: (req, res) => { + Test.remove({ _id: req.params.testId }, () => { + res.redirect('/ab'); + }); + }, + }, + + progress: (req, res) => { + Test.findById(req.params.testId, (err, test) => { + outcomeCtl.trialHits(test, (hitErr, result) => { + if (err || hitErr) res.render('pages/ab/progress.jade', { test, message: { error: err || hitErr } }); + else { + test.baselineHits = result.baseline.total; + test.featureHits = result.feature.total; + res.render('pages/ab/progress.jade', { test }); + } + }); + }); + }, + + rawProgress: (req, res) => { + Test.findById(req.params.testId, (err, test) => { + outcomeCtl.trialHits(test, (hitErr, result) => { + result.needed = test.conversionSampleSize; + res.send(result); + }); + }); + }, + + results: (req, res) => { + Test.findById(req.params.testId, (err, test) => { + outcomeCtl.trialHits(test, (hitErr, result) => { + if (err || hitErr) res.render('pages/ab/results.jade', { test, message: { error: err || hitErr } }); + else { + test.pValue = significance.pValue(0, result.baseline, 0, result.feature); + test.baselineHits = result.baseline.total; + test.baselineSuccesses = result.baseline.hits; + test.featureHits = result.feature.total; + test.featureSuccesses = result.feature.hits; + test.pValue = significance.pValue( + result.baseline.hits, result.baseline.total, + result.feature.hits, result.feature.total + ); + if (test.baselineHits > 0 && test.featureHits > 0) { + if (test.baselineSuccesses / test.baselineHits > + test.featureSuccesses / test.featureHits) { + test.verdict = 'Baseline is more successful than feature.'; + } else { + test.verdict = 'Feature is more successful than baseline.'; + } + if (test.pValue < 0.05) { + test.verdict += ` This is SIGNIFICANT with a p-value of ${test.pValue}.`; + } else if (test.pValue < 0.1) { + test.verdict += ` This is borderline significant with a p-value of ${test.pValue}.`; + } else { + test.verdict += ` This is NOT significant: p-value = ${test.pValue}.`; + } + } + res.render('pages/ab/results.jade', { test }); + } + }); + }); + }, + + pause: (req, res) => { + Test.findById(req.params.testId, (err, test) => { + test.statusId = statuses.paused.id; + test.save(() => { + res.redirect('/ab'); + }); + }); + }, + + start: (req, res) => { + Test.findById(req.params.testId, (err, test) => { + test.statusId = statuses.running.id; + if (!test.startDate) { + const now = new Date(); + now.setHours(2); + now.setMinutes(0); + now.setSeconds(0); + test.startDate = now; + } + test.save(() => { + res.redirect('/ab'); + }); + }); + }, + + stop: (req, res) => { + Test.findById(req.params.testId, (err, test) => { + test.statusId = statuses.completed.id; + test.endDate = new Date(); + test.save(() => { + res.redirect('/ab'); + }); + }); + }, + + archive: (req, res) => { + Test.findById(req.params.testId, (err, test) => { + test.statusId = statuses.archived.id; + test.save(() => { + res.redirect('/ab'); + }); + }); + }, + +}; diff --git a/server/controllers/ab/trials.js b/server/controllers/ab/trials.js new file mode 100644 index 0000000..2a0036a --- /dev/null +++ b/server/controllers/ab/trials.js @@ -0,0 +1,74 @@ +const Event = require('../../models/event'); + +const pageViews = (days, callback) => { + // Let's do baseline first +}; + +// const pageViews = (test, callback) => { +// const match = { +// $match: { +// date: { $gt: test.startDate }, +// type: 'navigate', +// url: { +// $nin: [ +// '#processIndicators', '#outcomeIndicators', +// '#indicator-trend', '#indicator-benchmark', '#indicator-patient-list', +// '#processIndicatorsQS', '#outcomeIndicatorsQS'], +// }, +// }, +// }; +// match.$match[`tests.${test.name}`] = { $exists: true }; +// const project = { $project: { _id: 0, groope: `$tests.${test.name}` } }; +// const group = { $group: { _id: '$groope', total: { $sum: 1 } } }; +// const query = [match, project, group]; + +// Event.aggregate(query, (err, output) => { +// let baseline = 0; +// let feature = 0; +// output.forEach((o) => { +// if (o._id === 'baseline') baseline = o.total; +// else feature = o.total; +// }); +// return callback(err, { baseline, feature }); +// }); +// }; +// #patient/ + +const patientViews = (test, callback) => { + const match = { + $match: { + date: { $gt: test.startDate }, + type: 'navigate', + url: { $regex: /#patient\// }, + }, + }; + match.$match[`tests.${test.name}`] = { $exists: true }; + const project = { $project: { _id: 0, groope: `$tests.${test.name}` } }; + const group = { $group: { _id: '$groope', total: { $sum: 1 } } }; + const query = [match, project, group]; + + Event.aggregate(query, (err, output) => { + let baseline = 0; + let feature = 0; + output.forEach((o) => { + if (o._id === 'baseline') baseline = o.total; + else feature = o.total; + }); + return callback(err, { baseline, feature }); + }); +}; + +module.exports = { + pageViews, + patientViews, + hits: (test, callback) => { + switch (test.trialId) { + case 'pageView': + return pageViews(test, callback); + case 'patientView': + return patientViews(test, callback); + default: + return callback(new Error('Unexpected trial')); + } + }, +}; diff --git a/server/models/ab/test.js b/server/models/ab/test.js new file mode 100644 index 0000000..2e53274 --- /dev/null +++ b/server/models/ab/test.js @@ -0,0 +1,48 @@ +const mongoose = require('mongoose'); +const { statuses, randomisationTypes, trials, outcomes } = require('../../../shared/ab/config'); +const testConfigs = require('../../../shared/ab/tests'); + +const { Schema } = mongoose; + +const notSet = { id: 'notSet', name: 'Not set', description: 'Not set' }; + +// For A/B tests +const TestSchema = new Schema({ + name: { type: String, index: { unique: true } }, + description: String, + researchQuestion: String, + startDate: Date, + endDate: Date, + days: Number, + outcomeBaselineValue: Number, + conversionMinDetectableEffect: Number, + conversionPower: Number, + conversionAlpha: Number, + conversionSampleSize: Number, + statusId: { type: String, enum: Object.keys(statuses), default: statuses.new.id }, + randomisationTypeId: { type: String, enum: Object.keys(randomisationTypes) }, + outcomeId: { type: String, enum: Object.keys(outcomes) }, + trialId: { type: String, enum: Object.keys(trials) }, +}); + +TestSchema.virtual('randomisationType').get(function () { + return this.randomisationTypeId ? randomisationTypes[this.randomisationTypeId] : notSet; +}); + +TestSchema.virtual('status').get(function () { + return this.statusId ? statuses[this.statusId] : notSet; +}); + +TestSchema.virtual('outcome').get(function () { + return this.outcomeId ? outcomes[this.outcomeId] : notSet; +}); + +TestSchema.virtual('trial').get(function () { + return this.trialId ? trials[this.trialId] : notSet; +}); + +TestSchema.virtual('readyToDeploy').get(function () { + return testConfigs[this.name] ? testConfigs[this.name].readyToDeploy === 'true' : false; +}); + +module.exports = mongoose.model('Test', TestSchema); diff --git a/server/models/event.js b/server/models/event.js index ec72001..5dcec77 100644 --- a/server/models/event.js +++ b/server/models/event.js @@ -10,6 +10,7 @@ const EventSchema = new Schema({ data: [{ _id: false, key: String, value: String }], url: String, xpath: String, + tests: Object, // the tests for the current user pageId: String, }); diff --git a/server/routes/ab.js b/server/routes/ab.js new file mode 100644 index 0000000..518dbed --- /dev/null +++ b/server/routes/ab.js @@ -0,0 +1,32 @@ +const ctl = require('../controllers/ab/tests'); +const benchCtl = require('../controllers/ab/outcomes'); +const { isAuthenticated, isAdmin } = require('./helpers'); + +module.exports = { + applyTo: (router) => { + router.get('/ab', isAuthenticated, isAdmin, ctl.index); + + router.get('/ab/new', isAuthenticated, isAdmin, ctl.create.get); + router.post('/ab/new', isAuthenticated, isAdmin, ctl.create.post); + + router.get('/ab/config/:testId', isAuthenticated, isAdmin, ctl.configure.get); + router.post('/ab/config/:testId', isAuthenticated, isAdmin, ctl.configure.post); + + router.get('/ab/remove/:testId', isAuthenticated, isAdmin, ctl.remove.get); + router.post('/ab/remove/:testId', isAuthenticated, isAdmin, ctl.remove.post); + + router.get('/ab/start/:testId', isAuthenticated, isAdmin, ctl.start); + router.get('/ab/stop/:testId', isAuthenticated, isAdmin, ctl.stop); + router.get('/ab/pause/:testId', isAuthenticated, isAdmin, ctl.pause); + router.get('/ab/archive/:testId', isAuthenticated, isAdmin, ctl.archive); + + router.get('/ab/progress/:testId', isAuthenticated, isAdmin, ctl.progress); + router.get('/ab/results/:testId', isAuthenticated, isAdmin, ctl.results); + + router.get('/ab/raw/progress/:testId', isAuthenticated, isAdmin, ctl.rawProgress); + + router.get('/ab/running', isAuthenticated, isAdmin, ctl.running); + + router.get('/ab/conversions/for/trial/:trialId/outcome/:outcomeId/over/:days/days', isAuthenticated, isAdmin, benchCtl.conversion); + }, +}; diff --git a/server/routes/index.js b/server/routes/index.js index 65f07eb..a4dc169 100644 --- a/server/routes/index.js +++ b/server/routes/index.js @@ -16,6 +16,7 @@ const text = require('../controllers/text.js'); const utils = require('../controllers/utils.js'); const config = require('../config'); const tutorials = require('../tutorials'); +const abRoutes = require('./ab'); const { isAuthenticated, isAdmin, isUserOkToViewPractice } = require('./helpers'); const router = express.Router(); @@ -169,6 +170,8 @@ module.exports = (passport) => { }); }); + abRoutes.applyTo(router); + // User forgets password router.get('/auth/reset', (req, res) => { res.render('pages/auth-forgot-password.jade'); @@ -387,6 +390,9 @@ module.exports = (passport) => { if (req.params.indicatorId) { evt.data.push({ key: 'indicatorId', value: req.params.indicatorId }); } + if (req.body.tests) { + evt.tests = req.body.tests; + } events.add(evt, () => { res.send(action); }); @@ -417,6 +423,9 @@ module.exports = (passport) => { user: req.user.email, pageId: req.body.pageId, }; + if (req.body.tests) { + evt.tests = req.body.tests; + } events.add(evt, () => { res.send(action); }); @@ -456,6 +465,9 @@ module.exports = (passport) => { if (req.body.url) { evt.url = req.body.url; } + if (req.body.tests) { + evt.tests = req.body.tests; + } if (req.body.action.agree === true) { evt.type = 'agree'; } else if (req.body.action.agree === false) { @@ -508,6 +520,9 @@ module.exports = (passport) => { if (req.params.indicatorId) { evt.data.push({ key: 'indicatorId', value: req.params.indicatorId }); } + if (req.body.tests) { + evt.tests = req.body.tests; + } if (req.body.action.agree === true) { evt.type = 'agree'; } else if (req.body.action.agree === false) { diff --git a/server/views/mixins/ab/test.jade b/server/views/mixins/ab/test.jade new file mode 100644 index 0000000..8f99064 --- /dev/null +++ b/server/views/mixins/ab/test.jade @@ -0,0 +1,211 @@ +mixin test(submitText, postTo) + if message && message.error + .alert.alert-danger.alert-dismissible.fade-in(role="alert") + button.close(type="button" data-dismiss="alert" aria-label="Close") + span(aria-hidden="true") × + p= message.error + if message && message.success + .alert.alert-success.alert-dismissible.fade-in(role="alert") + button.close(type="button" data-dismiss="alert" aria-label="Close") + span(aria-hidden="true") × + p= message.success + form(action=postTo, method='post') + h3 Initial config + .row.form-group + .col-sm-3(style="text-align:right") + label(style="font-weight:normal") Research question*: + .col-sm-9 + input.form-control(type='text', name='researchQuestion', placeholder='E.g. Do blue thumbs up get clicked more frequently than green thumbs?', autofocus='', value=test.researchQuestion) + .row.form-group + .col-sm-3(style="text-align:right") + label(style="font-weight:normal") Name*: + .col-sm-9 + input.form-control(type='text', name='name', placeholder='A short name', value=test.name) + .row.form-group + .col-sm-3(style="text-align:right") + label(style="font-weight:normal") Description (optional): + .col-sm-9 + input.form-control(type='text', name='description', placeholder='A longer description', value=test.description) + .row.form-group + .col-sm-3(style="text-align:right") + label(style="font-weight:normal") Randomisation type*: + .col-sm-9 + select.form-control.selectpicker(name='randomisationTypeId',title='Select randomisation type') + option(selected='', disabled='') Select randomisation type + each rt in randomisationTypeArray + option(value=rt.id, selected=test.randomisationTypeId===rt.id)= rt.name + .row.form-group + .col-sm-3(style="text-align:right") + label(style="font-weight:normal") What counts as a "trial"*: + .col-sm-9 + select#selTrial.form-control.selectpicker(name='trialId',title='Select what counts as a trial') + option(selected='', disabled='') Select what counts as a trial + each trial in trialArray + option(value=trial.id, selected=test.trialId===trial.id)= trial.name + ' (' + trial.description + ')' + .row.form-group + .col-sm-3(style="text-align:right") + label(for="day",style="font-weight:normal") Outcome (what we're measuring per trial)*: + .col-sm-9 + select#selOutcome.form-control.selectpicker(name='outcomeId',title='Select outcome') + option(selected='', disabled='') Select outcome + each outcome in outcomeArray + option(value=outcome.id, selected=test.outcomeId===outcome.id)= outcome.description + .row.form-group + .col-sm-3(style="text-align:right") + label(for="day",style="font-weight:normal") No. of previous days over which to measure the current conversion rate: + .col-sm-9 + input#days(name='days',value=test.days || 90, type='text', size="3", style="text-align: center") + .row.form-group + .col-sm-3(style="text-align:right") + label(for="day",style="font-weight:normal") Current conversion rate: + .col-sm-9 + input#conversionRateHidden(type='hidden', name='outcomeBaselineValue',value=test.outcomeBaselineValue) + label#conversionRate Will display when "trial", "outcome" and "days" are set. + .row.form-group + .col-sm-3(style="text-align:right") + label(for="day",style="font-weight:normal") Minimum detectable effect (absolute): + .col-sm-9 + input#minEffect(name='conversionMinDetectableEffect',value=test.conversionMinDetectableEffect || 5, type='text', size="3", style="text-align: center") + label  % + .row.form-group + .col-sm-3(style="text-align:right") + label(for="day",style="font-weight:normal") Statistical power 1-β: + .col-sm-9 + input#power(name='conversionPower',value=test.conversionPower || 80, type='text', size="3", style="text-align: center") + label  % - Percent of the time the minimum effect size will be detected, assuming it exists + .row.form-group + .col-sm-3(style="text-align:right") + label(for="day",style="font-weight:normal") Significance level α: + .col-sm-9 + input#alpha(name='conversionAlpha',value=test.conversionAlpha || 5, type='text', size="3", style="text-align: center") + label  % - Percent of the time a difference will be detected, assuming one does NOT exist + .row.form-group + .col-sm-3(style="text-align:right") + label(for="day",style="font-weight:normal") Sample size per variation: + .col-sm-9 + label#sampleSize + input#hiddenSampleSize(type='hidden',name='conversionSampleSize',value=test.conversionSampleSize) + .row.form-group + .col-sm-3(style="text-align:right") + label(style="font-weight:normal") Start date: + .col-sm-9 + input.form-control(type='text', name='startDate', placeholder='Start Date (yyyy-mm-dd)', value=test.startDate) + button.btn.btn-purple.btn-block.history(type='submit') + = submitText + | + span.fa.fa-arrow-circle-right +script(src='https://code.jquery.com/jquery-1.12.4.min.js', integrity='sha256-ZosEbRLbNQzLpnKIkEdrPv7lOy9C27hHQ+Xp8a4MxAQ=', crossorigin='anonymous') +script. + window.jQuery || document.write('