From e840ebc18856fafac84a3932025da8aaec61a0ec Mon Sep 17 00:00:00 2001 From: Richard Williams Date: Thu, 21 Jun 2018 15:35:23 +0100 Subject: [PATCH 01/24] More ab test stuff --- app/data.js | 3 + app/events.js | 34 ++-------- app/log.js | 19 +++++- app/main.js | 4 ++ app/script.js | 28 ++++++-- .../appserver/mongo_scripts/ab_test_test.js | 6 +- server/controllers/ab/benchmarks.js | 24 +++++-- server/controllers/ab/tests.js | 17 ++++- server/models/event.js | 1 + server/models/test.js | 22 +----- server/routes/ab.js | 2 + server/views/pages/ab/configure.jade | 6 +- server/views/pages/ab/index.jade | 4 +- shared/ab/config.js | 67 +++++++++++++++++++ shared/ab/tests.js | 16 +++++ shared/abConfig.js | 48 ------------- 16 files changed, 183 insertions(+), 118 deletions(-) create mode 100644 shared/ab/config.js create mode 100644 shared/ab/tests.js delete mode 100644 shared/abConfig.js diff --git a/app/data.js b/app/data.js index b7c7dcb..40b1ea5 100644 --- a/app/data.js +++ b/app/data.js @@ -23,6 +23,9 @@ const dt = { excludedPatients: {}, excludedPatientsByIndicator: {}, + // don't cache always fetch + getRunningTests: callback => $.getJSON('/ab/running', tests => callback(tests)), + getNHS: (practiceId, patientId) => { if ( dt.patLookup && diff --git a/app/events.js b/app/events.js index 3b8050d..b013b36 100644 --- a/app/events.js +++ b/app/events.js @@ -1,11 +1,10 @@ +const log = require('./log'); /** * Gets an XPath for an element which describes its hierarchical location. * Copied from firebug with a BSD licence - https://code.google.com/p/fbug/source/browse/branches/firebug1.6/content/firebug/lib.js?spec=svn12950&r=8828#1332 */ const SESSION_TIMEOUT = 2 * 3600 * 1000; -let eventFailCount = 0; - let timer = setTimeout(() => { window.location.href = '/signout'; }, SESSION_TIMEOUT); // set session logout to 2 hours @@ -51,13 +50,11 @@ const getElementXPath = (el) => { const logInfo = (event, type, href) => { refreshSession(); - const obj = {}; - obj.url = href || window.location.href; - obj.type = type; - obj.data = []; + const data = []; + let xpath = ''; let text = ''; if (event) { - obj.xpath = getElementXPath(event.target); + xpath = getElementXPath(event.target); const t = event.target; @@ -101,30 +98,11 @@ const logInfo = (event, type, href) => { } // if(event) if (text !== '') { - obj.data.push({ key: 'text', value: text }); + data.push({ key: 'text', value: text }); } // console.log(obj); setTimeout(() => { - const dataToSend = { event: obj }; - $.ajax({ - type: 'POST', - url: '/api/event', - data: JSON.stringify(dataToSend), - success() { - eventFailCount = 0; - }, - error() { - eventFailCount += 1; - if (eventFailCount > 2) { - // We've had too many errors from the back end - could be the server - // is down, or has restarted and the session has ended. Either way - // a page refresh might help - window.location.reload(); - } - }, - dataType: 'json', - contentType: 'application/json', - }); + log.event(type, href || window.location.href, data, xpath); }, 1); }; diff --git a/app/log.js b/app/log.js index 8ded79d..13a8d4c 100644 --- a/app/log.js +++ b/app/log.js @@ -2,6 +2,8 @@ const notify = require('./notify'); const data = require('./data'); const lookup = require('./lookup'); +let eventFailCount = 0; + const log = { reason: {}, @@ -9,14 +11,25 @@ const log = { log.event('navigate', toUrl, dataProp); }, - event(type, url, dataProp) { - const dataToSend = { event: { type, url, dataProp } }; + event(type, url, dataProp, xpath) { + const dataToSend = { event: { type, url, data: dataProp } }; + if (xpath && xpath.length > 0) dataToSend.event.xpath = xpath; + if (lookup.tests && Object.keys(lookup.tests).length > 0) dataToSend.event.tests = lookup.tests; $.ajax({ type: 'POST', url: '/api/event', data: JSON.stringify(dataToSend), success() { - // console.log(d); + eventFailCount = 0; + }, + error() { + eventFailCount += 1; + if (eventFailCount > 5) { + // We've had too many errors from the back end - could be the server + // is down, or has restarted and the session has ended. Either way + // a page refresh might help + window.location.reload(); + } }, dataType: 'json', contentType: 'application/json', diff --git a/app/main.js b/app/main.js index e927947..93ec673 100644 --- a/app/main.js +++ b/app/main.js @@ -21,6 +21,10 @@ const main = { main.wireUpPages(); }, + getRunningTests(callback) { + return data.getRunningTests(callback); + }, + getInitialData(callback) { data.get( state.selectedPractice ? state.selectedPractice._id : null, diff --git a/app/script.js b/app/script.js index 07bfea3..87e3cd2 100644 --- a/app/script.js +++ b/app/script.js @@ -8,7 +8,7 @@ require('datatables.net-buttons-bs')(window, $); require('datatables.net-buttons/js/buttons.colVis.js')(window, $); require('datatables.net-buttons/js/buttons.html5.js')(window, $); -const abConfig = require('../shared/abConfig'); +const abConfig = require('../shared/ab/config'); /* * For each disease area there will be 4 stages: Diagnosis, Monitoring, Treatment and Exclusions. @@ -112,12 +112,28 @@ const App = { getServerParameters(); + // Get all currently running A/B tests + main.getRunningTests((tests) => { + if (tests) { + lookup.tests = {}; // list of groups for the user + tests.forEach((test) => { + // assign the user to a group (might be already defined) + test.group = abConfig.assign(test); + // add to the state object + lookup.tests[test.name] = test.group; + // if the user is in the feature group execute + abConfig.init($, test); + console.log(`User ${lookup.userName} assigned to the '${test.group}' group for test '${test.name}'`); + }); + } + }); + // if (abConfig.abTestTest.isRunning) { - const group = abConfig.assignPerUser(abConfig.abTestTest, lookup.userName); - console.log(`User ${lookup.userName} is in group ${group}`); - if (group === abConfig.groups.feature) { - abConfig.abTestTest.featureCode($); - } + // const group = abConfig.assignPerUser(abConfig.abTestTest, lookup.userName); + // console.log(`User ${lookup.userName} is in group ${group}`); + // if (group === abConfig.groups.feature) { + // abConfig.abTestTest.featureCode($); + // } // } diff --git a/deploy/appserver/mongo_scripts/ab_test_test.js b/deploy/appserver/mongo_scripts/ab_test_test.js index ec2c311..e4d7d76 100644 --- a/deploy/appserver/mongo_scripts/ab_test_test.js +++ b/deploy/appserver/mongo_scripts/ab_test_test.js @@ -1,3 +1,7 @@ // execute the following to load the ab test test -db.tests.insert({ _id: 1, name: 'abTestTest' }); +db.tests.insert({ + _id: 1, + name: 'abTestTest', + description: 'Change colour of thumbs up from green to blue', +}); diff --git a/server/controllers/ab/benchmarks.js b/server/controllers/ab/benchmarks.js index f047ce3..a0cf0c0 100644 --- a/server/controllers/ab/benchmarks.js +++ b/server/controllers/ab/benchmarks.js @@ -1,5 +1,14 @@ const Event = require('../../models/event'); +const benchmarkNames = [ + 'Agrees clicked per session in last 90 days.', + 'Disagrees clicked per session in last 90 days.', + 'Thumbs clicked per session in last 90 days.', + 'Agrees clicked per minute in last 90 days.', + 'Disagrees clicked per minute in last 90 days.', + 'Thumbs clicked per minute in last 90 days.', +]; + const getNDaysAgo = (n) => { const today = new Date(); const nDaysAgo = new Date(); @@ -44,17 +53,17 @@ const eventsPerMinuteLastNDays = (eventTypes, n, id, description, callback) => { }; const agreesPerSessionLastNDays = (n, callback) => - eventsPerSessionLastNDays(['agree'], n, 1, `Agrees clicked per session in last ${n} days.`, callback); + eventsPerSessionLastNDays(['agree'], n, 1, benchmarkNames[0], callback); const disagreesPerSessionLastNDays = (n, callback) => - eventsPerSessionLastNDays(['disagree'], n, 2, `Disagrees clicked per session in last ${n} days.`, callback); + eventsPerSessionLastNDays(['disagree'], n, 2, benchmarkNames[1], callback); const thumbsPerSessionLastNDays = (n, callback) => - eventsPerSessionLastNDays(['agree', 'disagree'], n, 3, `Thumbs clicked per session in last ${n} days.`, callback); + eventsPerSessionLastNDays(['agree', 'disagree'], n, 3, benchmarkNames[2], callback); const agreesPerMinuteLastNDays = (n, callback) => - eventsPerMinuteLastNDays(['agree'], n, 4, `Agrees clicked per minute in last ${n} days.`, callback); + eventsPerMinuteLastNDays(['agree'], n, 4, benchmarkNames[3], callback); const disagreesPerMinuteLastNDays = (n, callback) => - eventsPerMinuteLastNDays(['disagree'], n, 5, `Disagrees clicked per minute in last ${n} days.`, callback); + eventsPerMinuteLastNDays(['disagree'], n, 5, benchmarkNames[4], callback); const thumbsPerMinuteLastNDays = (n, callback) => - eventsPerMinuteLastNDays(['agree', 'disagree'], n, 6, `Thumbs clicked per minute in last ${n} days.`, callback); + eventsPerMinuteLastNDays(['agree', 'disagree'], n, 6, benchmarkNames[5], callback); const go = (n, func) => { func(n, (err, output) => output.result); @@ -76,6 +85,7 @@ const getAll = (callback) => { }; module.exports = { + agreesPerSessionLastNDays: n => go(n, agreesPerSessionLastNDays), disagreesPerSessionLastNDays: n => go(n, disagreesPerSessionLastNDays), thumbsPerSessionLastNDays: n => go(n, thumbsPerSessionLastNDays), @@ -90,4 +100,6 @@ module.exports = { }, getAll, + + nameFromId: id => benchmarkNames[id - 1], }; diff --git a/server/controllers/ab/tests.js b/server/controllers/ab/tests.js index fd32635..4030dfc 100644 --- a/server/controllers/ab/tests.js +++ b/server/controllers/ab/tests.js @@ -1,11 +1,24 @@ -const { Test, statuses } = require('../../models/test'); +const Test = require('../../models/test'); +const { statuses } = require('../../../shared/ab/config'); const benchCtl = require('./benchmarks'); module.exports = { index: (req, res) => { Test.find({ status: { $ne: statuses.archived } }, (err, tests) => { - res.render('pages/ab/index.jade', { tests }); + const testMods = tests.map((test) => { + if (test.benchmarkId) { + test.benchmark = benchCtl.nameFromId(test.benchmarkId); + } + return test; + }); + res.render('pages/ab/index.jade', { tests: testMods }); + }); + }, + + running: (req, res) => { + Test.find({ status: statuses.running }, (err, tests) => { + res.send(tests); }); }, diff --git a/server/models/event.js b/server/models/event.js index 7deda99..c1c31b3 100644 --- a/server/models/event.js +++ b/server/models/event.js @@ -10,6 +10,7 @@ const EventSchema = new Schema({ data: [{ _id: false, key: String, value: String }], url: String, xpath: String, + tests: Object, // the tests for the current user }); module.exports = mongoose.model('Event', EventSchema); diff --git a/server/models/test.js b/server/models/test.js index 66f9ba1..a4b58b3 100644 --- a/server/models/test.js +++ b/server/models/test.js @@ -1,22 +1,8 @@ const mongoose = require('mongoose'); +const { statuses, randomisationTypes } = require('../../shared/ab/config'); const { Schema } = mongoose; -const statuses = { - new: 'new', - configured: 'configured', - running: 'running', - paused: 'paused', - complete: 'complete', - archived: 'archived', -}; - -const randomisationTypes = { - perSession: 'perSession', - perUser: 'perUser', - perPractice: 'perPractice', -}; - // For A/B tests const TestSchema = new Schema({ @@ -29,8 +15,4 @@ const TestSchema = new Schema({ benchmarkId: Number, }); -module.exports = { - statuses, - randomisationTypes, - Test: mongoose.model('Test', TestSchema), -}; +module.exports = mongoose.model('Test', TestSchema); diff --git a/server/routes/ab.js b/server/routes/ab.js index 7e63d89..14c5e5c 100644 --- a/server/routes/ab.js +++ b/server/routes/ab.js @@ -12,6 +12,8 @@ module.exports = { router.get('/ab/progress/:testId', isAuthenticated, isAdmin, ctl.progress); router.get('/ab/delete/:testId', isAuthenticated, isAdmin, ctl.delete); + router.get('/ab/running', isAuthenticated, isAdmin, ctl.running); + router.get('/benchmarks', isAuthenticated, isAdmin, benchCtl.all); }, }; diff --git a/server/views/pages/ab/configure.jade b/server/views/pages/ab/configure.jade index b08103b..7611e15 100644 --- a/server/views/pages/ab/configure.jade +++ b/server/views/pages/ab/configure.jade @@ -26,9 +26,9 @@ include ../../mixins/toolbar.jade i.fa.fa-stethoscope select#selRandomisation.form-control.selectpicker(name='randomisationType', required='',title='Select randomisation type') option(selected='', disabled='') Select randomisation type - option(value='perSession', selected=test.status==='perSession') Per Session - option(value='perUser', selected=test.status==='perUser') Per User - option(value='perPractice', selected=test.status==='perPractice') Per Practice + option(value='perSession', selected=test.randomisationType==='perSession') Per Session + option(value='perUser', selected=test.randomisationType==='perUser') Per User + option(value='perPractice', selected=test.randomisationType==='perPractice') Per Practice .form-group.input-group span.input-group-addon i.fa.fa-user diff --git a/server/views/pages/ab/index.jade b/server/views/pages/ab/index.jade index e3f08ce..bbeee33 100644 --- a/server/views/pages/ab/index.jade +++ b/server/views/pages/ab/index.jade @@ -18,6 +18,7 @@ include ../../mixins/toolbar.jade th Name th Description th Start date + th Benchmark th Type of randomisation th Status th(style='width:280px') Actions @@ -26,7 +27,8 @@ include ../../mixins/toolbar.jade tr td= test.name td= test.description - td= test.startDate + td= test.startDate.toISOString().substr(0,10) + td= test.benchmark td= test.randomisationType td= test.status td diff --git a/shared/ab/config.js b/shared/ab/config.js new file mode 100644 index 0000000..c22b87f --- /dev/null +++ b/shared/ab/config.js @@ -0,0 +1,67 @@ +const tests = require('./tests'); + +const groups = { + baseline: 'baseline', + feature: 'feature', +}; + +const statuses = { + new: 'new', + configured: 'configured', + running: 'running', + paused: 'paused', + complete: 'complete', + archived: 'archived', +}; + +const randomisationTypes = { + perSession: 'perSession', + perUser: 'perUser', + perPractice: 'perPractice', +}; + +// this should be in db +const userCache = {}; + +// load all test files + + +module.exports = { + + assign: (test) => { + switch (test.randomisationTypes) { + case randomisationTypes.perPractice: + return Math.random() < 0.5 ? groups.baseline : groups.feature; + case randomisationTypes.perUser: + return Math.random() < 0.5 ? groups.baseline : groups.feature; + default: // randomisationTypes.perSession: + return Math.random() < 0.5 ? groups.baseline : groups.feature; + } + }, + + init: ($, test) => { + if (test.group === groups.feature) { + tests[test.name].init($); + } + }, + + assignPerUser: (test, user) => { + if (!userCache[test.name]) { + console.log(`No cache for test: ${test.name}. Created.`); + userCache[test.name] = {}; + } + if (!userCache[test.name][user]) { + console.log(`No user cached for test: ${test.name}. Assigned.`); + if (Math.random() < 0.5) { + userCache[test.name][user] = groups.baseline; + } else { + userCache[test.name][user] = groups.feature; + } + } + return userCache[test.name][user]; + }, + + groups, + statuses, + randomisationTypes, +}; diff --git a/shared/ab/tests.js b/shared/ab/tests.js new file mode 100644 index 0000000..fceec33 --- /dev/null +++ b/shared/ab/tests.js @@ -0,0 +1,16 @@ +// Each test is the same as the name in the db +// Must contain an init method - which is executed on login +// init can be empty +// Can define other functions as well + +module.exports = { + abTestTest: { + init: ($) => { + if ($('#abTestCss').length === 0) { + $('head').append(''); + } + $('#abTestCss').text('.btn.btn-xs.btn-yes.btn-toggle.btn-success{ background-color: blue }'); + }, + }, +}; + diff --git a/shared/abConfig.js b/shared/abConfig.js deleted file mode 100644 index 4dfdc2e..0000000 --- a/shared/abConfig.js +++ /dev/null @@ -1,48 +0,0 @@ -const groups = { - baseline: 'baseline', - feature: 'feature', -}; - -// this should be in db -const userCache = {}; - -module.exports = { - abTestTest: { - name: 'abTestTest', - description: 'This is what we\'re going to do.. make the thumbs up colour blue and see if it effects the number of times it is clicked.', - featureCode: ($) => { - // Inject our style into the header if not alredy there - if ($('#abTestCss').length === 0) { - $('head').append(''); - } - - $('#abTestCss').text('.btn.btn-xs.btn-yes.btn-toggle.btn-success{ background-color: blue }'); - }, - }, - - assignPerPractice: () => { - - }, - - assignPerSession: () => { - - }, - - assignPerUser: (test, user) => { - if (!userCache[test.name]) { - console.log(`No cache for test: ${test.name}. Created.`); - userCache[test.name] = {}; - } - if (!userCache[test.name][user]) { - console.log(`No user cached for test: ${test.name}. Assigned.`); - if (Math.random() < 0.5) { - userCache[test.name][user] = groups.baseline; - } else { - userCache[test.name][user] = groups.feature; - } - } - return userCache[test.name][user]; - }, - - groups, -}; From 65e88ce996146ac880bc5f894f91a115a320cf7c Mon Sep 17 00:00:00 2001 From: Richard Williams Date: Wed, 27 Jun 2018 10:57:51 +0100 Subject: [PATCH 02/24] Added benchmarks --- .../{benchmarks.js => conversionMetrics.js} | 32 +-- server/controllers/ab/tests.js | 89 +++++++-- server/models/ab/metric.js | 11 + server/models/{ => ab}/test.js | 10 +- server/routes/ab.js | 17 +- server/views/mixins/ab/test.jade | 189 ++++++++++++++++++ server/views/mixins/toolbar.jade | 2 +- server/views/pages/ab/configure.jade | 38 +--- ...benchmarks.jade => conversionMetrics.jade} | 8 +- server/views/pages/ab/create.jade | 6 + server/views/pages/ab/index.jade | 34 ++-- server/views/pages/ab/remove.jade | 20 ++ server/views/partials/head.jade | 2 +- shared/ab/config.js | 2 +- 14 files changed, 368 insertions(+), 92 deletions(-) rename server/controllers/ab/{benchmarks.js => conversionMetrics.js} (74%) create mode 100644 server/models/ab/metric.js rename server/models/{ => ab}/test.js (58%) create mode 100644 server/views/mixins/ab/test.jade rename server/views/pages/ab/{benchmarks.jade => conversionMetrics.jade} (75%) create mode 100644 server/views/pages/ab/create.jade create mode 100644 server/views/pages/ab/remove.jade diff --git a/server/controllers/ab/benchmarks.js b/server/controllers/ab/conversionMetrics.js similarity index 74% rename from server/controllers/ab/benchmarks.js rename to server/controllers/ab/conversionMetrics.js index a0cf0c0..1f8ca05 100644 --- a/server/controllers/ab/benchmarks.js +++ b/server/controllers/ab/conversionMetrics.js @@ -1,12 +1,14 @@ const Event = require('../../models/event'); -const benchmarkNames = [ +const conversionMetricNames = [ 'Agrees clicked per session in last 90 days.', 'Disagrees clicked per session in last 90 days.', 'Thumbs clicked per session in last 90 days.', 'Agrees clicked per minute in last 90 days.', 'Disagrees clicked per minute in last 90 days.', 'Thumbs clicked per minute in last 90 days.', + 'Proportion of page views with thumb click in the last 90 days', + 'Proportion of page views with own action added in the last 90 days', ]; const getNDaysAgo = (n) => { @@ -29,7 +31,7 @@ const eventsPerSessionLastNDays = (eventTypes, n, id, description, callback) => { $group: { _id: '$sessionId', events: { $sum: '$hasEvent' } } }, { $group: { _id: null, result: { $avg: '$events' } } }, ]; - Event.aggregate(query, (err, output) => callback(err, { id, description, value: output[0].result })); + Event.aggregate(query, (err, output) => callback(err, { id, description, value: output[0].result * 100 })); }; const eventsPerMinuteLastNDays = (eventTypes, n, id, description, callback) => { @@ -49,21 +51,21 @@ const eventsPerMinuteLastNDays = (eventTypes, n, id, description, callback) => { { $project: { eventsPerMillisecond: { $divide: ['$totalEvents', '$totalDuration'] } } }, { $project: { result: { $multiply: [60000, '$eventsPerMillisecond'] } } }, ]; - Event.aggregate(query, (err, output) => callback(err, { id, description, value: output[0].result })); + Event.aggregate(query, (err, output) => callback(err, { id, description, value: output[0].result * 100 })); }; const agreesPerSessionLastNDays = (n, callback) => - eventsPerSessionLastNDays(['agree'], n, 1, benchmarkNames[0], callback); + eventsPerSessionLastNDays(['agree'], n, 1, conversionMetricNames[0], callback); const disagreesPerSessionLastNDays = (n, callback) => - eventsPerSessionLastNDays(['disagree'], n, 2, benchmarkNames[1], callback); + eventsPerSessionLastNDays(['disagree'], n, 2, conversionMetricNames[1], callback); const thumbsPerSessionLastNDays = (n, callback) => - eventsPerSessionLastNDays(['agree', 'disagree'], n, 3, benchmarkNames[2], callback); + eventsPerSessionLastNDays(['agree', 'disagree'], n, 3, conversionMetricNames[2], callback); const agreesPerMinuteLastNDays = (n, callback) => - eventsPerMinuteLastNDays(['agree'], n, 4, benchmarkNames[3], callback); + eventsPerMinuteLastNDays(['agree'], n, 4, conversionMetricNames[3], callback); const disagreesPerMinuteLastNDays = (n, callback) => - eventsPerMinuteLastNDays(['disagree'], n, 5, benchmarkNames[4], callback); + eventsPerMinuteLastNDays(['disagree'], n, 5, conversionMetricNames[4], callback); const thumbsPerMinuteLastNDays = (n, callback) => - eventsPerMinuteLastNDays(['agree', 'disagree'], n, 6, benchmarkNames[5], callback); + eventsPerMinuteLastNDays(['agree', 'disagree'], n, 6, conversionMetricNames[5], callback); const go = (n, func) => { func(n, (err, output) => output.result); @@ -72,13 +74,13 @@ const go = (n, func) => { const getAll = (callback) => { const need = 2; let done = 0; - const benchmarks = []; + const conversionMetrics = []; [thumbsPerSessionLastNDays, thumbsPerMinuteLastNDays].forEach((func) => { func(90, (err, output) => { done += 1; - benchmarks.push(output); + conversionMetrics.push(output); if (done === need) { - callback(null, benchmarks); + callback(null, conversionMetrics); } }); }); @@ -94,12 +96,12 @@ module.exports = { thumbsPerMinuteLastNDays: n => go(n, thumbsPerMinuteLastNDays), all: (req, res) => { - getAll((err, benchmarks) => { - res.render('pages/ab/benchmarks.jade', { benchmarks }); + getAll((err, conversionMetrics) => { + res.render('pages/ab/conversionMetrics.jade', { conversionMetrics }); }); }, getAll, - nameFromId: id => benchmarkNames[id - 1], + nameFromId: id => conversionMetricNames[id - 1], }; diff --git a/server/controllers/ab/tests.js b/server/controllers/ab/tests.js index 4030dfc..96ac18c 100644 --- a/server/controllers/ab/tests.js +++ b/server/controllers/ab/tests.js @@ -1,18 +1,21 @@ -const Test = require('../../models/test'); +const Test = require('../../models/ab/test'); const { statuses } = require('../../../shared/ab/config'); -const benchCtl = require('./benchmarks'); +const benchCtl = require('./conversionMetrics'); + +let localMetrics = []; module.exports = { index: (req, res) => { - Test.find({ status: { $ne: statuses.archived } }, (err, tests) => { + const query = req.query.showArchived ? {} : { status: { $ne: statuses.archived } }; + Test.find(query, (err, tests) => { const testMods = tests.map((test) => { - if (test.benchmarkId) { - test.benchmark = benchCtl.nameFromId(test.benchmarkId); + if (test.conversionMetricId) { + test.conversionMetric = benchCtl.nameFromId(test.conversionMetricId); } return test; }); - res.render('pages/ab/index.jade', { tests: testMods }); + res.render('pages/ab/index.jade', { tests: testMods, showArchived: req.query.showArchived }); }); }, @@ -22,20 +25,43 @@ module.exports = { }); }, + create: { + get: (req, res) => { + benchCtl.getAll((getAllErr, conversionMetrics) => { + localMetrics = conversionMetrics; + res.render('pages/ab/create.jade', { test: {}, conversionMetrics }); + }); + }, + post: (req, res) => { + const newTest = new Test(req.body); + newTest.save((err) => { + if (err) res.render('pages/ab/create.jade', { test: newTest, message: { error: err }, conversionMetrics: localMetrics }); + else res.redirect('/ab'); + }); + }, + }, + configure: { get: (req, res) => { - Test.findOne({ _id: +req.params.testId }, (err, test) => { - benchCtl.getAll((getAllErr, benchmarks) => { - res.render('pages/ab/configure.jade', { test, benchmarks }); + Test.findById(req.params.testId, (err, test) => { + benchCtl.getAll((getAllErr, conversionMetrics) => { + res.render('pages/ab/configure.jade', { test, conversionMetrics }); }); }); }, post: (req, res) => { - Test.findOne({ _id: +req.params.testId }, (err, test) => { + Test.findById(req.params.testId, (err, test) => { + test.name = req.body.name; + test.description = req.body.description; test.status = statuses.configured; test.randomisationType = req.body.randomisationType; - test.benchmarkId = req.body.benchmarkId; - test.startDate = new Date(req.body.startDate); + test.conversionMetricId = req.body.conversionMetricId; + test.conversionMetricBaselineValue = req.body.conversionMetricBaselineValue; + test.conversionMinDetectableEffect = req.body.conversionMinDetectableEffect; + test.conversionPower = req.body.conversionPower; + test.conversionAlpha = req.body.conversionAlpha; + test.conversionSampleSize = req.body.conversionSampleSize; + if (req.body.startDate) test.startDate = new Date(req.body.startDate); test.save(() => { res.redirect('/ab'); }); @@ -43,14 +69,33 @@ module.exports = { }, }, + remove: { + get: (req, res) => { + Test.findById(req.params.testId, (err, test) => { + res.render('pages/ab/remove.jade', { test }); + }); + }, + post: (req, res) => { + Test.remove({ _id: req.params.testId }, () => { + res.redirect('/ab'); + }); + }, + }, + progress: (req, res) => { - Test.findOne({ _id: +req.params.testId }, (err, test) => { + Test.findById(req.params.testId, (err, test) => { + res.render('pages/ab/progress.jade', { test }); + }); + }, + + results: (req, res) => { + Test.findById(req.params.testId, (err, test) => { res.render('pages/ab/progress.jade', { test }); }); }, pause: (req, res) => { - Test.findOne({ _id: +req.params.testId }, (err, test) => { + Test.findById(req.params.testId, (err, test) => { test.status = statuses.paused; test.save(() => { res.redirect('/ab'); @@ -59,7 +104,7 @@ module.exports = { }, start: (req, res) => { - Test.findOne({ _id: +req.params.testId }, (err, test) => { + Test.findById(req.params.testId, (err, test) => { test.status = statuses.running; test.save(() => { res.redirect('/ab'); @@ -67,12 +112,22 @@ module.exports = { }); }, - delete: (req, res) => { - Test.findOne({ _id: +req.params.testId }, (err, test) => { + stop: (req, res) => { + Test.findById(req.params.testId, (err, test) => { + test.status = statuses.completed; + test.save(() => { + res.redirect('/ab'); + }); + }); + }, + + archive: (req, res) => { + Test.findById(req.params.testId, (err, test) => { test.status = statuses.archived; test.save(() => { res.redirect('/ab'); }); }); }, + }; diff --git a/server/models/ab/metric.js b/server/models/ab/metric.js new file mode 100644 index 0000000..d73739d --- /dev/null +++ b/server/models/ab/metric.js @@ -0,0 +1,11 @@ +const mongoose = require('mongoose'); + +const { Schema } = mongoose; + +// Conversion metrics +const MetricSchema = new Schema({ + name: String, + description: String, +}); + +module.exports = mongoose.model('Metric', MetricSchema); diff --git a/server/models/test.js b/server/models/ab/test.js similarity index 58% rename from server/models/test.js rename to server/models/ab/test.js index a4b58b3..cc0614b 100644 --- a/server/models/test.js +++ b/server/models/ab/test.js @@ -1,18 +1,22 @@ const mongoose = require('mongoose'); -const { statuses, randomisationTypes } = require('../../shared/ab/config'); +const { statuses, randomisationTypes } = require('../../../shared/ab/config'); const { Schema } = mongoose; // For A/B tests const TestSchema = new Schema({ - _id: Number, name: String, description: String, status: { type: String, enum: Object.keys(statuses), default: statuses.new }, startDate: Date, randomisationType: { type: String, enum: Object.keys(randomisationTypes) }, - benchmarkId: Number, + conversionMetricId: Number, + conversionMetricBaselineValue: Number, + conversionMinDetectableEffect: Number, + conversionPower: Number, + conversionAlpha: Number, + conversionSampleSize: Number, }); module.exports = mongoose.model('Test', TestSchema); diff --git a/server/routes/ab.js b/server/routes/ab.js index 14c5e5c..e056f45 100644 --- a/server/routes/ab.js +++ b/server/routes/ab.js @@ -1,19 +1,30 @@ const ctl = require('../controllers/ab/tests'); -const benchCtl = require('../controllers/ab/benchmarks'); +const benchCtl = require('../controllers/ab/conversionMetrics'); const { isAuthenticated, isAdmin } = require('./helpers'); module.exports = { applyTo: (router) => { router.get('/ab', isAuthenticated, isAdmin, ctl.index); + + router.get('/ab/new', isAuthenticated, isAdmin, ctl.create.get); + router.post('/ab/new', isAuthenticated, isAdmin, ctl.create.post); + router.get('/ab/config/:testId', isAuthenticated, isAdmin, ctl.configure.get); router.post('/ab/config/:testId', isAuthenticated, isAdmin, ctl.configure.post); + + router.get('/ab/remove/:testId', isAuthenticated, isAdmin, ctl.remove.get); + router.post('/ab/remove/:testId', isAuthenticated, isAdmin, ctl.remove.post); + router.get('/ab/start/:testId', isAuthenticated, isAdmin, ctl.start); + router.get('/ab/stop/:testId', isAuthenticated, isAdmin, ctl.stop); router.get('/ab/pause/:testId', isAuthenticated, isAdmin, ctl.pause); + router.get('/ab/archive/:testId', isAuthenticated, isAdmin, ctl.archive); + router.get('/ab/progress/:testId', isAuthenticated, isAdmin, ctl.progress); - router.get('/ab/delete/:testId', isAuthenticated, isAdmin, ctl.delete); + router.get('/ab/results/:testId', isAuthenticated, isAdmin, ctl.results); router.get('/ab/running', isAuthenticated, isAdmin, ctl.running); - router.get('/benchmarks', isAuthenticated, isAdmin, benchCtl.all); + router.get('/conversionMetrics', isAuthenticated, isAdmin, benchCtl.all); }, }; diff --git a/server/views/mixins/ab/test.jade b/server/views/mixins/ab/test.jade new file mode 100644 index 0000000..d84e072 --- /dev/null +++ b/server/views/mixins/ab/test.jade @@ -0,0 +1,189 @@ +mixin test(submitText, postTo) + if message && message.error + .alert.alert-danger.alert-dismissible.fade-in(role="alert") + button.close(type="button" data-dismiss="alert" aria-label="Close") + span(aria-hidden="true") × + p= message.error + if message && message.success + .alert.alert-success.alert-dismissible.fade-in(role="alert") + button.close(type="button" data-dismiss="alert" aria-label="Close") + span(aria-hidden="true") × + p= message.success + form(action=postTo, method='post') + h3 Initial config + .row.form-group + .col-sm-3(style="text-align:right") + label(style="font-weight:normal") Name: + .col-sm-9 + input.form-control(type='text', name='name', placeholder='A short name', autofocus='', value=test.name) + .row.form-group + .col-sm-3(style="text-align:right") + label(style="font-weight:normal") Description: + .col-sm-9 + input.form-control(type='text', name='description', placeholder='A longer description', value=test.description) + + .row.form-group + .col-sm-3(style="text-align:right") + label(style="font-weight:normal") Randomisation type: + .col-sm-9 + select#selRandomisation.form-control.selectpicker(name='randomisationType',title='Select randomisation type') + option(selected='', disabled='') Select randomisation type + option(value='perSession', selected=test.randomisationType==='perSession') Per Session + option(value='perUser', selected=test.randomisationType==='perUser') Per User + option(value='perPractice', selected=test.randomisationType==='perPractice') Per Practice + .row.form-group + .col-sm-3(style="text-align:right") + label(style="font-weight:normal") Start date: + .col-sm-9 + input.form-control(type='text', name='startDate', placeholder='Start Date (yyyy-mm-dd)', value=test.startDate) + + h3 Conversion sample size calculator + .row.form-group + .col-sm-3(style="text-align:right") + label(for="day",style="font-weight:normal") Conversion metric: + .col-sm-9 + select#selConversionMetric.form-control.selectpicker(name='conversionMetricId',title='Select conversion metric') + option(selected='', disabled='') Select Conversion Metric + each conversionMetric in conversionMetrics + option(value=conversionMetric.id, selected=test.conversionMetricId===conversionMetric.id)= conversionMetric.description + .row.form-group + .col-sm-3(style="text-align:right") + label(for="day",style="font-weight:normal") Baseline conversion rate: + .col-sm-9 + input#conversionRateHidden(type='hidden', name='conversionMetricBaselineValue',value=test.conversionMetricBaselineValue) + label#conversionRate=test.conversionMetricBaselineValue + .row.form-group + .col-sm-3(style="text-align:right") + label(for="day",style="font-weight:normal") Minimum detectable effect (absolute): + .col-sm-9 + input#minEffect(name='conversionMinDetectableEffect',value=test.conversionMinDetectableEffect || 5, type='text', size="3", style="text-align: center") + label  % + .row.form-group + .col-sm-3(style="text-align:right") + label(for="day",style="font-weight:normal") Statistical power 1-β: + .col-sm-9 + input#power(name='conversionPower',value=test.conversionPower || 80, type='text', size="3", style="text-align: center") + label  % - Percent of the time the minimum effect size will be detected, assuming it exists + .row.form-group + .col-sm-3(style="text-align:right") + label(for="day",style="font-weight:normal") Significance level α: + .col-sm-9 + input#alpha(name='conversionAlpha',value=test.conversionAlpha || 5, type='text', size="3", style="text-align: center") + label  % - Percent of the time a difference will be detected, assuming one does NOT exist + .row.form-group + .col-sm-3(style="text-align:right") + label(for="day",style="font-weight:normal") Sample size per variation: + .col-sm-9 + label#sampleSize + input#hiddenSampleSize(type='hidden',name='conversionSampleSize',value=test.conversionSampleSize) + button.btn.btn-purple.btn-block.history(type='submit') + = submitText + | + span.fa.fa-arrow-circle-right +script(src='https://code.jquery.com/jquery-1.12.4.min.js', integrity='sha256-ZosEbRLbNQzLpnKIkEdrPv7lOy9C27hHQ+Xp8a4MxAQ=', crossorigin='anonymous') +script. + window.jQuery || document.write('