From 70f151178c1cbc87cfebff24ca8852f022029943 Mon Sep 17 00:00:00 2001 From: David Kaneda Date: Mon, 11 Aug 2014 00:45:34 -0700 Subject: [PATCH 01/28] Elasticsearch WIP MongoDB search is pretty meh. This branch is the first stab at leveraging Elasticsearch. More about Elasticsearch here: http://www.elasticsearch.org Experiment adds `query` and `search` params to Entry#findByParams (supported in both templates and REST API). `query` defers to Elasticsearch's "simple query string" (better for filtering/templates), while `search` uses a "fuzzy" query with some simple defaults (better for autocomplete). Ended up using query for the typeahead anyway (adding a * to the end by default). Add Entry#fetchWithParams and typeahead.js to frontend, and a quick demo to the dashboard. Issues: * Currently setting query or search will override and ignore all other params sent in. If people like this approach, we should port the function altogether to rely on Elasticsearch. (would be pretty straight-forward) --- app.json | 2 +- bower.json | 3 +- .../controllers/buckets_controller.coffee | 4 +- client/source/helpers.coffee | 2 +- client/source/models/entries.coffee | 3 + client/source/templates/buckets/dashboard.hbs | 3 + client/source/templates/entries/row.hbs | 2 +- .../source/templates/entries/suggestion.hbs | 1 + client/source/views/buckets/dashboard.coffee | 42 ++++++++++ client/source/views/entries/browser.coffee | 2 +- client/style/index.styl | 45 +++++++++- .../20140810052927866_sync_with_es.coffee | 22 +++++ package.json | 1 + server/config.coffee | 1 + server/models/entry.coffee | 59 +++++++++++-- test/server/models/bucket.coffee | 25 +++--- test/server/models/entry.coffee | 82 +++++++++++++++---- test/server/routes/api/buckets.coffee | 5 +- 18 files changed, 262 insertions(+), 42 deletions(-) create mode 100644 client/source/templates/entries/suggestion.hbs create mode 100644 migrations/20140810052927866_sync_with_es.coffee diff --git a/app.json b/app.json index 8018ea7..e19aff8 100644 --- a/app.json +++ b/app.json @@ -15,5 +15,5 @@ "BUILDPACK_URL": "https://github.com/mbuchetics/heroku-buildpack-nodejs-grunt.git", "NODE_ENV": "production" }, - "addons": ["mongohq", "mandrill"] + "addons": ["mongohq", "mandrill", "bonsai"] } diff --git a/bower.json b/bower.json index a254b0e..c0de0c6 100644 --- a/bower.json +++ b/bower.json @@ -27,7 +27,8 @@ "fastclick": "~1.0.2", "gsap": "~1.12.1", "seedrandom": "~2.3.6", - "Sortable": "~0.3.0" + "Sortable": "~0.3.0", + "typeahead.js": "~0.10.5" }, "exportsOverride": { "jquery.formParams": { diff --git a/client/source/controllers/buckets_controller.coffee b/client/source/controllers/buckets_controller.coffee index 86879b9..46b6a2f 100644 --- a/client/source/controllers/buckets_controller.coffee +++ b/client/source/controllers/buckets_controller.coffee @@ -55,7 +55,7 @@ module.exports = class BucketsController extends Controller compose: (options) -> @entries = new Entries - @entries.fetch( data: {bucket: bucket.get('slug'), status: ''}, processData: yes ).done => + @entries.fetchWithParams( bucket: bucket.get('slug'), status: '' ).done => @view = new EntriesBrowser collection: @entries bucket: bucket @@ -89,7 +89,7 @@ module.exports = class BucketsController extends Controller @listenToOnce bucket, 'sync', (bucket, data) => mediator.buckets.fetch(reset: yes) - if data.slug + if data?.slug toastr.success 'Bucket saved' @redirectTo 'buckets#browse', slug: data.slug else diff --git a/client/source/helpers.coffee b/client/source/helpers.coffee index e4bea9f..02b0003 100644 --- a/client/source/helpers.coffee +++ b/client/source/helpers.coffee @@ -37,7 +37,7 @@ Handlebars.registerHelper 'timeAgo', (dateTime) -> expanded = Handlebars.helpers.simpleDateTime dateTime new Handlebars.SafeString """ - + #{moment(dateTime).fromNow()} """ diff --git a/client/source/models/entries.coffee b/client/source/models/entries.coffee index 39e0e42..9c86a79 100644 --- a/client/source/models/entries.coffee +++ b/client/source/models/entries.coffee @@ -9,5 +9,8 @@ module.exports = class Entries extends Collection @url += "?#{$.param(bucketId: bucketId)}" @fetch() + fetchWithParams: (params={}) -> + @fetch({data: params}, processData: yes) + comparator: (entry) -> - new Date(entry.get('publishDate')).getTime() diff --git a/client/source/templates/buckets/dashboard.hbs b/client/source/templates/buckets/dashboard.hbs index ba191a3..9ea380e 100644 --- a/client/source/templates/buckets/dashboard.hbs +++ b/client/source/templates/buckets/dashboard.hbs @@ -3,5 +3,8 @@

Buckets is an open-source CMS, built in Node.js, which is being actively developed by the community at Assembly.

+ + +
diff --git a/client/source/templates/entries/row.hbs b/client/source/templates/entries/row.hbs index 31881f4..853114e 100644 --- a/client/source/templates/entries/row.hbs +++ b/client/source/templates/entries/row.hbs @@ -1,4 +1,4 @@

{{title}}

-

{{status}} {{author.name}} · {{timeAgo publishDate}}

+

{{status}} {{author.name}} · {{timeAgo publishDate}}

diff --git a/client/source/templates/entries/suggestion.hbs b/client/source/templates/entries/suggestion.hbs new file mode 100644 index 0000000..04ddc09 --- /dev/null +++ b/client/source/templates/entries/suggestion.hbs @@ -0,0 +1 @@ +{{icon bucket.icon}} {{title}}
{{timeAgo publishDate}} diff --git a/client/source/views/buckets/dashboard.coffee b/client/source/views/buckets/dashboard.coffee index 650b3ad..4b7a1a9 100644 --- a/client/source/views/buckets/dashboard.coffee +++ b/client/source/views/buckets/dashboard.coffee @@ -1,6 +1,48 @@ +_ = require 'underscore' +Chaplin = require 'chaplin' PageView = require 'views/base/page' +mediator = require 'mediator' + +suggestionTpl = require 'templates/entries/suggestion' tpl = require 'templates/buckets/dashboard' module.exports = class DashboardView extends PageView template: tpl + + events: + 'typeahead:selected': 'selectItem' + + render: -> + super + + engine = new Bloodhound + datumTokenizer: Bloodhound.tokenizers.obj.whitespace('value') + queryTokenizer: Bloodhound.tokenizers.whitespace + prefetch: '/api/entries' + remote: '/api/entries/?query=%QUERY*' + + engine.initialize() + engine.clearRemoteCache(); + + @$search = @$('[name="search"]').typeahead + highlight: yes + , + name: 'entries', + displayKey: 'title' + templates: { + empty: """ +
+ No matching entries. +
+ """ + suggestion: suggestionTpl + } + source: engine.ttAdapter() + + _.defer => @$search.focus() + + selectItem: (e, item) -> + Chaplin.utils.redirectTo 'buckets#browse', + slug: item.bucket.slug + entryID: item._id diff --git a/client/source/views/entries/browser.coffee b/client/source/views/entries/browser.coffee index a9940c1..6ca231f 100644 --- a/client/source/views/entries/browser.coffee +++ b/client/source/views/entries/browser.coffee @@ -57,7 +57,7 @@ module.exports = class EntriesBrowser extends PageView @renderDetailView() modelSaved: (entry, newData) => - if newData._id + if newData?._id toastr.success "You saved “#{entry.get('title')}”" @collection.add @model else diff --git a/client/style/index.styl b/client/style/index.styl index 2f19323..eb170cf 100644 --- a/client/style/index.styl +++ b/client/style/index.styl @@ -48,10 +48,9 @@ $sans = RobotoDraft, 'Avenir Next', Avenir, 'Helvetica Neue', Helvetica, 'Segoe body background-color #fcfcfc - display table width 100% + overflow-x hidden display flex - min-height 100vh flex-direction column #bkts-content @@ -190,3 +189,45 @@ input[type="text"].form-control, textarea.form-control .pageBreadcrumb margin 10px 0 background-color #f1f1f1 + + +.tt-query + width 300px + outline none + +.tt-hint + color: #999 + +.tt-highlight + background-color rgba(lighten($yellow, 30), .5) + +.tt-dropdown-menu + margin-top 12px + padding 2px 0 + background-color #fff + border 1px solid #ccc + border 1px solid rgba(0, 0, 0, 0.2) + border-radius 2px + width 100% + color #555 + + .icon + vertical-align middle + display inline-block + font-size 24px + +.tt-suggestion + padding: 3px 20px + font-size: 18px + line-height: 24px + +.tt-suggestion.tt-cursor + color #222 + background-color lighten($primary, 30) + cursor pointer + +.tt-suggestion p + margin: 0 + +.tt-empty + padding: 3px 20px diff --git a/migrations/20140810052927866_sync_with_es.coffee b/migrations/20140810052927866_sync_with_es.coffee new file mode 100644 index 0000000..f739aca --- /dev/null +++ b/migrations/20140810052927866_sync_with_es.coffee @@ -0,0 +1,22 @@ +Bucket = require '../server/models/bucket' +Entry = require '../server/models/entry' + +module.exports = + requiresDowntime: no # true or false + + up: (done) -> + Entry.update {content: null}, {content: {}}, {multi: yes}, (e, entries) -> + throw e if e + + stream = Entry.synchronize() + count = 0 + + stream.on 'data', (err, doc) -> count++ + stream.on 'close', -> + console.log('Indexed ' + count + ' entries into ElasticSearch') + done() + + stream.on 'error', (err) -> console.log err + + down: (done) -> + done() diff --git a/package.json b/package.json index 50fe12c..22ace15 100644 --- a/package.json +++ b/package.json @@ -86,6 +86,7 @@ "chai": "*", "grunt-contrib-testem": "*", "mocha": "*", + "mongoosastic": "^0.2.6", "rosie": "^0.3.1", "sinon": "*", "supertest": "*", diff --git a/server/config.coffee b/server/config.coffee index f3f39b3..4760670 100644 --- a/server/config.coffee +++ b/server/config.coffee @@ -14,6 +14,7 @@ config = pluginsPath: "#{__dirname}/../node_modules/" catchAll: yes db: "mongodb://localhost/buckets_#{env}" + elastic_search_index: "buckets_entries_#{env}" production: smtp: service: 'Mandrill' diff --git a/server/models/entry.coffee b/server/models/entry.coffee index 525de5b..6ffdd06 100755 --- a/server/models/entry.coffee +++ b/server/models/entry.coffee @@ -1,9 +1,11 @@ mongoose = require 'mongoose' +mongoosastic = require 'mongoosastic' _ = require 'underscore' chrono = require 'chrono-node' async = require 'async' getSlug = require 'speakingurl' +config = require '../config' db = require '../lib/database' # Add a parser to Chrono to understand "now" @@ -38,9 +40,15 @@ entrySchema = new Schema title: type: String required: yes - description: String + es_boost: 3.0 + es_indexed: yes + description: + type: String + es_boost: 2.0 + es_indexed: yes slug: type: String + es_indexed: yes status: type: String enum: ['draft', 'live', 'pending', 'rejected'] @@ -48,11 +56,15 @@ entrySchema = new Schema default: 'draft' lastModified: type: Date + es_type: 'date' + default: Date.now publishDate: type: Date + es_type: 'date' default: Date.now createdDate: type: Date + es_type: 'date' default: Date.now author: type: Schema.Types.ObjectId @@ -62,10 +74,16 @@ entrySchema = new Schema type: Schema.Types.ObjectId ref: 'Bucket' required: yes - keywords: [ - type: String - ] - content: {} + keywords: + type: [String] + es_indexed: yes + es_boost: 2.0 + content: + type: {} + default: {} + es_type : 'object' + es_indexed: yes + es_boost: 2.0 entrySchema.pre 'save', (next) -> @lastModified = Date.now() @@ -74,13 +92,15 @@ entrySchema.pre 'save', (next) -> entrySchema.pre 'validate', (next) -> @slug ?= getSlug @title - @model('Bucket').findOne _id: @bucket, (err, bkt) => + mongoose.model('Bucket').findOne _id: @bucket, (err, bkt) => @invalidate 'bucket', 'Must belong to a bucket' unless bkt for field in bkt?.fields or [] - if field.settings?.required and !@content[field.slug] + if field.settings?.required and !@content[field.slug]? @invalidate field.slug, 'required' + else if @content[field.slug] is '' + delete @content[field.slug] next() @@ -107,6 +127,8 @@ entrySchema.statics.findByParams = (params, callback) -> sort: '-publishDate' find: '' slug: null + search: null + query: null searchQuery = {} @@ -132,6 +154,27 @@ entrySchema.statics.findByParams = (params, callback) -> if settings.status?.length > 0 searchQuery.status = settings.status + if settings.search? + # Some default manipulation to make the search fuzzy + # and case insensitive by default + return @search query: + fuzzy: + _all: + value: "#{settings.search.toLowerCase()}*" + fuzziness: 5 + prefix_length: 2 + , {hydrate: yes, hydrateOptions: populate: 'bucket author'}, (err, elasticEntries) -> + throw err if err + callback null, elasticEntries.hits + + if settings.query? + return @search query: + simple_query_string: + query: settings.query + , {hydrate: yes, hydrateOptions: populate: 'bucket author'}, (err, elasticEntries) -> + throw err if err + callback null, elasticEntries.hits + if settings.since or settings.until searchQuery.publishDate = {} searchQuery.publishDate.$gt = new Date(chrono.parseDate settings.since) if settings.since @@ -152,4 +195,6 @@ entrySchema.statics.findByParams = (params, callback) -> entrySchema.set 'toJSON', virtuals: true +entrySchema.plugin mongoosastic, index: config.elastic_search_index + module.exports = db.model 'Entry', entrySchema diff --git a/test/server/models/bucket.coffee b/test/server/models/bucket.coffee index 9b78836..56f0e58 100644 --- a/test/server/models/bucket.coffee +++ b/test/server/models/bucket.coffee @@ -1,23 +1,28 @@ User = require('../../../server/models/user') Bucket = require('../../../server/models/bucket') db = require('../../../server/lib/database') + {assert} = require('chai') describe 'Bucket', -> + before (done) -> + db.connection.db.dropDatabase done afterEach (done) -> - for _, c of db.connection.collections - c.remove(->) - done() + db.connection.db.dropDatabase done describe '#getMembers', -> it 'returns members', (done) -> Bucket.create { name: 'Images', slug: 'images' }, (e, bucket) -> - u = new User({ name: 'Bucketer', email: 'hello@buckets.io', password: 'S3cr3ts' }) + throw e if e + User.create { name: 'Bucketer', email: 'hello@buckets.io', password: 'S3cr3ts' }, (e, u) -> + + u.upsertRole 'contributor', bucket, (e, user) -> + throw e if e - u.upsertRole 'contributor', bucket, (e, user) -> - bucket.getMembers (e, users) -> - assert.isArray(users) - assert.lengthOf(users, 1) - assert.equal(users[0].id, u.id) + bucket.getMembers (e, users) -> + throw e if e + assert.isArray(users) + assert.lengthOf(users, 1) + assert.equal(users[0].id, u.id) - done() + done() diff --git a/test/server/models/entry.coffee b/test/server/models/entry.coffee index 8af62d1..fbe7f60 100644 --- a/test/server/models/entry.coffee +++ b/test/server/models/entry.coffee @@ -1,4 +1,5 @@ db = require '../../../server/lib/database' +config = require '../../../server/config' Entry = require '../../../server/models/entry' Bucket = require '../../../server/models/bucket' @@ -6,10 +7,15 @@ User = require '../../../server/models/user' {expect} = require 'chai' +MongoosasticConfig = require 'mongoosastic/test/config' + describe 'Entry', -> user = null + after (done) -> + db.connection.db.dropDatabase done + before (done) -> User.create name: 'Bucketer' @@ -20,14 +26,6 @@ describe 'Entry', -> user = u done() - beforeEach (done) -> - for _, c of db.connection.collections - c.remove(->) - done() - - afterEach (done) -> - db.connection.db.dropDatabase done - describe 'Validation', -> it 'requires a bucket', (done) -> @@ -49,9 +47,13 @@ describe 'Entry', -> beforeEach (done) -> Bucket.create {name: 'Articles', slug: 'articles'}, (e, bucket) -> + throw e if e bucketId = bucket._id done() + afterEach (done) -> + Bucket.remove {}, done + it 'parses dates from strings', (done) -> Entry.create title: 'New Entry' @@ -72,7 +74,7 @@ describe 'Entry', -> describe '#findByParams', -> # Set up a bunch of entries to filter/search - beforeEach (done) -> + before (done) -> Bucket.create [ name: 'Articles' slug: 'articles' @@ -80,6 +82,7 @@ describe 'Entry', -> name: 'Photos' slug: 'photos' ], (e, articleBucket, photoBucket) -> + throw e if e Entry.create [ title: 'Test Article' bucket: articleBucket._id @@ -87,12 +90,14 @@ describe 'Entry', -> status: 'live' publishDate: '2 days ago' , - title: 'Test Photo' + title: 'Test Photoset' bucket: photoBucket._id author: user._id status: 'live' - ], -> - done() + ], done + + after (done) -> + Bucket.remove {}, -> Entry.remove {}, done it 'filters by bucket slug (empty)', (done) -> Entry.findByParams bucket: '', (e, entries) -> @@ -102,7 +107,7 @@ describe 'Entry', -> it 'filters by bucket slug', (done) -> Entry.findByParams bucket: 'photos', (e, entries) -> expect(entries).to.have.length 1 - expect(entries?[0]?.title).to.equal 'Test Photo' + expect(entries?[0]?.title).to.equal 'Test Photoset' done() it 'filters by multiple bucket slugs', (done) -> @@ -113,7 +118,7 @@ describe 'Entry', -> it 'filters with `since`', (done) -> Entry.findByParams since: 'yesterday', (e, entries) -> expect(entries).to.have.length 1 - expect(entries?[0]?.title).to.equal 'Test Photo' + expect(entries?[0]?.title).to.equal 'Test Photoset' done() it 'filters with `until`', (done) -> @@ -121,3 +126,52 @@ describe 'Entry', -> expect(entries).to.have.length 1 expect(entries?[0]?.title).to.equal 'Test Article' done() + + describe 'Search with #findByParams', -> + # We just create these once since we're only testing search + # (and we're using a delay to guarantee they're indexed) + before (done) -> + console.log config.elastic_search_index + MongoosasticConfig.deleteIndexIfExists [config.elastic_search_index], -> + Entry.synchronize().on 'close', -> + Bucket.create [ + name: 'Articles' + slug: 'articles' + , + name: 'Photos' + slug: 'photos' + ], (e, articleBucket, photoBucket) -> + throw e if e + + Entry.create [ + title: 'Test Article' + bucket: articleBucket._id + author: user._id + status: 'live' + publishDate: '2 days ago' + , + title: 'Test Photoset' + bucket: photoBucket._id + author: user._id + status: 'live' + keywords: ['summer'] + ], -> setTimeout done, 1100 # Gross, but ensures Elasticsearch index + + it 'performs a fuzzy search with `search`', (done) -> + Entry.findByParams search: 'photoste', (e, entries) -> + expect(entries).to.have.length 1 + expect(entries?[0]?.title).to.equal 'Test Photoset' + done() + + describe 'uses Elasticsearch simple query string with `query`', -> + it 'can negate a term', (done) -> + Entry.findByParams query: '-article', (e, entries) -> + expect(entries).to.have.length 1 + expect(entries?[0]?.title).to.equal 'Test Photoset' + done() + + it 'searches tags', (done) -> + Entry.findByParams query: 'summer', (e, entries) -> + expect(entries).to.have.length 1 + expect(entries?[0]?.title).to.equal 'Test Photoset' + done() diff --git a/test/server/routes/api/buckets.coffee b/test/server/routes/api/buckets.coffee index b05727e..0aa2ec9 100644 --- a/test/server/routes/api/buckets.coffee +++ b/test/server/routes/api/buckets.coffee @@ -7,13 +7,14 @@ db = require('../../../../server/lib/database') describe 'Buckets routes', -> before (done) -> - db.connection.on 'open', -> - db.connection.db.dropDatabase done + db.connection.db.dropDatabase done afterEach (done) -> db.connection.db.dropDatabase done describe 'GET /buckets/:bucket/members', -> + # STUB: Need to add auth + #it 'returns the members of a given bucket', (done) -> #Bucket.create { name: 'Products', slug: 'products', singular: 'product' }, (e, bucket) -> #u = new User({ name: 'Bucketer', email: 'hello@buckets.io', password: 'S3cr3ts' }) From a6b9dcad70a64e551990ff69d5622bfb03c0531b Mon Sep 17 00:00:00 2001 From: David Kaneda Date: Mon, 11 Aug 2014 00:53:40 -0700 Subject: [PATCH 02/28] Add elasticsearch to Travis --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 83caf4b..0bfc265 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,8 @@ language: node_js -services: mongodb +services: + - mongodb + - elasticsearch before_install: - npm install -g grunt-cli From 44c7bbb9623fe411698f650c397deeb0aa2d7846 Mon Sep 17 00:00:00 2001 From: David Kaneda Date: Tue, 12 Aug 2014 22:19:57 -0700 Subject: [PATCH 03/28] Remove entry via doc so middleware is called --- server/routes/api/entries.coffee | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/routes/api/entries.coffee b/server/routes/api/entries.coffee index e6e076a..b4b6117 100644 --- a/server/routes/api/entries.coffee +++ b/server/routes/api/entries.coffee @@ -51,12 +51,12 @@ app.route('/entries/:entryID') res.send 200, entry .delete (req, res) -> - delete req.body._id - Entry.remove _id: req.params.entryID, (err) -> - if err - res.send e: err, 400 - else - res.send {}, 200 + Entry.findOne _id: req.params.entryID, (err, entry) -> + entry.remove (err) -> + if err + res.send e: err, 400 + else + res.send {}, 200 app.route('/entries/keywords') .get (req, res) -> From f85aebb663612e34bfb47d25200e470de31b2213 Mon Sep 17 00:00:00 2001 From: David Kaneda Date: Tue, 12 Aug 2014 22:20:55 -0700 Subject: [PATCH 04/28] Try waiting for a the indexed callback before running test Still uses a timeout though :/ just want to try on Travis --- test/server/models/entry.coffee | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/test/server/models/entry.coffee b/test/server/models/entry.coffee index fbe7f60..a7f6108 100644 --- a/test/server/models/entry.coffee +++ b/test/server/models/entry.coffee @@ -131,7 +131,6 @@ describe 'Entry', -> # We just create these once since we're only testing search # (and we're using a delay to guarantee they're indexed) before (done) -> - console.log config.elastic_search_index MongoosasticConfig.deleteIndexIfExists [config.elastic_search_index], -> Entry.synchronize().on 'close', -> Bucket.create [ @@ -155,7 +154,25 @@ describe 'Entry', -> author: user._id status: 'live' keywords: ['summer'] - ], -> setTimeout done, 1100 # Gross, but ensures Elasticsearch index + ], (e, entry1, entry2) -> + throw e if e + + e1indexed = no + e2indexed = no + + entry1.on 'es-indexed', -> e1indexed = yes + entry2.on 'es-indexed', -> e2indexed = yes + + checkIndexed = -> + if e1indexed and e2indexed + # Even after being reported as indexed, + # docs may not be searchable right away + + # The testing gods are crying, I know + setTimeout done, 1100 + clearInterval interval + + interval = setInterval checkIndexed, 10 it 'performs a fuzzy search with `search`', (done) -> Entry.findByParams search: 'photoste', (e, entries) -> From e43f070a422984fc430292dbaf17a6842bcd71b7 Mon Sep 17 00:00:00 2001 From: David Kaneda Date: Tue, 12 Aug 2014 22:36:10 -0700 Subject: [PATCH 05/28] Set custom timeout for Search test 'before' --- test/server/models/entry.coffee | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/server/models/entry.coffee b/test/server/models/entry.coffee index a7f6108..4a8cce7 100644 --- a/test/server/models/entry.coffee +++ b/test/server/models/entry.coffee @@ -127,10 +127,13 @@ describe 'Entry', -> expect(entries?[0]?.title).to.equal 'Test Article' done() - describe 'Search with #findByParams', -> + describe 'Search', -> # We just create these once since we're only testing search # (and we're using a delay to guarantee they're indexed) + # Yeah, it's pretty gross. before (done) -> + @timeout 5000 + MongoosasticConfig.deleteIndexIfExists [config.elastic_search_index], -> Entry.synchronize().on 'close', -> Bucket.create [ From 815a341e565be88c5e0d204959bbf5bb74d4dac0 Mon Sep 17 00:00:00 2001 From: David Kaneda Date: Tue, 12 Aug 2014 22:47:29 -0700 Subject: [PATCH 06/28] Nix the synchronize call --- test/server/models/entry.coffee | 97 ++++++++++++++++----------------- 1 file changed, 47 insertions(+), 50 deletions(-) diff --git a/test/server/models/entry.coffee b/test/server/models/entry.coffee index 4a8cce7..4ea6ebd 100644 --- a/test/server/models/entry.coffee +++ b/test/server/models/entry.coffee @@ -135,47 +135,45 @@ describe 'Entry', -> @timeout 5000 MongoosasticConfig.deleteIndexIfExists [config.elastic_search_index], -> - Entry.synchronize().on 'close', -> - Bucket.create [ - name: 'Articles' - slug: 'articles' + Bucket.create [ + name: 'Articles' + slug: 'articles' + , + name: 'Photos' + slug: 'photos' + ], (e, articleBucket, photoBucket) -> + throw e if e + + Entry.create [ + title: 'Test Article' + bucket: articleBucket._id + author: user._id + status: 'live' + publishDate: '2 days ago' , - name: 'Photos' - slug: 'photos' - ], (e, articleBucket, photoBucket) -> + title: 'Test Photoset' + bucket: photoBucket._id + author: user._id + status: 'live' + keywords: ['summer'] + ], (e, entry1, entry2) -> throw e if e - Entry.create [ - title: 'Test Article' - bucket: articleBucket._id - author: user._id - status: 'live' - publishDate: '2 days ago' - , - title: 'Test Photoset' - bucket: photoBucket._id - author: user._id - status: 'live' - keywords: ['summer'] - ], (e, entry1, entry2) -> - throw e if e - - e1indexed = no - e2indexed = no - - entry1.on 'es-indexed', -> e1indexed = yes - entry2.on 'es-indexed', -> e2indexed = yes - - checkIndexed = -> - if e1indexed and e2indexed - # Even after being reported as indexed, - # docs may not be searchable right away - - # The testing gods are crying, I know - setTimeout done, 1100 - clearInterval interval - - interval = setInterval checkIndexed, 10 + e1indexed = no + e2indexed = no + + entry1.on 'es-indexed', -> e1indexed = yes + entry2.on 'es-indexed', -> e2indexed = yes + + checkIndexed = -> + if e1indexed and e2indexed + # Even after being reported as indexed, + # docs may not be searchable right away + # The testing gods are crying, I know + setTimeout done, 1100 + clearInterval interval + + interval = setInterval checkIndexed, 10 it 'performs a fuzzy search with `search`', (done) -> Entry.findByParams search: 'photoste', (e, entries) -> @@ -183,15 +181,14 @@ describe 'Entry', -> expect(entries?[0]?.title).to.equal 'Test Photoset' done() - describe 'uses Elasticsearch simple query string with `query`', -> - it 'can negate a term', (done) -> - Entry.findByParams query: '-article', (e, entries) -> - expect(entries).to.have.length 1 - expect(entries?[0]?.title).to.equal 'Test Photoset' - done() - - it 'searches tags', (done) -> - Entry.findByParams query: 'summer', (e, entries) -> - expect(entries).to.have.length 1 - expect(entries?[0]?.title).to.equal 'Test Photoset' - done() + it 'can negate a term', (done) -> + Entry.findByParams query: '-article', (e, entries) -> + expect(entries).to.have.length 1 + expect(entries?[0]?.title).to.equal 'Test Photoset' + done() + + it 'searches tags', (done) -> + Entry.findByParams query: 'summer', (e, entries) -> + expect(entries).to.have.length 1 + expect(entries?[0]?.title).to.equal 'Test Photoset' + done() From 8f6e6b12d16f3c2e6aa22c9bc07649fa2e6752f5 Mon Sep 17 00:00:00 2001 From: David Kaneda Date: Fri, 15 Aug 2014 22:38:36 -0700 Subject: [PATCH 07/28] Try a timeout on Entry remove --- test/server/models/entry.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/server/models/entry.coffee b/test/server/models/entry.coffee index 4ea6ebd..907915d 100644 --- a/test/server/models/entry.coffee +++ b/test/server/models/entry.coffee @@ -97,7 +97,7 @@ describe 'Entry', -> ], done after (done) -> - Bucket.remove {}, -> Entry.remove {}, done + Bucket.remove {}, -> Entry.remove {}, -> _.setTimeout done, 200 it 'filters by bucket slug (empty)', (done) -> Entry.findByParams bucket: '', (e, entries) -> From 88b891e809ad566c0eade54d84a1660014fbb623 Mon Sep 17 00:00:00 2001 From: David Kaneda Date: Fri, 15 Aug 2014 22:42:35 -0700 Subject: [PATCH 08/28] Oi --- test/server/models/entry.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/server/models/entry.coffee b/test/server/models/entry.coffee index 907915d..50f2cfa 100644 --- a/test/server/models/entry.coffee +++ b/test/server/models/entry.coffee @@ -97,7 +97,7 @@ describe 'Entry', -> ], done after (done) -> - Bucket.remove {}, -> Entry.remove {}, -> _.setTimeout done, 200 + Bucket.remove {}, -> Entry.remove {}, -> setTimeout done, 200 it 'filters by bucket slug (empty)', (done) -> Entry.findByParams bucket: '', (e, entries) -> From 40d54c99e4fafd7286df3e86728b447b93c71e93 Mon Sep 17 00:00:00 2001 From: David Kaneda Date: Fri, 15 Aug 2014 22:54:35 -0700 Subject: [PATCH 09/28] Last try --- test/server/models/entry.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/server/models/entry.coffee b/test/server/models/entry.coffee index 50f2cfa..e47f710 100644 --- a/test/server/models/entry.coffee +++ b/test/server/models/entry.coffee @@ -97,7 +97,7 @@ describe 'Entry', -> ], done after (done) -> - Bucket.remove {}, -> Entry.remove {}, -> setTimeout done, 200 + Bucket.remove {}, -> Entry.remove {}, -> setTimeout done, 1200 it 'filters by bucket slug (empty)', (done) -> Entry.findByParams bucket: '', (e, entries) -> From 4ee36ed2ae6d0db46d8360f85ac2d356ebc47a2e Mon Sep 17 00:00:00 2001 From: David Kaneda Date: Sat, 16 Aug 2014 12:30:25 -0700 Subject: [PATCH 10/28] Fix User roles test --- test/server/models/user.coffee | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/server/models/user.coffee b/test/server/models/user.coffee index f7c132a..101c668 100644 --- a/test/server/models/user.coffee +++ b/test/server/models/user.coffee @@ -67,15 +67,15 @@ describe 'User', -> assert.equal(user.roles[0].name, 'editor') done() - describe 'Updates existing roles', -> - beforeEach (done) -> - user.upsertRole('editor', bucket, done) - - it 'updates the role for a given resource', (done) -> - user.upsertRole 'contributor', bucket, (e, u) -> - assert.lengthOf(user.roles, 1) - assert.equal(user.roles[0].name, 'contributor') - done() + describe 'Updates existing roles', -> + beforeEach (done) -> + user.upsertRole('editor', bucket, done) + + it 'updates the role for a given resource', (done) -> + user.upsertRole 'contributor', bucket, (e, u) -> + assert.lengthOf(user.roles, 1) + assert.equal(user.roles[0].name, 'contributor') + done() describe '#removeRole', -> beforeEach (done) -> From 6b135e458eb2d82c4bd2cc5112a828ae48a5203d Mon Sep 17 00:00:00 2001 From: David Kaneda Date: Sat, 16 Aug 2014 12:31:56 -0700 Subject: [PATCH 11/28] Fix for Bucket#remove -> Entry#remove callback --- server/models/bucket.coffee | 41 ++++++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/server/models/bucket.coffee b/server/models/bucket.coffee index fbdc866..ee688bb 100644 --- a/server/models/bucket.coffee +++ b/server/models/bucket.coffee @@ -8,7 +8,6 @@ db = require '../lib/database' fieldSchema = new mongoose.Schema name: type: String - unique: no required: yes slug: type: String @@ -25,7 +24,19 @@ fieldSchema = new mongoose.Schema default: new Date fieldSchema.path('slug').validate (val) -> - val not in ['title', 'description', 'slug', 'status', 'lastModified', 'publishDate', 'createdAt', 'author', 'bucket', 'keywords', 'content'] + val not in [ + 'title' + 'description' + 'slug' + 'status' + 'lastModified' + 'publishDate' + 'createdAt' + 'author' + 'bucket' + 'keywords' + 'content' + ] , 'Sorry, that’s a reserved field slug.' bucketSchema = new mongoose.Schema @@ -47,12 +58,30 @@ bucketSchema = new mongoose.Schema required: yes icon: type: String - enum: ['edit', 'photos', 'calendar', 'movie', 'music-note', 'map-pin', 'quote', 'artboard', 'contacts-1'] + enum: [ + 'edit' + 'photos' + 'calendar' + 'movie' + 'music-note' + 'map-pin' + 'quote' + 'artboard' + 'contacts-1' + ] default: 'edit' required: yes color: type: String - enum: ['teal', 'purple', 'red', 'yellow', 'blue', 'orange', 'green'] + enum: [ + 'teal' + 'purple' + 'red' + 'yellow' + 'blue' + 'orange' + 'green' + ] default: 'teal' required: yes publishToSite: @@ -84,7 +113,9 @@ bucketSchema.path('urlPattern').validate (value) -> bucketSchema.plugin uniqueValidator, message: '“{VALUE}” is already taken.' bucketSchema.post 'remove', -> - @model('Entry').remove({bucket: @_id}).exec() + @model('Entry').find bucket: @_id, (e, doc) -> + throw e if e + doc.remove().exec() bucketSchema.methods.getMembers = (callback) -> @model('User').find From f3ea73177b4aecd370cc3dd71c01f4139a25a053 Mon Sep 17 00:00:00 2001 From: David Kaneda Date: Sat, 16 Aug 2014 12:35:16 -0700 Subject: [PATCH 12/28] Force rebuild From 20f25f4ce4d96d5cf3b3c46d5c8bb353224c313d Mon Sep 17 00:00:00 2001 From: David Kaneda Date: Sat, 16 Aug 2014 12:38:36 -0700 Subject: [PATCH 13/28] Move mongoosastic to production deps --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 22ace15..12120a1 100644 --- a/package.json +++ b/package.json @@ -68,6 +68,7 @@ "hbsfy": "^1.3.2", "inflection": "^1.3.6", "moment": "^2.7.0", + "mongoosastic": "^0.2.6", "mongoose": "^3.8.14", "mongoose-regexp": "0.0.1", "mongoose-unique-validator": "^0.3.0", @@ -86,7 +87,6 @@ "chai": "*", "grunt-contrib-testem": "*", "mocha": "*", - "mongoosastic": "^0.2.6", "rosie": "^0.3.1", "sinon": "*", "supertest": "*", From cd25859e3b5216f9c9242d3a21f616e94f95db03 Mon Sep 17 00:00:00 2001 From: David Kaneda Date: Sat, 16 Aug 2014 14:12:17 -0700 Subject: [PATCH 14/28] Add Bonsai support, change config format --- server/config.coffee | 4 +++- server/models/entry.coffee | 16 +++++++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/server/config.coffee b/server/config.coffee index 4760670..06067c8 100644 --- a/server/config.coffee +++ b/server/config.coffee @@ -14,7 +14,9 @@ config = pluginsPath: "#{__dirname}/../node_modules/" catchAll: yes db: "mongodb://localhost/buckets_#{env}" - elastic_search_index: "buckets_entries_#{env}" + elasticsearch: + url: process.env.BONSAI_URL || 'http://127.0.0.1:9200' + index: "buckets_entries_#{env}" production: smtp: service: 'Mandrill' diff --git a/server/models/entry.coffee b/server/models/entry.coffee index 6ffdd06..dbb4ef1 100755 --- a/server/models/entry.coffee +++ b/server/models/entry.coffee @@ -4,6 +4,7 @@ _ = require 'underscore' chrono = require 'chrono-node' async = require 'async' getSlug = require 'speakingurl' +url = require 'url' config = require '../config' db = require '../lib/database' @@ -163,7 +164,7 @@ entrySchema.statics.findByParams = (params, callback) -> value: "#{settings.search.toLowerCase()}*" fuzziness: 5 prefix_length: 2 - , {hydrate: yes, hydrateOptions: populate: 'bucket author'}, (err, elasticEntries) -> + , (err, elasticEntries) -> throw err if err callback null, elasticEntries.hits @@ -171,7 +172,7 @@ entrySchema.statics.findByParams = (params, callback) -> return @search query: simple_query_string: query: settings.query - , {hydrate: yes, hydrateOptions: populate: 'bucket author'}, (err, elasticEntries) -> + , (err, elasticEntries) -> throw err if err callback null, elasticEntries.hits @@ -195,6 +196,15 @@ entrySchema.statics.findByParams = (params, callback) -> entrySchema.set 'toJSON', virtuals: true -entrySchema.plugin mongoosastic, index: config.elastic_search_index +# Add Elasticsearch via mongoosastic +elasticConnection = url.parse config.elasticsearch.url +entrySchema.plugin mongoosastic, + index: config.elasticsearch.index + host: elasticConnection.hostname + auth: elasticConnection.auth + port: elasticConnection.port + hydrate: yes + hydrateOptions: + populate: 'bucket author' module.exports = db.model 'Entry', entrySchema From e7204b1f4116f65ecc0e2a0dd257c24447ebbe0f Mon Sep 17 00:00:00 2001 From: David Kaneda Date: Sat, 16 Aug 2014 14:51:19 -0700 Subject: [PATCH 15/28] More fixees for tests --- server/models/bucket.coffee | 4 +--- test/server/models/entry.coffee | 10 ++++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/server/models/bucket.coffee b/server/models/bucket.coffee index ee688bb..dbbcffa 100644 --- a/server/models/bucket.coffee +++ b/server/models/bucket.coffee @@ -113,9 +113,7 @@ bucketSchema.path('urlPattern').validate (value) -> bucketSchema.plugin uniqueValidator, message: '“{VALUE}” is already taken.' bucketSchema.post 'remove', -> - @model('Entry').find bucket: @_id, (e, doc) -> - throw e if e - doc.remove().exec() + @model('Entry').find(bucket: @_id).remove().exec() bucketSchema.methods.getMembers = (callback) -> @model('User').find diff --git a/test/server/models/entry.coffee b/test/server/models/entry.coffee index e47f710..c613020 100644 --- a/test/server/models/entry.coffee +++ b/test/server/models/entry.coffee @@ -62,7 +62,7 @@ describe 'Entry', -> author: user._id , (e, entry) -> expected = new Date - expected.setHours(21, 0, 0, 0) + expected.setHours 21, 0, 0, 0 # 9pm expect(expected.toISOString()).equal(entry.publishDate.toISOString()) done() @@ -83,6 +83,7 @@ describe 'Entry', -> slug: 'photos' ], (e, articleBucket, photoBucket) -> throw e if e + Entry.create [ title: 'Test Article' bucket: articleBucket._id @@ -97,7 +98,8 @@ describe 'Entry', -> ], done after (done) -> - Bucket.remove {}, -> Entry.remove {}, -> setTimeout done, 1200 + Bucket.find({}).remove().exec() + setTimeout done, 1200 it 'filters by bucket slug (empty)', (done) -> Entry.findByParams bucket: '', (e, entries) -> @@ -134,7 +136,7 @@ describe 'Entry', -> before (done) -> @timeout 5000 - MongoosasticConfig.deleteIndexIfExists [config.elastic_search_index], -> + MongoosasticConfig.deleteIndexIfExists [config.elasticsearch.index], -> Bucket.create [ name: 'Articles' slug: 'articles' @@ -170,7 +172,7 @@ describe 'Entry', -> # Even after being reported as indexed, # docs may not be searchable right away # The testing gods are crying, I know - setTimeout done, 1100 + setTimeout done, 1200 clearInterval interval interval = setInterval checkIndexed, 10 From 8544c4e48c0b73e003e1ec85b633ef1df31ce375 Mon Sep 17 00:00:00 2001 From: David Kaneda Date: Sat, 16 Aug 2014 15:32:16 -0700 Subject: [PATCH 16/28] Update mongoosastic --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 12120a1..014c620 100644 --- a/package.json +++ b/package.json @@ -68,7 +68,7 @@ "hbsfy": "^1.3.2", "inflection": "^1.3.6", "moment": "^2.7.0", - "mongoosastic": "^0.2.6", + "mongoosastic": "^0.3.0", "mongoose": "^3.8.14", "mongoose-regexp": "0.0.1", "mongoose-unique-validator": "^0.3.0", From d8ca5ecbc7357ae4f90a56e92af28c44eae38e85 Mon Sep 17 00:00:00 2001 From: David Kaneda Date: Sat, 16 Aug 2014 21:28:22 -0700 Subject: [PATCH 17/28] Abstract 'rest' function for tests Start to fill in Buckets route tests a bit --- server/routes/api/buckets.coffee | 2 + test/reset.coffee | 8 ++++ test/server/models/entry.coffee | 15 +++++--- test/server/routes/api/buckets.coffee | 55 ++++++++++++++++++++++----- 4 files changed, 65 insertions(+), 15 deletions(-) create mode 100644 test/reset.coffee diff --git a/server/routes/api/buckets.coffee b/server/routes/api/buckets.coffee index 5d68a58..876652e 100644 --- a/server/routes/api/buckets.coffee +++ b/server/routes/api/buckets.coffee @@ -7,6 +7,8 @@ module.exports = app = express() app.route('/buckets') .post (req, res) -> + return res.status(401).end() unless req.user + newBucket = new Bucket req.body newBucket.save (err, bucket) -> diff --git a/test/reset.coffee b/test/reset.coffee new file mode 100644 index 0000000..a067e4b --- /dev/null +++ b/test/reset.coffee @@ -0,0 +1,8 @@ +db = require '../server/lib/database' +config = require '../server/config' + +MongoosasticConfig = require 'mongoosastic/test/config' + +module.exports = (done) -> + db.connection.db.dropDatabase() + MongoosasticConfig.deleteIndexIfExists [config.elasticsearch.index], done diff --git a/test/server/models/entry.coffee b/test/server/models/entry.coffee index c613020..6872f1d 100644 --- a/test/server/models/entry.coffee +++ b/test/server/models/entry.coffee @@ -1,5 +1,6 @@ db = require '../../../server/lib/database' config = require '../../../server/config' +reset = require '../../reset' Entry = require '../../../server/models/entry' Bucket = require '../../../server/models/bucket' @@ -13,9 +14,6 @@ describe 'Entry', -> user = null - after (done) -> - db.connection.db.dropDatabase done - before (done) -> User.create name: 'Bucketer' @@ -26,6 +24,8 @@ describe 'Entry', -> user = u done() + after reset + describe 'Validation', -> it 'requires a bucket', (done) -> @@ -46,13 +46,16 @@ describe 'Entry', -> bucketId = null beforeEach (done) -> - Bucket.create {name: 'Articles', slug: 'articles'}, (e, bucket) -> + # Create a Bucket to put our Test Entries in + Bucket.create + name: 'Articles' + slug: 'articles' + , (e, bucket) -> throw e if e bucketId = bucket._id done() - afterEach (done) -> - Bucket.remove {}, done + afterEach reset it 'parses dates from strings', (done) -> Entry.create diff --git a/test/server/routes/api/buckets.coffee b/test/server/routes/api/buckets.coffee index 0aa2ec9..dbe97e9 100644 --- a/test/server/routes/api/buckets.coffee +++ b/test/server/routes/api/buckets.coffee @@ -1,18 +1,55 @@ -request = require('supertest') -User = require('../../../../server/models/user') -Bucket = require('../../../../server/models/bucket') -app = require('../../../../server') -db = require('../../../../server/lib/database') -{assert} = require('chai') +path = require 'path' +request = require 'supertest' + +serverPath = path.resolve __dirname, '../../../../server' +User = require "#{serverPath}/models/user" +Bucket = require "#{serverPath}/models/bucket" +reset = require '../../../reset' + +app = require serverPath + +{expect} = require 'chai' describe 'Buckets routes', -> + before (done) -> - db.connection.db.dropDatabase done + Bucket.create [ + name: 'Photos' + slug: 'photos' + , + name: 'Docs' + slug: 'docs' + ], done + + after reset - afterEach (done) -> - db.connection.db.dropDatabase done + describe 'GET /buckets', -> + it 'returns a 401 if unauthorized', (done) -> + request(app) + .get('/api/buckets') + .expect(401) + .end (e, res) -> + throw e if e + expect(res.body).to.be.an 'Object' + done() + + it 'returns 200 with buckets' + + describe 'POST /buckets', -> + it 'returns a 401 if not logged in', (done) -> + request(app) + .post('/api/buckets') + .expect(401) + .end (e, res) -> + throw e if e + expect(res.body).to.be.an 'Object' + done() + + it 'returns a 401 if not an admin' + it 'returns a 200 if bucket is created' describe 'GET /buckets/:bucket/members', -> + # STUB: Need to add auth #it 'returns the members of a given bucket', (done) -> From 86636c1133f83d67e5a95c11a86720fecc35bd62 Mon Sep 17 00:00:00 2001 From: David Kaneda Date: Sun, 17 Aug 2014 02:39:06 -0700 Subject: [PATCH 18/28] Add search to the frontend --- user/templates/index.hbs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/user/templates/index.hbs b/user/templates/index.hbs index 7fc19f5..0830037 100644 --- a/user/templates/index.hbs +++ b/user/templates/index.hbs @@ -1,5 +1,8 @@

Buckets

+
+ +
From 86b2b9003ebf4e203bfcc986f1aaf7befd03be65 Mon Sep 17 00:00:00 2001 From: David Kaneda Date: Sun, 17 Aug 2014 13:02:28 -0700 Subject: [PATCH 19/28] Separate DB and ES resets (and start a "prep" which waits for Mongo connection) --- test/reset.coffee | 21 ++++++++++++++--- test/server/models/activity.coffee | 9 +++---- test/server/models/bucket.coffee | 34 +++++++++++++++------------ test/server/models/entry.coffee | 7 +++--- test/server/routes/api/buckets.coffee | 2 +- 5 files changed, 44 insertions(+), 29 deletions(-) diff --git a/test/reset.coffee b/test/reset.coffee index a067e4b..28f131f 100644 --- a/test/reset.coffee +++ b/test/reset.coffee @@ -1,8 +1,23 @@ +async = require 'async' + db = require '../server/lib/database' +mongoose = require 'mongoose' config = require '../server/config' MongoosasticConfig = require 'mongoosastic/test/config' -module.exports = (done) -> - db.connection.db.dropDatabase() - MongoosasticConfig.deleteIndexIfExists [config.elasticsearch.index], done +dropDatabase = (done) -> + db.connection.db.dropDatabase done + +dropElastic = (done) -> + MongoosasticConfig.deleteIndexIfExists [config.elasticsearch.index], -> + setTimeout done, 1100 + +prep = (done) -> + mongoose.connection.on 'connected', done + +module.exports = + db: dropDatabase + prep: prep + all: (done) -> + async.parallel [dropElastic, dropDatabase], done diff --git a/test/server/models/activity.coffee b/test/server/models/activity.coffee index 7efd4e0..6277d9b 100644 --- a/test/server/models/activity.coffee +++ b/test/server/models/activity.coffee @@ -1,18 +1,15 @@ -db = require '../../../server/lib/database' mongoose = require 'mongoose' Activity = require '../../../server/models/activity' +reset = require '../../reset' {expect} = require 'chai' sinon = require 'sinon' moment = require 'moment' describe 'Activity', -> - - afterEach (done) -> - for _, c of db.connection.collections - c.remove(->) - done() + before reset.prep + afterEach reset.db describe 'Validation', -> it 'requires an actor', (done) -> diff --git a/test/server/models/bucket.coffee b/test/server/models/bucket.coffee index 56f0e58..86d10ba 100644 --- a/test/server/models/bucket.coffee +++ b/test/server/models/bucket.coffee @@ -1,28 +1,32 @@ User = require('../../../server/models/user') Bucket = require('../../../server/models/bucket') -db = require('../../../server/lib/database') +reset = require('../../reset') {assert} = require('chai') describe 'Bucket', -> - before (done) -> - db.connection.db.dropDatabase done - afterEach (done) -> - db.connection.db.dropDatabase done + before reset.db + afterEach reset.db describe '#getMembers', -> - it 'returns members', (done) -> - Bucket.create { name: 'Images', slug: 'images' }, (e, bucket) -> + bucket = null + user = null + + before (done) -> + Bucket.create { name: 'Images', slug: 'images' }, (e, bkt) -> throw e if e + bucket = bkt User.create { name: 'Bucketer', email: 'hello@buckets.io', password: 'S3cr3ts' }, (e, u) -> - - u.upsertRole 'contributor', bucket, (e, user) -> + user = u + u.upsertRole 'contributor', bucket, (e) -> throw e if e + done() - bucket.getMembers (e, users) -> - throw e if e - assert.isArray(users) - assert.lengthOf(users, 1) - assert.equal(users[0].id, u.id) + it 'returns members', (done) -> + bucket.getMembers (e, users) -> + throw e if e + assert.isArray(users) + assert.lengthOf(users, 1) + assert.equal(users[0].id, user.id) - done() + done() diff --git a/test/server/models/entry.coffee b/test/server/models/entry.coffee index 6872f1d..0046a9b 100644 --- a/test/server/models/entry.coffee +++ b/test/server/models/entry.coffee @@ -1,4 +1,3 @@ -db = require '../../../server/lib/database' config = require '../../../server/config' reset = require '../../reset' @@ -24,7 +23,7 @@ describe 'Entry', -> user = u done() - after reset + after reset.db describe 'Validation', -> @@ -55,7 +54,7 @@ describe 'Entry', -> bucketId = bucket._id done() - afterEach reset + afterEach reset.db it 'parses dates from strings', (done) -> Entry.create @@ -139,7 +138,7 @@ describe 'Entry', -> before (done) -> @timeout 5000 - MongoosasticConfig.deleteIndexIfExists [config.elasticsearch.index], -> + reset.all -> Bucket.create [ name: 'Articles' slug: 'articles' diff --git a/test/server/routes/api/buckets.coffee b/test/server/routes/api/buckets.coffee index dbe97e9..eef65b9 100644 --- a/test/server/routes/api/buckets.coffee +++ b/test/server/routes/api/buckets.coffee @@ -21,7 +21,7 @@ describe 'Buckets routes', -> slug: 'docs' ], done - after reset + after reset.db describe 'GET /buckets', -> it 'returns a 401 if unauthorized', (done) -> From 1bd7e8c3ebb2c42f4f15510a7f42053f8110c7a9 Mon Sep 17 00:00:00 2001 From: David Kaneda Date: Sun, 17 Aug 2014 20:38:01 -0700 Subject: [PATCH 20/28] Upgrade mongoosastic Discussed Model#refresh with mongoosastic contrib. Unfortunately still requires a timeout after initial indexing/refresh. Adds more tests. --- .../20140810052927866_sync_with_es.coffee | 23 ++++--- package.json | 4 +- server/config.coffee | 2 +- server/models/entry.coffee | 19 +++-- test/reset.coffee | 7 +- test/server/models/entry.coffee | 69 +++++++++++-------- 6 files changed, 71 insertions(+), 53 deletions(-) diff --git a/migrations/20140810052927866_sync_with_es.coffee b/migrations/20140810052927866_sync_with_es.coffee index f739aca..456a62b 100644 --- a/migrations/20140810052927866_sync_with_es.coffee +++ b/migrations/20140810052927866_sync_with_es.coffee @@ -1,5 +1,8 @@ -Bucket = require '../server/models/bucket' -Entry = require '../server/models/entry' +Bucket = require './server/models/bucket' +Entry = require './server/models/entry' + +http = require 'http' +config = require './server/config' module.exports = requiresDowntime: no # true or false @@ -8,15 +11,17 @@ module.exports = Entry.update {content: null}, {content: {}}, {multi: yes}, (e, entries) -> throw e if e - stream = Entry.synchronize() - count = 0 + Entry.createMapping (err) -> + throw e if e + stream = Entry.synchronize() + count = 0 - stream.on 'data', (err, doc) -> count++ - stream.on 'close', -> - console.log('Indexed ' + count + ' entries into ElasticSearch') - done() + stream.on 'data', (e, doc) -> count++ + stream.on 'close', -> + console.log('Indexed ' + count + ' entries into ElasticSearch') + done() - stream.on 'error', (err) -> console.log err + stream.on 'error', (e) -> throw e down: (done) -> done() diff --git a/package.json b/package.json index 014c620..4b65ec1 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ }, "author": "", "bugs": { - "url": "https://assemblymade.com/buckets/wips" + "url": "https://assembly.com/buckets/wips" }, "keywords": [ "cms", @@ -68,7 +68,7 @@ "hbsfy": "^1.3.2", "inflection": "^1.3.6", "moment": "^2.7.0", - "mongoosastic": "^0.3.0", + "mongoosastic": "^0.4.0", "mongoose": "^3.8.14", "mongoose-regexp": "0.0.1", "mongoose-unique-validator": "^0.3.0", diff --git a/server/config.coffee b/server/config.coffee index 06067c8..e57e326 100644 --- a/server/config.coffee +++ b/server/config.coffee @@ -15,7 +15,7 @@ config = catchAll: yes db: "mongodb://localhost/buckets_#{env}" elasticsearch: - url: process.env.BONSAI_URL || 'http://127.0.0.1:9200' + url: process.env.SEARCHBOX_SSL_URL || 'http://127.0.0.1:9200' index: "buckets_entries_#{env}" production: smtp: diff --git a/server/models/entry.coffee b/server/models/entry.coffee index dbb4ef1..b19da6d 100755 --- a/server/models/entry.coffee +++ b/server/models/entry.coffee @@ -42,14 +42,11 @@ entrySchema = new Schema type: String required: yes es_boost: 3.0 - es_indexed: yes description: type: String es_boost: 2.0 - es_indexed: yes slug: type: String - es_indexed: yes status: type: String enum: ['draft', 'live', 'pending', 'rejected'] @@ -77,13 +74,13 @@ entrySchema = new Schema required: yes keywords: type: [String] - es_indexed: yes + default: [] es_boost: 2.0 + es_type: 'string' content: type: {} default: {} es_type : 'object' - es_indexed: yes es_boost: 2.0 entrySchema.pre 'save', (next) -> @@ -149,9 +146,6 @@ entrySchema.statics.findByParams = (params, callback) -> if settings.slug searchQuery.slug = settings.slug - if settings.where - searchQuery.$where = settings.where - if settings.status?.length > 0 searchQuery.status = settings.status @@ -170,11 +164,14 @@ entrySchema.statics.findByParams = (params, callback) -> if settings.query? return @search query: - simple_query_string: + query_string: query: settings.query + , (err, elasticEntries) -> - throw err if err - callback null, elasticEntries.hits + if err + callback err, [] + else if elasticEntries.hits + callback null, elasticEntries.hits if settings.since or settings.until searchQuery.publishDate = {} diff --git a/test/reset.coffee b/test/reset.coffee index 28f131f..0c1deb3 100644 --- a/test/reset.coffee +++ b/test/reset.coffee @@ -3,15 +3,16 @@ async = require 'async' db = require '../server/lib/database' mongoose = require 'mongoose' config = require '../server/config' +Entry = require '../server/models/entry' MongoosasticConfig = require 'mongoosastic/test/config' dropDatabase = (done) -> db.connection.db.dropDatabase done -dropElastic = (done) -> +dropIndex = (done) -> MongoosasticConfig.deleteIndexIfExists [config.elasticsearch.index], -> - setTimeout done, 1100 + Entry.createMapping done prep = (done) -> mongoose.connection.on 'connected', done @@ -20,4 +21,4 @@ module.exports = db: dropDatabase prep: prep all: (done) -> - async.parallel [dropElastic, dropDatabase], done + async.parallel [dropIndex, dropDatabase], done diff --git a/test/server/models/entry.coffee b/test/server/models/entry.coffee index 0046a9b..8a69254 100644 --- a/test/server/models/entry.coffee +++ b/test/server/models/entry.coffee @@ -23,7 +23,7 @@ describe 'Entry', -> user = u done() - after reset.db + after reset.all describe 'Validation', -> @@ -74,9 +74,18 @@ describe 'Entry', -> expect(entry.slug).to.equal 'resumes-and-cvs' done() + it 'automatically sets the publishDate', (done) -> + Entry.create + title: 'New Entry' + bucket: bucketId + author: user._id + , (e, entry) -> + expect(entry.publishDate).to.be.a 'Date' + done() + describe '#findByParams', -> # Set up a bunch of entries to filter/search - before (done) -> + before (done) -> reset.all -> Bucket.create [ name: 'Articles' slug: 'articles' @@ -99,9 +108,7 @@ describe 'Entry', -> status: 'live' ], done - after (done) -> - Bucket.find({}).remove().exec() - setTimeout done, 1200 + after reset.db it 'filters by bucket slug (empty)', (done) -> Entry.findByParams bucket: '', (e, entries) -> @@ -114,6 +121,8 @@ describe 'Entry', -> expect(entries?[0]?.title).to.equal 'Test Photoset' done() + it 'filters by bucket slug (w/negation)' + it 'filters by multiple bucket slugs', (done) -> Entry.findByParams bucket: 'photos|articles', (e, entries) -> expect(entries).to.have.length 2 @@ -132,9 +141,6 @@ describe 'Entry', -> done() describe 'Search', -> - # We just create these once since we're only testing search - # (and we're using a delay to guarantee they're indexed) - # Yeah, it's pretty gross. before (done) -> @timeout 5000 @@ -160,24 +166,10 @@ describe 'Entry', -> author: user._id status: 'live' keywords: ['summer'] - ], (e, entry1, entry2) -> - throw e if e - - e1indexed = no - e2indexed = no - - entry1.on 'es-indexed', -> e1indexed = yes - entry2.on 'es-indexed', -> e2indexed = yes - - checkIndexed = -> - if e1indexed and e2indexed - # Even after being reported as indexed, - # docs may not be searchable right away - # The testing gods are crying, I know - setTimeout done, 1200 - clearInterval interval - - interval = setInterval checkIndexed, 10 + ], -> + # This is super painful, but only way I can + # think to test a live elasticsearch instance + Entry.synchronize -> Entry.refresh -> setTimeout done, 1500 it 'performs a fuzzy search with `search`', (done) -> Entry.findByParams search: 'photoste', (e, entries) -> @@ -191,8 +183,31 @@ describe 'Entry', -> expect(entries?[0]?.title).to.equal 'Test Photoset' done() - it 'searches tags', (done) -> + it 'can check for specific attribute', (done) -> + Entry.findByParams query: '_exists_:keywords', (e, entries) -> + expect(entries).to.have.length 1 + expect(entries?[0]?.title).to.equal 'Test Photoset' + done() + + it 'can search for specific attribute', (done) -> + Entry.findByParams query: 'keywords:summ*', (e, entries) -> + expect(entries).to.have.length 1 + expect(entries?[0]?.title).to.equal 'Test Photoset' + done() + + it 'doesn’t throw an exception for bad query', (done) -> + Entry.findByParams query: 'nokey:(6', (e, entries) -> + expect(entries).to.have.length 0 + done() + + it 'searches keywords', (done) -> Entry.findByParams query: 'summer', (e, entries) -> expect(entries).to.have.length 1 expect(entries?[0]?.title).to.equal 'Test Photoset' done() + + it 'searches phrases', (done) -> + Entry.findByParams query: '"Test Photoset"', (e, entries) -> + expect(entries).to.have.length 1 + expect(entries?[0]?.title).to.equal 'Test Photoset' + done() From 9631e628719160c3a8cc66f7031594683f3d6ce2 Mon Sep 17 00:00:00 2001 From: David Kaneda Date: Sat, 23 Aug 2014 21:13:30 -0700 Subject: [PATCH 21/28] Fix for migration --- migrations/20140810052927866_sync_with_es.coffee | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/migrations/20140810052927866_sync_with_es.coffee b/migrations/20140810052927866_sync_with_es.coffee index 456a62b..9c8c671 100644 --- a/migrations/20140810052927866_sync_with_es.coffee +++ b/migrations/20140810052927866_sync_with_es.coffee @@ -1,8 +1,8 @@ -Bucket = require './server/models/bucket' -Entry = require './server/models/entry' +Bucket = require '../server/models/bucket' +Entry = require '../server/models/entry' http = require 'http' -config = require './server/config' +config = require '../server/config' module.exports = requiresDowntime: no # true or false From 8c526d05179cbc1981cde8399ed7b4e00b9b6425 Mon Sep 17 00:00:00 2001 From: David Kaneda Date: Sat, 23 Aug 2014 21:34:57 -0700 Subject: [PATCH 22/28] Raising the timeout to be safe --- test/server/models/entry.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/server/models/entry.coffee b/test/server/models/entry.coffee index 8a69254..ab1acb0 100644 --- a/test/server/models/entry.coffee +++ b/test/server/models/entry.coffee @@ -169,7 +169,7 @@ describe 'Entry', -> ], -> # This is super painful, but only way I can # think to test a live elasticsearch instance - Entry.synchronize -> Entry.refresh -> setTimeout done, 1500 + Entry.synchronize -> Entry.refresh -> setTimeout done, 2000 it 'performs a fuzzy search with `search`', (done) -> Entry.findByParams search: 'photoste', (e, entries) -> From fc2a5aed6c8f2c24cdd0858beda429d37e3d9955 Mon Sep 17 00:00:00 2001 From: David Kaneda Date: Sat, 23 Aug 2014 21:50:18 -0700 Subject: [PATCH 23/28] Raise the timeout for the #findByParams test --- test/server/models/entry.coffee | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/server/models/entry.coffee b/test/server/models/entry.coffee index ab1acb0..eacdce9 100644 --- a/test/server/models/entry.coffee +++ b/test/server/models/entry.coffee @@ -84,6 +84,8 @@ describe 'Entry', -> done() describe '#findByParams', -> + @timeout 4000 # We do a network call to reset the ES index + # Set up a bunch of entries to filter/search before (done) -> reset.all -> Bucket.create [ From db2aa1cc7838f313db6cc6951ede35fbddd2e6bb Mon Sep 17 00:00:00 2001 From: David Kaneda Date: Sat, 23 Aug 2014 23:51:32 -0700 Subject: [PATCH 24/28] Adds Bucket negation via #findByParams --- server/models/entry.coffee | 28 +++++++++++++++++++--------- test/server/models/entry.coffee | 15 +++++++++++++-- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/server/models/entry.coffee b/server/models/entry.coffee index 828eccd..a1c07cf 100755 --- a/server/models/entry.coffee +++ b/server/models/entry.coffee @@ -137,16 +137,29 @@ entrySchema.statics.findByParams = (params, callback) -> async.parallel [ (callback) -> - if settings.bucket? - filteredBuckets = settings.bucket.split '|' - searchQuery.bucket = $in: [] + if settings.bucket?.length > 0 + bucketStrings = settings.bucket.split '|' - mongoose.model('Bucket').find {slug: $in: filteredBuckets}, (err, buckets) => - filteredBucketIDs = _.pluck _.filter(buckets, (bkt) -> bkt.slug in filteredBuckets), '_id' + return callback null if bucketStrings.length is 0 + + isNegated = (bkt) -> bkt.substr(0,1) is '-' + + rejectedBuckets = _.map _.filter(bucketStrings, isNegated), (bkt) -> + bkt.substr(1) + filteredBuckets = _.reject bucketStrings, isNegated + + $findParams = if rejectedBuckets.length + slug: $nin: rejectedBuckets + else + slug: $in: filteredBuckets + + mongoose.model('Bucket').find $findParams, (err, buckets) => + filteredBucketIDs = _.pluck buckets, '_id' searchQuery.bucket = $in: filteredBucketIDs callback null else callback null + ], => if settings.slug searchQuery.slug = settings.slug @@ -184,10 +197,7 @@ entrySchema.statics.findByParams = (params, callback) -> searchQuery.publishDate.$lte = new Date(chrono.parseDate settings.until) if settings.until @find searchQuery - .populate 'bucket' - .populate - path: 'author' - select: '-passwordDigest -resetPasswordToken -resetPasswordExpires' + .populate 'bucket author' .sort settings.sort .limit settings.limit .skip settings.skip diff --git a/test/server/models/entry.coffee b/test/server/models/entry.coffee index eacdce9..46e400b 100644 --- a/test/server/models/entry.coffee +++ b/test/server/models/entry.coffee @@ -114,16 +114,27 @@ describe 'Entry', -> it 'filters by bucket slug (empty)', (done) -> Entry.findByParams bucket: '', (e, entries) -> - expect(entries).to.have.length 0 + expect(entries).to.have.length 2 done() it 'filters by bucket slug', (done) -> Entry.findByParams bucket: 'photos', (e, entries) -> + throw e if e expect(entries).to.have.length 1 expect(entries?[0]?.title).to.equal 'Test Photoset' done() - it 'filters by bucket slug (w/negation)' + it 'filters by bucket slug (invalid)', (done) -> + Entry.findByParams bucket: 'asdf', (e, entries) -> + throw e if e + expect(entries).to.have.length 0 + done() + + it 'filters by bucket slug (w/negation)', (done) -> + Entry.findByParams bucket: '-articles', (e, entries) -> + expect(entries).to.have.length 1 + expect(entries?[0]?.title).to.equal 'Test Photoset' + done() it 'filters by multiple bucket slugs', (done) -> Entry.findByParams bucket: 'photos|articles', (e, entries) -> From 8259cda3cd62de5ffb02da0d0cc9e2bc02a05cca Mon Sep 17 00:00:00 2001 From: David Kaneda Date: Sun, 24 Aug 2014 15:51:36 -0700 Subject: [PATCH 25/28] Fix install, was overzealous with _id updates --- server/routes/api/install.coffee | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/routes/api/install.coffee b/server/routes/api/install.coffee index 11bac4c..6cd8f8f 100644 --- a/server/routes/api/install.coffee +++ b/server/routes/api/install.coffee @@ -31,8 +31,8 @@ app.post '/install', (req, res) -> Bucket.create bucketSeed ).then( (bucket) -> for entry in entrySeed - entry.bucket = bucket.id - entry.author = newUser.id + entry.bucket = bucket._id + entry.author = newUser._id Entry.create entrySeed , renderError).then -> req.login newUser, -> From cda4c864c094fa9700a35c80dad6e02210ac3210 Mon Sep 17 00:00:00 2001 From: David Kaneda Date: Sun, 24 Aug 2014 15:52:01 -0700 Subject: [PATCH 26/28] Express 4 syntax updates --- server/routes/api/routes.coffee | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/routes/api/routes.coffee b/server/routes/api/routes.coffee index 17b571e..eceb930 100644 --- a/server/routes/api/routes.coffee +++ b/server/routes/api/routes.coffee @@ -51,9 +51,9 @@ app.route('/routes') Route.find().sort(urlPattern: 1).exec (err, routes) -> if err - res.send err, 400 + res.status(400).send err else if routes - res.send routes + res.status(200).send routes .post (req, res) -> ### @@ -70,9 +70,9 @@ app.route('/routes') newRoute.save (err) -> if err - res.send 400, err + res.status(400).send err else - res.send 201, newRoute + res.status(201).send newRoute ### @api {post} /routes/:id Remove a Route From 10b3a236787c7684bf73d2757fb0d0918fcf1a66 Mon Sep 17 00:00:00 2001 From: David Kaneda Date: Sun, 24 Aug 2014 15:59:23 -0700 Subject: [PATCH 27/28] Switch bonsai to searchbox --- app.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.json b/app.json index e19aff8..3cd9314 100644 --- a/app.json +++ b/app.json @@ -15,5 +15,5 @@ "BUILDPACK_URL": "https://github.com/mbuchetics/heroku-buildpack-nodejs-grunt.git", "NODE_ENV": "production" }, - "addons": ["mongohq", "mandrill", "bonsai"] + "addons": ["mongohq", "mandrill", "searchbox"] } From 6a9aad042d95f1290de5c80bba098c6b294cc86b Mon Sep 17 00:00:00 2001 From: David Kaneda Date: Sun, 24 Aug 2014 17:10:20 -0700 Subject: [PATCH 28/28] Fix for search redirects --- client/source/views/buckets/dashboard.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/source/views/buckets/dashboard.coffee b/client/source/views/buckets/dashboard.coffee index 4b7a1a9..62a7ed8 100644 --- a/client/source/views/buckets/dashboard.coffee +++ b/client/source/views/buckets/dashboard.coffee @@ -45,4 +45,4 @@ module.exports = class DashboardView extends PageView selectItem: (e, item) -> Chaplin.utils.redirectTo 'buckets#browse', slug: item.bucket.slug - entryID: item._id + entryID: item.id