From aa660a4eed517041815f66e11ea3a9e8191c06dd Mon Sep 17 00:00:00 2001 From: Erick Fabian Date: Mon, 17 Jun 2019 16:28:12 -0500 Subject: [PATCH 01/14] Fix config file namespace access --- lib/dolly/configuration.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/dolly/configuration.rb b/lib/dolly/configuration.rb index 788d4a3..68eab02 100644 --- a/lib/dolly/configuration.rb +++ b/lib/dolly/configuration.rb @@ -4,7 +4,7 @@ module Dolly module Configuration def env - @env ||= configuration[db.to_s] + @env ||= configuration end def base_uri From 7c614eaa3e9be66024e251d2677de18f27e4ff0b Mon Sep 17 00:00:00 2001 From: Erick Fabian Date: Wed, 19 Jun 2019 10:50:34 -0500 Subject: [PATCH 02/14] Add couch base model and read me for migrating from couch 1.x to couch 2.x --- README.md | 43 +++++++++++++++++ lib/couch/base.rb | 96 ++++++++++++++++++++++++++++++++++++++ lib/dolly.rb | 2 + lib/dolly/callbacks.rb | 20 ++++++++ lib/dolly/configuration.rb | 3 +- 5 files changed, 163 insertions(+), 1 deletion(-) create mode 100644 lib/couch/base.rb create mode 100644 lib/dolly/callbacks.rb diff --git a/README.md b/README.md index 11c42e3..b3c0a3a 100644 --- a/README.md +++ b/README.md @@ -33,3 +33,46 @@ To install this gem onto your local machine, run `bundle exec rake install`. To ## Contributing Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/dolly3. + + +## Migrating from couch 1.x to 2.x + +[Official docs](https://docs.couchdb.org/en/2.3.1/install/index.html) + + +You will need to uninstall the couchdb service with brew + +`brew services stop couchdb` +`brew services uninstall couchdb` + +Download the application from the following source + +http://couchdb.apache.org/#download + +launch fauxton and check your installation + +Copy [this file](https://github.com/apache/couchdb/blob/master/rel/overlay/bin/couchup) into your filesystem + + +make it executable + +`chmod +x couchup.py` + +and run it + +`./couchup.py -h` + +You might need to install python 3 and pip3 and the following libs + +`pip3 install requests progressbar2` + +move your .couch files into the specified `database_dir` in your [fauxton config](http://127.0.0.1:5984/_utils/#_config/couchdb@localhost) + + +``` +$ ./couchup list # Shows your unmigrated 1.x databases +$ ./couchup replicate -a # Replicates your 1.x DBs to 2.x +$ ./couchup rebuild -a # Optional; starts rebuilding your views +$ ./couchup delete -a # Deletes your 1.x DBs (careful!) +$ ./couchup list # Should show no remaining databases! +``` diff --git a/lib/couch/base.rb b/lib/couch/base.rb new file mode 100644 index 0000000..950e358 --- /dev/null +++ b/lib/couch/base.rb @@ -0,0 +1,96 @@ +require 'dolly/callbacks' + +module Couch + class Base < Dolly::Document + extend ActiveModel::Translation + include ActiveModel::Conversion + include ActiveModel::Validations + include Dolly::Callbacks + + LAST_RECORD = '\u9999'.freeze + + def save + return false if invalid? || run_callbacks(:save) == false + super + end + + #TODO: This is needed as the adapter does not handle validations yet, and FactoryBot.create looks for save! + def save! + save + end + + #TODO: use Dolly native update_properties method + def update_attributes! attributes + attributes.each do |key, value| + send(:"#{key}=", value) + end + + save + end + + def update_attributes(attributes) + doc.merge(attributes) + save + end + + def base_id + self.class.base_id(id) + end + + def self.safe_find(id) + begin + find(id) + rescue Dolly::ResourceNotFound + nil + end + end + + alias_method :to_param, :base_id + + def persisted? + doc['_rev'].present? + end + + def self.raw_view(name, options = {}) + design = options.delete(:design) + design ||= 'av' + + connection.get("_design/#{design}/_view/#{name}", options) + end + + def self.last_record + LAST_RECORD + end + + def raw_view(name, options = {}) + self.class.raw_view(name, options) + end + + def view_value(name, options = {}) + raw_view(name, options).flat_map { |result| result['value'] } + end + + def reload + self.doc = self.class.find(id).doc + end + + def reload_rev! + return unless persisted? + doc['_rev'] = JSON.parse(connection.get CGI.escape(id))['_rev'] + end + + private + + def write_property name, value + instance_variable_set(:"@#{name}", value) + @doc[name.to_s] = value + end + + def read_property name + if instance_variable_get(:"@#{name}").nil? + write_property name, (doc[name.to_s] || self.properties[name].value) + end + instance_variable_get(:"@#{name}") + end + end +end diff --git a/lib/dolly.rb b/lib/dolly.rb index d824ee6..dafb83c 100644 --- a/lib/dolly.rb +++ b/lib/dolly.rb @@ -2,5 +2,7 @@ require "dolly/document" require 'railties/railtie' if defined?(Rails) +require 'couch/base' + module Dolly end diff --git a/lib/dolly/callbacks.rb b/lib/dolly/callbacks.rb new file mode 100644 index 0000000..b3f206c --- /dev/null +++ b/lib/dolly/callbacks.rb @@ -0,0 +1,20 @@ +module Dolly + module Callbacks + extend ActiveSupport::Concern + + module ClassMethods + include ActiveModel::Callbacks + end + + included do + include ActiveModel::Validations::Callbacks + + define_model_callbacks :initialize, :find, :touch, :only => :after + define_model_callbacks :save, :create, :update, :destroy + end + + def destroy(hard=true) + run_callbacks(:destroy) { super(hard) } + end + end +end diff --git a/lib/dolly/configuration.rb b/lib/dolly/configuration.rb index 68eab02..d799dad 100644 --- a/lib/dolly/configuration.rb +++ b/lib/dolly/configuration.rb @@ -1,10 +1,11 @@ # frozen_string_literal: true + require 'erb' module Dolly module Configuration def env - @env ||= configuration + @env ||= configuration[db.to_s] end def base_uri From 2b4618dcdbab4d4f0899efaee75def8077fae132 Mon Sep 17 00:00:00 2001 From: Erick Fabian Date: Thu, 20 Jun 2019 10:08:37 -0500 Subject: [PATCH 03/14] Add base mango querys --- lib/couch/base.rb | 9 ++---- lib/dolly/document.rb | 3 ++ lib/dolly/mango.rb | 64 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 7 deletions(-) create mode 100644 lib/dolly/mango.rb diff --git a/lib/couch/base.rb b/lib/couch/base.rb index 950e358..b5f8d67 100644 --- a/lib/couch/base.rb +++ b/lib/couch/base.rb @@ -19,18 +19,13 @@ def save! save end - #TODO: use Dolly native update_properties method def update_attributes! attributes - attributes.each do |key, value| - send(:"#{key}=", value) - end - + attributes.each(&update_attribute) save end def update_attributes(attributes) - doc.merge(attributes) - save + attributes.each(&update_attribute) end def base_id diff --git a/lib/dolly/document.rb b/lib/dolly/document.rb index fd4ef60..e26871b 100644 --- a/lib/dolly/document.rb +++ b/lib/dolly/document.rb @@ -1,3 +1,4 @@ +require 'dolly/mango' require 'dolly/query' require 'dolly/connection' require 'dolly/request' @@ -15,11 +16,13 @@ module Dolly class Document + extend Mango extend Query extend Request extend DepracatedDatabase extend Properties extend DocumentCreation + include PropertyManager include Timestamp include DocumentState diff --git a/lib/dolly/mango.rb b/lib/dolly/mango.rb new file mode 100644 index 0000000..810212f --- /dev/null +++ b/lib/dolly/mango.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +module Dolly + module Mango + + SELECTOR_SYMBOL = "$" + AVAILABLE_SELECTORS = %I[ + lt + lte + eq + ne + gte + gt + exists + type + in + nin + size + mod + regex + ].freeze + + DESIGN = "_find" + + def find_by(query, opts = {}) + opts.merge!(limit: 1) + perform_query(build_query(query, opts))[:docs].first + end + + def where(query, opts = {}) + perform_query(build_query(query, opts))[:docs] + end + + private + + def perform_query(structured_query) + connection.post(DESIGN, structured_query) + end + + def build_query(query, opts) + { + "selector" => build_selectors(query) + }.merge(opts) + end + + def build_selectors(query) + query.map do |key, values| + inner_key = values.keys.first + inner_value = values.values.first + next unless AVAILABLE_SELECTORS.include?(inner_key) + + { + "#{key}" => { + build_key(inner_key) => inner_value + } + } + end.compact.inject(:merge) + end + + def build_key(key) + "#{SELECTOR_SYMBOL}#{key}" + end + end +end From 7aea9a58cac233c16189a445c27bef18f046c53f Mon Sep 17 00:00:00 2001 From: Erick Fabian Date: Thu, 20 Jun 2019 10:25:56 -0500 Subject: [PATCH 04/14] Compact pr --- lib/couch/base.rb | 91 ------------------------------------------ lib/dolly.rb | 2 - lib/dolly/callbacks.rb | 20 ---------- 3 files changed, 113 deletions(-) delete mode 100644 lib/couch/base.rb delete mode 100644 lib/dolly/callbacks.rb diff --git a/lib/couch/base.rb b/lib/couch/base.rb deleted file mode 100644 index b5f8d67..0000000 --- a/lib/couch/base.rb +++ /dev/null @@ -1,91 +0,0 @@ -require 'dolly/callbacks' - -module Couch - class Base < Dolly::Document - extend ActiveModel::Translation - include ActiveModel::Conversion - include ActiveModel::Validations - include Dolly::Callbacks - - LAST_RECORD = '\u9999'.freeze - - def save - return false if invalid? || run_callbacks(:save) == false - super - end - - #TODO: This is needed as the adapter does not handle validations yet, and FactoryBot.create looks for save! - def save! - save - end - - def update_attributes! attributes - attributes.each(&update_attribute) - save - end - - def update_attributes(attributes) - attributes.each(&update_attribute) - end - - def base_id - self.class.base_id(id) - end - - def self.safe_find(id) - begin - find(id) - rescue Dolly::ResourceNotFound - nil - end - end - - alias_method :to_param, :base_id - - def persisted? - doc['_rev'].present? - end - - def self.raw_view(name, options = {}) - design = options.delete(:design) - design ||= 'av' - - connection.get("_design/#{design}/_view/#{name}", options) - end - - def self.last_record - LAST_RECORD - end - - def raw_view(name, options = {}) - self.class.raw_view(name, options) - end - - def view_value(name, options = {}) - raw_view(name, options).flat_map { |result| result['value'] } - end - - def reload - self.doc = self.class.find(id).doc - end - - def reload_rev! - return unless persisted? - doc['_rev'] = JSON.parse(connection.get CGI.escape(id))['_rev'] - end - - private - - def write_property name, value - instance_variable_set(:"@#{name}", value) - @doc[name.to_s] = value - end - - def read_property name - if instance_variable_get(:"@#{name}").nil? - write_property name, (doc[name.to_s] || self.properties[name].value) - end - instance_variable_get(:"@#{name}") - end - end -end diff --git a/lib/dolly.rb b/lib/dolly.rb index dafb83c..d824ee6 100644 --- a/lib/dolly.rb +++ b/lib/dolly.rb @@ -2,7 +2,5 @@ require "dolly/document" require 'railties/railtie' if defined?(Rails) -require 'couch/base' - module Dolly end diff --git a/lib/dolly/callbacks.rb b/lib/dolly/callbacks.rb deleted file mode 100644 index b3f206c..0000000 --- a/lib/dolly/callbacks.rb +++ /dev/null @@ -1,20 +0,0 @@ -module Dolly - module Callbacks - extend ActiveSupport::Concern - - module ClassMethods - include ActiveModel::Callbacks - end - - included do - include ActiveModel::Validations::Callbacks - - define_model_callbacks :initialize, :find, :touch, :only => :after - define_model_callbacks :save, :create, :update, :destroy - end - - def destroy(hard=true) - run_callbacks(:destroy) { super(hard) } - end - end -end From c8e9a847bc486ff6c3b5c3c554b04449f3912a6a Mon Sep 17 00:00:00 2001 From: Erick Fabian Date: Thu, 20 Jun 2019 12:29:19 -0500 Subject: [PATCH 05/14] Refactors and improvements --- lib/dolly/configuration.rb | 1 - lib/dolly/mango.rb | 27 ++++++++++++++++++++------- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/lib/dolly/configuration.rb b/lib/dolly/configuration.rb index d799dad..788d4a3 100644 --- a/lib/dolly/configuration.rb +++ b/lib/dolly/configuration.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true - require 'erb' module Dolly diff --git a/lib/dolly/mango.rb b/lib/dolly/mango.rb index 810212f..4556c5f 100644 --- a/lib/dolly/mango.rb +++ b/lib/dolly/mango.rb @@ -23,16 +23,30 @@ module Mango DESIGN = "_find" def find_by(query, opts = {}) + build_model_from_doc(find_doc_by(query, opts)) + end + + def find_doc_by(query, opts = {}) opts.merge!(limit: 1) perform_query(build_query(query, opts))[:docs].first end def where(query, opts = {}) + docs_where(query, opts).map do |doc| + build_model_from_doc(doc) + end + end + + def docs_where(query, opts = {}) perform_query(build_query(query, opts))[:docs] end private + def build_model_from_doc(doc) + self.new(doc.slice(*self.property_keys)) + end + def perform_query(structured_query) connection.post(DESIGN, structured_query) end @@ -44,17 +58,16 @@ def build_query(query, opts) end def build_selectors(query) - query.map do |key, values| - inner_key = values.keys.first - inner_value = values.values.first + query.each_with_object(Hash.new) do |(key, value), hsh| + inner_key = value.keys.first + inner_value = value.values.first next unless AVAILABLE_SELECTORS.include?(inner_key) - - { + hsh.merge!({ "#{key}" => { build_key(inner_key) => inner_value } - } - end.compact.inject(:merge) + }) + end end def build_key(key) From 16336c15204996ee117306bcbaa0a9a6f1896151 Mon Sep 17 00:00:00 2001 From: Erick Fabian Date: Thu, 20 Jun 2019 16:04:36 -0500 Subject: [PATCH 06/14] Support deeply nested operator combinations --- lib/dolly/mango.rb | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/lib/dolly/mango.rb b/lib/dolly/mango.rb index 4556c5f..f3136c0 100644 --- a/lib/dolly/mango.rb +++ b/lib/dolly/mango.rb @@ -3,8 +3,19 @@ module Dolly module Mango - SELECTOR_SYMBOL = "$" - AVAILABLE_SELECTORS = %I[ + SELECTOR_SYMBOL = '$' + + COMBINATION_OPERATORS = %I[ + and + or + not + nor + all + elemMatch + allMath + ].freeze + + CONDITION_OPERATORS = %I[ lt lte eq @@ -20,7 +31,7 @@ module Mango regex ].freeze - DESIGN = "_find" + DESIGN = '_find' def find_by(query, opts = {}) build_model_from_doc(find_doc_by(query, opts)) @@ -44,6 +55,7 @@ def docs_where(query, opts = {}) private def build_model_from_doc(doc) + return nil if doc.nil? self.new(doc.slice(*self.property_keys)) end @@ -53,25 +65,22 @@ def perform_query(structured_query) def build_query(query, opts) { - "selector" => build_selectors(query) + 'selector' => build_selectors(query) }.merge(opts) end def build_selectors(query) - query.each_with_object(Hash.new) do |(key, value), hsh| - inner_key = value.keys.first - inner_value = value.values.first - next unless AVAILABLE_SELECTORS.include?(inner_key) - hsh.merge!({ - "#{key}" => { - build_key(inner_key) => inner_value - } - }) + query.deep_transform_keys do |key| + is_operator?(key) ? build_key(key) : key end end def build_key(key) "#{SELECTOR_SYMBOL}#{key}" end + + def is_operator?(key) + COMBINATION_OPERATORS.include?(key) || CONDITION_OPERATORS.include?(key) + end end end From af875dfc16eef4f48b200b9d5d8738e6750f5c7e Mon Sep 17 00:00:00 2001 From: Erick Fabian Date: Thu, 20 Jun 2019 16:24:15 -0500 Subject: [PATCH 07/14] Better error handling --- lib/dolly/exceptions.rb | 10 ++++++++++ lib/dolly/mango.rb | 9 +++++++-- lib/dolly/properties.rb | 6 +++++- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/lib/dolly/exceptions.rb b/lib/dolly/exceptions.rb index 6e200ee..e0ce22b 100644 --- a/lib/dolly/exceptions.rb +++ b/lib/dolly/exceptions.rb @@ -15,6 +15,16 @@ def to_s end end + class InvalidMangoOperatorError < RuntimeError + def initialize msg + @msg = msg + end + + def to_s + "Invalid Mango operator: #{@msg.inspect}" + end + end + class InvalidConfigFileError < RuntimeError; end class InvalidProperty < RuntimeError; end class DocumentInvalidError < RuntimeError; end diff --git a/lib/dolly/mango.rb b/lib/dolly/mango.rb index f3136c0..cce0079 100644 --- a/lib/dolly/mango.rb +++ b/lib/dolly/mango.rb @@ -56,7 +56,7 @@ def docs_where(query, opts = {}) def build_model_from_doc(doc) return nil if doc.nil? - self.new(doc.slice(*self.property_keys)) + self.new(doc.slice(*self.all_property_keys)) end def perform_query(structured_query) @@ -71,7 +71,12 @@ def build_query(query, opts) def build_selectors(query) query.deep_transform_keys do |key| - is_operator?(key) ? build_key(key) : key + if is_operator?(key) + build_key(key) + else + raise "invalid operator #{key}" unless self.all_property_keys.include?(key) + key + end end end diff --git a/lib/dolly/properties.rb b/lib/dolly/properties.rb index 3af9d28..4a24ff7 100644 --- a/lib/dolly/properties.rb +++ b/lib/dolly/properties.rb @@ -20,8 +20,12 @@ def properties @properties ||= PropertySet.new end + def all_property_keys + properties.map(&:key) + SPECIAL_KEYS + end + def property_keys - properties.map(&:key) - SPECIAL_KEYS + all_property_keys - SPECIAL_KEYS end def property_clean_doc(doc) From fe796940531d8d36f2cd572229d2dc08d6c75a94 Mon Sep 17 00:00:00 2001 From: Erick Fabian Date: Thu, 20 Jun 2019 17:00:50 -0500 Subject: [PATCH 08/14] Add exception --- lib/dolly/mango.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/dolly/mango.rb b/lib/dolly/mango.rb index cce0079..6847deb 100644 --- a/lib/dolly/mango.rb +++ b/lib/dolly/mango.rb @@ -74,7 +74,7 @@ def build_selectors(query) if is_operator?(key) build_key(key) else - raise "invalid operator #{key}" unless self.all_property_keys.include?(key) + raise InvalidMangoOperatorError unless self.all_property_keys.include?(key) key end end From 16e3060d530377c2fa24c32295161af077e3f5f2 Mon Sep 17 00:00:00 2001 From: Erick Fabian Date: Mon, 24 Jun 2019 10:40:33 -0500 Subject: [PATCH 09/14] Add mango index management --- lib/dolly.rb | 1 + lib/dolly/document.rb | 1 + lib/dolly/exceptions.rb | 1 + lib/dolly/mango.rb | 18 ++++++++++- lib/dolly/mango_index.rb | 49 +++++++++++++++++++++++++++++ lib/refinements/hash_refinements.rb | 17 ++++++++++ lib/tasks/db.rake | 12 +++++++ 7 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 lib/dolly/mango_index.rb create mode 100644 lib/refinements/hash_refinements.rb diff --git a/lib/dolly.rb b/lib/dolly.rb index d824ee6..3bd1093 100644 --- a/lib/dolly.rb +++ b/lib/dolly.rb @@ -1,5 +1,6 @@ require "dolly/version" require "dolly/document" +require 'dolly/mango_index' require 'railties/railtie' if defined?(Rails) module Dolly diff --git a/lib/dolly/document.rb b/lib/dolly/document.rb index e26871b..fb71e2d 100644 --- a/lib/dolly/document.rb +++ b/lib/dolly/document.rb @@ -1,4 +1,5 @@ require 'dolly/mango' +require 'dolly/mango_index' require 'dolly/query' require 'dolly/connection' require 'dolly/request' diff --git a/lib/dolly/exceptions.rb b/lib/dolly/exceptions.rb index e0ce22b..b5515f2 100644 --- a/lib/dolly/exceptions.rb +++ b/lib/dolly/exceptions.rb @@ -25,6 +25,7 @@ def to_s end end + class IndexNotFound < RuntimeError; end class InvalidConfigFileError < RuntimeError; end class InvalidProperty < RuntimeError; end class DocumentInvalidError < RuntimeError; end diff --git a/lib/dolly/mango.rb b/lib/dolly/mango.rb index 6847deb..eda177d 100644 --- a/lib/dolly/mango.rb +++ b/lib/dolly/mango.rb @@ -1,7 +1,11 @@ # frozen_string_literal: true +require 'dolly/mango_index' +require 'refinements/hash_refinements' + module Dolly module Mango + using HashRefinements SELECTOR_SYMBOL = '$' @@ -38,6 +42,7 @@ def find_by(query, opts = {}) end def find_doc_by(query, opts = {}) + raise IndexNotFound unless index_exists?(query) opts.merge!(limit: 1) perform_query(build_query(query, opts))[:docs].first end @@ -49,6 +54,7 @@ def where(query, opts = {}) end def docs_where(query, opts = {}) + raise IndexNotFound unless index_exists?(query) perform_query(build_query(query, opts))[:docs] end @@ -74,7 +80,7 @@ def build_selectors(query) if is_operator?(key) build_key(key) else - raise InvalidMangoOperatorError unless self.all_property_keys.include?(key) + raise InvalidMangoOperatorError.new(key) unless self.all_property_keys.include?(key) key end end @@ -87,5 +93,15 @@ def build_key(key) def is_operator?(key) COMBINATION_OPERATORS.include?(key) || CONDITION_OPERATORS.include?(key) end + + def index_exists?(query) + Dolly::MangoIndex.find_by_fields(fetch_fields(query)) + end + + def fetch_fields(query) + query.deep_keys.reject do |key| + is_operator?(key) + end + end end end diff --git a/lib/dolly/mango_index.rb b/lib/dolly/mango_index.rb new file mode 100644 index 0000000..e027a49 --- /dev/null +++ b/lib/dolly/mango_index.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +require 'dolly/document' + +module Dolly + class MangoIndex + class << self + DESIGN = '_index' + + def all + Dolly::Document.connection.get(DESIGN)[:indexes] + end + + def create(name, fields, type = 'json') + puts name + puts fields + Dolly::Document.connection.post(DESIGN, build_index_structure(name, fields, type)) + end + + def find_by_fields(fields) + all.find do |index_doc| + index_doc.dig(:def, :fields).map(&:keys).flatten == fields + end + end + + def delete_all + all.each do |index_doc| + delete(index_doc) + end + end + + def delete(doc) + + end + + private + + def build_index_structure(name, fields, type) + { + index: { + fields: fields + }, + name: name, + type: type + } + end + end + end +end diff --git a/lib/refinements/hash_refinements.rb b/lib/refinements/hash_refinements.rb new file mode 100644 index 0000000..25373e9 --- /dev/null +++ b/lib/refinements/hash_refinements.rb @@ -0,0 +1,17 @@ +module HashRefinements + + refine Hash do + # File 'lib/github_api/core_ext/hash.rb', line 54 + + def deep_keys + keys = self.keys + keys.each do |key| + if self[key].is_a?(Hash) + keys << self[key].deep_keys.compact.flatten + next + end + end + keys.flatten + end + end +end diff --git a/lib/tasks/db.rake b/lib/tasks/db.rake index f60281b..d1b5ca6 100644 --- a/lib/tasks/db.rake +++ b/lib/tasks/db.rake @@ -59,5 +59,17 @@ namespace :db do end end + namespace :index do + desc "Creates indexes for mango querys located in db/indexes/*.json" + task create: :environment do + indexes_dir = Rails.root.join('db', 'indexes') + files = Dir.glob File.join(indexes_dir, '**', '*.json') + + files.each do |file| + index_data = JSON.parse(File.read(file)) + Dolly::MangoIndex.create(index_data['name'], index_data['fields']) + end + end + end end From 4af2bbe27a23cda67db6ff2572e6bc4a16031ae8 Mon Sep 17 00:00:00 2001 From: Erick Fabian Date: Mon, 24 Jun 2019 14:38:09 -0500 Subject: [PATCH 10/14] Separate mango indexes for another pr --- lib/dolly.rb | 1 - lib/dolly/document.rb | 2 -- lib/dolly/exceptions.rb | 1 - lib/dolly/mango.rb | 15 --------- lib/dolly/mango_index.rb | 49 ----------------------------- lib/refinements/hash_refinements.rb | 17 ---------- lib/tasks/db.rake | 13 -------- 7 files changed, 98 deletions(-) delete mode 100644 lib/dolly/mango_index.rb delete mode 100644 lib/refinements/hash_refinements.rb diff --git a/lib/dolly.rb b/lib/dolly.rb index 3bd1093..d824ee6 100644 --- a/lib/dolly.rb +++ b/lib/dolly.rb @@ -1,6 +1,5 @@ require "dolly/version" require "dolly/document" -require 'dolly/mango_index' require 'railties/railtie' if defined?(Rails) module Dolly diff --git a/lib/dolly/document.rb b/lib/dolly/document.rb index fb71e2d..6c71c4b 100644 --- a/lib/dolly/document.rb +++ b/lib/dolly/document.rb @@ -1,5 +1,4 @@ require 'dolly/mango' -require 'dolly/mango_index' require 'dolly/query' require 'dolly/connection' require 'dolly/request' @@ -17,7 +16,6 @@ module Dolly class Document - extend Mango extend Query extend Request extend DepracatedDatabase diff --git a/lib/dolly/exceptions.rb b/lib/dolly/exceptions.rb index b5515f2..e0ce22b 100644 --- a/lib/dolly/exceptions.rb +++ b/lib/dolly/exceptions.rb @@ -25,7 +25,6 @@ def to_s end end - class IndexNotFound < RuntimeError; end class InvalidConfigFileError < RuntimeError; end class InvalidProperty < RuntimeError; end class DocumentInvalidError < RuntimeError; end diff --git a/lib/dolly/mango.rb b/lib/dolly/mango.rb index eda177d..2168a42 100644 --- a/lib/dolly/mango.rb +++ b/lib/dolly/mango.rb @@ -1,8 +1,5 @@ # frozen_string_literal: true -require 'dolly/mango_index' -require 'refinements/hash_refinements' - module Dolly module Mango using HashRefinements @@ -42,7 +39,6 @@ def find_by(query, opts = {}) end def find_doc_by(query, opts = {}) - raise IndexNotFound unless index_exists?(query) opts.merge!(limit: 1) perform_query(build_query(query, opts))[:docs].first end @@ -54,7 +50,6 @@ def where(query, opts = {}) end def docs_where(query, opts = {}) - raise IndexNotFound unless index_exists?(query) perform_query(build_query(query, opts))[:docs] end @@ -93,15 +88,5 @@ def build_key(key) def is_operator?(key) COMBINATION_OPERATORS.include?(key) || CONDITION_OPERATORS.include?(key) end - - def index_exists?(query) - Dolly::MangoIndex.find_by_fields(fetch_fields(query)) - end - - def fetch_fields(query) - query.deep_keys.reject do |key| - is_operator?(key) - end - end end end diff --git a/lib/dolly/mango_index.rb b/lib/dolly/mango_index.rb deleted file mode 100644 index e027a49..0000000 --- a/lib/dolly/mango_index.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true - -require 'dolly/document' - -module Dolly - class MangoIndex - class << self - DESIGN = '_index' - - def all - Dolly::Document.connection.get(DESIGN)[:indexes] - end - - def create(name, fields, type = 'json') - puts name - puts fields - Dolly::Document.connection.post(DESIGN, build_index_structure(name, fields, type)) - end - - def find_by_fields(fields) - all.find do |index_doc| - index_doc.dig(:def, :fields).map(&:keys).flatten == fields - end - end - - def delete_all - all.each do |index_doc| - delete(index_doc) - end - end - - def delete(doc) - - end - - private - - def build_index_structure(name, fields, type) - { - index: { - fields: fields - }, - name: name, - type: type - } - end - end - end -end diff --git a/lib/refinements/hash_refinements.rb b/lib/refinements/hash_refinements.rb deleted file mode 100644 index 25373e9..0000000 --- a/lib/refinements/hash_refinements.rb +++ /dev/null @@ -1,17 +0,0 @@ -module HashRefinements - - refine Hash do - # File 'lib/github_api/core_ext/hash.rb', line 54 - - def deep_keys - keys = self.keys - keys.each do |key| - if self[key].is_a?(Hash) - keys << self[key].deep_keys.compact.flatten - next - end - end - keys.flatten - end - end -end diff --git a/lib/tasks/db.rake b/lib/tasks/db.rake index d1b5ca6..8abd51b 100644 --- a/lib/tasks/db.rake +++ b/lib/tasks/db.rake @@ -58,18 +58,5 @@ namespace :db do Dolly::Document.connection.request :put, design_doc_name, view_doc if will_save end end - - namespace :index do - desc "Creates indexes for mango querys located in db/indexes/*.json" - task create: :environment do - indexes_dir = Rails.root.join('db', 'indexes') - files = Dir.glob File.join(indexes_dir, '**', '*.json') - - files.each do |file| - index_data = JSON.parse(File.read(file)) - Dolly::MangoIndex.create(index_data['name'], index_data['fields']) - end - end - end end From 821304a4eab5191db9328cc6811bb73dce178208 Mon Sep 17 00:00:00 2001 From: Erick Fabian Date: Tue, 25 Jun 2019 10:54:03 -0500 Subject: [PATCH 11/14] Add tests for mango querys --- Gemfile.lock | 58 ++++++++++ dolly.gemspec | 2 +- lib/dolly/connection.rb | 2 +- lib/dolly/document.rb | 1 + lib/dolly/mango.rb | 2 + lib/refinements/hash_refinements.rb | 27 +++++ test/document_test.rb | 2 - test/mango_test.rb | 164 ++++++++++++++++++++++++++++ test/test_helper.rb | 2 + 9 files changed, 256 insertions(+), 4 deletions(-) create mode 100644 Gemfile.lock create mode 100644 lib/refinements/hash_refinements.rb create mode 100644 test/mango_test.rb diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 0000000..d638023 --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,58 @@ +PATH + remote: . + specs: + dolly (3.0.0) + oj + +GEM + remote: https://rubygems.org/ + specs: + addressable (2.6.0) + public_suffix (>= 2.0.2, < 4.0) + crack (0.4.3) + safe_yaml (~> 1.0.0) + hashdiff (0.4.0) + metaclass (0.0.4) + mocha (1.9.0) + metaclass (~> 0.0.1) + oj (3.7.12) + power_assert (1.1.4) + public_suffix (3.1.0) + rake (10.5.0) + rr (1.2.1) + safe_yaml (1.0.5) + test-unit (3.3.3) + power_assert + test-unit-context (0.5.1) + test-unit (>= 2.4.0) + test-unit-full (0.0.5) + test-unit + test-unit-context + test-unit-notify + test-unit-rr + test-unit-runner-tap + test-unit-notify (1.0.4) + test-unit (>= 2.4.9) + test-unit-rr (1.0.5) + rr (>= 1.1.1) + test-unit (>= 2.5.2) + test-unit-runner-tap (1.1.2) + test-unit + webmock (3.6.0) + addressable (>= 2.3.6) + crack (>= 0.3.2) + hashdiff (>= 0.4.0, < 2.0.0) + +PLATFORMS + ruby + +DEPENDENCIES + bundler + dolly! + mocha + rake (~> 10.0) + test-unit-full + webmock + +BUNDLED WITH + 1.17.1 diff --git a/dolly.gemspec b/dolly.gemspec index dc9299e..5a92fb5 100644 --- a/dolly.gemspec +++ b/dolly.gemspec @@ -30,7 +30,7 @@ Gem::Specification.new do |spec| spec.require_paths = ["lib"] spec.add_dependency "oj" - spec.add_development_dependency "bundler", "~> 2.0" + spec.add_development_dependency "bundler" spec.add_development_dependency "rake", "~> 10.0" spec.add_development_dependency "test-unit-full" spec.add_development_dependency "webmock" diff --git a/lib/dolly/connection.rb b/lib/dolly/connection.rb index 6a72e69..13bd234 100644 --- a/lib/dolly/connection.rb +++ b/lib/dolly/connection.rb @@ -72,7 +72,7 @@ def request(method, resource, data = {}) def start_request(req) Net::HTTP.start(req.uri.hostname, req.uri.port) do |http| - req.basic_auth env['username'], env['password'] if env['username'].present? + req.basic_auth env['username'], env['password'] if env['username']&.present? http.request(req) end end diff --git a/lib/dolly/document.rb b/lib/dolly/document.rb index 6c71c4b..e26871b 100644 --- a/lib/dolly/document.rb +++ b/lib/dolly/document.rb @@ -16,6 +16,7 @@ module Dolly class Document + extend Mango extend Query extend Request extend DepracatedDatabase diff --git a/lib/dolly/mango.rb b/lib/dolly/mango.rb index 2168a42..3052de4 100644 --- a/lib/dolly/mango.rb +++ b/lib/dolly/mango.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'refinements/hash_refinements' + module Dolly module Mango using HashRefinements diff --git a/lib/refinements/hash_refinements.rb b/lib/refinements/hash_refinements.rb new file mode 100644 index 0000000..9afdcf9 --- /dev/null +++ b/lib/refinements/hash_refinements.rb @@ -0,0 +1,27 @@ +module HashRefinements + refine Hash do + # File activesupport/lib/active_support/core_ext/hash/keys.rb, line 82 + def deep_transform_keys(&block) + _deep_transform_keys_in_object(self, &block) + end + + def slice(*keys) + keys.each_with_object(Hash.new) { |k, hash| hash[k] = self[k] if has_key?(k) } + end + + private + + def _deep_transform_keys_in_object(object, &block) + case object + when Hash + object.each_with_object({}) do |(key, value), result| + result[yield(key)] = _deep_transform_keys_in_object(value, &block) + end + when Array + object.map { |e| _deep_transform_keys_in_object(e, &block) } + else + object + end + end + end +end diff --git a/test/document_test.rb b/test/document_test.rb index 4ddafcf..09eda44 100644 --- a/test/document_test.rb +++ b/test/document_test.rb @@ -1,7 +1,5 @@ require 'test_helper' -class BaseDolly < Dolly::Document; end - class BarFoo < BaseDolly property :a, :b, :c, :d, :e, :f, :g, :h, :i, :j, :k, :l, :m, :n, :persist end diff --git a/test/mango_test.rb b/test/mango_test.rb new file mode 100644 index 0000000..b9c0942 --- /dev/null +++ b/test/mango_test.rb @@ -0,0 +1,164 @@ +require 'test_helper' + +class FooBar < BaseDolly + property :foo, :bar + property :with_default, default: 1 + property :boolean, class_name: TrueClass, default: true + property :date, class_name: Date + property :time, class_name: Time + property :datetime, class_name: DateTime + property :is_nil, class_name: NilClass, default: nil + + timestamps! +end + +class MangoTest < Test::Unit::TestCase + DB_BASE_PATH = "http://localhost:5984/test".freeze + + def setup + data = {foo: 'Foo', bar: 'Bar', type: 'foo_bar'} + + all_docs = [ {foo: 'Foo B', bar: 'Bar B', type: 'foo_bar'}, {foo: 'Foo A', bar: 'Bar A', type: 'foo_bar'}] + + view_resp = build_view_response [data] + empty_resp = build_view_response [] + not_found_resp = generic_response [{ key: "foo_bar/2", error: "not_found" }] + @multi_resp = build_view_response all_docs + @multi_type_resp = build_view_collation_response all_docs + + build_request [["foo_bar","1"]], view_resp + build_request [["foo_bar","2"]], empty_resp + build_request [["foo_bar","1"],["foo_bar","2"]], @multi_resp + + stub_request(:get, "#{query_base_path}?startkey=%22foo_bar%2F%22&endkey=%22foo_bar%2F%EF%BF%B0%22&include_docs=true"). + to_return(body: @multi_resp.to_json) + end + + test '#find_by' do + #TODO: clean up all the fake request creation + resp = { docs: [{ foo: 'bar', id: "foo_bar/1"} ] } + + stub_request(:post, query_base_path). + to_return(body: resp.to_json) + + assert_equal(FooBar.find_by(foo: 'bar').class, FooBar) + end + + test '#find_by with no returned data' do + resp = { docs: [] } + + stub_request(:post, query_base_path). + to_return(body: resp.to_json) + + assert_equal(FooBar.find_by(foo: 'bar'), nil) + end + + test '#find_doc_by' do + #TODO: clean up all the fake request creation + resp = { docs: [{ foo: 'bar', id: "foo_bar/1"} ] } + + stub_request(:post, query_base_path). + to_return(body: resp.to_json) + + assert_equal(FooBar.find_doc_by(foo: 'bar').class, Hash) + end + + test '#where' do + #TODO: clean up all the fake request creation + resp = { docs: [{ foo: 'bar', id: "foo_bar/1"} ] } + + stub_request(:post, query_base_path). + to_return(body: resp.to_json) + + assert_equal(FooBar.where(foo: { eq: 'bar' }).map(&:class).uniq, [FooBar]) + end + + test '#where with no returned data' do + resp = { docs: [] } + + stub_request(:post, query_base_path). + to_return(body: resp.to_json) + + assert_equal(FooBar.where(foo: 'bar'), []) + end + + test '#docs_where' do + #TODO: clean up all the fake request creation + resp = { docs: [{ foo: 'bar', id: "foo_bar/1"} ] } + + stub_request(:post, query_base_path). + to_return(body: resp.to_json) + + assert_equal(FooBar.docs_where(foo: { eq: 'bar' }).map(&:class).uniq, [Hash]) + end + + test '#build_query' do + query = { and: [{ _id: { eq: 'foo_bar/1' } } , { foo: { eq: 'bar'}} ] } + opts = {} + expected = {"selector"=>{"$and"=>[{:_id=>{"$eq"=>"foo_bar/1"}}, {:foo=>{"$eq"=>"bar"}}]}} + + assert_equal(FooBar.send(:build_query, query, opts), expected) + end + + test '#build_query with options' do + query = { and: [{ _id: { eq: 'foo_bar/1' } } , { foo: { eq: 'bar'}} ] } + opts = { limit: 1, fields: ['foo']} + expected = {"selector"=>{"$and"=>[{:_id=>{"$eq"=>"foo_bar/1"}}, {:foo=>{"$eq"=>"bar"}}]}, limit: 1, fields: ['foo']} + + assert_equal(FooBar.send(:build_query, query, opts), expected) + end + + test '#build_selectors with invalid operator' do + query = { _id: { eeeq: 'foo_bar/1' } } + + assert_raise Dolly::InvalidMangoOperatorError do + FooBar.send(:build_selectors, query) + end + end + + private + def generic_response rows, count = 1 + {total_rows: count, offset:0, rows: rows} + end + + def build_view_response properties + rows = properties.map.with_index do |v, i| + { + id: "foo_bar/#{i}", + key: "foo_bar", + value: 1, + doc: {_id: "foo_bar/#{i}", _rev: SecureRandom.hex}.merge!(v) + } + end + generic_response rows, properties.count + end + + def build_view_collation_response properties + rows = properties.map.with_index do |v, i| + id = i.zero? ? "foo_bar/#{i}" : "baz/#{i}" + { + id: id, + key: "foo_bar", + value: 1, + doc: {_id: id, _rev: SecureRandom.hex}.merge!(v) + } + end + generic_response rows, properties.count + end + + + def build_request keys, body, view_name = 'foo_bar' + query = "keys=#{CGI::escape keys.to_s.gsub(' ','')}&" unless keys&.empty? + stub_request(:get, "#{query_base_path}?#{query.to_s}include_docs=true"). + to_return(body: body.to_json) + end + + def query_base_path + "#{DB_BASE_PATH}/_find" + end + + def build_save_request(obj) + stub_request(:put, "#{DB_BASE_PATH}/#{CGI.escape(obj.id)}"). + to_return(body: {ok: true, id: obj.id, rev: "FF0000" }.to_json) + end +end diff --git a/test/test_helper.rb b/test/test_helper.rb index 69681c7..61766bf 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -29,3 +29,5 @@ def base_path %r{http://.*:5984/#{DEFAULT_DB}} end end + +class BaseDolly < Dolly::Document; end From c1f554a9e5d2a01f50ec826bd322aafba2199dd9 Mon Sep 17 00:00:00 2001 From: Erick Fabian Date: Tue, 25 Jun 2019 10:56:25 -0500 Subject: [PATCH 12/14] fix bundler merge conflict --- Gemfile.lock | 4 ++-- dolly.gemspec | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index d638023..c1dad03 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -47,7 +47,7 @@ PLATFORMS ruby DEPENDENCIES - bundler + bundler (~> 2.0) dolly! mocha rake (~> 10.0) @@ -55,4 +55,4 @@ DEPENDENCIES webmock BUNDLED WITH - 1.17.1 + 2.0.2 diff --git a/dolly.gemspec b/dolly.gemspec index 5a92fb5..dc9299e 100644 --- a/dolly.gemspec +++ b/dolly.gemspec @@ -30,7 +30,7 @@ Gem::Specification.new do |spec| spec.require_paths = ["lib"] spec.add_dependency "oj" - spec.add_development_dependency "bundler" + spec.add_development_dependency "bundler", "~> 2.0" spec.add_development_dependency "rake", "~> 10.0" spec.add_development_dependency "test-unit-full" spec.add_development_dependency "webmock" From af03c183fceaca6c815e422e5d59b8fe67e00d0e Mon Sep 17 00:00:00 2001 From: Erick Fabian Date: Tue, 25 Jun 2019 10:58:24 -0500 Subject: [PATCH 13/14] Remove unrelated space change --- lib/tasks/db.rake | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/tasks/db.rake b/lib/tasks/db.rake index 8abd51b..f60281b 100644 --- a/lib/tasks/db.rake +++ b/lib/tasks/db.rake @@ -58,5 +58,6 @@ namespace :db do Dolly::Document.connection.request :put, design_doc_name, view_doc if will_save end end + end From ad07fba3ea9993f034328800f5cf1032f969c5b0 Mon Sep 17 00:00:00 2001 From: Erick Fabian Date: Tue, 25 Jun 2019 11:22:43 -0500 Subject: [PATCH 14/14] Attend comments --- lib/dolly/mango.rb | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/lib/dolly/mango.rb b/lib/dolly/mango.rb index 3052de4..85bbb5c 100644 --- a/lib/dolly/mango.rb +++ b/lib/dolly/mango.rb @@ -34,6 +34,8 @@ module Mango regex ].freeze + ALL_OPERATORS = COMBINATION_OPERATORS + CONDITION_OPERATORS + DESIGN = '_find' def find_by(query, opts = {}) @@ -59,7 +61,7 @@ def docs_where(query, opts = {}) def build_model_from_doc(doc) return nil if doc.nil? - self.new(doc.slice(*self.all_property_keys)) + new(doc.slice(*all_property_keys)) end def perform_query(structured_query) @@ -67,19 +69,14 @@ def perform_query(structured_query) end def build_query(query, opts) - { - 'selector' => build_selectors(query) - }.merge(opts) + { 'selector' => build_selectors(query) }.merge(opts) end def build_selectors(query) query.deep_transform_keys do |key| - if is_operator?(key) - build_key(key) - else - raise InvalidMangoOperatorError.new(key) unless self.all_property_keys.include?(key) - key - end + next build_key(key) if is_operator?(key) + raise InvalidMangoOperatorError.new(key) unless self.all_property_keys.include?(key) + key end end @@ -88,7 +85,7 @@ def build_key(key) end def is_operator?(key) - COMBINATION_OPERATORS.include?(key) || CONDITION_OPERATORS.include?(key) + ALL_OPERATORS.include?(key) end end end