From ca8bc87efa20d8e3c8f43f71c6a2acf2a0288a3f Mon Sep 17 00:00:00 2001 From: Luca Marturana Date: Sat, 5 Jul 2014 19:39:46 +0200 Subject: [PATCH 1/5] Initial draft of redis_ha connection --- lib/redic_ha.rb | 83 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 lib/redic_ha.rb diff --git a/lib/redic_ha.rb b/lib/redic_ha.rb new file mode 100644 index 0000000..26c6980 --- /dev/null +++ b/lib/redic_ha.rb @@ -0,0 +1,83 @@ +require_relative "redic/client" +require_relative "redic" + +class RedicHA + attr :sentinel_url + attr :master_name + attr :client + attr :timeout + attr :db + attr :queue + + def initialize(master_name, sentinel_url = "redis://127.0.0.1:26379", db = 0, timeout = 10_000_000) + @sentinel_url = sentinel_url + @master_name = master_name + @timeout = timeout + @client = Redic::Client.new(get_master_url, timeout) + @db = db + @queue = [] + end + + def call(*args) + exec_with_retry do + @client.connect do + @client.write(args) + @client.read + end + end + end + + def queue(*args) + @queue << args + end + + def commit + exec_with_retry do + @client.connect do + @queue.each do |args| + @client.write(args) + end + + @queue.map do + @client.read + end + end + end + ensure + @queue.clear + end + + def timeout + @client.timeout + end + + private + + def get_master_url + sentinel = Redic.new self.sentinel_url + reply = sentinel.call "SENTINEL", "masters" + masters = reply.inject({}) do |m, el| + m[el[1]] = { + :ip => el[3], + :port => el[5] + } + m + end + master = masters[self.master_name] + return "redis://#{master[:ip]}:#{master[:port]}/#{@db}" + end + + def exec_with_retry(&block) + retries = 20 + begin + block.call + rescue Errno::ECONNREFUSED + if retries >= 0 + retries -= 1 + sleep 0.1 + @client = Redic::Client.new(get_master_url, @timeout) + retry + end + end + end +end \ No newline at end of file From c1ce8f9eb9f7c966342be72a814c9b0c050f017f Mon Sep 17 00:00:00 2001 From: Luca Marturana Date: Wed, 9 Jul 2014 10:59:11 +0200 Subject: [PATCH 2/5] Raise exception if we are writing on readonly slave --- lib/redic/client.rb | 7 ++++++- lib/redic_ha.rb | 16 +++++----------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/lib/redic/client.rb b/lib/redic/client.rb index 4e7d542..ecf0751 100644 --- a/lib/redic/client.rb +++ b/lib/redic/client.rb @@ -13,7 +13,12 @@ def initialize(url, timeout) end def read - @connection.read + ret = @connection.read + if ret.is_a? RuntimeError and ret.to_s.start_with? "READONLY" + raise ret + else + return ret + end end def write(command) diff --git a/lib/redic_ha.rb b/lib/redic_ha.rb index 26c6980..0347947 100644 --- a/lib/redic_ha.rb +++ b/lib/redic_ha.rb @@ -55,28 +55,22 @@ def timeout def get_master_url sentinel = Redic.new self.sentinel_url - reply = sentinel.call "SENTINEL", "masters" - masters = reply.inject({}) do |m, el| - m[el[1]] = { - :ip => el[3], - :port => el[5] - } - m - end - master = masters[self.master_name] - return "redis://#{master[:ip]}:#{master[:port]}/#{@db}" + master_ip, master_port = sentinel.call "SENTINEL", "get-master-addr-by-name", master_name + return "redis://#{master_ip}:#{master_port}/#{@db}" end def exec_with_retry(&block) retries = 20 begin block.call - rescue Errno::ECONNREFUSED + rescue Errno::ECONNREFUSED, RuntimeError if retries >= 0 retries -= 1 sleep 0.1 @client = Redic::Client.new(get_master_url, @timeout) retry + else + raise e end end end From 6ff261ddc57a529e50dae0be82f3df2328084ec2 Mon Sep 17 00:00:00 2001 From: Luca Marturana Date: Wed, 9 Jul 2014 11:33:26 +0200 Subject: [PATCH 3/5] Add tests for redisHA --- makefile => Makefile | 0 lib/redic_ha.rb | 4 ++-- tests/redic_ha_test.rb | 25 +++++++++++++++++++++++++ tests/redic_test.rb | 2 ++ 4 files changed, 29 insertions(+), 2 deletions(-) rename makefile => Makefile (100%) create mode 100644 tests/redic_ha_test.rb diff --git a/makefile b/Makefile similarity index 100% rename from makefile rename to Makefile diff --git a/lib/redic_ha.rb b/lib/redic_ha.rb index 0347947..0baccd8 100644 --- a/lib/redic_ha.rb +++ b/lib/redic_ha.rb @@ -63,10 +63,10 @@ def exec_with_retry(&block) retries = 20 begin block.call - rescue Errno::ECONNREFUSED, RuntimeError + rescue Errno::ECONNREFUSED, RuntimeError => e if retries >= 0 retries -= 1 - sleep 0.1 + sleep 0.5 @client = Redic::Client.new(get_master_url, @timeout) retry else diff --git a/tests/redic_ha_test.rb b/tests/redic_ha_test.rb new file mode 100644 index 0000000..50c5215 --- /dev/null +++ b/tests/redic_ha_test.rb @@ -0,0 +1,25 @@ +require "cutest" +require_relative "../lib/redic_ha" +require_relative "../lib/redic" + +prepare do + RedicHA.new(ENV['MASTER_NAME'], ENV['SENTINEL_URL']).call("FLUSHDB") +end + +setup do + RedicHA.new(ENV['MASTER_NAME'], ENV['SENTINEL_URL']) +end + +test "test_failover_handling" do |c| + 10.times do + assert_equal "OK", c.call(:set, :foo, :bar) + end + + # Simulate failover + c2 = Redic.new ENV['SENTINEL_URL'] + c2.call :sentinel, :failover, :test + + 10.times do + assert_equal "OK", c.call(:set, :foo, :bar) + end +end \ No newline at end of file diff --git a/tests/redic_test.rb b/tests/redic_test.rb index c09cb62..92c9806 100644 --- a/tests/redic_test.rb +++ b/tests/redic_test.rb @@ -1,3 +1,5 @@ +# coding=utf-8 + require "cutest" require_relative "../lib/redic" From a6a26bd0d34ed891c5fe72ed95ab1c3647651f96 Mon Sep 17 00:00:00 2001 From: Luca Marturana Date: Wed, 9 Jul 2014 14:23:53 +0200 Subject: [PATCH 4/5] Correct initialization order --- lib/redic_ha.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/redic_ha.rb b/lib/redic_ha.rb index 0baccd8..fb2c6a4 100644 --- a/lib/redic_ha.rb +++ b/lib/redic_ha.rb @@ -13,8 +13,8 @@ def initialize(master_name, sentinel_url = "redis://127.0.0.1:26379", db = 0, ti @sentinel_url = sentinel_url @master_name = master_name @timeout = timeout - @client = Redic::Client.new(get_master_url, timeout) @db = db + @client = Redic::Client.new(get_master_url, timeout) @queue = [] end From 626d58478be961cbe8ef1db286303df93e9003f3 Mon Sep 17 00:00:00 2001 From: Luca Marturana Date: Wed, 9 Jul 2014 18:03:41 +0200 Subject: [PATCH 5/5] Add method for compatibility with Redic object --- lib/redic_ha.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/redic_ha.rb b/lib/redic_ha.rb index fb2c6a4..e938d8d 100644 --- a/lib/redic_ha.rb +++ b/lib/redic_ha.rb @@ -51,6 +51,10 @@ def timeout @client.timeout end + def url + @sentinel_url + end + private def get_master_url