json and k8s

Readme.md

@@ -32,6 +32,28 @@ Pipeable Ruby - forget about grep / sed / awk / wc ... use pure, readable Ruby!
     # select lines longer than 5 letters, then join with commas
     ls -1 | pru 'size > 5' 'join(",")'
 
+## JSON - each parsed value
+
+Parse a stream of JSON values (newline-delimited or pretty-printed multiline)
+into items. Hash/Array results are pretty-printed, everything else is printed
+plainly so it pipes nicely into other tools.
+
+    # pull a field out of each object
+    printf '{"a":1}\n{"a":2}\n' | pru --json 'self["a"]'
+
+    # filter objects
+    printf '{"n":1}\n{"n":5}\n' | pru --json 'self["n"] > 3'
+
+    # map then reduce
+    printf '{"a":1}\n{"a":2}\n' | pru --json 'self["a"]' 'sum'
+
+## Kubernetes - each item
+
+Like `--json`, but iterates `"items"` key (as `kubectl ... -o json` returns).
+
+    # list pod names
+    kubectl get pods -A -o json | pru --k8s 'dig("metadata", "name")'
+
 ## Inplace edit
 
     pru -i Gemfile 'sub /ruby/, "foo"'
@@ -51,6 +73,8 @@ curl https://rubinjam.herokuapp.com/pack/pru > pru && chmod +x pru
 ## Options
 
     -r, --reduce CODE                reduce via CODE
+    -j, --json                       Parse input as a JSON stream, each value is an item
+    -k, --k8s                        Like --json, but iterate the "items" array if present (kubectl -o json)
 
     -I, --libdir DIR                 Add DIR to load path
         --require LIB                Require LIB (also comma-separated)

bin/pru

@@ -7,10 +7,9 @@ $LOAD_PATH << "#{root}/lib" if File.exist?("#{root}/Gemfile")
 
 require 'pru'
 
-usage = nil
 options = {}
 
-OptionParser.new do |opts|
+parser = OptionParser.new do |opts|
   opts.banner = <<-BANNER.gsub(/^    /, "")
     Pipeable Ruby
 
@@ -28,37 +27,60 @@ OptionParser.new do |opts|
     Options:
   BANNER
   opts.on("-r", "--reduce CODE","reduce via CODE") {|code| options[:reduce] = code }
+  opts.on("-j", "--json","Parse input as a JSON stream, each value is an item") { options[:json] = true }
+  opts.on("-k", "--k8s","Like --json, but iterate the \"items\" array if present (kubectl -o json)") { options[:json] = true; options[:k8s] = true }
   opts.separator ''
   opts.on('-I', '--libdir DIR', 'Add DIR to load path') { |dir| $LOAD_PATH << dir }
   opts.on('--require LIB', 'Require LIB (also comma-separated)') { |lib| lib.split(',').each{|l| require l } }
   opts.on('-i', '--inplace-edit FILE', 'Edit FILE inplace') { |file| options[:file] = file }
   opts.separator ''
   opts.on("-h", "--help","Show this.") { puts opts; exit }
   opts.on('-v', '--version','Show Version'){ require 'pru/version'; puts Pru::VERSION; exit}
-  usage = opts
-end.parse!
-
-if ARGV.empty? && options.empty? # no arguments -> show usage
-  puts usage
-  exit
 end
+parser.parse!
 
-abort "Too many arguments, see --help" if ARGV.size > 2
+# bad arguments -> fail
+if ARGV.empty? && options.empty?
+  abort parser.to_s
+end
+if ARGV.size > 2
+  abort "Too many arguments, see --help"
+end
+if ARGV.size == 2 && options[:reduce]
+  abort "Cannot combine a reduce argument with --reduce, see --help"
+end
+if options[:file] && ARGV.empty? && !options[:reduce]
+  abort "No code given for --inplace-edit, see --help"
+end
 
 map, reduce = ARGV
 reduce ||= options[:reduce]
 map = 'true' if !map || map.empty?
 
+# reading from file or stdin ?
 if options[:file]
   output_lines = []
   input = File.read(options[:file])
   newline = input[/\r\n|\r|\n/]
-  trailing_newline = (input =~ /#{newline}\Z/)
+  trailing_newline = (newline && input.end_with?(newline))
 else
   input = $stdin
 end
 
+# output as JSON ?
+if options[:json]
+  require 'json' # also used by --k8s, which sets :json
+  output_format = lambda do |out|
+    case out
+    when Hash, Array then JSON.pretty_generate(out)
+    else out
+    end
+  end
+end
+
+# collect results into list or print them
 collector = lambda do |line|
+  line = output_format.call(line) if output_format
   if output_lines
     output_lines << line
   else
@@ -70,14 +92,27 @@ collector = lambda do |line|
   end
 end
 
+# find all items (ideally lazy so we can stream)
+items = if options[:k8s]
+  # single document (e.g. `kubectl get ... -o json`) -> parse once and iterate its items
+  JSON.parse(input.read).fetch("items")
+elsif options[:json]
+  # lazy stream of JSON values
+  Pru.each_json(input)
+else
+  # plain lines, chomped, lazily so streaming/head still work
+  input.each_line.lazy.map { |line| line.chomp }
+end
+
 if reduce
   results = []
-  Pru.map(input, map) { |out| results << out }
+  Pru.map(items, map) { |out| results << out }
   collector.call Pru.reduce(results, reduce)
 else
-  Pru.map(input, map) { |out| collector.call out }
+  Pru.map(items, map) { |out| collector.call out }
 end
 
+# write back to file if requested
 if options[:file]
   content = output_lines.join(newline)
   content << newline if trailing_newline

lib/pru.rb

@@ -2,34 +2,50 @@
 
 module Pru
   class << self
-    def map(io, code)
-      String.class_eval <<-RUBY, __FILE__, __LINE__ + 1
-        def _pru(i)
-          #{code}
-        end
-      RUBY
-
+    def map(items, code)
+      block = compile(code)
       i = 0
-      io.each_line do |line|
+      items.each do |item|
         i += 1
-        line.chomp!
-        result = line._pru(i) or next
+        result = item.instance_exec(i, &block) || next
 
         case result
-        when true then yield line
-        when Regexp then yield line if line =~ result
+        when true then yield item
+        when Regexp then yield item if item =~ result
         else yield result
         end
       end
     end
 
     def reduce(array, code)
-      Array.class_eval <<-RUBY, __FILE__, __LINE__ + 1
-        def _pru
-          #{code}
+      array.instance_exec(&compile(code))
+    end
+
+    # An enumerable of JSON values parsed from a stream (newline-delimited or multiline),
+    # parsed lazily so we process as input arrives, by accumulating lines until the buffer
+    # forms a complete value.
+    # TODO: this is not very efficient, but keeping track of opening/closing braces might be ugly too
+    def each_json(io)
+      Enumerator.new do |yielder|
+        buffer = +""
+        io.each_line do |line|
+          buffer << line
+          begin
+            item = JSON.parse(buffer)
+          rescue JSON::ParserError
+            next
+          end
+          yielder << item
+          buffer = +""
         end
-      RUBY
-      array._pru
+        raise JSON::ParserError, "unexpected trailing input: #{buffer.strip}" unless buffer.strip.empty?
+      end
+    end
+
+    private
+
+    def compile(code)
+      eval("proc { |i| #{code} }", TOPLEVEL_BINDING, __FILE__, __LINE__)
     end
   end
 end

spec/pru_spec.rb

@@ -6,8 +6,9 @@
     Pru::VERSION.should =~ /^\d+\.\d+\.\d+$/
   end
 
-  it "shows help when no arguments are given" do
-    `./bin/pru`.should include('Usage:')
+  it "shows usage and fails when no arguments are given" do
+    `./bin/pru 2>&1`.should include('Usage:')
+    $?.success?.should == false
   end
 
   it 'shows -v' do
@@ -110,6 +111,69 @@
     it "selects with empty string and reduces" do
       `cat spec/test.txt | ./bin/pru '' 'size'`.should == "5\n"
     end
+
+    it "fails when combining a reduce argument with --reduce" do
+      `echo x | ./bin/pru self size --reduce size 2>&1`.should include("Cannot combine a reduce argument with --reduce")
+      $?.success?.should == false
+    end
+  end
+
+  describe '--json' do
+    it "pretty-prints hashes" do
+      `printf '{"a":1}\n' | ./bin/pru --json`.should == "{\n  \"a\": 1\n}\n"
+    end
+
+    it "parses multiple values from one line" do
+      `printf '{"a":1}\n{"b":2}\n' | ./bin/pru --json 'keys.first'`.should == "a\nb\n"
+    end
+
+    it "parses values spanning multiple lines" do
+      `printf '{\n  "a": 1\n}\n' | ./bin/pru --json 'self["a"]'`.should == "1\n"
+    end
+
+    it "outputs strings plainly without quotes" do
+      `printf '{"a":"foo"}\n' | ./bin/pru --json 'self["a"]'`.should == "foo\n"
+    end
+
+    it "outputs numbers plainly" do
+      `printf '{"a":1}\n{"a":2}\n' | ./bin/pru --json 'self["a"]'`.should == "1\n2\n"
+    end
+
+    it "selects via true" do
+      `printf '{"n":1}\n{"n":5}\n' | ./bin/pru --json 'self["n"] > 3'`.should == "{\n  \"n\": 5\n}\n"
+    end
+
+    it "reduces" do
+      `printf '{"a":1}\n{"a":2}\n' | ./bin/pru --json 'self["a"]' 'sum'`.should == "3\n"
+    end
+
+    it "pretty-prints array reduce results" do
+      `printf '{"a":1}\n{"a":2}\n' | ./bin/pru --json '' 'map { |h| h["a"] }'`.should == "[\n  1,\n  2\n]\n"
+    end
+
+    it "works with inplace-edit" do
+      Tempfile.create do |f|
+        f.write "{\"a\":1}\n{\"a\":2}\n"
+        f.close
+        `./bin/pru --inplace-edit #{f.path} --json 'self["a"]'`.should == ''
+        File.read(f.path).should == "1\n2\n"
+      end
+    end
+  end
+
+  describe '--k8s' do
+    it "iterates the items array" do
+      `printf '{"items":[{"metadata":{"name":"a"}},{"metadata":{"name":"b"}}]}\n' | ./bin/pru --k8s 'dig("metadata", "name")'`.should == "a\nb\n"
+    end
+
+    it "fails when there is no items key" do
+      `printf '{"metadata":{"name":"solo"}}\n' | ./bin/pru --k8s 'dig("metadata", "name")' 2>&1`.should include("key not found")
+      $?.success?.should == false
+    end
+
+    it "reduces over the items" do
+      `printf '{"items":[{"metadata":{"name":"a"}},{"metadata":{"name":"b"}}]}\n' | ./bin/pru --k8s 'dig("metadata", "name")' 'size'`.should == "2\n"
+    end
   end
 
   describe '-I / --libdir' do
@@ -143,6 +207,11 @@
       `./bin/pru --inplace-edit xxx size 2>&1`.sub(' @ rb_sysopen', '').should include('No such file or directory - xxx')
     end
 
+    it "fails when no code is given" do
+      `./bin/pru --inplace-edit xxx 2>&1`.should include('No code given for --inplace-edit')
+      $?.success?.should == false
+    end
+
     it "keeps line separators when modifying" do
       File.open('xxx','w'){|f| f.write "abc\r\nab\r\na" }
       `./bin/pru --inplace-edit xxx size`.should == ''