diff --git a/Readme.md b/Readme.md index 9215143..1077ae7 100644 --- a/Readme.md +++ b/Readme.md @@ -32,6 +32,28 @@ Pipeable Ruby - forget about grep / sed / awk / wc ... use pure, readable Ruby! # select lines longer than 5 letters, then join with commas ls -1 | pru 'size > 5' 'join(",")' +## JSON - each parsed value + +Parse a stream of JSON values (newline-delimited or pretty-printed multiline) +into items. Hash/Array results are pretty-printed, everything else is printed +plainly so it pipes nicely into other tools. + + # pull a field out of each object + printf '{"a":1}\n{"a":2}\n' | pru --json 'self["a"]' + + # filter objects + printf '{"n":1}\n{"n":5}\n' | pru --json 'self["n"] > 3' + + # map then reduce + printf '{"a":1}\n{"a":2}\n' | pru --json 'self["a"]' 'sum' + +## Kubernetes - each item + +Like `--json`, but iterates `"items"` key (as `kubectl ... -o json` returns). + + # list pod names + kubectl get pods -A -o json | pru --k8s 'dig("metadata", "name")' + ## Inplace edit pru -i Gemfile 'sub /ruby/, "foo"' @@ -51,6 +73,8 @@ curl https://rubinjam.herokuapp.com/pack/pru > pru && chmod +x pru ## Options -r, --reduce CODE reduce via CODE + -j, --json Parse input as a JSON stream, each value is an item + -k, --k8s Like --json, but iterate the "items" array if present (kubectl -o json) -I, --libdir DIR Add DIR to load path --require LIB Require LIB (also comma-separated) diff --git a/bin/pru b/bin/pru index ecaf4de..6d18b82 100755 --- a/bin/pru +++ b/bin/pru @@ -7,10 +7,9 @@ $LOAD_PATH << "#{root}/lib" if File.exist?("#{root}/Gemfile") require 'pru' -usage = nil options = {} -OptionParser.new do |opts| +parser = OptionParser.new do |opts| opts.banner = <<-BANNER.gsub(/^ /, "") Pipeable Ruby @@ -28,6 +27,8 @@ OptionParser.new do |opts| Options: BANNER opts.on("-r", "--reduce CODE","reduce via CODE") {|code| options[:reduce] = code } + opts.on("-j", "--json","Parse input as a JSON stream, each value is an item") { options[:json] = true } + opts.on("-k", "--k8s","Like --json, but iterate the \"items\" array if present (kubectl -o json)") { options[:json] = true; options[:k8s] = true } opts.separator '' opts.on('-I', '--libdir DIR', 'Add DIR to load path') { |dir| $LOAD_PATH << dir } opts.on('--require LIB', 'Require LIB (also comma-separated)') { |lib| lib.split(',').each{|l| require l } } @@ -35,30 +36,51 @@ OptionParser.new do |opts| opts.separator '' opts.on("-h", "--help","Show this.") { puts opts; exit } opts.on('-v', '--version','Show Version'){ require 'pru/version'; puts Pru::VERSION; exit} - usage = opts -end.parse! - -if ARGV.empty? && options.empty? # no arguments -> show usage - puts usage - exit end +parser.parse! -abort "Too many arguments, see --help" if ARGV.size > 2 +# bad arguments -> fail +if ARGV.empty? && options.empty? + abort parser.to_s +end +if ARGV.size > 2 + abort "Too many arguments, see --help" +end +if ARGV.size == 2 && options[:reduce] + abort "Cannot combine a reduce argument with --reduce, see --help" +end +if options[:file] && ARGV.empty? && !options[:reduce] + abort "No code given for --inplace-edit, see --help" +end map, reduce = ARGV reduce ||= options[:reduce] map = 'true' if !map || map.empty? +# reading from file or stdin ? if options[:file] output_lines = [] input = File.read(options[:file]) newline = input[/\r\n|\r|\n/] - trailing_newline = (input =~ /#{newline}\Z/) + trailing_newline = (newline && input.end_with?(newline)) else input = $stdin end +# output as JSON ? +if options[:json] + require 'json' # also used by --k8s, which sets :json + output_format = lambda do |out| + case out + when Hash, Array then JSON.pretty_generate(out) + else out + end + end +end + +# collect results into list or print them collector = lambda do |line| + line = output_format.call(line) if output_format if output_lines output_lines << line else @@ -70,14 +92,27 @@ collector = lambda do |line| end end +# find all items (ideally lazy so we can stream) +items = if options[:k8s] + # single document (e.g. `kubectl get ... -o json`) -> parse once and iterate its items + JSON.parse(input.read).fetch("items") +elsif options[:json] + # lazy stream of JSON values + Pru.each_json(input) +else + # plain lines, chomped, lazily so streaming/head still work + input.each_line.lazy.map { |line| line.chomp } +end + if reduce results = [] - Pru.map(input, map) { |out| results << out } + Pru.map(items, map) { |out| results << out } collector.call Pru.reduce(results, reduce) else - Pru.map(input, map) { |out| collector.call out } + Pru.map(items, map) { |out| collector.call out } end +# write back to file if requested if options[:file] content = output_lines.join(newline) content << newline if trailing_newline diff --git a/lib/pru.rb b/lib/pru.rb index 50c2022..8ca2a3c 100644 --- a/lib/pru.rb +++ b/lib/pru.rb @@ -2,34 +2,50 @@ module Pru class << self - def map(io, code) - String.class_eval <<-RUBY, __FILE__, __LINE__ + 1 - def _pru(i) - #{code} - end - RUBY - + def map(items, code) + block = compile(code) i = 0 - io.each_line do |line| + items.each do |item| i += 1 - line.chomp! - result = line._pru(i) or next + result = item.instance_exec(i, &block) || next case result - when true then yield line - when Regexp then yield line if line =~ result + when true then yield item + when Regexp then yield item if item =~ result else yield result end end end def reduce(array, code) - Array.class_eval <<-RUBY, __FILE__, __LINE__ + 1 - def _pru - #{code} + array.instance_exec(&compile(code)) + end + + # An enumerable of JSON values parsed from a stream (newline-delimited or multiline), + # parsed lazily so we process as input arrives, by accumulating lines until the buffer + # forms a complete value. + # TODO: this is not very efficient, but keeping track of opening/closing braces might be ugly too + def each_json(io) + Enumerator.new do |yielder| + buffer = +"" + io.each_line do |line| + buffer << line + begin + item = JSON.parse(buffer) + rescue JSON::ParserError + next + end + yielder << item + buffer = +"" end - RUBY - array._pru + raise JSON::ParserError, "unexpected trailing input: #{buffer.strip}" unless buffer.strip.empty? + end + end + + private + + def compile(code) + eval("proc { |i| #{code} }", TOPLEVEL_BINDING, __FILE__, __LINE__) end end end diff --git a/spec/pru_spec.rb b/spec/pru_spec.rb index cf34188..1a1bd0d 100644 --- a/spec/pru_spec.rb +++ b/spec/pru_spec.rb @@ -6,8 +6,9 @@ Pru::VERSION.should =~ /^\d+\.\d+\.\d+$/ end - it "shows help when no arguments are given" do - `./bin/pru`.should include('Usage:') + it "shows usage and fails when no arguments are given" do + `./bin/pru 2>&1`.should include('Usage:') + $?.success?.should == false end it 'shows -v' do @@ -110,6 +111,69 @@ it "selects with empty string and reduces" do `cat spec/test.txt | ./bin/pru '' 'size'`.should == "5\n" end + + it "fails when combining a reduce argument with --reduce" do + `echo x | ./bin/pru self size --reduce size 2>&1`.should include("Cannot combine a reduce argument with --reduce") + $?.success?.should == false + end + end + + describe '--json' do + it "pretty-prints hashes" do + `printf '{"a":1}\n' | ./bin/pru --json`.should == "{\n \"a\": 1\n}\n" + end + + it "parses multiple values from one line" do + `printf '{"a":1}\n{"b":2}\n' | ./bin/pru --json 'keys.first'`.should == "a\nb\n" + end + + it "parses values spanning multiple lines" do + `printf '{\n "a": 1\n}\n' | ./bin/pru --json 'self["a"]'`.should == "1\n" + end + + it "outputs strings plainly without quotes" do + `printf '{"a":"foo"}\n' | ./bin/pru --json 'self["a"]'`.should == "foo\n" + end + + it "outputs numbers plainly" do + `printf '{"a":1}\n{"a":2}\n' | ./bin/pru --json 'self["a"]'`.should == "1\n2\n" + end + + it "selects via true" do + `printf '{"n":1}\n{"n":5}\n' | ./bin/pru --json 'self["n"] > 3'`.should == "{\n \"n\": 5\n}\n" + end + + it "reduces" do + `printf '{"a":1}\n{"a":2}\n' | ./bin/pru --json 'self["a"]' 'sum'`.should == "3\n" + end + + it "pretty-prints array reduce results" do + `printf '{"a":1}\n{"a":2}\n' | ./bin/pru --json '' 'map { |h| h["a"] }'`.should == "[\n 1,\n 2\n]\n" + end + + it "works with inplace-edit" do + Tempfile.create do |f| + f.write "{\"a\":1}\n{\"a\":2}\n" + f.close + `./bin/pru --inplace-edit #{f.path} --json 'self["a"]'`.should == '' + File.read(f.path).should == "1\n2\n" + end + end + end + + describe '--k8s' do + it "iterates the items array" do + `printf '{"items":[{"metadata":{"name":"a"}},{"metadata":{"name":"b"}}]}\n' | ./bin/pru --k8s 'dig("metadata", "name")'`.should == "a\nb\n" + end + + it "fails when there is no items key" do + `printf '{"metadata":{"name":"solo"}}\n' | ./bin/pru --k8s 'dig("metadata", "name")' 2>&1`.should include("key not found") + $?.success?.should == false + end + + it "reduces over the items" do + `printf '{"items":[{"metadata":{"name":"a"}},{"metadata":{"name":"b"}}]}\n' | ./bin/pru --k8s 'dig("metadata", "name")' 'size'`.should == "2\n" + end end describe '-I / --libdir' do @@ -143,6 +207,11 @@ `./bin/pru --inplace-edit xxx size 2>&1`.sub(' @ rb_sysopen', '').should include('No such file or directory - xxx') end + it "fails when no code is given" do + `./bin/pru --inplace-edit xxx 2>&1`.should include('No code given for --inplace-edit') + $?.success?.should == false + end + it "keeps line separators when modifying" do File.open('xxx','w'){|f| f.write "abc\r\nab\r\na" } `./bin/pru --inplace-edit xxx size`.should == ''