From 65844244e2fdf9a839a5bc6c0010822b4435c4e2 Mon Sep 17 00:00:00 2001 From: Michael Grosser Date: Thu, 25 Jun 2026 17:15:35 -0700 Subject: [PATCH 01/10] json mode --- Readme.md | 16 ++++++++++++++++ bin/pru | 22 +++++++++++++++++++++- lib/pru.rb | 42 ++++++++++++++++++++++++++++++++++++++++++ spec/pru_spec.rb | 34 ++++++++++++++++++++++++++++++++++ 4 files changed, 113 insertions(+), 1 deletion(-) diff --git a/Readme.md b/Readme.md index 9215143..5ab857f 100644 --- a/Readme.md +++ b/Readme.md @@ -32,6 +32,21 @@ Pipeable Ruby - forget about grep / sed / awk / wc ... use pure, readable Ruby! # select lines longer than 5 letters, then join with commas ls -1 | pru 'size > 5' 'join(",")' +## JSON - each parsed value + +Parse a stream of JSON values (newline-delimited or pretty-printed multiline) +into items. Hash/Array results are pretty-printed, everything else is printed +plainly so it pipes nicely into other tools. + + # pull a field out of each object + printf '{"a":1}\n{"a":2}\n' | pru --json 'self["a"]' + + # filter objects + printf '{"n":1}\n{"n":5}\n' | pru --json 'self["n"] > 3' + + # map then reduce + printf '{"a":1}\n{"a":2}\n' | pru --json 'self["a"]' 'sum' + ## Inplace edit pru -i Gemfile 'sub /ruby/, "foo"' @@ -51,6 +66,7 @@ curl https://rubinjam.herokuapp.com/pack/pru > pru && chmod +x pru ## Options -r, --reduce CODE reduce via CODE + -j, --json Parse input as a JSON stream, each value is an item -I, --libdir DIR Add DIR to load path --require LIB Require LIB (also comma-separated) diff --git a/bin/pru b/bin/pru index ecaf4de..2ee8655 100755 --- a/bin/pru +++ b/bin/pru @@ -28,6 +28,7 @@ OptionParser.new do |opts| Options: BANNER opts.on("-r", "--reduce CODE","reduce via CODE") {|code| options[:reduce] = code } + opts.on("-j", "--json","Parse input as a JSON stream, each value is an item") { options[:json] = true } opts.separator '' opts.on('-I', '--libdir DIR', 'Add DIR to load path') { |dir| $LOAD_PATH << dir } opts.on('--require LIB', 'Require LIB (also comma-separated)') { |lib| lib.split(',').each{|l| require l } } @@ -58,7 +59,18 @@ else input = $stdin end +if options[:json] + require 'json' + format = lambda do |out| + case out + when Hash, Array then JSON.pretty_generate(out) + else out + end + end +end + collector = lambda do |line| + line = format.call(line) if format if output_lines output_lines << line else @@ -70,7 +82,15 @@ collector = lambda do |line| end end -if reduce +if options[:json] + if reduce + results = [] + Pru.json_map(input, map) { |out| results << out } + collector.call Pru.json_reduce(results, reduce) + else + Pru.json_map(input, map) { |out| collector.call out } + end +elsif reduce results = [] Pru.map(input, map) { |out| results << out } collector.call Pru.reduce(results, reduce) diff --git a/lib/pru.rb b/lib/pru.rb index 50c2022..fbdf029 100644 --- a/lib/pru.rb +++ b/lib/pru.rb @@ -31,5 +31,47 @@ def _pru RUBY array._pru end + + def json_map(io, code) + block = compile(code) + i = 0 + each_json(io) do |item| + i += 1 + result = item.instance_exec(i, &block) or next + + case result + when true then yield item + else yield result + end + end + end + + def json_reduce(array, code) + array.instance_exec(&compile(code)) + end + + private + + def compile(code) + eval("proc { |i| #{code} }", TOPLEVEL_BINDING, __FILE__, __LINE__) + end + + # Parse a stream of concatenated JSON values (newline-delimited or multiline) + # by accumulating lines until the buffer forms a complete value. + def each_json(io) + require 'json' + buffer = +"" + io.each_line do |line| + buffer << line + begin + item = JSON.parse(buffer) + rescue JSON::ParserError + next + end + yield item + buffer = +"" + end + raise JSON::ParserError, "unexpected trailing input: #{buffer.strip}" unless buffer.strip.empty? + end end end diff --git a/spec/pru_spec.rb b/spec/pru_spec.rb index cf34188..4ed24a6 100644 --- a/spec/pru_spec.rb +++ b/spec/pru_spec.rb @@ -112,6 +112,40 @@ end end + describe '--json' do + it "pretty-prints hashes" do + `printf '{"a":1}\n' | ./bin/pru --json`.should == "{\n \"a\": 1\n}\n" + end + + it "parses multiple values from one line" do + `printf '{"a":1}\n{"b":2}\n' | ./bin/pru --json 'keys.first'`.should == "a\nb\n" + end + + it "parses values spanning multiple lines" do + `printf '{\n "a": 1\n}\n' | ./bin/pru --json 'self["a"]'`.should == "1\n" + end + + it "outputs strings plainly without quotes" do + `printf '{"a":"foo"}\n' | ./bin/pru --json 'self["a"]'`.should == "foo\n" + end + + it "outputs numbers plainly" do + `printf '{"a":1}\n{"a":2}\n' | ./bin/pru --json 'self["a"]'`.should == "1\n2\n" + end + + it "selects via true" do + `printf '{"n":1}\n{"n":5}\n' | ./bin/pru --json 'self["n"] > 3'`.should == "{\n \"n\": 5\n}\n" + end + + it "reduces" do + `printf '{"a":1}\n{"a":2}\n' | ./bin/pru --json 'self["a"]' 'sum'`.should == "3\n" + end + + it "pretty-prints array reduce results" do + `printf '{"a":1}\n{"a":2}\n' | ./bin/pru --json '' 'map { |h| h["a"] }'`.should == "[\n 1,\n 2\n]\n" + end + end + describe '-I / --libdir' do it "adds a folder to the load-path" do `echo 1 | ./bin/pru -I spec --reduce 'require "a_test"; ATest.to_s'`.should == "ATest\n" From e2e382f214645f5184be2565ab6b881ca3dc54a5 Mon Sep 17 00:00:00 2001 From: Michael Grosser Date: Thu, 25 Jun 2026 17:20:21 -0700 Subject: [PATCH 02/10] k8s --- Readme.md | 9 +++++++++ bin/pru | 5 +++-- lib/pru.rb | 18 ++++++++++++++++-- spec/pru_spec.rb | 14 ++++++++++++++ 4 files changed, 42 insertions(+), 4 deletions(-) diff --git a/Readme.md b/Readme.md index 5ab857f..b1f9526 100644 --- a/Readme.md +++ b/Readme.md @@ -47,6 +47,14 @@ plainly so it pipes nicely into other tools. # map then reduce printf '{"a":1}\n{"a":2}\n' | pru --json 'self["a"]' 'sum' +## Kubernetes - each item + +Like `--json`, but if the first value has an `"items"` key (as `kubectl ... -o json` +returns) its elements are iterated instead. + + # list pod names + kubectl get pods -A -o json | pru --k8s 'dig("metadata", "name")' + ## Inplace edit pru -i Gemfile 'sub /ruby/, "foo"' @@ -67,6 +75,7 @@ curl https://rubinjam.herokuapp.com/pack/pru > pru && chmod +x pru -r, --reduce CODE reduce via CODE -j, --json Parse input as a JSON stream, each value is an item + -k, --k8s Like --json, but iterate the "items" array if present (kubectl -o json) -I, --libdir DIR Add DIR to load path --require LIB Require LIB (also comma-separated) diff --git a/bin/pru b/bin/pru index 2ee8655..433b945 100755 --- a/bin/pru +++ b/bin/pru @@ -29,6 +29,7 @@ OptionParser.new do |opts| BANNER opts.on("-r", "--reduce CODE","reduce via CODE") {|code| options[:reduce] = code } opts.on("-j", "--json","Parse input as a JSON stream, each value is an item") { options[:json] = true } + opts.on("-k", "--k8s","Like --json, but iterate the \"items\" array if present (kubectl -o json)") { options[:json] = true; options[:k8s] = true } opts.separator '' opts.on('-I', '--libdir DIR', 'Add DIR to load path') { |dir| $LOAD_PATH << dir } opts.on('--require LIB', 'Require LIB (also comma-separated)') { |lib| lib.split(',').each{|l| require l } } @@ -85,10 +86,10 @@ end if options[:json] if reduce results = [] - Pru.json_map(input, map) { |out| results << out } + Pru.json_map(input, map, k8s: options[:k8s]) { |out| results << out } collector.call Pru.json_reduce(results, reduce) else - Pru.json_map(input, map) { |out| collector.call out } + Pru.json_map(input, map, k8s: options[:k8s]) { |out| collector.call out } end elsif reduce results = [] diff --git a/lib/pru.rb b/lib/pru.rb index fbdf029..300425c 100644 --- a/lib/pru.rb +++ b/lib/pru.rb @@ -32,10 +32,10 @@ def _pru array._pru end - def json_map(io, code) + def json_map(io, code, k8s: false) block = compile(code) i = 0 - each_json(io) do |item| + each_item(io, k8s) do |item| i += 1 result = item.instance_exec(i, &block) or next @@ -56,6 +56,20 @@ def compile(code) eval("proc { |i| #{code} }", TOPLEVEL_BINDING, __FILE__, __LINE__) end + # Yield items from a JSON stream. In k8s mode, if the first value has an + # "items" key (e.g. `kubectl get ... -o json`) its elements are yielded instead. + def each_item(io, k8s) + first = true + each_json(io) do |item| + if k8s && first && item.is_a?(Hash) && item.key?("items") + item.fetch("items").each { |i| yield i } + else + yield item + end + first = false + end + end + # Parse a stream of concatenated JSON values (newline-delimited or multiline) # by accumulating lines until the buffer forms a complete value. def each_json(io) diff --git a/spec/pru_spec.rb b/spec/pru_spec.rb index 4ed24a6..11b9f82 100644 --- a/spec/pru_spec.rb +++ b/spec/pru_spec.rb @@ -146,6 +146,20 @@ end end + describe '--k8s' do + it "iterates the items array" do + `printf '{"items":[{"metadata":{"name":"a"}},{"metadata":{"name":"b"}}]}\n' | ./bin/pru --k8s 'dig("metadata", "name")'`.should == "a\nb\n" + end + + it "behaves like --json when there is no items key" do + `printf '{"metadata":{"name":"solo"}}\n' | ./bin/pru --k8s 'dig("metadata", "name")'`.should == "solo\n" + end + + it "reduces over the items" do + `printf '{"items":[{"metadata":{"name":"a"}},{"metadata":{"name":"b"}}]}\n' | ./bin/pru --k8s 'dig("metadata", "name")' 'size'`.should == "2\n" + end + end + describe '-I / --libdir' do it "adds a folder to the load-path" do `echo 1 | ./bin/pru -I spec --reduce 'require "a_test"; ATest.to_s'`.should == "ATest\n" From b496f99f1291ef4ae4a597f0c12b548188ff3f74 Mon Sep 17 00:00:00 2001 From: Michael Grosser Date: Thu, 25 Jun 2026 17:22:07 -0700 Subject: [PATCH 03/10] simplify usage --- bin/pru | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/bin/pru b/bin/pru index 433b945..8d4c29d 100755 --- a/bin/pru +++ b/bin/pru @@ -7,10 +7,9 @@ $LOAD_PATH << "#{root}/lib" if File.exist?("#{root}/Gemfile") require 'pru' -usage = nil options = {} -OptionParser.new do |opts| +parser = OptionParser.new do |opts| opts.banner = <<-BANNER.gsub(/^ /, "") Pipeable Ruby @@ -37,15 +36,19 @@ OptionParser.new do |opts| opts.separator '' opts.on("-h", "--help","Show this.") { puts opts; exit } opts.on('-v', '--version','Show Version'){ require 'pru/version'; puts Pru::VERSION; exit} - usage = opts -end.parse! +end +parser.parse! -if ARGV.empty? && options.empty? # no arguments -> show usage - puts usage +# no arguments -> show usage +if ARGV.empty? && options.empty? + puts parser exit end -abort "Too many arguments, see --help" if ARGV.size > 2 +# bad arguments -> fail +if ARGV.size > 2 + abort "Too many arguments, see --help" +end map, reduce = ARGV reduce ||= options[:reduce] From ef73276cddf1ceae4d5f926505ce4be18fbf033b Mon Sep 17 00:00:00 2001 From: Michael Grosser Date: Thu, 25 Jun 2026 17:28:36 -0700 Subject: [PATCH 04/10] usage --- bin/pru | 8 +++++--- spec/pru_spec.rb | 10 ++++++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/bin/pru b/bin/pru index 8d4c29d..60b7301 100755 --- a/bin/pru +++ b/bin/pru @@ -39,16 +39,18 @@ parser = OptionParser.new do |opts| end parser.parse! -# no arguments -> show usage +# no arguments -> show usage and fail if ARGV.empty? && options.empty? - puts parser - exit + abort parser.to_s end # bad arguments -> fail if ARGV.size > 2 abort "Too many arguments, see --help" end +if ARGV.size == 2 && options[:reduce] + abort "Cannot combine a reduce argument with --reduce, see --help" +end map, reduce = ARGV reduce ||= options[:reduce] diff --git a/spec/pru_spec.rb b/spec/pru_spec.rb index 11b9f82..355daa8 100644 --- a/spec/pru_spec.rb +++ b/spec/pru_spec.rb @@ -6,8 +6,9 @@ Pru::VERSION.should =~ /^\d+\.\d+\.\d+$/ end - it "shows help when no arguments are given" do - `./bin/pru`.should include('Usage:') + it "shows usage and fails when no arguments are given" do + `./bin/pru 2>&1`.should include('Usage:') + $?.success?.should == false end it 'shows -v' do @@ -110,6 +111,11 @@ it "selects with empty string and reduces" do `cat spec/test.txt | ./bin/pru '' 'size'`.should == "5\n" end + + it "fails when combining a reduce argument with --reduce" do + `echo x | ./bin/pru self size --reduce size 2>&1`.should include("Cannot combine a reduce argument with --reduce") + $?.success?.should == false + end end describe '--json' do From cdd3525cc82cdf689fa500adf782f335e990ce5f Mon Sep 17 00:00:00 2001 From: Michael Grosser Date: Thu, 25 Jun 2026 17:32:35 -0700 Subject: [PATCH 05/10] inplace vs no code --- bin/pru | 7 ++++--- spec/pru_spec.rb | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/bin/pru b/bin/pru index 60b7301..4db2137 100755 --- a/bin/pru +++ b/bin/pru @@ -39,18 +39,19 @@ parser = OptionParser.new do |opts| end parser.parse! -# no arguments -> show usage and fail +# bad arguments -> fail if ARGV.empty? && options.empty? abort parser.to_s end - -# bad arguments -> fail if ARGV.size > 2 abort "Too many arguments, see --help" end if ARGV.size == 2 && options[:reduce] abort "Cannot combine a reduce argument with --reduce, see --help" end +if options[:file] && ARGV.empty? && !options[:reduce] + abort "No code given for --inplace-edit, see --help" +end map, reduce = ARGV reduce ||= options[:reduce] diff --git a/spec/pru_spec.rb b/spec/pru_spec.rb index 355daa8..25260b3 100644 --- a/spec/pru_spec.rb +++ b/spec/pru_spec.rb @@ -150,6 +150,15 @@ it "pretty-prints array reduce results" do `printf '{"a":1}\n{"a":2}\n' | ./bin/pru --json '' 'map { |h| h["a"] }'`.should == "[\n 1,\n 2\n]\n" end + + it "works with inplace-edit" do + Tempfile.create do |f| + f.write "{\"a\":1}\n{\"a\":2}\n" + f.close + `./bin/pru --inplace-edit #{f.path} --json 'self["a"]'`.should == '' + File.read(f.path).should == "1\n2\n" + end + end end describe '--k8s' do @@ -197,6 +206,11 @@ `./bin/pru --inplace-edit xxx size 2>&1`.sub(' @ rb_sysopen', '').should include('No such file or directory - xxx') end + it "fails when no code is given" do + `./bin/pru --inplace-edit xxx 2>&1`.should include('No code given for --inplace-edit') + $?.success?.should == false + end + it "keeps line separators when modifying" do File.open('xxx','w'){|f| f.write "abc\r\nab\r\na" } `./bin/pru --inplace-edit xxx size`.should == '' From d853d9d25abdf9ea370fad3ef5806fec1e04319b Mon Sep 17 00:00:00 2001 From: Michael Grosser Date: Thu, 25 Jun 2026 17:40:06 -0700 Subject: [PATCH 06/10] styl --- bin/pru | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bin/pru b/bin/pru index 4db2137..a46aa4b 100755 --- a/bin/pru +++ b/bin/pru @@ -61,14 +61,15 @@ if options[:file] output_lines = [] input = File.read(options[:file]) newline = input[/\r\n|\r|\n/] - trailing_newline = (input =~ /#{newline}\Z/) + trailing_newline = (newline && input.end_with?(newline)) else input = $stdin end +# dump write as JSON ? if options[:json] require 'json' - format = lambda do |out| + output_format = lambda do |out| case out when Hash, Array then JSON.pretty_generate(out) else out @@ -77,7 +78,7 @@ if options[:json] end collector = lambda do |line| - line = format.call(line) if format + line = output_format.call(line) if output_format if output_lines output_lines << line else From 747ef186a1b41ca1c2aba938c8a224781977b1d0 Mon Sep 17 00:00:00 2001 From: Michael Grosser Date: Thu, 25 Jun 2026 18:24:40 -0700 Subject: [PATCH 07/10] unify --- bin/pru | 20 +++++++++----------- lib/pru.rb | 54 +++++++++++++----------------------------------------- 2 files changed, 22 insertions(+), 52 deletions(-) diff --git a/bin/pru b/bin/pru index a46aa4b..3b1b331 100755 --- a/bin/pru +++ b/bin/pru @@ -90,20 +90,18 @@ collector = lambda do |line| end end -if options[:json] - if reduce - results = [] - Pru.json_map(input, map, k8s: options[:k8s]) { |out| results << out } - collector.call Pru.json_reduce(results, reduce) - else - Pru.json_map(input, map, k8s: options[:k8s]) { |out| collector.call out } - end -elsif reduce +items = if options[:json] + Pru.each_json_item(input, k8s: options[:k8s]) +else + input.each_line.lazy.map { |line| line.chomp } +end + +if reduce results = [] - Pru.map(input, map) { |out| results << out } + Pru.map(items, map) { |out| results << out } collector.call Pru.reduce(results, reduce) else - Pru.map(input, map) { |out| collector.call out } + Pru.map(items, map) { |out| collector.call out } end if options[:file] diff --git a/lib/pru.rb b/lib/pru.rb index 300425c..2d0f618 100644 --- a/lib/pru.rb +++ b/lib/pru.rb @@ -2,63 +2,29 @@ module Pru class << self - def map(io, code) - String.class_eval <<-RUBY, __FILE__, __LINE__ + 1 - def _pru(i) - #{code} - end - RUBY - - i = 0 - io.each_line do |line| - i += 1 - line.chomp! - result = line._pru(i) or next - - case result - when true then yield line - when Regexp then yield line if line =~ result - else yield result - end - end - end - - def reduce(array, code) - Array.class_eval <<-RUBY, __FILE__, __LINE__ + 1 - def _pru - #{code} - end - RUBY - array._pru - end - - def json_map(io, code, k8s: false) + def map(items, code) block = compile(code) i = 0 - each_item(io, k8s) do |item| + items.each do |item| i += 1 - result = item.instance_exec(i, &block) or next + result = item.instance_exec(i, &block) || next case result when true then yield item + when Regexp then yield item if item =~ result else yield result end end end - def json_reduce(array, code) + def reduce(array, code) array.instance_exec(&compile(code)) end - private - - def compile(code) - eval("proc { |i| #{code} }", TOPLEVEL_BINDING, __FILE__, __LINE__) - end - # Yield items from a JSON stream. In k8s mode, if the first value has an # "items" key (e.g. `kubectl get ... -o json`) its elements are yielded instead. - def each_item(io, k8s) + def each_json_item(io, k8s: false) + return enum_for(:each_json_item, io, k8s: k8s) unless block_given? first = true each_json(io) do |item| if k8s && first && item.is_a?(Hash) && item.key?("items") @@ -70,6 +36,12 @@ def each_item(io, k8s) end end + private + + def compile(code) + eval("proc { |i| #{code} }", TOPLEVEL_BINDING, __FILE__, __LINE__) + end + # Parse a stream of concatenated JSON values (newline-delimited or multiline) # by accumulating lines until the buffer forms a complete value. def each_json(io) From b351508190039cd3a0cdc8982eb5e6d241adc1e6 Mon Sep 17 00:00:00 2001 From: Michael Grosser Date: Thu, 25 Jun 2026 18:28:26 -0700 Subject: [PATCH 08/10] first --- Readme.md | 2 +- lib/pru.rb | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/Readme.md b/Readme.md index b1f9526..f09eda4 100644 --- a/Readme.md +++ b/Readme.md @@ -49,7 +49,7 @@ plainly so it pipes nicely into other tools. ## Kubernetes - each item -Like `--json`, but if the first value has an `"items"` key (as `kubectl ... -o json` +Like `--json`, but if the value has an `"items"` key (as `kubectl ... -o json` returns) its elements are iterated instead. # list pod names diff --git a/lib/pru.rb b/lib/pru.rb index 2d0f618..2382f5a 100644 --- a/lib/pru.rb +++ b/lib/pru.rb @@ -21,18 +21,16 @@ def reduce(array, code) array.instance_exec(&compile(code)) end - # Yield items from a JSON stream. In k8s mode, if the first value has an + # Yield items from a JSON stream. In k8s mode, if the value has an # "items" key (e.g. `kubectl get ... -o json`) its elements are yielded instead. def each_json_item(io, k8s: false) return enum_for(:each_json_item, io, k8s: k8s) unless block_given? - first = true each_json(io) do |item| - if k8s && first && item.is_a?(Hash) && item.key?("items") + if k8s && item.is_a?(Hash) && item.key?("items") item.fetch("items").each { |i| yield i } else yield item end - first = false end end @@ -44,8 +42,8 @@ def compile(code) # Parse a stream of concatenated JSON values (newline-delimited or multiline) # by accumulating lines until the buffer forms a complete value. + # TODO: this is not very efficient, but keeping track of opening/closing braces might be ugly too def each_json(io) - require 'json' buffer = +"" io.each_line do |line| buffer << line From a9a7fcfc57c2135b49eada107131c84c18d67f1e Mon Sep 17 00:00:00 2001 From: Michael Grosser Date: Thu, 25 Jun 2026 18:35:44 -0700 Subject: [PATCH 09/10] clean --- bin/pru | 2 +- lib/pru.rb | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/bin/pru b/bin/pru index 3b1b331..9a03502 100755 --- a/bin/pru +++ b/bin/pru @@ -91,7 +91,7 @@ collector = lambda do |line| end items = if options[:json] - Pru.each_json_item(input, k8s: options[:k8s]) + Pru.json_items(input, k8s: options[:k8s]) else input.each_line.lazy.map { |line| line.chomp } end diff --git a/lib/pru.rb b/lib/pru.rb index 2382f5a..69d6b7e 100644 --- a/lib/pru.rb +++ b/lib/pru.rb @@ -21,15 +21,16 @@ def reduce(array, code) array.instance_exec(&compile(code)) end - # Yield items from a JSON stream. In k8s mode, if the value has an - # "items" key (e.g. `kubectl get ... -o json`) its elements are yielded instead. - def each_json_item(io, k8s: false) - return enum_for(:each_json_item, io, k8s: k8s) unless block_given? - each_json(io) do |item| - if k8s && item.is_a?(Hash) && item.key?("items") - item.fetch("items").each { |i| yield i } - else - yield item + # An enumerable of items parsed from a JSON stream. In k8s mode, if a value + # has an "items" key (e.g. `kubectl get ... -o json`) its elements are the items instead. + def json_items(io, k8s: false) + Enumerator.new do |yielder| + each_json(io) do |item| + if k8s && item.is_a?(Hash) && item.key?("items") + item.fetch("items").each { |i| yielder << i } + else + yielder << item + end end end end From 30f67cbe6b352ce18ecd6aca0222ece8091f70e5 Mon Sep 17 00:00:00 2001 From: Michael Grosser Date: Thu, 25 Jun 2026 18:46:03 -0700 Subject: [PATCH 10/10] fast --- Readme.md | 3 +-- bin/pru | 17 +++++++++++++---- lib/pru.rb | 41 +++++++++++++++-------------------------- spec/pru_spec.rb | 5 +++-- 4 files changed, 32 insertions(+), 34 deletions(-) diff --git a/Readme.md b/Readme.md index f09eda4..1077ae7 100644 --- a/Readme.md +++ b/Readme.md @@ -49,8 +49,7 @@ plainly so it pipes nicely into other tools. ## Kubernetes - each item -Like `--json`, but if the value has an `"items"` key (as `kubectl ... -o json` -returns) its elements are iterated instead. +Like `--json`, but iterates `"items"` key (as `kubectl ... -o json` returns). # list pod names kubectl get pods -A -o json | pru --k8s 'dig("metadata", "name")' diff --git a/bin/pru b/bin/pru index 9a03502..6d18b82 100755 --- a/bin/pru +++ b/bin/pru @@ -57,6 +57,7 @@ map, reduce = ARGV reduce ||= options[:reduce] map = 'true' if !map || map.empty? +# reading from file or stdin ? if options[:file] output_lines = [] input = File.read(options[:file]) @@ -66,9 +67,9 @@ else input = $stdin end -# dump write as JSON ? +# output as JSON ? if options[:json] - require 'json' + require 'json' # also used by --k8s, which sets :json output_format = lambda do |out| case out when Hash, Array then JSON.pretty_generate(out) @@ -77,6 +78,7 @@ if options[:json] end end +# collect results into list or print them collector = lambda do |line| line = output_format.call(line) if output_format if output_lines @@ -90,9 +92,15 @@ collector = lambda do |line| end end -items = if options[:json] - Pru.json_items(input, k8s: options[:k8s]) +# find all items (ideally lazy so we can stream) +items = if options[:k8s] + # single document (e.g. `kubectl get ... -o json`) -> parse once and iterate its items + JSON.parse(input.read).fetch("items") +elsif options[:json] + # lazy stream of JSON values + Pru.each_json(input) else + # plain lines, chomped, lazily so streaming/head still work input.each_line.lazy.map { |line| line.chomp } end @@ -104,6 +112,7 @@ else Pru.map(items, map) { |out| collector.call out } end +# write back to file if requested if options[:file] content = output_lines.join(newline) content << newline if trailing_newline diff --git a/lib/pru.rb b/lib/pru.rb index 69d6b7e..8ca2a3c 100644 --- a/lib/pru.rb +++ b/lib/pru.rb @@ -21,17 +21,24 @@ def reduce(array, code) array.instance_exec(&compile(code)) end - # An enumerable of items parsed from a JSON stream. In k8s mode, if a value - # has an "items" key (e.g. `kubectl get ... -o json`) its elements are the items instead. - def json_items(io, k8s: false) + # An enumerable of JSON values parsed from a stream (newline-delimited or multiline), + # parsed lazily so we process as input arrives, by accumulating lines until the buffer + # forms a complete value. + # TODO: this is not very efficient, but keeping track of opening/closing braces might be ugly too + def each_json(io) Enumerator.new do |yielder| - each_json(io) do |item| - if k8s && item.is_a?(Hash) && item.key?("items") - item.fetch("items").each { |i| yielder << i } - else - yielder << item + buffer = +"" + io.each_line do |line| + buffer << line + begin + item = JSON.parse(buffer) + rescue JSON::ParserError + next end + yielder << item + buffer = +"" end + raise JSON::ParserError, "unexpected trailing input: #{buffer.strip}" unless buffer.strip.empty? end end @@ -40,23 +47,5 @@ def json_items(io, k8s: false) def compile(code) eval("proc { |i| #{code} }", TOPLEVEL_BINDING, __FILE__, __LINE__) end - - # Parse a stream of concatenated JSON values (newline-delimited or multiline) - # by accumulating lines until the buffer forms a complete value. - # TODO: this is not very efficient, but keeping track of opening/closing braces might be ugly too - def each_json(io) - buffer = +"" - io.each_line do |line| - buffer << line - begin - item = JSON.parse(buffer) - rescue JSON::ParserError - next - end - yield item - buffer = +"" - end - raise JSON::ParserError, "unexpected trailing input: #{buffer.strip}" unless buffer.strip.empty? - end end end diff --git a/spec/pru_spec.rb b/spec/pru_spec.rb index 25260b3..1a1bd0d 100644 --- a/spec/pru_spec.rb +++ b/spec/pru_spec.rb @@ -166,8 +166,9 @@ `printf '{"items":[{"metadata":{"name":"a"}},{"metadata":{"name":"b"}}]}\n' | ./bin/pru --k8s 'dig("metadata", "name")'`.should == "a\nb\n" end - it "behaves like --json when there is no items key" do - `printf '{"metadata":{"name":"solo"}}\n' | ./bin/pru --k8s 'dig("metadata", "name")'`.should == "solo\n" + it "fails when there is no items key" do + `printf '{"metadata":{"name":"solo"}}\n' | ./bin/pru --k8s 'dig("metadata", "name")' 2>&1`.should include("key not found") + $?.success?.should == false end it "reduces over the items" do