diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 09cd91c04..d0676dffe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,12 +34,19 @@ jobs: complete: if: always() name: complete - needs: [build, test, action-using-artifact, push, action-using-registry] + needs: [lint, build, test, action-using-artifact, push, action-using-registry] runs-on: ubuntu-latest steps: - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') run: exit 1 + lint: + name: lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - run: make lint # runs rubocop in a ruby container; docker is preinstalled on runners + setup: name: 1 setup runs-on: ubuntu-latest diff --git a/.rubocop.yml b/.rubocop.yml new file mode 100644 index 000000000..8244b36e7 --- /dev/null +++ b/.rubocop.yml @@ -0,0 +1,34 @@ +# RuboCop keeps the scripts in .scripts/ written in concise, idiomatic Ruby. +# Method calls still use parentheses (no bracket-less calls), but otherwise the +# scripts lean on Ruby's terse forms: guard clauses, modifier if/unless, and +# Enumerable methods. + +AllCops: + TargetRubyVersion: 3.2 + NewCops: enable + SuggestExtensions: false + Include: + - ".scripts/*" + - "**/*.rb" + Exclude: + - "Gemfile" + +# Always use parentheses around method-call arguments (no "bracket-less" calls). +Style/MethodCallWithArgsParentheses: + Enabled: true + EnforcedStyle: require_parentheses + +# Use double quotes everywhere so quoting is one less thing to think about. +Style/StringLiterals: + EnforcedStyle: double_quotes + +Style/StringLiteralsInInterpolation: + EnforcedStyle: double_quotes + +# These are top-level scripts; top-level method definitions are expected. +Style/TopLevelMethodDefinition: + Enabled: false + +# Don't force these small scripts to be split up by length/complexity limits. +Metrics: + Enabled: false diff --git a/.scripts/auto-update b/.scripts/auto-update index fbee2a9b4..460f3a60c 100755 --- a/.scripts/auto-update +++ b/.scripts/auto-update @@ -1,8 +1,8 @@ -#!/usr/bin/env python3 +#!/usr/bin/env ruby +# frozen_string_literal: true -import json -import subprocess -import sys +require "json" +require "open3" # Auto-update script for images.json # @@ -11,65 +11,55 @@ import sys # For 'stable': uses GitHub's latest stable release # For 'rc': uses the newest release (prerelease or stable) -def main(): - if len(sys.argv) != 3 or sys.argv[2] not in ('stable', 'rc'): - print("Usage: auto-update ", file=sys.stderr) - sys.exit(1) +def main + abort("Usage: auto-update ") unless ARGV.length == 2 && %w[stable rc].include?(ARGV[1]) + image_tag, mode = ARGV - image_tag, mode = sys.argv[1], sys.argv[2] + images = JSON.parse(File.read("images.json")) + target = images.find { |image| image["tag"] == image_tag } + abort("Error: image '#{image_tag}' not found") unless target - with open('images.json', 'r') as f: - images = json.load(f) + updated = false + target["deps"].each do |dep| + name, repo, ref = dep.values_at("name", "repo", "ref") - # Find target image - target = next((img for img in images if img['tag'] == image_tag), None) - if not target: - print(f"Error: image '{image_tag}' not found", file=sys.stderr) - sys.exit(1) + # Skip non-release refs (branches, SHAs). + unless ref.start_with?("v") + puts("#{name}: skipping, ref is not a version") + next + end - updated = False - for dep in target['deps']: - name, repo, current = dep['name'], dep['repo'], dep['ref'] + latest = latest_release(repo, mode == "rc") + if latest.nil? + puts("#{name}: skipping, no release found") + elsif latest != ref + puts("#{name}: #{ref} -> #{latest}") + dep["ref"] = latest + updated = true + else + puts("#{name}: matches latest release") + end + end - # Skip non-release refs (branches, SHAs) - if not current.startswith('v'): - print(f"{name}: skipping, ref is not a version") - continue + File.write("images.json", "#{JSON.pretty_generate(images)}\n") if updated +end - latest = get_latest_release(repo, include_prerelease=(mode == 'rc')) - if not latest: - print(f"{name}: skipping, no release found") - elif latest != current: - print(f"{name}: {current} -> {latest}") - dep['ref'] = latest - updated = True - else: - print(f"{name}: matches latest release") +# Get the latest release tag, or nil if there is none. When include_prerelease +# is false, prereleases are skipped. +def latest_release(repo, include_prerelease) + release = releases(repo).find do |rel| + (include_prerelease && rel["tag"].include?("rc")) || !rel["prerelease"] + end + release&.fetch("tag") +end - if updated: - with open('images.json', 'w') as f: - json.dump(images, f, indent=2) - f.write('\n') +# Get all releases from a repo, newest first. +def releases(repo) + stdout, stderr, status = Open3.capture3( + "gh", "api", "repos/#{repo}/releases", "--jq", "[.[] | {tag: .tag_name, prerelease: .prerelease}]" + ) + abort("Error: failed to get releases for #{repo}: #{stderr}") unless status.success? + JSON.parse(stdout) +end -def get_latest_release(repo, include_prerelease=False): - """Get the latest release. If include_prerelease is False, skip prereleases.""" - for rel in get_releases(repo): - if include_prerelease and 'rc' in rel['tag']: - return rel['tag'] - if not rel['prerelease']: - return rel['tag'] - return None - -def get_releases(repo): - """Get all releases from a repo.""" - result = subprocess.run( - ['gh', 'api', f'repos/{repo}/releases', '--jq', '[.[] | {tag: .tag_name, prerelease: .prerelease}]'], - capture_output=True, text=True - ) - if result.returncode != 0: - print(f"Error: failed to get releases for {repo}: {result.stderr}", file=sys.stderr) - sys.exit(1) - return json.loads(result.stdout) - -if __name__ == '__main__': - main() +main diff --git a/.scripts/images-additional-tests b/.scripts/images-additional-tests index e34ede5b1..a4d9cf2cb 100755 --- a/.scripts/images-additional-tests +++ b/.scripts/images-additional-tests @@ -1,19 +1,15 @@ -#!/usr/bin/env python3 +#!/usr/bin/env ruby +# frozen_string_literal: true -import json -import sys +require "json" # Accepts as stdin a JSON object in the format of images.json. Outputs an array # of all the additional test cases defined in the images merged together. # Usage: < images.json ./.scripts/images-additional-tests -images = json.load(sys.stdin) -tests = [] -for image in images: - tag = image['tag'] - for test in image.get('additional-tests', []): - tests.append({'image': image, **test}) - for test in image.get('tests', {}).get('additional-tests', []): - tests.append({'image': image, **test}) - -print(json.dumps(tests)) +images = JSON.parse($stdin.read) +tests = images.flat_map do |image| + defined_tests = image.fetch("additional-tests", []) + image.fetch("tests", {}).fetch("additional-tests", []) + defined_tests.map { |test| { "image" => image }.merge(test) } +end +puts(JSON.generate(tests)) diff --git a/.scripts/images-deps b/.scripts/images-deps index 8d1d5e776..4759b29f8 100755 --- a/.scripts/images-deps +++ b/.scripts/images-deps @@ -1,11 +1,12 @@ -#!/usr/bin/env bash +#!/usr/bin/env ruby +# frozen_string_literal: true -set -e -set -u -set -o pipefail +require "json" # Accepts as stdin a JSON object in the format of images.json. Outputs an array # of all dependencies that need to be built across all the images. # Usage: < images.json ./.scripts/images-deps -jq -c '[ .[] | .deps[] ] | unique' +images = JSON.parse($stdin.read) +deps = images.flat_map { |image| image["deps"] }.uniq +puts(JSON.generate(deps)) diff --git a/.scripts/images-resolve-inherits b/.scripts/images-resolve-inherits index 48632ae1a..64cf51d9f 100755 --- a/.scripts/images-resolve-inherits +++ b/.scripts/images-resolve-inherits @@ -1,4 +1,7 @@ -#!/usr/bin/env python3 +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "json" # Accepts as stdin a JSON object in the format of images.json. # Resolves any 'inherit' fields by copying deps and config from the referenced image. @@ -20,83 +23,52 @@ # < images.json ./.scripts/images-resolve-inherits # < images.json ./.scripts/images-resolve-inherits parents.json -import json -import sys - -def main(): - """Read images from stdin, resolve all inherits, and output to stdout.""" - images = json.load(sys.stdin) - - # Load parent images from file if provided - parents = [] - if len(sys.argv) > 1: - with open(sys.argv[1], 'r') as f: - parents = json.load(f) - # Resolve any inherits within the parent images first - while has_inherits(parents): - parents = resolve_inherits(parents) - - while has_inherits(images): - images = resolve_inherits(images, parents) - print(json.dumps(images, separators=(',', ':'))) - -def has_inherits(images): - """Return True if any image has an 'inherit' field.""" - return any("inherit" in img for img in images) - -def resolve_inherits(images, parents=None): - """Resolve one level of inheritance for all images. - - For each image with an 'inherit' field: - - Copy deps from the parent that aren't already defined in the child - - Merge config from the parent (child values override parent values) - - If the parent also has an 'inherit' field, copy it (for nested resolution) - - Remove the child's 'inherit' field once resolved - - Parents can be looked up from either the input images or the optional parents list. - Input images take precedence over parents with the same tag. - - Exits with error if an image inherits from an unknown tag. - """ - if parents is None: - parents = [] - lookup = build_lookup(images) - parent_lookup = build_lookup(parents) - for image in images: - if "inherit" in image: - parent_tag = image["inherit"] - # Look up in input images first, then fall back to parents - parent = lookup.get(parent_tag) or parent_lookup.get(parent_tag) - if parent is None: - print(f"Error: image '{image['tag']}' inherits from unknown image '{parent_tag}'", file=sys.stderr) - sys.exit(1) - - # Get existing dep names in child - child_dep_names = {dep["name"] for dep in image.get("deps", [])} - - # Copy deps from parent that aren't in child - inherited_deps = [dep for dep in parent.get("deps", []) if dep["name"] not in child_dep_names] - image["deps"] = inherited_deps + image.get("deps", []) - - # Merge config from parent (child values override parent) - parent_config = parent.get("config", {}) - child_config = image.get("config", {}) - if parent_config or child_config: - merged_config = {**parent_config, **child_config} - image["config"] = merged_config - - # Remove child's inherit since it has been resolved - del image["inherit"] - - # Copy parent's inherit if present (for nested resolution) - if "inherit" in parent: - image["inherit"] = parent["inherit"] - - return images - -def build_lookup(images): - """Return a dict mapping image tags to their image objects.""" - return {img["tag"]: img for img in images} - -if __name__ == "__main__": - main() +def main + images = JSON.parse($stdin.read) + parents = ARGV.any? ? JSON.parse(File.read(ARGV[0])) : [] + parents = resolve_inherits(parents, []) while inherits?(parents) + images = resolve_inherits(images, parents) while inherits?(images) + puts(JSON.generate(images)) +end + +# Resolve one level of inheritance for all images. For each image with an +# 'inherit' field, copy the parent's deps that aren't already defined in the +# child, merge config (child overrides parent), and carry over the parent's own +# 'inherit' for nested resolution. Parents are looked up in the input images +# first, then in the optional parents list. Exits if a tag is unknown. +def resolve_inherits(images, parents) + lookup = lookup_by_tag(images) + parent_lookup = lookup_by_tag(parents) + + images.each do |image| + next unless image.key?("inherit") + + parent_tag = image["inherit"] + parent = lookup[parent_tag] || parent_lookup[parent_tag] + abort("Error: image '#{image["tag"]}' inherits from unknown image '#{parent_tag}'") unless parent + + child_dep_names = image.fetch("deps", []).map { |dep| dep["name"] } + inherited_deps = parent.fetch("deps", []).reject { |dep| child_dep_names.include?(dep["name"]) } + image["deps"] = inherited_deps + image.fetch("deps", []) + + merged_config = parent.fetch("config", {}).merge(image.fetch("config", {})) + image["config"] = merged_config unless merged_config.empty? + + image.delete("inherit") + image["inherit"] = parent["inherit"] if parent.key?("inherit") + end + + images +end + +# Return true if any image has an 'inherit' field. +def inherits?(images) + images.any? { |image| image.key?("inherit") } +end + +# Return a hash mapping image tags to their image objects. +def lookup_by_tag(images) + images.to_h { |image| [image["tag"], image] } +end + +main diff --git a/.scripts/images-with-extras b/.scripts/images-with-extras index 9b9ed3b8b..5643bc321 100755 --- a/.scripts/images-with-extras +++ b/.scripts/images-with-extras @@ -1,14 +1,14 @@ -#!/usr/bin/env python3 +#!/usr/bin/env ruby +# frozen_string_literal: true -import json -import sys -import subprocess -import hashlib +require "json" +require "open3" +require "digest" # Accepts as stdin a JSON object in the format of images.json. # And adds some calculatble elements used during the build. # -# 1. Resolves any 'ref' values in the JSON to a revision sha. +# 1. Resolves any 'ref' values in the JSON to a revision sha. # 2. Hashes the entire dep details and injects the hash as an id into the deps # details. The id can be used to uniquely identify a dep configuration. # @@ -16,34 +16,30 @@ import hashlib # # Usage: < images.json ./.scripts/images-with-extras -images = json.load(sys.stdin) +def main + images = JSON.parse($stdin.read) + cache = {} -cache = {} - -for image in images: + images.each do |image| tag = image["tag"] - for dep in image["deps"]: - name = dep["name"] - repo = dep["repo"] - ref = dep["ref"] - print(f"{tag} {name} {repo} {ref} ...", file=sys.stderr) - key = (name, repo, ref) - if key in cache: - sha = cache[key] - else: - sha = subprocess.run( - ["gh", "api", f"repos/{repo}/commits/{ref}", "--jq", ".sha"], - capture_output=True, - text=True, - check=True - ).stdout.strip() - cache[key] = sha - print(f" • revision sha = {sha}", file=sys.stderr) - dep["sha"] = sha + image["deps"].each do |dep| + name, repo, ref = dep.values_at("name", "repo", "ref") + warn("#{tag} #{name} #{repo} #{ref} ...") + dep["sha"] = cache[[repo, ref]] ||= commit_sha(repo, ref) + warn(" • revision sha = #{dep["sha"]}") + dep["id"] = Digest::SHA256.hexdigest(JSON.generate(dep)) + warn(" • id = #{dep["id"]}") + end + end + + puts(JSON.generate(images)) +end - dep_str = json.dumps(dep, separators=(',', ':')) - id = hashlib.sha256(dep_str.encode()).hexdigest() - dep["id"] = id - print(f" • id = {id}", file=sys.stderr) +# Resolve a repo ref to its commit sha via the GitHub API. +def commit_sha(repo, ref) + stdout, stderr, status = Open3.capture3("gh", "api", "repos/#{repo}/commits/#{ref}", "--jq", ".sha") + abort("Error: failed to get commit for #{repo}/#{ref}: #{stderr}") unless status.success? + stdout.strip +end -print(json.dumps(images, separators=(',', ':'))) +main diff --git a/Gemfile b/Gemfile new file mode 100644 index 000000000..7ae700656 --- /dev/null +++ b/Gemfile @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +gem "rubocop", "~> 1.60" diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 000000000..09c4e1d08 --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,42 @@ +GEM + remote: https://rubygems.org/ + specs: + ast (2.4.3) + json (2.19.7) + language_server-protocol (3.17.0.5) + lint_roller (1.1.0) + parallel (1.28.0) + parser (3.3.11.1) + ast (~> 2.4.1) + racc + prism (1.9.0) + racc (1.8.1) + rainbow (3.1.1) + regexp_parser (2.12.0) + rubocop (1.86.2) + json (~> 2.3) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.1.0) + parallel (>= 1.10) + parser (>= 3.3.0.2) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 2.9.3, < 3.0) + rubocop-ast (>= 1.49.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 4.0) + rubocop-ast (1.49.1) + parser (>= 3.3.7.2) + prism (~> 1.7) + ruby-progressbar (1.13.0) + unicode-display_width (3.2.0) + unicode-emoji (~> 4.1) + unicode-emoji (4.2.0) + +PLATFORMS + aarch64-linux + +DEPENDENCIES + rubocop (~> 1.60) + +BUNDLED WITH + 2.4.19 diff --git a/Makefile b/Makefile index a8e6083b8..ce75a2620 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -__PHONY__: run logs console build build-deps build-deps-xdr build-deps-core build-deps-horizon build-deps-friendbot build-deps-rpc build-deps-lab test +__PHONY__: run logs console build build-deps build-deps-xdr build-deps-core build-deps-horizon build-deps-friendbot build-deps-rpc build-deps-lab test lint CONTAINER_RUNTIME?=docker REVISION=$(shell git -c core.abbrev=no describe --always --exclude='*' --long --dirty) @@ -59,3 +59,11 @@ test: go run tests/test_friendbot.go go run tests/test_stellar_rpc_up.go go run tests/test_stellar_rpc_healthy.go + +# Lint the Ruby scripts in .scripts/ with rubocop. +lint: + $(CONTAINER_RUNTIME) run --rm \ + -v "$(CURDIR)":/work -w /work \ + -v quickstart-bundle:/usr/local/bundle \ + ruby:3.2 \ + sh -c "bundle install && bundle exec rubocop"