diff --git a/.github/workflows/build-bundles.yml b/.github/workflows/build-bundles.yml index ba5f48c..dc6da57 100644 --- a/.github/workflows/build-bundles.yml +++ b/.github/workflows/build-bundles.yml @@ -1,6 +1,8 @@ name: build-bundles -# Builds the light and heavy RTrace bundles +# Builds the light and heavy RTrace bundles via the top-level Makefile (which +# drives packaging/build-native.sh + build-bundle.sh). The bundled-Python pin +# lives in the Makefile, the single source of truth for the build. # All submodules are public and cloned over https, so no secrets are required. on: @@ -8,11 +10,6 @@ on: push: tags: ["v*"] -env: - # Pinned relocatable interpreter (astral-sh/python-build-standalone, install_only - # = relocatable). Verify/update this pin when bumping the bundled Python. - PYTHON_BUILD_STANDALONE_URL: "https://github.com/astral-sh/python-build-standalone/releases/download/20250106/cpython-3.11.11+20250106-x86_64-unknown-linux-gnu-install_only.tar.gz" - jobs: build: runs-on: ubuntu-latest @@ -49,26 +46,13 @@ jobs: with: submodules: recursive - - name: Build native artifacts + - name: Build & package ${{ matrix.edition }} bundle run: | set -eux + # `make tarball` runs build-native.sh + build-bundle.sh and writes + # rtrace--linux-x64.tar.gz(+.sha256) to the workspace root. # 'sudo' is a no-op shim under the root container but build-native.sh calls it. - packaging/build-native.sh --prefix "$GITHUB_WORKSPACE/staging/rtrace" - - - name: Assemble ${{ matrix.edition }} bundle - run: | - set -eux - packaging/build-bundle.sh \ - --prefix "$GITHUB_WORKSPACE/staging/rtrace" \ - --edition "${{ matrix.edition }}" \ - --python-url "$PYTHON_BUILD_STANDALONE_URL" - - - name: Package tarball + checksum - run: | - set -eux - OUT="rtrace-${{ matrix.edition }}-linux-x64.tar.gz" - tar -C "$GITHUB_WORKSPACE/staging" -czf "$OUT" rtrace - sha256sum "$OUT" > "$OUT.sha256" + make tarball EDITION="${{ matrix.edition }}" - name: Upload artifact uses: actions/upload-artifact@v4 diff --git a/.gitignore b/.gitignore index 8be3fe3..5ed2766 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,7 @@ __pycache__/ *.egg-info/ dist/ .rtrace-cache/ +# `make` build outputs +/staging/ +/rtrace-*-linux-x64.tar.gz +/rtrace-*-linux-x64.tar.gz.sha256 diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..29a42cf --- /dev/null +++ b/Makefile @@ -0,0 +1,108 @@ +# RTrace build & install -- the `make` front-end to the packaging scripts. +# +# This drives the same two-step build the release workflow uses +# (.github/workflows/build-bundles.yml): build the shared native artifacts, then +# assemble a relocatable, self-contained $RTRACE_HOME bundle for one edition. +# The build logic itself lives in packaging/build-native.sh and +# packaging/build-bundle.sh; this Makefile only wires them together and adds an +# `install` target. See DISTRIBUTION.md / docs/packaging.md for the full design. +# +# Quick start: +# make # build the light bundle into ./staging/rtrace +# make install # build + install to ~/.local/share/rtrace, symlink rtrace +# make EDITION=heavy install # same, heavy edition (adds angr for rich mode) +# make tarball # build + produce rtrace--linux-x64.tar.gz(+.sha256) +# make clean # remove build outputs +# +# Prerequisites: the build deps listed in docs/packaging.md (cmake, +# build-essential, g++-multilib, python3-dev, the zlib/unwind/snappy/lz4/xxhash +# and binutils -dev packages, dotnet-sdk-6.0) and initialized submodules +# (`make` initializes them on first use). Building capstone runs `sudo make +# install` so the system loader can find libcapstone for nucleus. + +# --- knobs (override on the command line, e.g. `make EDITION=heavy`) ---------- + +# Edition: light (light mode only) or heavy (light + rich; adds angr). +EDITION ?= light + +# Where the bundle is assembled (the tarball's top-level `rtrace/` dir). +PREFIX ?= $(CURDIR)/staging/rtrace + +# Pinned relocatable interpreter (astral-sh/python-build-standalone, install_only +# = relocatable). This is the single source of truth for the bundled Python; +# verify/update the pin when bumping it (see docs/packaging.md). +PYTHON_URL ?= https://github.com/astral-sh/python-build-standalone/releases/download/20250106/cpython-3.11.11+20250106-x86_64-unknown-linux-gnu-install_only.tar.gz + +# Install destination (relocatable, so this can be anywhere) and the dir the +# `rtrace` launcher is symlinked into. +INSTALL_PREFIX ?= $(HOME)/.local/share/rtrace +BINDIR ?= $(HOME)/.local/bin + +# Tarball name, matching the release workflow / install.sh convention. +TARBALL := rtrace-$(EDITION)-linux-x64.tar.gz + +.DEFAULT_GOAL := bundle +.PHONY: all bundle native tarball install uninstall submodules clean help + +all: bundle + +# Initialize submodules on first use (idempotent; only runs when missing). +submodules: + @if [ ! -e submodules/dynamorio/CMakeLists.txt ]; then \ + echo "==> initializing submodules"; \ + git submodule update --init --recursive; \ + fi + +# Build the shared native artifacts (DynamoRIO runtime, librtrace.so, capstone, +# nucleus sources, FunSeeker) into PREFIX. +native: submodules + packaging/build-native.sh --prefix "$(PREFIX)" + +# Assemble the relocatable bundle for EDITION on top of the native tree: bundled +# Python, the rtrace package + deps, nucleus bindings, and the launcher. +bundle: native + packaging/build-bundle.sh \ + --prefix "$(PREFIX)" \ + --edition "$(EDITION)" \ + --python-url "$(PYTHON_URL)" + @echo "==> bundle ready: $(PREFIX) (edition=$(EDITION))" + +# Package the bundle into a tarball + sha256 (what the release workflow uploads). +tarball: bundle + tar -C "$(dir $(PREFIX))" -czf "$(TARBALL)" "$(notdir $(PREFIX))" + sha256sum "$(TARBALL)" > "$(TARBALL).sha256" + @echo "==> wrote $(TARBALL) (+ .sha256)" + +# Install the freshly built bundle to INSTALL_PREFIX and symlink the launcher +# into BINDIR. The bundle is relocatable, so this is just a copy + symlink. +install: bundle + rm -rf "$(INSTALL_PREFIX)" + mkdir -p "$(dir $(INSTALL_PREFIX))" + cp -a "$(PREFIX)" "$(INSTALL_PREFIX)" + mkdir -p "$(BINDIR)" + ln -sf "$(INSTALL_PREFIX)/bin/rtrace" "$(BINDIR)/rtrace" + @echo "==> installed rtrace ($(EDITION)) to $(INSTALL_PREFIX)" + @echo " symlinked $(BINDIR)/rtrace -> $(INSTALL_PREFIX)/bin/rtrace" + @case ":$$PATH:" in *":$(BINDIR):"*) ;; *) echo " note: $(BINDIR) is not on PATH";; esac + +uninstall: + rm -rf "$(INSTALL_PREFIX)" + rm -f "$(BINDIR)/rtrace" + @echo "==> removed $(INSTALL_PREFIX) and $(BINDIR)/rtrace" + +clean: + rm -rf "$(CURDIR)/staging" + rm -f rtrace-*-linux-x64.tar.gz rtrace-*-linux-x64.tar.gz.sha256 + rm -rf src/build submodules/dynamorio/build + @echo "==> cleaned build outputs" + +help: + @echo "RTrace make targets:" + @echo " bundle (default) build the EDITION bundle into PREFIX" + @echo " install build + install to INSTALL_PREFIX, symlink into BINDIR" + @echo " tarball build + package rtrace--linux-x64.tar.gz(+.sha256)" + @echo " uninstall remove an installed bundle + its launcher symlink" + @echo " clean remove build outputs" + @echo "" + @echo "Variables: EDITION=$(EDITION) PREFIX=$(PREFIX)" + @echo " INSTALL_PREFIX=$(INSTALL_PREFIX) BINDIR=$(BINDIR)" diff --git a/README.md b/README.md index b2fd27f..66cd79c 100644 --- a/README.md +++ b/README.md @@ -82,23 +82,17 @@ sudo apt-get install -y git curl ca-certificates cmake build-essential \ g++-multilib python3 python3-dev zlib1g-dev libunwind-dev libsnappy-dev \ liblz4-dev libxxhash-dev binutils-dev binutils-multiarch-dev dotnet-sdk-6.0 -git clone --recurse-submodules https://github.com/negativa-ai/rtrace.git +git clone https://github.com/negativa-ai/rtrace.git cd rtrace -# build the native artifacts (DynamoRIO, the tracer client, FunSeeker, ...) -packaging/build-native.sh --prefix /tmp/staging/rtrace - -# assemble a relocatable bundle around them (downloads a standalone Python) -packaging/build-bundle.sh --prefix /tmp/staging/rtrace --edition light \ - --python-url "https://github.com/astral-sh/python-build-standalone/releases/download/20250106/cpython-3.11.11+20250106-x86_64-unknown-linux-gnu-install_only.tar.gz" - -# install -mkdir -p ~/.local/share ~/.local/bin -cp -a /tmp/staging/rtrace ~/.local/share/rtrace -ln -sf ~/.local/share/rtrace/bin/rtrace ~/.local/bin/rtrace +# build a relocatable bundle and install it to ~/.local/share/rtrace, +# symlinking the `rtrace` launcher into ~/.local/bin (submodules are +# initialized automatically on first build) +make install ``` -Pass `--edition heavy` to `build-bundle.sh` for the heavy edition. See +Pass `EDITION=heavy` for the heavy edition (`make EDITION=heavy install`), and +`make help` for the other targets (`tarball`, `uninstall`, ...). See [docs/packaging.md](docs/packaging.md) for how the bundles are put together. ## Development diff --git a/_docker/Dockerfile b/_docker/Dockerfile index 485b3be..318fd2b 100644 --- a/_docker/Dockerfile +++ b/_docker/Dockerfile @@ -1,13 +1,16 @@ -FROM ubuntu:24.04 +FROM ubuntu:22.04 ARG USER_ID ARG GROUP_ID RUN apt update && apt install sudo -y -# existence check for user and group -RUN getent group ${GROUP_ID} -RUN getent passwd ${USER_ID} +# Ubuntu 22.04 has no default `ubuntu` user (24.04 does), so create one matching +# the host UID/GID passed by _docker/build.sh -- this keeps the bind-mounted repo +# (-v $PWD:...) owned correctly. Reuse the group/user if those IDs already exist. +RUN getent group ${GROUP_ID} || groupadd -g ${GROUP_ID} ubuntu +RUN getent passwd ${USER_ID} || useradd -m -u ${USER_ID} -g ${GROUP_ID} -s /bin/bash ubuntu +RUN usermod -aG sudo ubuntu RUN echo "%sudo ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers USER ubuntu @@ -37,15 +40,10 @@ RUN sudo apt install ubuntu-dbgsym-keyring -y RUN sudo apt install debian-goodies -y RUN sudo apt install cmake g++ g++-multilib doxygen git zlib1g-dev libunwind-dev libsnappy-dev liblz4-dev libxxhash-dev -y -# Build env for FunSeeker -RUN wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb -O /tmp/packages-microsoft-prod.deb && \ - sudo dpkg -i /tmp/packages-microsoft-prod.deb && \ - rm /tmp/packages-microsoft-prod.deb - -RUN sudo apt-get update; \ - sudo apt-get install -y apt-transport-https && \ - sudo apt-get update && \ - DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC sudo apt-get install -y dotnet-sdk-6.0 +# Build env for FunSeeker. Ubuntu 22.04 ships dotnet-sdk-6.0 in its own archive, +# so no packages.microsoft.com repo is needed (unlike 24.04, whose archive only +# has dotnet 8). This matches the build-bundles.yml CI base image. +RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC sudo apt-get install -y dotnet-sdk-6.0 ENV DYNAMORIO_HOME=/home/ubuntu/repos/rtrace/submodules/dynamorio/build diff --git a/dev.sh b/dev.sh index 08dd465..76251c0 100755 --- a/dev.sh +++ b/dev.sh @@ -32,11 +32,11 @@ then docker run -d --privileged --name $devname --gpus all -v /tmp/:/tmp -v $PWD:/home/ubuntu/repos/rtrace -w /home/ubuntu/repos/rtrace $devname tail -f /dev/null echo "====================install CUDA================" docker exec -i $devname bash <-linux-x64.tar.gz` (+ `.sha256`). +- `make help` lists the targets and variables. + +The Makefile holds the pinned bundled-Python URL (single source of truth) and +delegates the actual work to two scripts: - **`packaging/build-native.sh --prefix `** — builds the shared native artifacts (DynamoRIO, `librtrace.so`, capstone, nucleus, FunSeeker) into the @@ -45,7 +56,7 @@ ABI and keeps the end-user install build-free. See the `build-bundle.sh` header. ## CI [`../.github/workflows/build-bundles.yml`](../.github/workflows/build-bundles.yml) -runs the scripts in an `ubuntu:22.04` container across an +runs `make tarball` in an `ubuntu:22.04` container across an `edition: [light, heavy]` matrix, on `workflow_dispatch` and on `v*` tags, and uploads the tarballs + `.sha256` as artifacts. @@ -85,7 +96,7 @@ git push origin v0.1.0 ## Notes / knobs - **No secrets required** — all submodules are public and cloned over https. -- **Pinned interpreter** — `PYTHON_BUILD_STANDALONE_URL` in the workflow pins a +- **Pinned interpreter** — the `PYTHON_URL` variable in the `Makefile` pins a `python-build-standalone` release (currently CPython 3.11). When bumping it, confirm the URL resolves and that `angr==9.2.102` installs on that minor version (3.11 is the safe choice for the heavy edition's wheels). @@ -103,7 +114,17 @@ git push origin v0.1.0 - FunSeeker output was verified byte-identical with and without `InvariantGlobalization=true` + `DebugType=None` on `/bin/ls` and libc. -## Local build (if you have the submodules + toolchain) +## Local build (if you have the toolchain) + +```bash +make # build the light bundle into ./staging/rtrace +make install # ... and install to ~/.local/share/rtrace, symlink rtrace +make EDITION=heavy tarball # build + package the heavy tarball +``` + +`make` initializes the submodules on first use. To override where the bundle is +assembled or installed, set `PREFIX` / `INSTALL_PREFIX` (see `make help`). The +underlying scripts can still be invoked directly if you need finer control: ```bash git submodule update --init --recursive