Embed the release tarball into the botan-src, instead of using a submodule

.ci/build.py

@@ -62,13 +62,13 @@ def main(args = None):
         if "SCCACHE_MAXSIZE" not in os.environ:
             os.environ["SCCACHE_MAXSIZE"] = "2G"
 
-    KNOWN_TOOLCHAINS = ['stable', 'nightly', '1.64.0']
+    KNOWN_TOOLCHAINS = ['stable', 'nightly', '1.85.1']
     KNOWN_FEATURES = ['vendored', 'git', 'no-std']
 
     toolchain = args[1]
 
     if toolchain not in KNOWN_TOOLCHAINS:
-        print("ERROR: Unknown toolchain %s" % (toolchain))
+        print("ERROR: Unknown toolchain %s (need to update .ci/build.py?)" % (toolchain))
         return 1
 
     features = [] if len(args) < 3 else args[2].split('+')
@@ -102,20 +102,22 @@ def main(args = None):
         return 1
 
     if 'vendored' in features:
-        run_command(['git', 'submodule', 'update', '--init', '--depth', '3'])
+        run_command([sys.executable, './botan-src/scripts/fetch.py'])
     elif 'git' in features:
         nproc = multiprocessing.cpu_count()
         botan_src = 'botan-git'
         run_command(['git', 'clone', '--depth', '1', 'https://github.com/randombit/botan.git', botan_src])
 
         import tomllib
-        excludes = tomllib.loads(open(os.path.join('botan-src', 'Cargo.toml')).read())['package']['exclude']
-        for exclude in excludes:
-            path = exclude.replace('botan/', botan_src + '/')
-            if os.path.isdir(path):
-                shutil.rmtree(path)
-            elif os.path.isfile(path):
-                os.remove(path)
+        cargo_toml = tomllib.loads(open(os.path.join('botan-src', 'Cargo.toml')).read())['package']
+
+        if 'excludes' in cargo_toml:
+            for exclude in cargo_toml['excludes']:
+                path = exclude.replace('botan/', botan_src + '/')
+                if os.path.isdir(path):
+                    shutil.rmtree(path)
+                elif os.path.isfile(path):
+                    os.remove(path)
 
         run_command(['./configure.py',
                      '--compiler-cache=%s' % (options.compiler_cache),
@@ -133,6 +135,8 @@ def main(args = None):
         os.environ["RUSTFLAGS"] = "-D warnings -L/opt/homebrew/lib"
         os.environ["RUSTDOCFLAGS"] = "-D warnings -L/opt/homebrew/lib"
         os.environ["DYLD_LIBRARY_PATH"] = homebrew_dir
+    elif os.access('/usr/bin/apt-get', os.R_OK):
+        run_command(['sudo', 'apt-get', 'install', 'libbotan-2-dev'])
 
     run_command(['rustc', '--version'])
 
@@ -143,10 +147,7 @@ def main(args = None):
     run_command(['cargo', 'test'] + sys_features, 'botan-sys')
 
     run_command(['cargo', 'build'] + lib_features, 'botan')
-
-    # Can't build tests with 1.64 due to serde_json having MSRV of 1.71.0 now
-    if toolchain != '1.64.0':
-        run_command(['cargo', 'test'] + lib_features, 'botan')
+    run_command(['cargo', 'test'] + lib_features, 'botan')
 
     run_command([options.compiler_cache, '--show-stats'])
 

.github/workflows/ci.yml

@@ -31,6 +31,7 @@ jobs:
 
       - uses: actions/checkout@v4
 
+      - run: ./botan-src/scripts/fetch.py
       - run: cargo +nightly clippy -- --deny warnings
   ci:
     runs-on: ubuntu-24.04
@@ -50,15 +51,15 @@ jobs:
             features: git
           - toolchain: stable
             features: vendored
-          - toolchain: 1.64.0 # MSRV no-std
+          - toolchain: 1.85.1 # MSRV no-std
             features: no-std+git
-          - toolchain: 1.64.0 # MSRV
+          - toolchain: 1.85.1 # MSRV
           - toolchain: nightly
           - toolchain: nightly
             features: no-std+git
 
     steps:
-      - run: sudo apt-get -qq install ccache libbotan-2-dev
+      - run: sudo apt-get -qq install ccache
       - uses: actions/checkout@v4
       - uses: actions/cache@v4
         with:
@@ -90,7 +91,7 @@ jobs:
           - toolchain: nightly
 
     steps:
-      - run: sudo apt-get -qq install ccache libbotan-2-dev
+      - run: sudo apt-get -qq install ccache
       - uses: actions/checkout@v4
       - uses: actions/cache@v4
         with:

NEWS.md

@@ -1,3 +1,23 @@
+## 0.13.0 Not Yet Released
+
+- Minor version bump due to MSRV increase
+- Bump MSRV to 1.85.0
+- Bump Rust Edition to 2024
+- Add support for new FFI interfaces in Botan 3.10, 3.11, 3.12
+- The `botan-src` crate now ships the upstream Botan release tarball
+  (`.tar.xz`) inside the published crate and extracts it at build time
+  rather than building from a git submodule.
+- Add locking to Privkey to work around bugs in versions of Botan prior to 3.11.0.
+  This requires `std` support; thus `no_std` builds now require at least Botan 3.11.0
+- Update `botan-src` to Botan 3.12.0
+- Implement `rand::TryRngCore` for `RandomNumberGenerator`
+- Add `Signer::signature_length`
+- Add `Pubkey::load_rsa_pkcs1` for loading RSA public keys in PKCS#1 form
+- Add various interfaces for CRL generation and handling
+- Add additional X.509 certificate getters: `ocsp_responders`,
+  `issuer_dn`, `subject_dn`, `is_ca`, `path_limit`, `pem_encode`, `der_encode`
+- Fix some tests that hit behavior changes in Botan 3.12
+
 ## 0.12.0 2025-08-05
 
 - Minor version bump due to removing a feature

README.md

@@ -18,7 +18,7 @@ PRs and comments/issues happily accepted.
 MSRV
 -----
 
-The Minimum Supported Rust Version (MSRV) of this crate is Rust 1.64.0.
+The Minimum Supported Rust Version (MSRV) of this crate is Rust 1.85.0.
 
 Any future increase in the MSRV will be accompanied by increasing the minor
 version number.

botan-src/Cargo.toml

@@ -1,32 +1,32 @@
 [package]
 name = "botan-src"
-version = "0.30900.2"
-authors = ["Rodolphe Breard <rodolphe@what.tf>", "Jack Lloyd <jack@randombit.net>"]
+version = "0.31200.0"
+authors = ["Jack Lloyd <jack@randombit.net>", "Rodolphe Breard <rodolphe@what.tf>"]
 description = "Sources of Botan cryptography library"
 license = "MIT"
-edition = "2021"
+edition = "2024"
 homepage = "https://botan.randombit.net/"
 repository = "https://github.com/randombit/botan-rs"
 readme = "README.md"
 categories = ["cryptography"]
-rust-version = "1.64"
+rust-version = "1.85"
+build = "build.rs"
 
-exclude = ["botan/doc",
-           "botan/src/bogo_shim",
-           "botan/src/cli",
-           "botan/src/ct_selftest",
-           "botan/src/examples",
-           "botan/src/fuzzer",
-           "botan/src/lib/compat/sodium",
-           "botan/src/lib/filters",
-           "botan/src/lib/prov/pkcs11",
-           "botan/src/lib/prov/tpm",
-           "botan/src/lib/prov/tpm2",
-           "botan/src/lib/tls",
-           "botan/src/python",
-           "botan/src/tests"]
+include = [
+    "Cargo.toml",
+    "README.md",
+    "build.rs",
+    "release.toml",
+    "src/**/*.rs",
+    "examples/**/*.rs",
+    "scripts/fetch.py",
+    "vendor/Botan-*.tar.xz",
+]
 
 [dependencies]
+lzma-rs = { version = "0.3", default-features = false }
+sha2 = { version = "0.10", default-features = false }
+tar = { version = "0.4", default-features = false }
 
 [[example]]
 name = "build"

botan-src/README.md

@@ -7,3 +7,17 @@ library. It is supposed to be used by the
 
 A high level Rust interface built on this library is included in the
 [botan](https://crates.io/crates/botan) crate.
+
+## Building against a custom Botan tree
+
+The crate ships the upstream Botan release tarball in `vendor/` and
+extracts it at build time. Two environment variables let you override
+that, which is useful when testing a fork or pre-release:
+
+- `BOTAN_SRC_DIR=/path/to/checkout` — build from an existing source tree
+  (e.g. a git checkout). No extraction, no checksum.
+- `BOTAN_SRC_TARBALL=/path/to/foo.tar.xz` — extract this `.tar.xz`
+  instead of the bundled one. No checksum.
+
+If neither is set, the bundled tarball is extracted and verified against
+the pinned SHA-256.

botan-src/build.rs

@@ -0,0 +1,107 @@
+// Parses release.toml and exposes its values as compile-time `env!`
+// strings inside src/lib.rs. The hand-rolled parser handles the
+// trivial subset of TOML actually used in release.toml — three
+// top-level `key = "value"` lines plus comments — to avoid pulling in
+// a TOML build-dependency.
+use std::fs;
+use std::path::Path;
+
+fn extract(content: &str, key: &str) -> Option<String> {
+    for line in content.lines() {
+        let line = line.split('#').next().unwrap_or("").trim();
+        if line.is_empty() {
+            continue;
+        }
+        if let Some((k, v)) = line.split_once('=') {
+            if k.trim() == key {
+                return Some(v.trim().trim_matches('"').to_string());
+            }
+        }
+    }
+    None
+}
+
+fn parse_botan_version(s: &str) -> Option<(u32, u32, u32)> {
+    let mut parts = s.split('.').map(|p| p.parse::<u32>().ok());
+    Some((parts.next()??, parts.next()??, parts.next()??))
+}
+
+// botan-src crate version is `0.MNNPP.X` where M.NN.PP is the bundled
+// Botan version. Cross-check that against release.toml so we never
+// publish e.g. 0.31100.0 with a Botan 3.11.1 tarball inside.
+fn parse_crate_botan_version(crate_version: &str) -> Option<(u32, u32, u32)> {
+    let mut parts = crate_version.split('.');
+    parts.next()?; // leading "0"
+    let encoded: u32 = parts.next()?.parse().ok()?;
+    Some((encoded / 10000, (encoded / 100) % 100, encoded % 100))
+}
+
+fn main() {
+    let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR");
+    let manifest_dir = Path::new(&manifest_dir);
+    let release_path = manifest_dir.join("release.toml");
+    println!("cargo:rerun-if-changed=release.toml");
+    let content = fs::read_to_string(&release_path)
+        .unwrap_or_else(|e| panic!("read {}: {e}", release_path.display()));
+
+    let pairs = [
+        ("version", "BOTAN_VERSION"),
+        ("sha256", "BOTAN_TARBALL_SHA256"),
+        ("url", "BOTAN_TARBALL_URL"),
+    ];
+    let mut values = std::collections::HashMap::new();
+    for (key, _) in pairs {
+        let value =
+            extract(&content, key).unwrap_or_else(|| panic!("release.toml missing key `{key}`"));
+        values.insert(key, value);
+    }
+    // Expand `{key}` references between fields (so `url` can refer to `{version}`).
+    let raw: Vec<(&str, String)> = values.iter().map(|(k, v)| (*k, v.clone())).collect();
+    for v in values.values_mut() {
+        for (k, replacement) in &raw {
+            *v = v.replace(&format!("{{{k}}}"), replacement);
+        }
+    }
+    for (key, env_var) in pairs {
+        println!("cargo:rustc-env={env_var}={}", values[key]);
+    }
+
+    let botan_version = parse_botan_version(&values["version"]).unwrap_or_else(|| {
+        panic!(
+            "release.toml `version` not in M.N.P form: {}",
+            values["version"]
+        )
+    });
+    let crate_version = std::env::var("CARGO_PKG_VERSION").expect("CARGO_PKG_VERSION");
+    let crate_encoded = parse_crate_botan_version(&crate_version).unwrap_or_else(|| {
+        panic!("can't decode botan version from crate version `{crate_version}`")
+    });
+    if crate_encoded != botan_version {
+        panic!(
+            "\nbotan-src crate version `{crate_version}` encodes Botan {}.{}.{},\n\
+             but release.toml says Botan {}.{}.{}. Bump one or the other so they agree.\n",
+            crate_encoded.0,
+            crate_encoded.1,
+            crate_encoded.2,
+            botan_version.0,
+            botan_version.1,
+            botan_version.2,
+        );
+    }
+
+    // Refuse to build (or publish, via the default `cargo publish` verify
+    // step) without the upstream tarball staged in vendor/. Catches the
+    // "publish with empty vendor/" footgun.
+    let tarball = manifest_dir
+        .join("vendor")
+        .join(format!("Botan-{}.tar.xz", values["version"]));
+    println!("cargo:rerun-if-changed={}", tarball.display());
+    if !tarball.exists() {
+        panic!(
+            "\nBotan source tarball missing at {}.\n\
+             Run `python3 scripts/fetch.py` from botan-src/ to download it\n\
+             before building or publishing this crate.\n",
+            tarball.display()
+        );
+    }
+}

botan-src/release.toml

@@ -0,0 +1,4 @@
+# Pinned release, override using BOTAN_SRC_DIR or BOTAN_SRC_TARBALL
+version = "3.12.0"
+sha256 = "5370f98dc15f8c222ee1ce52cd61c8756a53be0dc57cc4c1b0714d5a09ad74fb"
+url = "https://botan.randombit.net/releases/Botan-{version}.tar.xz"

botan-src/scripts/fetch.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+"""Download and verify the upstream Botan release tarball into vendor/.
+
+Run before `cargo publish -p botan-src`. Reads release.toml so the
+version/sha/URL stays in lockstep with what build.rs sees.
+
+Requires Python 3.11+ (for tomllib).
+"""
+
+import hashlib
+import sys
+import tomllib
+import urllib.request
+from pathlib import Path
+
+
+def main() -> int:
+    repo = Path(__file__).resolve().parent.parent
+    raw = tomllib.loads((repo / "release.toml").read_text())
+    # Expand `{key}` references between fields so e.g. url can refer to {version}.
+    info = {k: v.format(**raw) if isinstance(v, str) else v for k, v in raw.items()}
+    vendor = repo / "vendor"
+    vendor.mkdir(exist_ok=True)
+    tarball = vendor / f"Botan-{info['version']}.tar.xz"
+
+    if not tarball.exists():
+        print(f"downloading {info['url']}")
+        urllib.request.urlretrieve(info["url"], tarball)
+
+    actual = hashlib.sha256(tarball.read_bytes()).hexdigest()
+    if actual != info["sha256"]:
+        tarball.unlink()
+        print(
+            f"sha256 mismatch for {tarball.name}: "
+            f"expected {info['sha256']}, got {actual}",
+            file=sys.stderr,
+        )
+        return 1
+    print(f"ok: {tarball}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())