feat(curl): config-driven URL allowlist to bypass schema-mode rewrite

`rtk curl` rewrites every `curl URL` to `rtk curl URL` and pipes the
response through `rtk json --schema`, which produces field-type literals
(`field: int`) and a `(N)` array-length suffix. That's a token-savings
win for human-facing exploration of arbitrary third-party APIs, but it
ACTIVELY BREAKS downstream JSON parsing (jq, `python json.load`,
agent-side filtering, anything that round-trips JSON) for private or
internal APIs whose responses are consumed by parsers rather than read
by humans.

Today the only escape hatch is `[hooks] exclude_commands = ["curl"]`,
which disables curl rewriting entirely — losing the token savings on the
3rd-party APIs that motivated the rewrite in the first place.

This change adds a narrow opt-in mechanism: a substring allowlist that
opts specific URLs out of the rewrite while leaving everything else
rewriting as before.

  [curl]
  bypass_url_markers = [
    "localhost:8080/api/",
    "//internal.example.com/v1/",
  ]

Each marker is matched as a substring against the full command segment
(after env-prefix stripping), so it composes cleanly with curl's flag
positioning (`-X POST`, `-H`, `-d`, etc). Bypass is per-segment for
compound commands: `a && curl <internal> | jq` bypasses the curl segment
while `a` and the pipe target still rewrite as normal.

Default `bypass_url_markers = []` preserves historical behavior — users
who don't configure anything see no change.

API surface
-----------
* New `crate::core::config::CurlConfig { bypass_url_markers: Vec<String> }`
* New `crate::discover::registry::RewriteOptions { curl_bypass_url_markers }`
* New `rewrite_command_with_options(cmd, excluded, &opts)` — takes
  options explicitly, useful for tests and callers that already have
  config in hand.
* Existing `rewrite_command(cmd, excluded)` is unchanged for callers but
  now reads `Config.curl.bypass_url_markers` internally and forwards.

Mirrors the existing #196 bypass shape for `gh --json/--jq/--template`:
detect a structured-output consumer and skip schema-mode filtering.

Tests
-----
* 4 new config tests (default empty / [curl] roundtrip / missing
  section).
* 9 new registry tests covering: localhost / loopback / hostname
  marker bypass; POST with headers + payload; multi-marker OR;
  default-empty preserves rewrite; unmatched URL still rewrites;
  port-specific narrowness; per-segment behavior in compound commands.
* Full suite: 1699 passed, 0 failed, 6 ignored.

Author: globalsecurepayments

src/core/config.rs

@@ -21,6 +21,8 @@ pub struct Config {
     pub hooks: HooksConfig,
     #[serde(default)]
     pub limits: LimitsConfig,
+    #[serde(default)]
+    pub curl: CurlConfig,
 }
 
 #[derive(Debug, Serialize, Deserialize, Default)]
@@ -31,6 +33,32 @@ pub struct HooksConfig {
     pub exclude_commands: Vec<String>,
 }
 
+#[derive(Debug, Serialize, Deserialize, Default)]
+pub struct CurlConfig {
+    /// URL substring markers that opt-out a `curl` command from RTK's
+    /// `rtk curl ... --schema` rewrite. Use this for private/internal
+    /// JSON APIs whose responses are consumed by parsers (jq, agents,
+    /// downstream code) that need raw JSON rather than RTK's schema
+    /// summary.
+    ///
+    /// Each marker is a substring match against the full command text;
+    /// any match in any segment of a compound command bypasses the
+    /// rewrite for that segment only. Defaults to empty — RTK's default
+    /// behavior of rewriting all `curl` invocations is unchanged unless
+    /// you opt in.
+    ///
+    /// Example:
+    /// ```toml
+    /// [curl]
+    /// bypass_url_markers = [
+    ///   "localhost:8080/api/",
+    ///   "//internal.example.com/",
+    /// ]
+    /// ```
+    #[serde(default)]
+    pub bypass_url_markers: Vec<String>,
+}
+
 #[derive(Debug, Serialize, Deserialize)]
 pub struct TrackingConfig {
     pub enabled: bool,
@@ -233,6 +261,39 @@ enabled = true
         assert!(config.telemetry.consent_given.is_none());
     }
 
+    #[test]
+    fn test_curl_config_default_empty() {
+        let config = Config::default();
+        assert!(
+            config.curl.bypass_url_markers.is_empty(),
+            "default curl.bypass_url_markers must be empty (opt-in only)"
+        );
+    }
+
+    #[test]
+    fn test_curl_config_deserialize() {
+        let toml = r#"
+[curl]
+bypass_url_markers = ["localhost:8080/", "//api.example.com/v1/"]
+"#;
+        let config: Config = toml::from_str(toml).expect("valid toml");
+        assert_eq!(
+            config.curl.bypass_url_markers,
+            vec!["localhost:8080/", "//api.example.com/v1/"]
+        );
+    }
+
+    #[test]
+    fn test_curl_config_missing_section_is_valid() {
+        let toml = r#"
+[tracking]
+enabled = true
+history_days = 90
+"#;
+        let config: Config = toml::from_str(toml).expect("valid toml");
+        assert!(config.curl.bypass_url_markers.is_empty());
+    }
+
     #[test]
     fn test_telemetry_consent_roundtrip() {
         let toml = r#"

src/discover/registry.rs

@@ -433,12 +433,44 @@ fn strip_trailing_redirects(cmd: &str) -> (&str, &str) {
     (cmd_part, redir_part)
 }
 
+/// Optional knobs for `rewrite_command_with_options`. Default values reproduce
+/// the historical behavior of `rewrite_command(cmd, excluded)`.
+#[derive(Debug, Default, Clone)]
+pub struct RewriteOptions {
+    /// URL substring markers that opt-out `curl` invocations from being
+    /// rewritten to `rtk curl … | rtk json --schema`. See
+    /// `crate::core::config::CurlConfig::bypass_url_markers`.
+    pub curl_bypass_url_markers: Vec<String>,
+}
+
 /// Returns `None` if the command is unsupported or ignored (hook should pass through).
 ///
 /// Handles compound commands (`&&`, `||`, `;`) by rewriting each segment independently.
 /// For pipes (`|`), only rewrites the left-hand command (pipe targets stay raw),
 /// but continues rewriting segments after subsequent `&&`/`||`/`;` operators.
+///
+/// Reads `[curl] bypass_url_markers` from `config.toml` to decide whether a
+/// `curl` segment should be passed through unchanged. For explicit control
+/// over that list (e.g. in tests), use `rewrite_command_with_options`.
 pub fn rewrite_command(cmd: &str, excluded: &[String]) -> Option<String> {
+    let opts = RewriteOptions {
+        curl_bypass_url_markers: crate::core::config::Config::load()
+            .map(|c| c.curl.bypass_url_markers)
+            .unwrap_or_default(),
+    };
+    rewrite_command_with_options(cmd, excluded, &opts)
+}
+
+/// Same as `rewrite_command`, but lets the caller supply `RewriteOptions`
+/// directly instead of reading from the on-disk config. Tests use this to
+/// pin a specific `curl_bypass_url_markers` set without touching the user's
+/// config file; production callers that already have config in hand can
+/// avoid a second `Config::load()` round-trip.
+pub fn rewrite_command_with_options(
+    cmd: &str,
+    excluded: &[String],
+    opts: &RewriteOptions,
+) -> Option<String> {
     let trimmed = cmd.trim();
     if trimmed.is_empty() {
         return None;
@@ -462,11 +494,15 @@ pub fn rewrite_command(cmd: &str, excluded: &[String]) -> Option<String> {
         return Some(trimmed.to_string());
     }
 
-    rewrite_compound(trimmed, &compiled)
+    rewrite_compound(trimmed, &compiled, &opts.curl_bypass_url_markers)
 }
 
 /// Rewrite a compound command (with `&&`, `||`, `;`, `|`) by rewriting each segment.
-fn rewrite_compound(cmd: &str, excluded: &[ExcludePattern]) -> Option<String> {
+fn rewrite_compound(
+    cmd: &str,
+    excluded: &[ExcludePattern],
+    curl_bypass: &[String],
+) -> Option<String> {
     let tokens = tokenize(cmd);
     let mut result = String::with_capacity(cmd.len() + 32);
     let mut any_changed = false;
@@ -479,7 +515,8 @@ fn rewrite_compound(cmd: &str, excluded: &[ExcludePattern]) -> Option<String> {
         match tok.kind {
             TokenKind::Operator => {
                 let seg = cmd[seg_start..tok.offset].trim();
-                let rewritten = rewrite_segment(seg, excluded).unwrap_or_else(|| seg.to_string());
+                let rewritten = rewrite_segment(seg, excluded, curl_bypass)
+                    .unwrap_or_else(|| seg.to_string());
                 if rewritten != seg {
                     any_changed = true;
                 }
@@ -509,7 +546,8 @@ fn rewrite_compound(cmd: &str, excluded: &[ExcludePattern]) -> Option<String> {
                 let rewritten = if is_pipe_incompatible {
                     seg.to_string()
                 } else {
-                    rewrite_segment(seg, excluded).unwrap_or_else(|| seg.to_string())
+                    rewrite_segment(seg, excluded, curl_bypass)
+                        .unwrap_or_else(|| seg.to_string())
                 };
                 if rewritten != seg {
                     any_changed = true;
@@ -537,7 +575,8 @@ fn rewrite_compound(cmd: &str, excluded: &[ExcludePattern]) -> Option<String> {
             }
             TokenKind::Shellism if tok.value == "&" => {
                 let seg = cmd[seg_start..tok.offset].trim();
-                let rewritten = rewrite_segment(seg, excluded).unwrap_or_else(|| seg.to_string());
+                let rewritten = rewrite_segment(seg, excluded, curl_bypass)
+                    .unwrap_or_else(|| seg.to_string());
                 if rewritten != seg {
                     any_changed = true;
                 }
@@ -553,7 +592,8 @@ fn rewrite_compound(cmd: &str, excluded: &[ExcludePattern]) -> Option<String> {
     }
 
     let seg = cmd[seg_start..].trim();
-    let rewritten = rewrite_segment(seg, excluded).unwrap_or_else(|| seg.to_string());
+    let rewritten =
+        rewrite_segment(seg, excluded, curl_bypass).unwrap_or_else(|| seg.to_string());
     if rewritten != seg {
         any_changed = true;
     }
@@ -634,8 +674,12 @@ fn compile_exclude_patterns(patterns: &[String]) -> Vec<ExcludePattern> {
         .collect()
 }
 
-fn rewrite_segment(seg: &str, excluded: &[ExcludePattern]) -> Option<String> {
-    rewrite_segment_inner(seg, excluded, 0)
+fn rewrite_segment(
+    seg: &str,
+    excluded: &[ExcludePattern],
+    curl_bypass: &[String],
+) -> Option<String> {
+    rewrite_segment_inner(seg, excluded, curl_bypass, 0)
 }
 
 fn is_excluded(cmd: &str, excluded: &[ExcludePattern]) -> bool {
@@ -645,7 +689,12 @@ fn is_excluded(cmd: &str, excluded: &[ExcludePattern]) -> bool {
     })
 }
 
-fn rewrite_segment_inner(seg: &str, excluded: &[ExcludePattern], depth: usize) -> Option<String> {
+fn rewrite_segment_inner(
+    seg: &str,
+    excluded: &[ExcludePattern],
+    curl_bypass: &[String],
+    depth: usize,
+) -> Option<String> {
     let trimmed = seg.trim();
     if trimmed.is_empty() {
         return None;
@@ -660,7 +709,7 @@ fn rewrite_segment_inner(seg: &str, excluded: &[ExcludePattern], depth: usize) -
             if rest.is_empty() {
                 return None;
             }
-            return match rewrite_segment_inner(rest, excluded, depth + 1) {
+            return match rewrite_segment_inner(rest, excluded, curl_bypass, depth + 1) {
                 Some(rewritten) => Some(format!("{} {}", prefix, rewritten)),
                 None => None,
             };
@@ -746,6 +795,22 @@ fn rewrite_segment_inner(seg: &str, excluded: &[ExcludePattern], depth: usize) -
         }
     }
 
+    // `rtk curl` pipes responses through `rtk json --schema`, which produces
+    // field-type literals (`field: int`, `field: string`) and a `(N)`
+    // array-length suffix. That's a token-savings win for arbitrary
+    // third-party APIs, but actively breaks downstream JSON parsing (jq,
+    // python `json.load`, agent-side filtering) for private/internal APIs
+    // whose responses are consumed as raw JSON.
+    //
+    // Skip the rewrite when the URL contains any user-configured marker.
+    // Empty list = unchanged historical behavior (every curl gets rewritten).
+    // Configure via `[curl] bypass_url_markers` in `~/.config/rtk/config.toml`.
+    if rule.rtk_cmd == "rtk curl" && !curl_bypass.is_empty() {
+        if curl_bypass.iter().any(|marker| cmd_clean.contains(marker)) {
+            return None;
+        }
+    }
+
     // Try each rewrite prefix (longest first) with word-boundary check
     for &prefix in rule.rewrite_prefixes {
         if let Some(rest) = strip_word_prefix(cmd_clean, prefix) {
@@ -2963,6 +3028,153 @@ mod tests {
         assert!(rewrite_command("curl https://api.example.com", &excluded).is_some());
     }
 
+    // `[curl] bypass_url_markers` lets users opt private / internal JSON APIs
+    // out of the `rtk curl … --schema` rewrite when the rewritten output would
+    // break a downstream parser. Tests below pass markers explicitly via
+    // `rewrite_command_with_options` so behavior is independent of the on-disk
+    // user config.
+    fn curl_bypass_opts(markers: &[&str]) -> RewriteOptions {
+        RewriteOptions {
+            curl_bypass_url_markers: markers.iter().map(|s| s.to_string()).collect(),
+        }
+    }
+
+    #[test]
+    fn test_rewrite_curl_bypasses_localhost_marker() {
+        // curl to a configured localhost API returns None (no rewrite), so the
+        // original curl runs verbatim and the caller gets real JSON instead of
+        // schema-mode literals.
+        let opts = curl_bypass_opts(&["localhost:3300/"]);
+        assert_eq!(
+            rewrite_command_with_options(
+                "curl http://localhost:3300/api/change-requests",
+                &[],
+                &opts
+            ),
+            None
+        );
+    }
+
+    #[test]
+    fn test_rewrite_curl_bypasses_loopback_marker() {
+        // Loopback variant — separate marker entry.
+        let opts = curl_bypass_opts(&["127.0.0.1:3300/"]);
+        assert_eq!(
+            rewrite_command_with_options(
+                "curl -s http://127.0.0.1:3300/api/health",
+                &[],
+                &opts
+            ),
+            None
+        );
+    }
+
+    #[test]
+    fn test_rewrite_curl_bypasses_post_with_headers_and_payload() {
+        // Real-world POST variant: -X POST, -H header, -d body. Bypass must
+        // trigger regardless of curl flag positioning, since markers are a
+        // substring match against the full segment.
+        let opts = curl_bypass_opts(&["localhost:3300/"]);
+        assert_eq!(
+            rewrite_command_with_options(
+                "curl -s -X POST -H 'x-api-key: foo' -d '{}' http://localhost:3300/api/session-ack",
+                &[],
+                &opts
+            ),
+            None
+        );
+    }
+
+    #[test]
+    fn test_rewrite_curl_bypasses_https_hostname_marker() {
+        // Hostname-based marker (e.g. an internal Tailscale or VPN-only API).
+        let opts = curl_bypass_opts(&["//api.internal.example/"]);
+        assert_eq!(
+            rewrite_command_with_options(
+                "curl https://api.internal.example/v1/projects",
+                &[],
+                &opts
+            ),
+            None
+        );
+    }
+
+    #[test]
+    fn test_rewrite_curl_bypasses_multiple_markers() {
+        // Multiple markers act as OR — any matching substring bypasses.
+        let opts = curl_bypass_opts(&["localhost:8090/", "localhost:11434/"]);
+        assert_eq!(
+            rewrite_command_with_options("curl http://localhost:8090/v1/models", &[], &opts),
+            None
+        );
+        assert_eq!(
+            rewrite_command_with_options("curl http://localhost:11434/api/tags", &[], &opts),
+            None
+        );
+    }
+
+    #[test]
+    fn test_rewrite_curl_default_empty_bypass_still_rewrites() {
+        // Default behavior — empty bypass markers — leaves the historical
+        // rewrite-everything semantics intact. This is the core upstream
+        // contract: opt-in only, no behavior change for users who haven't
+        // configured anything.
+        let opts = RewriteOptions::default();
+        assert_eq!(
+            rewrite_command_with_options(
+                "curl http://localhost:3300/api/change-requests",
+                &[],
+                &opts
+            ),
+            Some("rtk curl http://localhost:3300/api/change-requests".into())
+        );
+    }
+
+    #[test]
+    fn test_rewrite_curl_still_rewrites_unmatched_url() {
+        // Third-party / unconfigured URL is not in the marker list, so the
+        // rtk curl rewrite still fires. Token-savings premise preserved for
+        // everyone except the user's own opt-in endpoints.
+        let opts = curl_bypass_opts(&["localhost:3300/"]);
+        assert_eq!(
+            rewrite_command_with_options("curl https://api.github.com/repos/foo/bar", &[], &opts),
+            Some("rtk curl https://api.github.com/repos/foo/bar".into())
+        );
+        assert_eq!(
+            rewrite_command_with_options("curl https://example.com/api/data", &[], &opts),
+            Some("rtk curl https://example.com/api/data".into())
+        );
+    }
+
+    #[test]
+    fn test_rewrite_curl_marker_is_port_specific() {
+        // Subtle: localhost on a port NOT in the marker list (e.g. someone
+        // running a third-party JSON server on :4000) STILL gets rewritten.
+        // The bypass is intentionally narrow — markers include port to keep
+        // collateral surface small.
+        let opts = curl_bypass_opts(&["localhost:3300/"]);
+        assert_eq!(
+            rewrite_command_with_options("curl http://localhost:4000/api/foo", &[], &opts),
+            Some("rtk curl http://localhost:4000/api/foo".into())
+        );
+    }
+
+    #[test]
+    fn test_rewrite_compound_with_bypassed_curl_skips_only_that_segment() {
+        // In a compound command, the bypassed-curl segment passes through
+        // unchanged but other rewritable segments (git status) still get
+        // rewritten. Confirms the bypass is per-segment, not all-or-nothing.
+        let opts = curl_bypass_opts(&["localhost:3300/"]);
+        assert_eq!(
+            rewrite_command_with_options(
+                "git status && curl http://localhost:3300/api/change-requests",
+                &[],
+                &opts
+            ),
+            Some("rtk git status && curl http://localhost:3300/api/change-requests".into())
+        );
+    }
+
     #[test]
     fn test_rewrite_compound_partial_exclude() {
         // curl excluded but git still rewrites