Skip to content

Commit 9eda406

Browse files
iceboundrockclaude
andauthored
Improve URL parsing in VortexMultiFileScan to handle file paths more … (#6842)
## Summary This PR fixes the issue that relative paths cause Neither URL nor path error Closes: #6841 This PR utilizes AI (codex from OpenAI) to generate code. ## API Changes None ## Testing `cargo test -p vortex-duckdb --lib` --------- Signed-off-by: Ruoshi Li <iceboundrock@gmail.com> Signed-off-by: iceboundrock <iceboundrock@outlook.com> Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent 20a3978 commit 9eda406

1 file changed

Lines changed: 162 additions & 8 deletions

File tree

vortex-duckdb/src/multi_file.rs

Lines changed: 162 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,38 @@ use crate::duckdb::ClientContextRef;
2020
use crate::duckdb::LogicalType;
2121
use crate::filesystem::resolve_filesystem;
2222

23+
/// Parse a glob string into a [`Url`].
24+
///
25+
/// Accepts full URLs (e.g. `s3://bucket/prefix/*.vortex`, `file:///data/*.vortex`) as well as
26+
/// bare file paths. For bare paths, the path is made absolute (without requiring it to exist)
27+
/// so that relative paths such as `./data/*.vortex` or `../data/*.vortex` are resolved correctly.
28+
fn parse_glob_url(glob_url_str: &str) -> VortexResult<Url> {
29+
Url::parse(glob_url_str).or_else(|_| {
30+
let path = absolute(Path::new(glob_url_str))
31+
.map_err(|e| vortex_err!("Failed making {glob_url_str} absolute: {e}"))?;
32+
// `absolute()` does not normalize `..` components, so `/a/b/../c` stays as-is.
33+
// Normalizing manually avoids `..` being percent-encoded in the resulting URL.
34+
let path = normalize_path(path);
35+
Url::from_file_path(path).map_err(|_| vortex_err!("Neither URL nor path: {glob_url_str}"))
36+
})
37+
}
38+
39+
/// Normalize a path by resolving `.` and `..` components without accessing the filesystem.
40+
fn normalize_path(path: std::path::PathBuf) -> std::path::PathBuf {
41+
use std::path::Component;
42+
let mut normalized = std::path::PathBuf::new();
43+
for component in path.components() {
44+
match component {
45+
Component::CurDir => {}
46+
Component::ParentDir => {
47+
normalized.pop();
48+
}
49+
c => normalized.push(c),
50+
}
51+
}
52+
normalized
53+
}
54+
2355
/// Vortex multi-file scan table function (`vortex_scan` / `read_vortex`).
2456
///
2557
/// Takes a file glob parameter and resolves it into a [`MultiFileDataSource`].
@@ -39,14 +71,9 @@ impl DataSourceTableFunction for VortexMultiFileScan {
3971
.ok_or_else(|| vortex_err!("Missing file glob parameter"))?;
4072

4173
// Parse the URL and separate the base URL (keep scheme, host, etc.) from the path.
42-
let glob_url_str = glob_url_parameter.as_string();
43-
44-
let glob_url = Url::parse(&glob_url_str).or_else(|_| {
45-
let glob_url = glob_url_str.as_str();
46-
let path = absolute(Path::new(glob_url));
47-
let path = path.map_err(|e| vortex_err!("Failed making {glob_url} absolute: {e}"))?;
48-
Url::from_file_path(path).map_err(|_| vortex_err!("Neither URL nor path: {glob_url}"))
49-
})?;
74+
let glob_url_string = glob_url_parameter.as_string();
75+
let glob_url_str = glob_url_string.as_str();
76+
let glob_url = parse_glob_url(glob_url_str)?;
5077

5178
let mut base_url = glob_url.clone();
5279
base_url.set_path("");
@@ -62,3 +89,130 @@ impl DataSourceTableFunction for VortexMultiFileScan {
6289
})
6390
}
6491
}
92+
93+
#[cfg(test)]
94+
mod tests {
95+
#[allow(clippy::wildcard_imports)]
96+
use rstest::rstest;
97+
98+
use super::*;
99+
100+
#[test]
101+
fn test_parse_glob_url_s3() -> VortexResult<()> {
102+
let url = parse_glob_url("s3://my-bucket/prefix/*.vortex")?;
103+
assert_eq!(url.scheme(), "s3");
104+
assert_eq!(url.host_str(), Some("my-bucket"));
105+
assert_eq!(url.path(), "/prefix/*.vortex");
106+
Ok(())
107+
}
108+
109+
#[test]
110+
fn test_parse_glob_url_file_scheme() -> VortexResult<()> {
111+
let url = parse_glob_url("file:///absolute/path/data.vortex")?;
112+
assert_eq!(url.scheme(), "file");
113+
assert_eq!(url.path(), "/absolute/path/data.vortex");
114+
Ok(())
115+
}
116+
117+
#[test]
118+
fn test_parse_glob_url_absolute_glob_path() -> VortexResult<()> {
119+
let tmpdir = tempfile::tempdir().unwrap();
120+
let glob = format!("{}/*.vortex", tmpdir.path().display());
121+
let url = parse_glob_url(&glob)?;
122+
assert_eq!(url.scheme(), "file");
123+
assert!(url.path().ends_with("/*.vortex"));
124+
Ok(())
125+
}
126+
127+
#[test]
128+
fn test_parse_glob_url_absolute_existing_path() -> VortexResult<()> {
129+
let tmpfile = tempfile::NamedTempFile::new().unwrap();
130+
let canonical = std::fs::canonicalize(tmpfile.path()).unwrap();
131+
let path_str = canonical.to_str().unwrap();
132+
let url = parse_glob_url(path_str)?;
133+
assert_eq!(url.scheme(), "file");
134+
assert_eq!(url.path(), path_str);
135+
Ok(())
136+
}
137+
138+
#[test]
139+
fn test_parse_glob_url_relative_path() -> VortexResult<()> {
140+
// Create a tempfile in the current working directory so we can refer to it
141+
// by a relative name (just the filename, without any directory component).
142+
let tmpfile = tempfile::NamedTempFile::new_in(".").unwrap();
143+
let filename = tmpfile.path().file_name().unwrap().to_str().unwrap();
144+
145+
let url = parse_glob_url(filename)?;
146+
assert_eq!(url.scheme(), "file");
147+
// The relative name must have been resolved to an absolute path.
148+
assert!(url.path().ends_with(filename));
149+
assert!(url.path().starts_with('/'));
150+
Ok(())
151+
}
152+
153+
#[test]
154+
fn test_parse_glob_url_relative_glob_path() -> VortexResult<()> {
155+
// A relative path with a glob character (e.g. `./data/*.vortex`) must also resolve
156+
// correctly.
157+
let tmpdir = tempfile::tempdir_in(".").unwrap();
158+
let dir_name = tmpdir.path().file_name().unwrap().to_str().unwrap();
159+
let glob = format!("./{dir_name}/*.vortex");
160+
let url = parse_glob_url(&glob)?;
161+
assert_eq!(url.scheme(), "file");
162+
assert!(url.path().starts_with('/'));
163+
assert!(url.path().ends_with("/*.vortex"));
164+
Ok(())
165+
}
166+
167+
#[test]
168+
fn test_parse_glob_url_nonexistent_path() -> VortexResult<()> {
169+
// absolute() does not require the path to exist, so a non-existent path succeeds.
170+
let url = parse_glob_url("/nonexistent/path/file.vortex")?;
171+
assert_eq!(url.scheme(), "file");
172+
assert_eq!(url.path(), "/nonexistent/path/file.vortex");
173+
Ok(())
174+
}
175+
176+
#[test]
177+
fn test_parse_glob_url_parent_relative_path() -> VortexResult<()> {
178+
// A path starting with `..` must be resolved to an absolute path without
179+
// percent-encoding the `..` component in the resulting URL.
180+
let tmpfile = tempfile::NamedTempFile::new_in("..").unwrap();
181+
let filename = tmpfile.path().file_name().unwrap().to_str().unwrap();
182+
let relative = format!("../{filename}");
183+
184+
let url = parse_glob_url(&relative)?;
185+
assert_eq!(url.scheme(), "file");
186+
// The resolved path must be absolute and must not contain encoded dots.
187+
assert!(url.path().starts_with('/'));
188+
assert!(
189+
!url.path().contains("%2E"),
190+
"path must not contain percent-encoded dots"
191+
);
192+
assert!(url.path().ends_with(filename));
193+
Ok(())
194+
}
195+
196+
// Use absolute paths so the expected result is cwd-independent.
197+
#[rstest]
198+
#[case("/a/./b", "/a/b")]
199+
#[case("/a/b/./c", "/a/b/c")]
200+
#[case("/a/../b", "/b")]
201+
#[case("/a/b/../c", "/a/c")]
202+
#[case("/a/b/../../c", "/c")]
203+
#[case("/a/./b/.././c", "/a/c")]
204+
#[case("/a/b/../..", "/")]
205+
fn test_parse_glob_url_dot_normalization(
206+
#[case] input: &str,
207+
#[case] expected_path: &str,
208+
) -> VortexResult<()> {
209+
let url = parse_glob_url(input)?;
210+
assert_eq!(url.scheme(), "file");
211+
assert_eq!(
212+
url.path(),
213+
expected_path,
214+
"input {input:?} should normalize to {expected_path:?}"
215+
);
216+
Ok(())
217+
}
218+
}

0 commit comments

Comments
 (0)