Skip to content

perf: cache CryptoProvider/RootCertStore/BrowserEmulator, optimize Python iterators, fix Random OS consistency #148

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
deedy5 merged 3 commits into from
May 19, 2026
Merged

perf: cache CryptoProvider/RootCertStore/BrowserEmulator, optimize Python iterators, fix Random OS consistency #148

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
81f4e35
perf: cache CryptoProvider and RootCertStore, use Arc<BrowserEmulator…
deedy5 May 19, 2026
b75c8fd
perf(python): cache encoding per iterator, pre-alloc buffers, elimina…
deedy5 May 19, 2026
58215c5
fix: resolve ImpersonateOS::Random once per build_*_settings for cons…
deedy5 May 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions crates/primp-python/src/async/client.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,12 @@ impl AsyncClient {
client_guard.clone()
};

// Restore redirect policy immediately after cloning if it was changed
if follow_redirects.is_some() {
let mut client_guard = self.client.write().unwrap_or_else(|e| e.into_inner());
client_guard.set_redirect_policy(::primp::redirect::Policy::limited(20));
}

let future = async move {
// Create request builder
let mut request_builder = client.request(method, &resolved_url);
Expand Down Expand Up @@ -324,12 +330,6 @@ impl AsyncClient {
Ok::<(PrimpResponse, String, u16), PrimpErrorEnum>((resp, url, status_code))
};

// Restore redirect policy if it was changed
if follow_redirects.is_some() {
let mut client_guard = self.client.write().unwrap_or_else(|e| e.into_inner());
client_guard.set_redirect_policy(::primp::redirect::Policy::limited(20));
}

// Convert Rust future to Python awaitable
if stream {
let py_future = async move {
Expand Down
44 changes: 21 additions & 23 deletions crates/primp-python/src/async/response.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -332,7 +332,7 @@ impl AsyncBytesIterator {
AsyncBytesIterator {
resp,
chunk_size,
buffer: Arc::new(TMutex::new(Vec::new())),
buffer: Arc::new(TMutex::new(Vec::with_capacity(chunk_size * 2))),
}
}
}
Expand Down Expand Up @@ -412,7 +412,7 @@ impl AsyncBytesIterator {
#[pyclass]
pub struct AsyncTextIterator {
resp: Arc<TMutex<Option<::primp::Response>>>,
encoding: String,
encoding: &'static encoding_rs::Encoding,
chunk_size: usize,
buffer: Arc<TMutex<Vec<u8>>>,
}
Expand All @@ -423,11 +423,13 @@ impl AsyncTextIterator {
encoding: String,
chunk_size: usize,
) -> Self {
let encoding =
encoding_rs::Encoding::for_label(encoding.as_bytes()).unwrap_or(encoding_rs::UTF_8);
AsyncTextIterator {
resp,
encoding,
chunk_size,
buffer: Arc::new(TMutex::new(Vec::new())),
buffer: Arc::new(TMutex::new(Vec::with_capacity(chunk_size * 2))),
}
}
}
Expand All @@ -442,7 +444,7 @@ impl AsyncTextIterator {
use pyo3_async_runtimes::tokio::future_into_py;

let resp = Arc::clone(&self.resp);
let encoding = self.encoding.clone();
let encoding = self.encoding;
let chunk_size = self.chunk_size;
let buffer = Arc::clone(&self.buffer);

Expand All @@ -451,8 +453,7 @@ impl AsyncTextIterator {
let mut buf = buffer.lock().await;
if buf.len() >= chunk_size {
let chunk: Vec<u8> = buf.drain(..chunk_size).collect();
let enc = Encoding::for_label(encoding.as_bytes()).unwrap_or(UTF_8);
let (text, _, _) = enc.decode(&chunk);
let (text, _, _) = encoding.decode(&chunk);
return Ok::<String, PyErr>(text.to_string());
}
}
Expand All @@ -465,13 +466,11 @@ impl AsyncTextIterator {
buf.extend_from_slice(&data);
if buf.len() >= chunk_size {
let result: Vec<u8> = buf.drain(..chunk_size).collect();
let enc = Encoding::for_label(encoding.as_bytes()).unwrap_or(UTF_8);
let (text, _, _) = enc.decode(&result);
let (text, _, _) = encoding.decode(&result);
Ok(text.to_string())
} else if !buf.is_empty() {
let result: Vec<u8> = std::mem::take(&mut *buf);
let enc = Encoding::for_label(encoding.as_bytes()).unwrap_or(UTF_8);
let (text, _, _) = enc.decode(&result);
let (text, _, _) = encoding.decode(&result);
Ok(text.to_string())
} else {
Ok(String::new())
Expand All @@ -481,8 +480,7 @@ impl AsyncTextIterator {
let mut buf = buffer.lock().await;
if !buf.is_empty() {
let result: Vec<u8> = std::mem::take(&mut *buf);
let enc = Encoding::for_label(encoding.as_bytes()).unwrap_or(UTF_8);
let (text, _, _) = enc.decode(&result);
let (text, _, _) = encoding.decode(&result);
Ok(text.to_string())
} else {
Err(PyErr::new::<pyo3::exceptions::PyStopAsyncIteration, _>(
Expand All @@ -496,8 +494,7 @@ impl AsyncTextIterator {
let mut buf = buffer.lock().await;
if !buf.is_empty() {
let result: Vec<u8> = std::mem::take(&mut *buf);
let enc = Encoding::for_label(encoding.as_bytes()).unwrap_or(UTF_8);
let (text, _, _) = enc.decode(&result);
let (text, _, _) = encoding.decode(&result);
Ok(text.to_string())
} else {
Err(PyErr::new::<pyo3::exceptions::PyStopAsyncIteration, _>(
Expand All @@ -516,15 +513,15 @@ impl AsyncTextIterator {
#[pyclass]
pub struct AsyncLinesIterator {
resp: Arc<TMutex<Option<::primp::Response>>>,
buffer: Arc<TMutex<String>>,
buffer: Arc<TMutex<Vec<u8>>>,
done: Arc<TMutex<bool>>,
}

impl AsyncLinesIterator {
fn new(resp: Arc<TMutex<Option<::primp::Response>>>) -> Self {
AsyncLinesIterator {
resp,
buffer: Arc::new(TMutex::new(String::new())),
buffer: Arc::new(TMutex::new(Vec::with_capacity(8192))),
done: Arc::new(TMutex::new(false)),
}
}
Expand All @@ -547,10 +544,11 @@ impl AsyncLinesIterator {
loop {
{
let mut buf = buffer.lock().await;
if let Some(newline_pos) = buf.find('\n') {
let line: String = buf.drain(..=newline_pos).collect();
if let Some(newline_pos) = buf.iter().position(|&b| b == b'\n') {
let line_bytes: Vec<u8> = buf.drain(..=newline_pos).collect();
let line = String::from_utf8_lossy(&line_bytes);
let line = line.trim_end_matches('\r').trim_end_matches('\n');
return Ok::<String, PyErr>(line.to_string());
return Ok::<String, PyErr>(line.to_owned());
}
}

Expand All @@ -559,8 +557,9 @@ impl AsyncLinesIterator {
if is_done {
let mut buf = buffer.lock().await;
if !buf.is_empty() {
let line = std::mem::take(&mut *buf);
return Ok(line);
let remaining = std::mem::take(&mut *buf);
let line = String::from_utf8_lossy(&remaining);
return Ok::<String, PyErr>(line.into_owned());
}
return Err(PyErr::new::<pyo3::exceptions::PyStopAsyncIteration, _>(
"Stream exhausted",
Expand All @@ -573,8 +572,7 @@ impl AsyncLinesIterator {
Some(r) => match r.chunk().await {
Ok(Some(data)) => {
let mut buf = buffer.lock().await;
let text = String::from_utf8_lossy(&data);
buf.push_str(&text);
buf.extend_from_slice(&data);
}
Ok(None) => {
let mut is_done = done.lock().await;
Expand Down
28 changes: 14 additions & 14 deletions crates/primp-python/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -369,6 +369,12 @@ impl Client {
// Clone the inner client to avoid holding the RwLock across await points
let client_clone = client.read().unwrap_or_else(|e| e.into_inner()).clone();

// Restore redirect policy immediately after cloning if it was changed
if follow_redirects.is_some() {
let mut client_guard = client.write().unwrap_or_else(|e| e.into_inner());
client_guard.set_redirect_policy(::primp::redirect::Policy::limited(20));
}

let future = async move {
// Create request builder using the cloned client
let mut request_builder = client_clone.request(method, &resolved_url);
Expand Down Expand Up @@ -453,12 +459,6 @@ impl Client {
let response: Result<(PrimpResponse, String, u16), PrimpErrorEnum> =
py.detach(move || runtime.block_on(future));

// Restore redirect policy if it was changed
if follow_redirects.is_some() {
let mut client_guard = client.write().unwrap_or_else(|e| e.into_inner());
client_guard.set_redirect_policy(::primp::redirect::Policy::limited(20));
}

let result = response?;
let resp = result.0;
let url = result.1;
Expand Down Expand Up @@ -829,7 +829,7 @@ fn get(
None,
None,
None,
headers.clone(),
headers,
None,
None,
None,
Expand Down Expand Up @@ -918,7 +918,7 @@ fn head(
None,
None,
None,
headers.clone(),
headers,
None,
None,
None,
Expand Down Expand Up @@ -1007,7 +1007,7 @@ fn options(
None,
None,
None,
headers.clone(),
headers,
None,
None,
None,
Expand Down Expand Up @@ -1096,7 +1096,7 @@ fn delete(
None,
None,
None,
headers.clone(),
headers,
None,
None,
None,
Expand Down Expand Up @@ -1185,7 +1185,7 @@ fn post(
None,
None,
None,
headers.clone(),
headers,
None,
None,
None,
Expand Down Expand Up @@ -1274,7 +1274,7 @@ fn put(
None,
None,
None,
headers.clone(),
headers,
None,
None,
None,
Expand Down Expand Up @@ -1363,7 +1363,7 @@ fn patch(
None,
None,
None,
headers.clone(),
headers,
None,
None,
None,
Expand Down Expand Up @@ -1454,7 +1454,7 @@ fn request(
None,
None,
None,
headers.clone(),
headers,
None,
None,
None,
Expand Down
38 changes: 18 additions & 20 deletions crates/primp-python/src/response.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,7 @@ impl BytesIterator {
BytesIterator {
resp,
chunk_size,
buffer: Vec::new(),
buffer: Vec::with_capacity(chunk_size * 2),
}
}
}
Expand Down Expand Up @@ -293,7 +293,7 @@ impl BytesIterator {
#[pyclass]
pub struct TextIterator {
resp: Arc<TMutex<Option<::primp::Response>>>,
encoding: String,
encoding: &'static Encoding,
chunk_size: usize,
buffer: Vec<u8>,
}
Expand All @@ -304,11 +304,12 @@ impl TextIterator {
encoding: String,
chunk_size: usize,
) -> Self {
let encoding = Encoding::for_label(encoding.as_bytes()).unwrap_or(UTF_8);
TextIterator {
resp,
encoding,
chunk_size,
buffer: Vec::new(),
buffer: Vec::with_capacity(chunk_size * 2),
}
}
}
Expand All @@ -322,8 +323,7 @@ impl TextIterator {
fn __next__<'rs>(&mut self, py: Python<'rs>) -> PyResult<Option<Bound<'rs, PyString>>> {
if self.buffer.len() >= self.chunk_size {
let chunk: Vec<u8> = self.buffer.drain(..self.chunk_size).collect();
let encoding = Encoding::for_label(self.encoding.as_bytes()).unwrap_or(UTF_8);
let (text, _, _) = encoding.decode(&chunk);
let (text, _, _) = self.encoding.decode(&chunk);
return Ok(Some(text.into_pyobject_or_pyerr(py)?));
}

Expand All @@ -348,13 +348,11 @@ impl TextIterator {
self.buffer.extend_from_slice(&data);
if self.buffer.len() >= self.chunk_size {
let result: Vec<u8> = self.buffer.drain(..self.chunk_size).collect();
let encoding = Encoding::for_label(self.encoding.as_bytes()).unwrap_or(UTF_8);
let (text, _, _) = encoding.decode(&result);
let (text, _, _) = self.encoding.decode(&result);
Ok(Some(text.into_pyobject_or_pyerr(py)?))
} else if !self.buffer.is_empty() {
let result: Vec<u8> = std::mem::take(&mut self.buffer);
let encoding = Encoding::for_label(self.encoding.as_bytes()).unwrap_or(UTF_8);
let (text, _, _) = encoding.decode(&result);
let (text, _, _) = self.encoding.decode(&result);
Ok(Some(text.into_pyobject_or_pyerr(py)?))
} else {
Ok(None)
Expand All @@ -363,8 +361,7 @@ impl TextIterator {
None => {
if !self.buffer.is_empty() {
let result: Vec<u8> = std::mem::take(&mut self.buffer);
let encoding = Encoding::for_label(self.encoding.as_bytes()).unwrap_or(UTF_8);
let (text, _, _) = encoding.decode(&result);
let (text, _, _) = self.encoding.decode(&result);
Ok(Some(text.into_pyobject_or_pyerr(py)?))
} else {
Err(pyo3::exceptions::PyStopIteration::new_err(
Expand All @@ -380,15 +377,15 @@ impl TextIterator {
#[pyclass]
pub struct LinesIterator {
resp: Arc<TMutex<Option<::primp::Response>>>,
buffer: String,
buffer: Vec<u8>,
done: bool,
}

impl LinesIterator {
fn new(resp: Arc<TMutex<Option<::primp::Response>>>) -> Self {
LinesIterator {
resp,
buffer: String::new(),
buffer: Vec::with_capacity(8192),
done: false,
}
}
Expand All @@ -402,16 +399,18 @@ impl LinesIterator {

fn __next__<'rs>(&mut self, py: Python<'rs>) -> PyResult<Option<Bound<'rs, PyString>>> {
loop {
if let Some(newline_pos) = self.buffer.find('\n') {
let line: String = self.buffer.drain(..=newline_pos).collect();
if let Some(newline_pos) = self.buffer.iter().position(|&b| b == b'\n') {
let line_bytes: Vec<u8> = self.buffer.drain(..=newline_pos).collect();
let line = String::from_utf8_lossy(&line_bytes);
let line = line.trim_end_matches('\r').trim_end_matches('\n');
return Ok(Some(line.to_string().into_pyobject_or_pyerr(py)?));
return Ok(Some(line.to_owned().into_pyobject_or_pyerr(py)?));
}

if self.done {
if !self.buffer.is_empty() {
let line = std::mem::take(&mut self.buffer);
return Ok(Some(line.into_pyobject_or_pyerr(py)?));
let remaining = std::mem::take(&mut self.buffer);
let line = String::from_utf8_lossy(&remaining);
return Ok(Some(line.into_owned().into_pyobject_or_pyerr(py)?));
}
return Err(pyo3::exceptions::PyStopIteration::new_err(
"Stream exhausted",
Expand All @@ -436,8 +435,7 @@ impl LinesIterator {

match chunk {
Some(data) => {
let text = String::from_utf8_lossy(&data);
self.buffer.push_str(&text);
self.buffer.extend_from_slice(&data);
}
None => {
self.done = true;
Expand Down
Loading
Loading