diff --git a/crates/primp-python/src/async/client.rs b/crates/primp-python/src/async/client.rs index 998c965..30e7287 100644 --- a/crates/primp-python/src/async/client.rs +++ b/crates/primp-python/src/async/client.rs @@ -246,6 +246,12 @@ impl AsyncClient { client_guard.clone() }; + // Restore redirect policy immediately after cloning if it was changed + if follow_redirects.is_some() { + let mut client_guard = self.client.write().unwrap_or_else(|e| e.into_inner()); + client_guard.set_redirect_policy(::primp::redirect::Policy::limited(20)); + } + let future = async move { // Create request builder let mut request_builder = client.request(method, &resolved_url); @@ -324,12 +330,6 @@ impl AsyncClient { Ok::<(PrimpResponse, String, u16), PrimpErrorEnum>((resp, url, status_code)) }; - // Restore redirect policy if it was changed - if follow_redirects.is_some() { - let mut client_guard = self.client.write().unwrap_or_else(|e| e.into_inner()); - client_guard.set_redirect_policy(::primp::redirect::Policy::limited(20)); - } - // Convert Rust future to Python awaitable if stream { let py_future = async move { diff --git a/crates/primp-python/src/async/response.rs b/crates/primp-python/src/async/response.rs index 203759a..1e78d4e 100644 --- a/crates/primp-python/src/async/response.rs +++ b/crates/primp-python/src/async/response.rs @@ -332,7 +332,7 @@ impl AsyncBytesIterator { AsyncBytesIterator { resp, chunk_size, - buffer: Arc::new(TMutex::new(Vec::new())), + buffer: Arc::new(TMutex::new(Vec::with_capacity(chunk_size * 2))), } } } @@ -412,7 +412,7 @@ impl AsyncBytesIterator { #[pyclass] pub struct AsyncTextIterator { resp: Arc>>, - encoding: String, + encoding: &'static encoding_rs::Encoding, chunk_size: usize, buffer: Arc>>, } @@ -423,11 +423,13 @@ impl AsyncTextIterator { encoding: String, chunk_size: usize, ) -> Self { + let encoding = + encoding_rs::Encoding::for_label(encoding.as_bytes()).unwrap_or(encoding_rs::UTF_8); AsyncTextIterator { resp, encoding, chunk_size, - buffer: Arc::new(TMutex::new(Vec::new())), + buffer: Arc::new(TMutex::new(Vec::with_capacity(chunk_size * 2))), } } } @@ -442,7 +444,7 @@ impl AsyncTextIterator { use pyo3_async_runtimes::tokio::future_into_py; let resp = Arc::clone(&self.resp); - let encoding = self.encoding.clone(); + let encoding = self.encoding; let chunk_size = self.chunk_size; let buffer = Arc::clone(&self.buffer); @@ -451,8 +453,7 @@ impl AsyncTextIterator { let mut buf = buffer.lock().await; if buf.len() >= chunk_size { let chunk: Vec = buf.drain(..chunk_size).collect(); - let enc = Encoding::for_label(encoding.as_bytes()).unwrap_or(UTF_8); - let (text, _, _) = enc.decode(&chunk); + let (text, _, _) = encoding.decode(&chunk); return Ok::(text.to_string()); } } @@ -465,13 +466,11 @@ impl AsyncTextIterator { buf.extend_from_slice(&data); if buf.len() >= chunk_size { let result: Vec = buf.drain(..chunk_size).collect(); - let enc = Encoding::for_label(encoding.as_bytes()).unwrap_or(UTF_8); - let (text, _, _) = enc.decode(&result); + let (text, _, _) = encoding.decode(&result); Ok(text.to_string()) } else if !buf.is_empty() { let result: Vec = std::mem::take(&mut *buf); - let enc = Encoding::for_label(encoding.as_bytes()).unwrap_or(UTF_8); - let (text, _, _) = enc.decode(&result); + let (text, _, _) = encoding.decode(&result); Ok(text.to_string()) } else { Ok(String::new()) @@ -481,8 +480,7 @@ impl AsyncTextIterator { let mut buf = buffer.lock().await; if !buf.is_empty() { let result: Vec = std::mem::take(&mut *buf); - let enc = Encoding::for_label(encoding.as_bytes()).unwrap_or(UTF_8); - let (text, _, _) = enc.decode(&result); + let (text, _, _) = encoding.decode(&result); Ok(text.to_string()) } else { Err(PyErr::new::( @@ -496,8 +494,7 @@ impl AsyncTextIterator { let mut buf = buffer.lock().await; if !buf.is_empty() { let result: Vec = std::mem::take(&mut *buf); - let enc = Encoding::for_label(encoding.as_bytes()).unwrap_or(UTF_8); - let (text, _, _) = enc.decode(&result); + let (text, _, _) = encoding.decode(&result); Ok(text.to_string()) } else { Err(PyErr::new::( @@ -516,7 +513,7 @@ impl AsyncTextIterator { #[pyclass] pub struct AsyncLinesIterator { resp: Arc>>, - buffer: Arc>, + buffer: Arc>>, done: Arc>, } @@ -524,7 +521,7 @@ impl AsyncLinesIterator { fn new(resp: Arc>>) -> Self { AsyncLinesIterator { resp, - buffer: Arc::new(TMutex::new(String::new())), + buffer: Arc::new(TMutex::new(Vec::with_capacity(8192))), done: Arc::new(TMutex::new(false)), } } @@ -547,10 +544,11 @@ impl AsyncLinesIterator { loop { { let mut buf = buffer.lock().await; - if let Some(newline_pos) = buf.find('\n') { - let line: String = buf.drain(..=newline_pos).collect(); + if let Some(newline_pos) = buf.iter().position(|&b| b == b'\n') { + let line_bytes: Vec = buf.drain(..=newline_pos).collect(); + let line = String::from_utf8_lossy(&line_bytes); let line = line.trim_end_matches('\r').trim_end_matches('\n'); - return Ok::(line.to_string()); + return Ok::(line.to_owned()); } } @@ -559,8 +557,9 @@ impl AsyncLinesIterator { if is_done { let mut buf = buffer.lock().await; if !buf.is_empty() { - let line = std::mem::take(&mut *buf); - return Ok(line); + let remaining = std::mem::take(&mut *buf); + let line = String::from_utf8_lossy(&remaining); + return Ok::(line.into_owned()); } return Err(PyErr::new::( "Stream exhausted", @@ -573,8 +572,7 @@ impl AsyncLinesIterator { Some(r) => match r.chunk().await { Ok(Some(data)) => { let mut buf = buffer.lock().await; - let text = String::from_utf8_lossy(&data); - buf.push_str(&text); + buf.extend_from_slice(&data); } Ok(None) => { let mut is_done = done.lock().await; diff --git a/crates/primp-python/src/lib.rs b/crates/primp-python/src/lib.rs index c5a10fa..04c812c 100644 --- a/crates/primp-python/src/lib.rs +++ b/crates/primp-python/src/lib.rs @@ -369,6 +369,12 @@ impl Client { // Clone the inner client to avoid holding the RwLock across await points let client_clone = client.read().unwrap_or_else(|e| e.into_inner()).clone(); + // Restore redirect policy immediately after cloning if it was changed + if follow_redirects.is_some() { + let mut client_guard = client.write().unwrap_or_else(|e| e.into_inner()); + client_guard.set_redirect_policy(::primp::redirect::Policy::limited(20)); + } + let future = async move { // Create request builder using the cloned client let mut request_builder = client_clone.request(method, &resolved_url); @@ -453,12 +459,6 @@ impl Client { let response: Result<(PrimpResponse, String, u16), PrimpErrorEnum> = py.detach(move || runtime.block_on(future)); - // Restore redirect policy if it was changed - if follow_redirects.is_some() { - let mut client_guard = client.write().unwrap_or_else(|e| e.into_inner()); - client_guard.set_redirect_policy(::primp::redirect::Policy::limited(20)); - } - let result = response?; let resp = result.0; let url = result.1; @@ -829,7 +829,7 @@ fn get( None, None, None, - headers.clone(), + headers, None, None, None, @@ -918,7 +918,7 @@ fn head( None, None, None, - headers.clone(), + headers, None, None, None, @@ -1007,7 +1007,7 @@ fn options( None, None, None, - headers.clone(), + headers, None, None, None, @@ -1096,7 +1096,7 @@ fn delete( None, None, None, - headers.clone(), + headers, None, None, None, @@ -1185,7 +1185,7 @@ fn post( None, None, None, - headers.clone(), + headers, None, None, None, @@ -1274,7 +1274,7 @@ fn put( None, None, None, - headers.clone(), + headers, None, None, None, @@ -1363,7 +1363,7 @@ fn patch( None, None, None, - headers.clone(), + headers, None, None, None, @@ -1454,7 +1454,7 @@ fn request( None, None, None, - headers.clone(), + headers, None, None, None, diff --git a/crates/primp-python/src/response.rs b/crates/primp-python/src/response.rs index e641a09..717b98e 100644 --- a/crates/primp-python/src/response.rs +++ b/crates/primp-python/src/response.rs @@ -229,7 +229,7 @@ impl BytesIterator { BytesIterator { resp, chunk_size, - buffer: Vec::new(), + buffer: Vec::with_capacity(chunk_size * 2), } } } @@ -293,7 +293,7 @@ impl BytesIterator { #[pyclass] pub struct TextIterator { resp: Arc>>, - encoding: String, + encoding: &'static Encoding, chunk_size: usize, buffer: Vec, } @@ -304,11 +304,12 @@ impl TextIterator { encoding: String, chunk_size: usize, ) -> Self { + let encoding = Encoding::for_label(encoding.as_bytes()).unwrap_or(UTF_8); TextIterator { resp, encoding, chunk_size, - buffer: Vec::new(), + buffer: Vec::with_capacity(chunk_size * 2), } } } @@ -322,8 +323,7 @@ impl TextIterator { fn __next__<'rs>(&mut self, py: Python<'rs>) -> PyResult>> { if self.buffer.len() >= self.chunk_size { let chunk: Vec = self.buffer.drain(..self.chunk_size).collect(); - let encoding = Encoding::for_label(self.encoding.as_bytes()).unwrap_or(UTF_8); - let (text, _, _) = encoding.decode(&chunk); + let (text, _, _) = self.encoding.decode(&chunk); return Ok(Some(text.into_pyobject_or_pyerr(py)?)); } @@ -348,13 +348,11 @@ impl TextIterator { self.buffer.extend_from_slice(&data); if self.buffer.len() >= self.chunk_size { let result: Vec = self.buffer.drain(..self.chunk_size).collect(); - let encoding = Encoding::for_label(self.encoding.as_bytes()).unwrap_or(UTF_8); - let (text, _, _) = encoding.decode(&result); + let (text, _, _) = self.encoding.decode(&result); Ok(Some(text.into_pyobject_or_pyerr(py)?)) } else if !self.buffer.is_empty() { let result: Vec = std::mem::take(&mut self.buffer); - let encoding = Encoding::for_label(self.encoding.as_bytes()).unwrap_or(UTF_8); - let (text, _, _) = encoding.decode(&result); + let (text, _, _) = self.encoding.decode(&result); Ok(Some(text.into_pyobject_or_pyerr(py)?)) } else { Ok(None) @@ -363,8 +361,7 @@ impl TextIterator { None => { if !self.buffer.is_empty() { let result: Vec = std::mem::take(&mut self.buffer); - let encoding = Encoding::for_label(self.encoding.as_bytes()).unwrap_or(UTF_8); - let (text, _, _) = encoding.decode(&result); + let (text, _, _) = self.encoding.decode(&result); Ok(Some(text.into_pyobject_or_pyerr(py)?)) } else { Err(pyo3::exceptions::PyStopIteration::new_err( @@ -380,7 +377,7 @@ impl TextIterator { #[pyclass] pub struct LinesIterator { resp: Arc>>, - buffer: String, + buffer: Vec, done: bool, } @@ -388,7 +385,7 @@ impl LinesIterator { fn new(resp: Arc>>) -> Self { LinesIterator { resp, - buffer: String::new(), + buffer: Vec::with_capacity(8192), done: false, } } @@ -402,16 +399,18 @@ impl LinesIterator { fn __next__<'rs>(&mut self, py: Python<'rs>) -> PyResult>> { loop { - if let Some(newline_pos) = self.buffer.find('\n') { - let line: String = self.buffer.drain(..=newline_pos).collect(); + if let Some(newline_pos) = self.buffer.iter().position(|&b| b == b'\n') { + let line_bytes: Vec = self.buffer.drain(..=newline_pos).collect(); + let line = String::from_utf8_lossy(&line_bytes); let line = line.trim_end_matches('\r').trim_end_matches('\n'); - return Ok(Some(line.to_string().into_pyobject_or_pyerr(py)?)); + return Ok(Some(line.to_owned().into_pyobject_or_pyerr(py)?)); } if self.done { if !self.buffer.is_empty() { - let line = std::mem::take(&mut self.buffer); - return Ok(Some(line.into_pyobject_or_pyerr(py)?)); + let remaining = std::mem::take(&mut self.buffer); + let line = String::from_utf8_lossy(&remaining); + return Ok(Some(line.into_owned().into_pyobject_or_pyerr(py)?)); } return Err(pyo3::exceptions::PyStopIteration::new_err( "Stream exhausted", @@ -436,8 +435,7 @@ impl LinesIterator { match chunk { Some(data) => { - let text = String::from_utf8_lossy(&data); - self.buffer.push_str(&text); + self.buffer.extend_from_slice(&data); } None => { self.done = true; diff --git a/crates/primp/src/imp/chrome/mod.rs b/crates/primp/src/imp/chrome/mod.rs index 0901ccc..affd4dd 100644 --- a/crates/primp/src/imp/chrome/mod.rs +++ b/crates/primp/src/imp/chrome/mod.rs @@ -33,6 +33,11 @@ pub(crate) fn build_chrome_settings( chrome: Impersonate, os: crate::imp::ImpersonateOS, ) -> crate::imp::BrowserSettings { + let os = if matches!(os, crate::imp::ImpersonateOS::Random) { + crate::imp::random_impersonate_os() + } else { + os + }; let user_agent = build_user_agent(chrome, os); let sec_ch_ua = build_sec_ch_ua(chrome, os); @@ -82,7 +87,7 @@ fn build_user_agent(chrome: Impersonate, os: crate::imp::ImpersonateOS) -> &'sta crate::imp::ImpersonateOS::Linux => "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36", crate::imp::ImpersonateOS::Android => "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Mobile Safari/537.36", crate::imp::ImpersonateOS::IOS => "Mozilla/5.0 (iPhone; CPU iPhone OS 18_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/144.0.0.0 Mobile/15E148 Safari/604.1", - _ => build_user_agent(chrome, crate::imp::random_impersonate_os()), + _ => unreachable!(), }, Impersonate::ChromeV145 => match os { crate::imp::ImpersonateOS::Windows => "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36", @@ -90,7 +95,7 @@ fn build_user_agent(chrome: Impersonate, os: crate::imp::ImpersonateOS) -> &'sta crate::imp::ImpersonateOS::Linux => "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36", crate::imp::ImpersonateOS::Android => "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Mobile Safari/537.36", crate::imp::ImpersonateOS::IOS => "Mozilla/5.0 (iPhone; CPU iPhone OS 18_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/145.0.0.0 Mobile/15E148 Safari/604.1", - _ => build_user_agent(chrome, crate::imp::random_impersonate_os()), + _ => unreachable!(), }, Impersonate::ChromeV146 => match os { crate::imp::ImpersonateOS::Windows => "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36", @@ -98,7 +103,7 @@ fn build_user_agent(chrome: Impersonate, os: crate::imp::ImpersonateOS) -> &'sta crate::imp::ImpersonateOS::Linux => "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36", crate::imp::ImpersonateOS::Android => "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Mobile Safari/537.36", crate::imp::ImpersonateOS::IOS => "Mozilla/5.0 (iPhone; CPU iPhone OS 18_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/146.0.0.0 Mobile/15E148 Safari/604.1", - _ => build_user_agent(chrome, crate::imp::random_impersonate_os()), + _ => unreachable!(), }, Impersonate::ChromeV147 => match os { crate::imp::ImpersonateOS::Windows => "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36", @@ -106,7 +111,7 @@ fn build_user_agent(chrome: Impersonate, os: crate::imp::ImpersonateOS) -> &'sta crate::imp::ImpersonateOS::Linux => "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36", crate::imp::ImpersonateOS::Android => "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Mobile Safari/537.36", crate::imp::ImpersonateOS::IOS => "Mozilla/5.0 (iPhone; CPU iPhone OS 18_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/147.0.0.0 Mobile/15E148 Safari/604.1", - _ => build_user_agent(chrome, crate::imp::random_impersonate_os()), + _ => unreachable!(), }, Impersonate::ChromeV148 => match os { crate::imp::ImpersonateOS::Windows => "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36", @@ -114,7 +119,7 @@ fn build_user_agent(chrome: Impersonate, os: crate::imp::ImpersonateOS) -> &'sta crate::imp::ImpersonateOS::Linux => "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36", crate::imp::ImpersonateOS::Android => "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Mobile Safari/537.36", crate::imp::ImpersonateOS::IOS => "Mozilla/5.0 (iPhone; CPU iPhone OS 18_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/148.0.0.0 Mobile/15E148 Safari/604.1", - _ => build_user_agent(chrome, crate::imp::random_impersonate_os()), + _ => unreachable!(), }, _ => unreachable!(), } diff --git a/crates/primp/src/imp/edge/mod.rs b/crates/primp/src/imp/edge/mod.rs index 23e7c57..053a82e 100644 --- a/crates/primp/src/imp/edge/mod.rs +++ b/crates/primp/src/imp/edge/mod.rs @@ -34,6 +34,11 @@ pub(crate) fn build_edge_settings( edge: Impersonate, os: crate::imp::ImpersonateOS, ) -> crate::imp::BrowserSettings { + let os = if matches!(os, crate::imp::ImpersonateOS::Random) { + crate::imp::random_impersonate_os() + } else { + os + }; let user_agent = build_user_agent(edge, os); let sec_ch_ua = build_sec_ch_ua(edge, os); @@ -83,7 +88,7 @@ fn build_user_agent(edge: Impersonate, os: crate::imp::ImpersonateOS) -> &'stati crate::imp::ImpersonateOS::Linux => "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36 Edg/144.0.0.0", crate::imp::ImpersonateOS::Android => "Mozilla/5.0 (Linux; Android 10; Pixel 3 XL) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Mobile Safari/537.36 EdgA/144.0.0.0", crate::imp::ImpersonateOS::IOS => "Mozilla/5.0 (iPhone; CPU iPhone OS 18_7_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 EdgiOS/144.0.0.0 Mobile/15E148 Safari/605.1.15", - _ => build_user_agent(edge, crate::imp::random_impersonate_os()), + _ => unreachable!(), }, Impersonate::EdgeV145 => match os { crate::imp::ImpersonateOS::Windows => "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36 Edg/145.0.0.0", @@ -91,7 +96,7 @@ fn build_user_agent(edge: Impersonate, os: crate::imp::ImpersonateOS) -> &'stati crate::imp::ImpersonateOS::Linux => "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36 Edg/145.0.0.0", crate::imp::ImpersonateOS::Android => "Mozilla/5.0 (Linux; Android 10; Pixel 3 XL) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Mobile Safari/537.36 EdgA/145.0.0.0", crate::imp::ImpersonateOS::IOS => "Mozilla/5.0 (iPhone; CPU iPhone OS 18_7_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 EdgiOS/145.0.0.0 Mobile/15E148 Safari/605.1.15", - _ => build_user_agent(edge, crate::imp::random_impersonate_os()), + _ => unreachable!(), }, Impersonate::EdgeV146 => match os { crate::imp::ImpersonateOS::Windows => "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36 Edg/146.0.0.0", @@ -99,7 +104,7 @@ fn build_user_agent(edge: Impersonate, os: crate::imp::ImpersonateOS) -> &'stati crate::imp::ImpersonateOS::Linux => "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36 Edg/146.0.0.0", crate::imp::ImpersonateOS::Android => "Mozilla/5.0 (Linux; Android 10; Pixel 3 XL) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Mobile Safari/537.36 EdgA/146.0.0.0", crate::imp::ImpersonateOS::IOS => "Mozilla/5.0 (iPhone; CPU iPhone OS 18_7_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 EdgiOS/146.0.0.0 Mobile/15E148 Safari/605.1.15", - _ => build_user_agent(edge, crate::imp::random_impersonate_os()), + _ => unreachable!(), }, Impersonate::EdgeV147 => match os { crate::imp::ImpersonateOS::Windows => "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36 Edg/147.0.3912.51", @@ -107,7 +112,7 @@ fn build_user_agent(edge: Impersonate, os: crate::imp::ImpersonateOS) -> &'stati crate::imp::ImpersonateOS::Linux => "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36 Edg/147.0.3912.51", crate::imp::ImpersonateOS::Android => "Mozilla/5.0 (Linux; Android 10; SM-G973F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Mobile Safari/537.36 EdgA/147.0.3912.51", crate::imp::ImpersonateOS::IOS => "Mozilla/5.0 (iPhone; CPU iPhone OS 18_7_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 EdgiOS/147.0.3912.51 Mobile/15E148 Safari/605.1.15", - _ => build_user_agent(edge, crate::imp::random_impersonate_os()), + _ => unreachable!(), }, Impersonate::EdgeV148 => match os { crate::imp::ImpersonateOS::Windows => "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36 Edg/148.0.0.0", @@ -115,7 +120,7 @@ fn build_user_agent(edge: Impersonate, os: crate::imp::ImpersonateOS) -> &'stati crate::imp::ImpersonateOS::Linux => "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36 Edg/148.0.0.0", crate::imp::ImpersonateOS::Android => "Mozilla/5.0 (Linux; Android 10; Pixel 3 XL) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Mobile Safari/537.36 EdgA/148.0.0.0", crate::imp::ImpersonateOS::IOS => "Mozilla/5.0 (iPhone; CPU iPhone OS 18_7_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 EdgiOS/148.0.0.0 Mobile/15E148 Safari/605.1.15", - _ => build_user_agent(edge, crate::imp::random_impersonate_os()), + _ => unreachable!(), }, _ => unreachable!(), } diff --git a/crates/primp/src/imp/firefox/mod.rs b/crates/primp/src/imp/firefox/mod.rs index 7557a9c..33fc006 100644 --- a/crates/primp/src/imp/firefox/mod.rs +++ b/crates/primp/src/imp/firefox/mod.rs @@ -10,6 +10,11 @@ pub(crate) fn build_firefox_settings( firefox: Impersonate, os: crate::imp::ImpersonateOS, ) -> crate::imp::BrowserSettings { + let os = if matches!(os, crate::imp::ImpersonateOS::Random) { + crate::imp::random_impersonate_os() + } else { + os + }; let user_agent = build_user_agent(firefox, os); let headers = build_headers(user_agent); @@ -66,7 +71,7 @@ fn build_user_agent(firefox: Impersonate, os: crate::imp::ImpersonateOS) -> &'st crate::imp::ImpersonateOS::Linux => "Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0", crate::imp::ImpersonateOS::Android => "Mozilla/5.0 (Android 14; Mobile; rv:140.0) Gecko/140.0 Firefox/140.0", crate::imp::ImpersonateOS::IOS => "Mozilla/5.0 (iPhone; CPU iPhone OS 17_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/140.0 Mobile/15E148 Safari/605.1", - _ => build_user_agent(firefox, crate::imp::random_impersonate_os()), + _ => unreachable!(), }, Impersonate::FirefoxV146 => match os { crate::imp::ImpersonateOS::Windows => "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:146.0) Gecko/20100101 Firefox/146.0", @@ -74,7 +79,7 @@ fn build_user_agent(firefox: Impersonate, os: crate::imp::ImpersonateOS) -> &'st crate::imp::ImpersonateOS::Linux => "Mozilla/5.0 (X11; Linux x86_64; rv:146.0) Gecko/20100101 Firefox/146.0", crate::imp::ImpersonateOS::Android => "Mozilla/5.0 (Android 14; Mobile; rv:146.0) Gecko/146.0 Firefox/146.0", crate::imp::ImpersonateOS::IOS => "Mozilla/5.0 (iPhone; CPU iPhone OS 17_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/146.0 Mobile/15E148 Safari/605.1", - _ => build_user_agent(firefox, crate::imp::random_impersonate_os()), + _ => unreachable!(), }, Impersonate::FirefoxV147 => match os { crate::imp::ImpersonateOS::Windows => "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:147.0) Gecko/20100101 Firefox/147.0", @@ -82,7 +87,7 @@ fn build_user_agent(firefox: Impersonate, os: crate::imp::ImpersonateOS) -> &'st crate::imp::ImpersonateOS::Linux => "Mozilla/5.0 (X11; Linux x86_64; rv:147.0) Gecko/20100101 Firefox/147.0", crate::imp::ImpersonateOS::Android => "Mozilla/5.0 (Android 14; Mobile; rv:147.0) Gecko/147.0 Firefox/147.0", crate::imp::ImpersonateOS::IOS => "Mozilla/5.0 (iPhone; CPU iPhone OS 17_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/147.0 Mobile/15E148 Safari/605.1", - _ => build_user_agent(firefox, crate::imp::random_impersonate_os()), + _ => unreachable!(), }, Impersonate::FirefoxV148 => match os { crate::imp::ImpersonateOS::Windows => "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:148.0) Gecko/20100101 Firefox/148.0", @@ -90,7 +95,7 @@ fn build_user_agent(firefox: Impersonate, os: crate::imp::ImpersonateOS) -> &'st crate::imp::ImpersonateOS::Linux => "Mozilla/5.0 (X11; Linux x86_64; rv:148.0) Gecko/20100101 Firefox/148.0", crate::imp::ImpersonateOS::Android => "Mozilla/5.0 (Android 14; Mobile; rv:148.0) Gecko/148.0 Firefox/148.0", crate::imp::ImpersonateOS::IOS => "Mozilla/5.0 (iPhone; CPU iPhone OS 17_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/148.0 Mobile/15E148 Safari/605.1", - _ => build_user_agent(firefox, crate::imp::random_impersonate_os()), + _ => unreachable!(), }, _ => unreachable!(), } diff --git a/crates/primp/src/imp/mod.rs b/crates/primp/src/imp/mod.rs index 6772cb0..f266b6a 100644 --- a/crates/primp/src/imp/mod.rs +++ b/crates/primp/src/imp/mod.rs @@ -383,13 +383,18 @@ pub fn random_impersonate_os() -> ImpersonateOS { /// Returns the OS-specific sec-ch-ua-platform header value. pub(crate) fn os_platform(os: ImpersonateOS) -> &'static str { + let os = if matches!(os, ImpersonateOS::Random) { + random_impersonate_os() + } else { + os + }; match os { ImpersonateOS::Windows => r#""Windows""#, ImpersonateOS::MacOS => r#""macOS""#, ImpersonateOS::Linux => r#""Linux""#, ImpersonateOS::Android => r#""Android""#, ImpersonateOS::IOS => r#""iOS""#, - ImpersonateOS::Random => os_platform(random_impersonate_os()), + ImpersonateOS::Random => unreachable!(), } } diff --git a/crates/primp/src/imp/opera/mod.rs b/crates/primp/src/imp/opera/mod.rs index 833da1e..fb81b2a 100644 --- a/crates/primp/src/imp/opera/mod.rs +++ b/crates/primp/src/imp/opera/mod.rs @@ -34,6 +34,11 @@ pub(crate) fn build_opera_settings( opera: Impersonate, os: crate::imp::ImpersonateOS, ) -> crate::imp::BrowserSettings { + let os = if matches!(os, crate::imp::ImpersonateOS::Random) { + crate::imp::random_impersonate_os() + } else { + os + }; let user_agent = build_user_agent(opera, os); let sec_ch_ua = build_sec_ch_ua(opera, os); @@ -89,7 +94,7 @@ fn build_user_agent(opera: Impersonate, os: crate::imp::ImpersonateOS) -> &'stat crate::imp::ImpersonateOS::Linux => "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36 OPR/126.0.0.0", crate::imp::ImpersonateOS::Android => "Mozilla/5.0 (Linux; Android 10; SM-G970F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Mobile Safari/537.36 OPR/126.0.0.0", crate::imp::ImpersonateOS::IOS => "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) OPiOS/126.0.0.0 Mobile/15E148 Safari/605.1.15", - _ => build_user_agent(opera, crate::imp::random_impersonate_os()), + _ => unreachable!(), }, // Opera 127 is based on Chrome 143 Impersonate::OperaV127 => match os { @@ -98,7 +103,7 @@ fn build_user_agent(opera: Impersonate, os: crate::imp::ImpersonateOS) -> &'stat crate::imp::ImpersonateOS::Linux => "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36 OPR/127.0.0.0", crate::imp::ImpersonateOS::Android => "Mozilla/5.0 (Linux; Android 10; SM-G970F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Mobile Safari/537.36 OPR/127.0.0.0", crate::imp::ImpersonateOS::IOS => "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) OPiOS/127.0.0.0 Mobile/15E148 Safari/605.1.15", - _ => build_user_agent(opera, crate::imp::random_impersonate_os()), + _ => unreachable!(), }, // Opera 128 is based on Chrome 144 Impersonate::OperaV128 => match os { @@ -107,7 +112,7 @@ fn build_user_agent(opera: Impersonate, os: crate::imp::ImpersonateOS) -> &'stat crate::imp::ImpersonateOS::Linux => "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36 OPR/128.0.0.0", crate::imp::ImpersonateOS::Android => "Mozilla/5.0 (Linux; Android 10; SM-G970F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Mobile Safari/537.36 OPR/128.0.0.0", crate::imp::ImpersonateOS::IOS => "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) OPiOS/128.0.0.0 Mobile/15E148 Safari/605.1.15", - _ => build_user_agent(opera, crate::imp::random_impersonate_os()), + _ => unreachable!(), }, // Opera 129 is based on Chrome 145 Impersonate::OperaV129 => match os { @@ -116,7 +121,7 @@ fn build_user_agent(opera: Impersonate, os: crate::imp::ImpersonateOS) -> &'stat crate::imp::ImpersonateOS::Linux => "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36 OPR/129.0.0.0", crate::imp::ImpersonateOS::Android => "Mozilla/5.0 (Linux; Android 10; SM-G970F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Mobile Safari/537.36 OPR/129.0.0.0", crate::imp::ImpersonateOS::IOS => "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) OPiOS/129.0.0.0 Mobile/15E148 Safari/605.1.15", - _ => build_user_agent(opera, crate::imp::random_impersonate_os()), + _ => unreachable!(), }, // Opera 130 is based on Chrome 146 Impersonate::OperaV130 => match os { @@ -125,7 +130,7 @@ fn build_user_agent(opera: Impersonate, os: crate::imp::ImpersonateOS) -> &'stat crate::imp::ImpersonateOS::Linux => "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36 OPR/130.0.0.0", crate::imp::ImpersonateOS::Android => "Mozilla/5.0 (Linux; Android 10; SM-G970F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Mobile Safari/537.36 OPR/130.0.0.0", crate::imp::ImpersonateOS::IOS => "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) OPiOS/130.0.0.0 Mobile/15E148 Safari/605.1.15", - _ => build_user_agent(opera, crate::imp::random_impersonate_os()), + _ => unreachable!(), }, // Opera 131 is based on Chrome 147 Impersonate::OperaV131 => match os { @@ -134,7 +139,7 @@ fn build_user_agent(opera: Impersonate, os: crate::imp::ImpersonateOS) -> &'stat crate::imp::ImpersonateOS::Linux => "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36 OPR/131.0.0.0", crate::imp::ImpersonateOS::Android => "Mozilla/5.0 (Linux; Android 10; SM-G970F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Mobile Safari/537.36 OPR/131.0.0.0", crate::imp::ImpersonateOS::IOS => "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) OPiOS/131.0.0.0 Mobile/15E148 Safari/605.1.15", - _ => build_user_agent(opera, crate::imp::random_impersonate_os()), + _ => unreachable!(), }, _ => unreachable!(), } diff --git a/crates/primp/src/imp/safari/mod.rs b/crates/primp/src/imp/safari/mod.rs index ac72c77..3254d31 100644 --- a/crates/primp/src/imp/safari/mod.rs +++ b/crates/primp/src/imp/safari/mod.rs @@ -32,6 +32,11 @@ pub(crate) fn build_safari_settings( safari: Impersonate, os: crate::imp::ImpersonateOS, ) -> crate::imp::BrowserSettings { + let os = if matches!(os, crate::imp::ImpersonateOS::Random) { + crate::imp::random_impersonate_os() + } else { + os + }; let user_agent = build_user_agent(safari, os); let headers = build_safari_base_headers(user_agent); @@ -42,16 +47,15 @@ pub(crate) fn build_safari_settings( _ => BrowserEmulatorOS::MacOS, }; - // Get cached browser emulator for Safari (avoids Vec allocations on each call) + // Get cached browser emulator for Safari (Arc clone = cheap refcount increment) let mut browser_emulator = safari_emulator(safari, browser_os); - - // Set OS type on the cloned emulator - browser_emulator.os_type = Some(browser_os); + // Set OS type on our unique clone (cached emulator retains None) + Arc::make_mut(&mut browser_emulator).os_type = Some(browser_os); let http2 = build_http2_settings(); crate::imp::BrowserSettings { - browser_emulator: Arc::new(browser_emulator), + browser_emulator, http2, headers, gzip: true, @@ -62,23 +66,26 @@ pub(crate) fn build_safari_settings( } /// Builds a User-Agent string for a Safari version and OS. +/// Safari only supports MacOS and iOS; other OSes default to MacOS. fn build_user_agent(safari: Impersonate, os: crate::imp::ImpersonateOS) -> &'static str { + // Random is resolved before this is called; only MacOS and IOS reach here + let os = match os { + crate::imp::ImpersonateOS::IOS => crate::imp::ImpersonateOS::IOS, + _ => crate::imp::ImpersonateOS::MacOS, + }; match safari { Impersonate::SafariV18_5 => match os { - crate::imp::ImpersonateOS::MacOS => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.5 Safari/605.1.15", crate::imp::ImpersonateOS::IOS => "Mozilla/5.0 (iPhone; CPU iPhone OS 18_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.5 Mobile/15E148 Safari/604.1", - _ => build_user_agent(safari, crate::imp::random_impersonate_os()), + _ => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.5 Safari/605.1.15", }, Impersonate::SafariV26 => match os { - crate::imp::ImpersonateOS::MacOS => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.0 Safari/605.1.15", crate::imp::ImpersonateOS::IOS => "Mozilla/5.0 (iPhone; CPU iPhone OS 26_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.0 Mobile/15E148 Safari/604.1", - _ => build_user_agent(safari, crate::imp::random_impersonate_os()), + _ => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.0 Safari/605.1.15", }, // Safari 26.3: macOS uses Safari 26 UA, iOS uses Safari 18.5 UA Impersonate::SafariV26_3 => match os { - crate::imp::ImpersonateOS::MacOS => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.3 Safari/605.1.15", crate::imp::ImpersonateOS::IOS => "Mozilla/5.0 (iPhone; CPU iPhone OS 18_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.5 Mobile/15E148 Safari/604.1", - _ => build_user_agent(safari, crate::imp::random_impersonate_os()), + _ => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.3 Safari/605.1.15", }, _ => unreachable!(), } @@ -109,24 +116,28 @@ fn build_http2_settings() -> crate::imp::Http2Data { } } -fn safari_emulator(safari: Impersonate, browser_os: BrowserEmulatorOS) -> BrowserEmulator { +fn safari_emulator(safari: Impersonate, browser_os: BrowserEmulatorOS) -> Arc { match safari { Impersonate::SafariV18_5 => { - static EMU: OnceLock = OnceLock::new(); - EMU.get_or_init(new_safari_18_5_emulator).clone() + static EMU: OnceLock> = OnceLock::new(); + EMU.get_or_init(|| Arc::new(new_safari_18_5_emulator())) + .clone() } Impersonate::SafariV26 => { - static EMU: OnceLock = OnceLock::new(); - EMU.get_or_init(new_safari_26_emulator).clone() + static EMU: OnceLock> = OnceLock::new(); + EMU.get_or_init(|| Arc::new(new_safari_26_emulator())) + .clone() } Impersonate::SafariV26_3 => { if browser_os == BrowserEmulatorOS::IOS { - static EMU_IOS: OnceLock = OnceLock::new(); - EMU_IOS.get_or_init(new_safari_26_3_ios_emulator).clone() + static EMU_IOS: OnceLock> = OnceLock::new(); + EMU_IOS + .get_or_init(|| Arc::new(new_safari_26_3_ios_emulator())) + .clone() } else { - static EMU_MACOS: OnceLock = OnceLock::new(); + static EMU_MACOS: OnceLock> = OnceLock::new(); EMU_MACOS - .get_or_init(new_safari_26_3_macos_emulator) + .get_or_init(|| Arc::new(new_safari_26_3_macos_emulator())) .clone() } } @@ -159,6 +170,7 @@ fn new_safari_26_3_ios_emulator() -> BrowserEmulator { emulator.named_groups = Some(emulation::named_groups::SAFARI.to_vec()); emulator.signature_algorithms = Some(emulation::signature_algorithms::SAFARI.to_vec()); emulator.extension_order_seed = Some(emulation::extension_order::SAFARI_26); + emulator.os_type = Some(BrowserEmulatorOS::IOS); emulator } @@ -169,6 +181,7 @@ fn new_safari_26_3_macos_emulator() -> BrowserEmulator { emulator.signature_algorithms = Some(emulation::signature_algorithms::SAFARI.to_vec()); emulator.extension_order_seed = Some(emulation::extension_order::SAFARI_18_5); emulator.include_status_request_v2 = true; + emulator.os_type = Some(BrowserEmulatorOS::MacOS); emulator } diff --git a/crates/primp/src/lib.rs b/crates/primp/src/lib.rs index 902213c..3049d83 100644 --- a/crates/primp/src/lib.rs +++ b/crates/primp/src/lib.rs @@ -598,18 +598,26 @@ fn build_impersonate_tls_config( custom_certs: &[reqwest::Certificate], ) -> crate::Result { use rustls::client::EchMode; - use std::sync::Arc; + use std::sync::{Arc, OnceLock}; - let provider = rustls::crypto::CryptoProvider::get_default() - .cloned() - .unwrap_or_else(|| Arc::new(rustls::crypto::aws_lc_rs::default_provider())); + // Cache the CryptoProvider to avoid repeated lookups and allocations + static CRYPTO_PROVIDER: OnceLock> = OnceLock::new(); + let provider = CRYPTO_PROVIDER + .get_or_init(|| { + rustls::crypto::CryptoProvider::get_default() + .cloned() + .unwrap_or_else(|| Arc::new(rustls::crypto::aws_lc_rs::default_provider())) + }) + .clone(); - // Use cached root store with both webpki roots and native OS root CAs. - // If custom certs are provided, merge them in. + // Cache the default root store to avoid cloning hundreds of certs on every build + static DEFAULT_ROOT_STORE: OnceLock> = OnceLock::new(); let root_store = if custom_certs.is_empty() { - reqwest::tls::default_root_store().clone() + DEFAULT_ROOT_STORE + .get_or_init(|| Arc::new(reqwest::tls::default_root_store().clone())) + .clone() } else { - reqwest::tls::merged_root_store(custom_certs.to_vec())? + Arc::new(reqwest::tls::merged_root_store(custom_certs.to_vec())?) }; // ECH GREASE is only used for Chrome-based browsers and Firefox to match JA4 fingerprint