From 49f883dabcb3c15bfdbd35299db62064a9813400 Mon Sep 17 00:00:00 2001 From: Giuseppe Fabiano Date: Mon, 21 Jul 2025 22:39:15 +0200 Subject: [PATCH 1/2] cli: added support to sparse images flashing Signed-off-by: Giuseppe Fabiano --- Cargo.lock | 70 ++++++++++++++++++++++++++++++++++++++-- cli/Cargo.toml | 1 + cli/src/programfile.rs | 72 +++++++++++++++++++++++++++++++++++++++++- 3 files changed, 140 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 76e08e5..3bba7ba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,18 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "android-sparse-image" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7bd95f94be90346dd3df1342039d4b8201f86237af02a1bc8aa28b778bf156f" +dependencies = [ + "bytes", + "log", + "strum", + "thiserror 2.0.12", +] + [[package]] name = "anstream" version = "0.6.18" @@ -79,6 +91,12 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" +[[package]] +name = "bytes" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" + [[package]] name = "cc" version = "1.2.17" @@ -201,7 +219,7 @@ dependencies = [ "crc", "nix", "serde", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -259,6 +277,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "log" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" + [[package]] name = "nix" version = "0.27.1" @@ -338,6 +362,7 @@ dependencies = [ name = "qdl-rs" version = "0.1.0" dependencies = [ + "android-sparse-image", "anyhow", "clap", "clap-num", @@ -432,6 +457,27 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "syn" version = "2.0.100" @@ -449,7 +495,16 @@ version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +dependencies = [ + "thiserror-impl 2.0.12", ] [[package]] @@ -463,6 +518,17 @@ dependencies = [ "syn", ] +[[package]] +name = "thiserror-impl" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "unicode-ident" version = "1.0.18" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index a408511..abfee6b 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -14,6 +14,7 @@ publish = false # TODO maintenance = { status = "actively-developed" } [dependencies] +android-sparse-image = "0.1.3" anyhow = "1.0.89" clap = { version = "4.5.18", features = ["derive"] } clap-num = "1.1.1" diff --git a/cli/src/programfile.rs b/cli/src/programfile.rs index 6aee111..dce2b13 100644 --- a/cli/src/programfile.rs +++ b/cli/src/programfile.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: BSD-3-Clause // Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. -use anyhow::bail; +use anyhow::{Ok, bail}; use indexmap::IndexMap; use std::{ fs, @@ -14,6 +14,10 @@ use qdl::{ types::QdlChan, }; +use android_sparse_image::{ + ChunkHeader, ChunkHeaderBytes, ChunkType, FILE_HEADER_BYTES_LEN, FileHeader, FileHeaderBytes, +}; + fn parse_read_cmd( channel: &mut T, out_dir: &Path, @@ -141,7 +145,73 @@ fn parse_program_cmd( } } + let sparse = attrs + .get("sparse") + .unwrap_or(&"false".to_owned()) + .parse::() + .unwrap_or(false); + let mut buf = fs::File::open(file_path)?; + + if sparse { + let mut header_bytes: FileHeaderBytes = [0; FILE_HEADER_BYTES_LEN]; + buf.read_exact(&mut header_bytes)?; + let header = FileHeader::from_bytes(&header_bytes)?; + + let mut offset: usize = 0; + let start_sector = start_sector.parse::()?; + for index in 0..header.chunks { + let label_sparse = format!("{label}_{index}"); + let mut chunk_bytes = ChunkHeaderBytes::default(); + buf.read_exact(&mut chunk_bytes)?; + let chunk = ChunkHeader::from_bytes(&chunk_bytes)?; + + let out_size = chunk.out_size(&header); + let num_sectors = out_size / sector_size; + let start_offset = start_sector + offset; + match chunk.chunk_type { + ChunkType::Raw => { + firehose_program_storage( + channel, + &mut buf, + &label_sparse, + num_sectors, + phys_part_idx, + start_offset.to_string().as_str(), + )?; + } + ChunkType::Fill => { + let mut fill_value = [0u8; 4]; + buf.read_exact(&mut fill_value)?; + + let mut fill_vec = Vec::::with_capacity(out_size); + for _ in 0..out_size / 4 { + fill_vec.extend_from_slice(&fill_value[..]); + } + + firehose_program_storage( + channel, + &mut &fill_vec[..], + &label_sparse, + num_sectors, + phys_part_idx, + start_offset.to_string().as_str(), + )?; + } + ChunkType::DontCare => { + // Don't Care, skip + } + ChunkType::Crc32 => { + // Not supported, on qcom tools is ignored, seek if present + buf.seek_relative(4)?; + } + } + + offset += out_size; + } + return Ok(()); + } + buf.seek(SeekFrom::Current( sector_size as i64 * file_sector_offset as i64, ))?; From 42cd9cb8b6c8125ca4327f21f3eceed7ba353038 Mon Sep 17 00:00:00 2001 From: Giuseppe Fabiano Date: Fri, 20 Feb 2026 14:28:01 +0100 Subject: [PATCH 2/2] cli: changed the logic to speed up transfer and not hog speed The previous logics tried to flash each useful chunk one by one in 4KB chunks but this hogged the speed due to firehose transfers that took ~20-50ms for each XML command ACK cycle. Inspired by official qcom tool I merged different type of sparsed image chunks filling also the gaps of don't care with zeroes and try to keep large contiguous transfers. This allowed the flash to speedup from minutes to some seconds on sparse images like persist. Signed-off-by: Giuseppe Fabiano --- cli/src/programfile.rs | 136 ++++++++++++++++++++++++++++++++--------- 1 file changed, 106 insertions(+), 30 deletions(-) diff --git a/cli/src/programfile.rs b/cli/src/programfile.rs index dce2b13..9523e73 100644 --- a/cli/src/programfile.rs +++ b/cli/src/programfile.rs @@ -159,56 +159,132 @@ fn parse_program_cmd( let header = FileHeader::from_bytes(&header_bytes)?; let mut offset: usize = 0; - let start_sector = start_sector.parse::()?; + let start_sector_base = start_sector.parse::()?; + + let mut agg_data = Vec::new(); + let mut agg_start_sector = 0; + let mut agg_num_sectors = 0; + for index in 0..header.chunks { - let label_sparse = format!("{label}_{index}"); let mut chunk_bytes = ChunkHeaderBytes::default(); buf.read_exact(&mut chunk_bytes)?; let chunk = ChunkHeader::from_bytes(&chunk_bytes)?; let out_size = chunk.out_size(&header); let num_sectors = out_size / sector_size; - let start_offset = start_sector + offset; + let current_start_sector = start_sector_base + offset; + match chunk.chunk_type { - ChunkType::Raw => { - firehose_program_storage( - channel, - &mut buf, - &label_sparse, - num_sectors, - phys_part_idx, - start_offset.to_string().as_str(), - )?; - } - ChunkType::Fill => { - let mut fill_value = [0u8; 4]; - buf.read_exact(&mut fill_value)?; + ChunkType::Raw | ChunkType::Fill => { + let is_large = out_size > channel.fh_config().send_buffer_size; + let is_contiguous = agg_num_sectors > 0 + && (agg_start_sector + agg_num_sectors == current_start_sector); + let would_overflow = + agg_data.len() + out_size > channel.fh_config().send_buffer_size; - let mut fill_vec = Vec::::with_capacity(out_size); - for _ in 0..out_size / 4 { - fill_vec.extend_from_slice(&fill_value[..]); + if !is_contiguous || would_overflow || is_large { + if agg_num_sectors > 0 { + firehose_program_storage( + channel, + &mut &agg_data[..], + &format!("{label}_merged"), + agg_num_sectors, + phys_part_idx, + agg_start_sector.to_string().as_str(), + )?; + agg_data.clear(); + agg_num_sectors = 0; + } } - firehose_program_storage( - channel, - &mut &fill_vec[..], - &label_sparse, - num_sectors, - phys_part_idx, - start_offset.to_string().as_str(), - )?; + if is_large { + if chunk.chunk_type == ChunkType::Raw { + firehose_program_storage( + channel, + &mut buf, + &format!("{label}_{index}"), + num_sectors, + phys_part_idx, + current_start_sector.to_string().as_str(), + )?; + } else { + let mut fill_value = [0u8; 4]; + buf.read_exact(&mut fill_value)?; + + let mut fill_vec = Vec::::with_capacity(out_size); + for _ in 0..out_size / 4 { + fill_vec.extend_from_slice(&fill_value[..]); + } + + firehose_program_storage( + channel, + &mut &fill_vec[..], + &format!("{label}_{index}"), + num_sectors, + phys_part_idx, + current_start_sector.to_string().as_str(), + )?; + } + } else { + if agg_num_sectors == 0 { + agg_start_sector = current_start_sector; + } + if chunk.chunk_type == ChunkType::Raw { + let mut tmp = vec![0u8; out_size]; + buf.read_exact(&mut tmp)?; + agg_data.extend(tmp); + } else { + let mut fill_value = [0u8; 4]; + buf.read_exact(&mut fill_value)?; + for _ in 0..out_size / 4 { + agg_data.extend_from_slice(&fill_value[..]); + } + } + agg_num_sectors += num_sectors; + } } ChunkType::DontCare => { - // Don't Care, skip + // Fill gaps up to 256KB + let is_small_gap = out_size <= 256 * 1024; + let would_overflow = + agg_data.len() + out_size > channel.fh_config().send_buffer_size; + + if agg_num_sectors > 0 && is_small_gap && !would_overflow { + // Fill gap with zeros to keep aggregation going + agg_data.resize(agg_data.len() + out_size, 0); + agg_num_sectors += num_sectors; + } else if agg_num_sectors > 0 { + firehose_program_storage( + channel, + &mut &agg_data[..], + &format!("{label}_merged"), + agg_num_sectors, + phys_part_idx, + agg_start_sector.to_string().as_str(), + )?; + agg_data.clear(); + agg_num_sectors = 0; + } } ChunkType::Crc32 => { - // Not supported, on qcom tools is ignored, seek if present buf.seek_relative(4)?; } } - offset += out_size; + offset += num_sectors; } + + if agg_num_sectors > 0 { + firehose_program_storage( + channel, + &mut &agg_data[..], + &format!("{label}_merged"), + agg_num_sectors, + phys_part_idx, + agg_start_sector.to_string().as_str(), + )?; + } + return Ok(()); }