Skip to content

Support configurable sector size for raw/WIC images #156

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
vdskvortsov wants to merge 1 commit into
base: trixie
Choose a base branch
from
+165 −41
Open

Support configurable sector size for raw/WIC images #156

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 9 additions & 4 deletions tcbuilder/backend/combine.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
from tcbuilder.backend.common import \
(set_output_ownership, check_licence_acceptance, run_with_loading_animation, open_disk_image,
get_tar_compress_program_options, DOCKER_BUNDLE_TARNAME, TAR_EXT_TO_PROGRAM,
OSTREE_SOTA_DIR_PATH)
OSTREE_SOTA_DIR_PATH, DEFAULT_RAW_SECTOR_SIZE)
from tcbuilder.errors import InvalidStateError, InvalidDataError, TorizonCoreBuilderError

log = logging.getLogger("torizon." + __name__)
Expand Down Expand Up @@ -290,7 +290,9 @@ def combine_tezi_image(image_dir, bundle_dir, output_directory, tezi_props, forc
combine_single_tezi_image(**combine_params)


def combine_raw_image(image_path, bundle_dir, output_path, rootfs_label, force):
# pylint: disable-next=too-many-positional-arguments,too-many-locals
def combine_raw_image(image_path, bundle_dir, output_path, rootfs_label, force,
sector_size=DEFAULT_RAW_SECTOR_SIZE):
"""
Combine a container bundle to a raw disk image by copying its contents to the image root
filesystem. If the size of the container bundle exceeds the available space in root,
Expand All @@ -301,6 +303,7 @@ def combine_raw_image(image_path, bundle_dir, output_path, rootfs_label, force):
:param output_path: Path where the resulting raw disk image will be created.
:param rootfs_label: Filesystem label of the root partition.
:param force: Overwrite output file if it already exists.
:param sector_size: Logical sector size of the image, in bytes.
"""

delete_on_error = True
Expand Down Expand Up @@ -335,7 +338,8 @@ def combine_raw_image(image_path, bundle_dir, output_path, rootfs_label, force):

req_space_kb = get_bundle_size_kb(files_to_add, bundle_dir)

with open_disk_image(output_path, delete_on_error=delete_on_error) as gfs:
with open_disk_image(output_path, delete_on_error=delete_on_error,
sector_size=sector_size) as gfs:
root_partition = gfs.findfs_label(rootfs_label)
gfs.mount(root_partition, "/")
statvfs_dict = gfs.statvfs("/")
Expand Down Expand Up @@ -374,7 +378,8 @@ def combine_raw_image(image_path, bundle_dir, output_path, rootfs_label, force):
log.debug("Deleting '%s'", tmp_image)
os.remove(tmp_image)

with open_disk_image(output_path, delete_on_error=delete_on_error) as gfs:
with open_disk_image(output_path, delete_on_error=delete_on_error,
sector_size=sector_size) as gfs:
gfs.mount(root_partition, "/")
copy_to_mounted_root(gfs, files_to_add, bundle_dir)

Expand Down
28 changes: 23 additions & 5 deletions tcbuilder/backend/common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,9 @@

DEFAULT_RAW_ROOTFS_LABEL = "otaroot"

DEFAULT_RAW_SECTOR_SIZE = 512
SUPPORTED_RAW_SECTOR_SIZES = (512, 4096)

TEZI_PROP_TO_ARGNAME = {
"name": "--image-name",
"description": "--image-description",
Expand All @@ -63,11 +66,13 @@
}

RAW_PROP_TO_ARGNAME = {
"raw_rootfs_label": "--raw-rootfs-label"
"raw_rootfs_label": "--raw-rootfs-label",
"raw_sector_size": "--raw-sector-size"
}

RAW_PROP_DEFAULTS = {
"raw_rootfs_label" : DEFAULT_RAW_ROOTFS_LABEL
"raw_rootfs_label" : DEFAULT_RAW_ROOTFS_LABEL,
"raw_sector_size" : DEFAULT_RAW_SECTOR_SIZE
}

TAR_EXT_TO_PROGRAM = {
Expand Down Expand Up @@ -305,6 +310,12 @@ def add_common_raw_image_arguments(subparser):
default=None) # Default is in RAW_PROP_DEFAULTS. Default should only be
# set if arg is used in a raw image.
# Arg value should remain None if a tezi image is used.
subparser.add_argument(RAW_PROP_TO_ARGNAME["raw_sector_size"], dest="raw_sector_size",
metavar="BYTES", type=int, choices=SUPPORTED_RAW_SECTOR_SIZES,
help="(raw images only) logical sector size of the source WIC/raw "
"image, in bytes. Use 4096 for 4Kn media (e.g. some UFS "
f"targets). (default: {DEFAULT_RAW_SECTOR_SIZE})",
default=None) # Default is in RAW_PROP_DEFAULTS; see raw_rootfs_label.
Comment thread
jsrc27 marked this conversation as resolved.


def add_ssh_arguments(subparser):
Expand Down Expand Up @@ -1058,12 +1069,19 @@ def is_file_type_fit(file_path):


@contextmanager
def open_disk_image(image_path, *, delete_on_error=False, readonly=False, loading_msg=None):
"""Create a context manager to open and manipulate raw disk images."""
def open_disk_image(image_path, *, delete_on_error=False, readonly=False, loading_msg=None,
sector_size=DEFAULT_RAW_SECTOR_SIZE):
"""Create a context manager to open and manipulate raw disk images.

:param sector_size: Logical sector size of the image, in bytes. Must be
passed for 4Kn media so libguestfs can read the GPT.
"""

try:
gfs = guestfs.GuestFS(python_return_dict=True)
gfs.add_drive_opts(image_path, format="raw", readonly=readonly)
# blocksize optarg needs libguestfs >= 1.44; omit it on the default path.

@lucas-akira lucas-akira Jul 17, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

From what I found the blocksize parameter was added to add_drive_opts in libguestfs 1.42, not 1.44:

https://libguestfs.org/guestfs-release-notes-1.42.1.html

extra = {"blocksize": sector_size} if sector_size != DEFAULT_RAW_SECTOR_SIZE else {}
gfs.add_drive_opts(image_path, format="raw", readonly=readonly, **extra)
run_with_loading_animation(
func=gfs.launch,
loading_msg=loading_msg or "Initializing image...")
Expand Down
75 changes: 67 additions & 8 deletions tcbuilder/backend/deploy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@
from tcbuilder.backend import ostree
from tcbuilder.backend.common import (get_rootfs_tarball, resolve_remote_host,
run_with_loading_animation, open_disk_image,
REMOTE_CMD_TIMEOUT, SECBOOT_ARTIFACTS_DIR)
REMOTE_CMD_TIMEOUT, SECBOOT_ARTIFACTS_DIR,
DEFAULT_RAW_SECTOR_SIZE)
from tcbuilder.backend.rforward import reverse_forward_tunnel, request_port_forward
from tcbuilder.backend.secboot import FUSE_CMD_TXT_NAME
from tcbuilder.errors import TorizonCoreBuilderError, InvalidDataError
Expand Down Expand Up @@ -324,8 +325,48 @@ def deploy_tezi_image(tezi_dir, src_sysroot_dir, src_ostree_archive_dir,
copy_signed_artifacts(commit_dir, output_dir)


def grow_last_partition(raw_img, added_size_kb, sector_size, rootfs_partition):
"""Enlarge a raw image and extend its last partition to fill the new space.

For 4Kn images, where virt-resize cannot operate. Preserves the partition's
GPT identity (name, type, GUID, attributes) so it still boots; the rootfs
must be the last partition, which is checked.
"""
# Round up to a whole sector; a non-sector-multiple size can be rejected at 4Kn.
new_size = os.path.getsize(raw_img) + int(added_size_kb) * 1024
new_size = (new_size + sector_size - 1) // sector_size * sector_size
subprocess.check_output(["truncate", "-s", str(new_size), raw_img])

with open_disk_image(raw_img, sector_size=sector_size) as gfs:
dev = "/dev/sda"
gfs.part_expand_gpt(dev) # relocate the GPT backup header to the new end
partnum = len(gfs.list_partitions())
if gfs.part_to_partnum(rootfs_partition) != partnum:
raise TorizonCoreBuilderError(
"the rootfs must be the last partition to grow a 4Kn raw image.")
start_sector = gfs.part_list(dev)[-1]["part_start"] // sector_size

name = gfs.part_get_name(dev, partnum)
gpt_type = gfs.part_get_gpt_type(dev, partnum)
gpt_guid = gfs.part_get_gpt_guid(dev, partnum)
gpt_attributes = gfs.part_get_gpt_attributes(dev, partnum)

# Stop short of the disk end to clear the GPT backup (33 LBAs of 512 B).
gpt_tail = (33 * 512 + sector_size - 1) // sector_size
end_sector = os.path.getsize(raw_img) // sector_size - 1 - gpt_tail
gfs.part_del(dev, partnum)
gfs.part_add(dev, "primary", start_sector, end_sector)

gfs.part_set_name(dev, partnum, name)
gfs.part_set_gpt_type(dev, partnum, gpt_type)
gfs.part_set_gpt_guid(dev, partnum, gpt_guid)
gfs.part_set_gpt_attributes(dev, partnum, gpt_attributes)


# pylint: disable-next=too-many-positional-arguments
def create_output_raw_image(base_raw_img, output_raw_img, base_rootfs_partition,
other_partitions_size_kb, rootfs_size_kb):
other_partitions_size_kb, rootfs_size_kb,
sector_size=DEFAULT_RAW_SECTOR_SIZE):
"""Create a new raw disk image based on an existing one.

The result is a copy of the base disk image with all partitions except
Expand All @@ -337,6 +378,21 @@ def create_output_raw_image(base_raw_img, output_raw_img, base_rootfs_partition,
IMAGE_OVERHEAD_FACTOR*(rootfs_size_kb + other_partitions_size_kb))
out_size_kb += EXTRA_ROOTFS_SIZE_KB

if sector_size != DEFAULT_RAW_SECTOR_SIZE:
# virt-resize drives libguestfs at the default 512-byte sector size and
# cannot open a 4Kn disk, so grow the image with libguestfs directly and
# let write_rootfs_to_raw_image() reformat the enlarged last partition.
shutil.copy2(base_raw_img, output_raw_img)
if out_size_kb > base_img_size_kb:
added_size_kb = out_size_kb - base_img_size_kb
log.info(f"Adding {added_size_kb/1024:.2f} MiB to output image.")
log.info(f"Size of output image will be: {out_size_kb/1024/1024:.2f} GiB")
grow_last_partition(output_raw_img, added_size_kb, sector_size,
base_rootfs_partition)
else:
log.info("Output image will have the same size as the base one.")
return

# Copy of the base disk will be the starting point of the output disk
shutil.copy2(base_raw_img, output_raw_img)
log.debug(f"Created output image: {output_raw_img}")
Expand All @@ -361,15 +417,16 @@ def create_output_raw_image(base_raw_img, output_raw_img, base_rootfs_partition,
print("------------------------------------------------------------")


def write_rootfs_to_raw_image(raw_disk_img, rootfs_label, rootfs_dir):
def write_rootfs_to_raw_image(raw_disk_img, rootfs_label, rootfs_dir,
sector_size=DEFAULT_RAW_SECTOR_SIZE):
"""Writes unpacked rootfs contents to a raw disk image

Creates a new rootfs ext4 partition with a given label on the last partition
of a raw disk and writes the contents of the provided rootfs directory.
"""

loading_msg="Initializing output image..."
with open_disk_image(raw_disk_img, loading_msg=loading_msg) as gfs:
with open_disk_image(raw_disk_img, loading_msg=loading_msg, sector_size=sector_size) as gfs:
# virt-resize rearranged all existing partitions and generated a new empty partition at the
# end of the disk. We will format it to ext4 and put the unpacked rootfs contents in it.
# Its partition number (/dev/sda1, /dev/sda2, etc.) is equal to the number of partitions
Expand All @@ -395,7 +452,8 @@ def write_rootfs_to_raw_image(raw_disk_img, rootfs_label, rootfs_dir):

# pylint: disable-next=too-many-positional-arguments
def deploy_raw_image(base_raw_img, src_sysroot_dir, src_ostree_archive_dir,
output_raw_img, dst_sysroot_dir, rootfs_label, *, ref=None):
output_raw_img, dst_sysroot_dir, rootfs_label, *, ref=None,
sector_size=DEFAULT_RAW_SECTOR_SIZE):
"""Deploys a WIC image with given OSTree reference

Creates a new WIC image with an OSTree deployment of the
Expand All @@ -404,7 +462,8 @@ def deploy_raw_image(base_raw_img, src_sysroot_dir, src_ostree_archive_dir,
deploy_ostree_local(src_sysroot_dir, src_ostree_archive_dir, dst_sysroot_dir, ref)

loading_msg="Initializing base WIC/raw image..."
with open_disk_image(base_raw_img, readonly=True, loading_msg=loading_msg) as gfs:
with open_disk_image(base_raw_img, readonly=True, loading_msg=loading_msg,
sector_size=sector_size) as gfs:
# Get partition number from ext4 fs called rootfs_label in disk image (.wic/.img)
rootfs_partition = gfs.findfs_label(rootfs_label)
log.info(f" '{rootfs_label}' partition found: {rootfs_partition}")
Expand All @@ -424,9 +483,9 @@ def deploy_raw_image(base_raw_img, src_sysroot_dir, src_ostree_archive_dir,
log.info(f"Unpacked rootfs size: {rootfs_size_kb/1024/1024:.2f} GiB")

create_output_raw_image(base_raw_img, output_raw_img, rootfs_partition,
other_partitions_size_kb, rootfs_size_kb)
other_partitions_size_kb, rootfs_size_kb, sector_size)

write_rootfs_to_raw_image(output_raw_img, rootfs_label, dst_sysroot_dir)
write_rootfs_to_raw_image(output_raw_img, rootfs_label, dst_sysroot_dir, sector_size)

log.info(f"Image {os.path.basename(output_raw_img)} created successfully!")

Expand Down
39 changes: 25 additions & 14 deletions tcbuilder/backend/images.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
set_output_ownership, run_with_loading_animation,
get_tezi_image_version, open_disk_image, make_feed_url,
fetch_remote, RAW_PROP_TO_ARGNAME, DEFAULT_RAW_ROOTFS_LABEL,
DEFAULT_RAW_SECTOR_SIZE,
SECBOOT_ARTIFACTS_DIR, REMOTE_CMD_TIMEOUT, TAR_EXT_TO_PROGRAM,
OSTREE_SOTA_DIR_PATH, LEGACY_VARIANT_PREFIX, VARIANT_PREFIX,
SYNAIMG_MACHINES)
Expand Down Expand Up @@ -312,13 +313,14 @@ def unpack_local_tezi_image(image_dir_or_file, tezi_dir):
raise TorizonCoreBuilderError(f"Image does not exist: {image_dir_or_file}")


def unpack_local_raw_image(image_dir, sysroot_dir, raw_rootfs_label):
def unpack_local_raw_image(image_dir, sysroot_dir, raw_rootfs_label,
sector_size=DEFAULT_RAW_SECTOR_SIZE):
"""Extract the root fs from the image into the sysroot directory"""

if raw_rootfs_label is None:
raw_rootfs_label = DEFAULT_RAW_ROOTFS_LABEL

with open_disk_image(image_dir, readonly=True) as gfs:
with open_disk_image(image_dir, readonly=True, sector_size=sector_size) as gfs:
# Get partition number from ext4 fs called raw_rootfs_label in disk image (.wic/.img)
rootfs_partition = gfs.findfs_label(raw_rootfs_label)
log.info(f"'{raw_rootfs_label}' partition found: {rootfs_partition}")
Expand All @@ -338,8 +340,9 @@ def _make_tezi_extract_dir(tezi_dir):
return extract_dir


# pylint: disable-next=too-many-positional-arguments,too-many-locals
def import_local_image(image_dir_or_file, tezi_dir, src_sysroot_dir, src_ostree_archive_dir,
raw_rootfs_label=None):
raw_rootfs_label=None, raw_sector_size=None):
"""Import local raw/WIC or Toradex Easy Installer image.

Import local raw/WIC or Toradex Easy installer image (archive file or unpacked
Expand All @@ -350,12 +353,14 @@ def import_local_image(image_dir_or_file, tezi_dir, src_sysroot_dir, src_ostree_

if ((image_dir_or_file.lower().endswith(".wic") or
image_dir_or_file.lower().endswith(".img")) and os.path.isfile(image_dir_or_file)):
unpack_local_raw_image(image_dir_or_file, src_sysroot_dir, raw_rootfs_label)
unpack_local_raw_image(image_dir_or_file, src_sysroot_dir, raw_rootfs_label,
raw_sector_size or DEFAULT_RAW_SECTOR_SIZE)
else:
unpack_local_tezi_image(image_dir_or_file, tezi_dir)

common_raw_props_args = {
"raw_rootfs_label": raw_rootfs_label
"raw_rootfs_label": raw_rootfs_label,
"raw_sector_size": raw_sector_size
}
# pylint: disable-next=consider-using-dict-items
for prop in common_raw_props_args:
Expand Down Expand Up @@ -425,7 +430,8 @@ def check_provdata_presence_in_raw_image(gfs):
return True


def _prov_get_image_version_raw_image(disk_img, rootfs_label):
def _prov_get_image_version_raw_image(disk_img, rootfs_label,
sector_size=DEFAULT_RAW_SECTOR_SIZE):
"""Check disk image for any existing provisioning data, also returning the img version.

:param disk_img: path to disk image.
Expand All @@ -437,7 +443,7 @@ def _prov_get_image_version_raw_image(disk_img, rootfs_label):
the tuple entry will be 'None'.
"""

with open_disk_image(disk_img, readonly=True) as gfs:
with open_disk_image(disk_img, readonly=True, sector_size=sector_size) as gfs:
rootfs_partition = gfs.findfs_label(rootfs_label)
log.info(f"'{rootfs_label}' partition found: {rootfs_partition}")
gfs.mount_ro(rootfs_partition, "/")
Expand Down Expand Up @@ -577,7 +583,8 @@ def prov_check_supported_img_features(img_version, online_data, hibernated, flee
log.debug("Found image version: %d.%d", img_major, img_minor)


def prov_check_input(input_path, output_path, rootfs_label):
def prov_check_input(input_path, output_path, rootfs_label,
sector_size=DEFAULT_RAW_SECTOR_SIZE):
"""Check provisioning inputs passed by the user."""

is_tezi_img = False
Expand All @@ -594,7 +601,7 @@ def prov_check_input(input_path, output_path, rootfs_label):
if rootfs_label is None:
rootfs_label = DEFAULT_RAW_ROOTFS_LABEL

img_version = _prov_get_image_version_raw_image(input_path, rootfs_label)
img_version = _prov_get_image_version_raw_image(input_path, rootfs_label, sector_size)

# If TEZI image
else:
Expand All @@ -616,7 +623,8 @@ def prov_check_input(input_path, output_path, rootfs_label):


def provision(input_path, output_path, rootfs_label, prov_data, *,
hibernated=False, fleets=None, force=False):
hibernated=False, fleets=None, force=False,
sector_size=DEFAULT_RAW_SECTOR_SIZE):
"""Generate OS image with added provisioning data

:param input_path: Path containing Easy Installer directory or disk image.
Expand All @@ -633,7 +641,8 @@ def provision(input_path, output_path, rootfs_label, prov_data, *,
"""

# Basic validations:
img_version, is_tezi_img, rootfs_label = prov_check_input(input_path, output_path, rootfs_label)
img_version, is_tezi_img, rootfs_label = prov_check_input(
input_path, output_path, rootfs_label, sector_size)

prov_check_supported_img_features(img_version, prov_data['online'], hibernated, fleets)

Expand Down Expand Up @@ -665,7 +674,7 @@ def provision(input_path, output_path, rootfs_label, prov_data, *,
if is_tezi_img:
prov_tezi_img(output_path, prov_data, hibernated, fleets)
else:
prov_raw_img(rootfs_label, output_path, prov_data, hibernated, fleets)
prov_raw_img(rootfs_label, output_path, prov_data, hibernated, fleets, sector_size)

log.info("Image successfully provisioned.")

Expand All @@ -692,11 +701,13 @@ def prov_tezi_img(output_path, prov_data, hibernated, fleets):
set_output_ownership(output_path)


def prov_raw_img(rootfs_label, output_path, prov_data, hibernated, fleets):
# pylint: disable-next=too-many-positional-arguments
def prov_raw_img(rootfs_label, output_path, prov_data, hibernated, fleets,
sector_size=DEFAULT_RAW_SECTOR_SIZE):
"""Create provisioning data and add it to the root filesystem of the disk image."""

msg = f"Initializing {output_path}..."
with (open_disk_image(output_path, loading_msg=msg) as gfs,
with (open_disk_image(output_path, loading_msg=msg, sector_size=sector_size) as gfs,
tempfile.TemporaryDirectory() as tmpdir):
rootfs_partition = gfs.findfs_label(rootfs_label)
gfs.mount(rootfs_partition, "/")
Expand Down
Loading
Loading