gardenlinux: add PoC for pre-copy auto-converge

docs/live_migration.md

@@ -171,7 +171,13 @@ After a few seconds the VM should be up and you can interact with it.
 Initiate the Migration over TCP:
 
 ```console
-src $ ch-remote --api-socket=/tmp/api send-migration  tcp:{dst}:{port}
+src $ ch-remote --api-socket=/tmp/api send-migration tcp:{dst}:{port}
+```
+
+With migration parameters:
+
+```console
+src $ ch-remote --api-socket=/tmp/api send-migration tcp:{dst}:{port} --migration-timeout 60 --downtime 5000
 ```
 
 > Replace {dst}:{port} with the actual IP address and port of your destination host.
@@ -180,3 +186,24 @@ After completing the above commands, the source VM will be migrated to
 the destination host and continue running there. The source VM instance
 will terminate normally. All ongoing processes and connections within
 the VM should remain intact after the migration.
+
+#### Migration Parameters
+
+Cloud Hypervisor supports additional parameters to control the
+migration process:
+
+- `migration-timeout <seconds>`
+Sets the maximum time (in seconds) allowed for the migration process.
+If the migration takes longer than this timeout, it will be aborted. A
+value of 0 means no timeout limit.
+- `downtime <milliseconds>`
+Sets the maximum acceptable downtime (in milliseconds) during the
+migration. This parameter helps control the trade-off between migration
+time and VM downtime.
+
+> The downtime limit is related to the cost of serialization
+(deserialization) of vCPU and device state. Therefore, the expected
+downtime is always shorter than the actual downtime.
+
+These parameters can be used with the `send-migration` command to
+fine-tune the migration behavior according to your requirements.

src/bin/ch-remote.rs

@@ -478,6 +478,16 @@ fn rest_api_do_command(matches: &ArgMatches, socket: &mut UnixStream) -> ApiResu
                     .subcommand_matches("send-migration")
                     .unwrap()
                     .get_flag("send_migration_local"),
+                *matches
+                    .subcommand_matches("send-migration")
+                    .unwrap()
+                    .get_one::<u64>("downtime")
+                    .unwrap_or(&500),
+                *matches
+                    .subcommand_matches("send-migration")
+                    .unwrap()
+                    .get_one::<u64>("migration_timeout")
+                    .unwrap_or(&3600),
             );
             simple_api_command(socket, "PUT", "send-migration", Some(&send_migration_data))
                 .map_err(Error::HttpApiClient)
@@ -692,6 +702,16 @@ fn dbus_api_do_command(matches: &ArgMatches, proxy: &DBusApi1ProxyBlocking<'_>)
                     .subcommand_matches("send-migration")
                     .unwrap()
                     .get_flag("send_migration_local"),
+                *matches
+                    .subcommand_matches("send-migration")
+                    .unwrap()
+                    .get_one::<u64>("downtime")
+                    .unwrap_or(&500),
+                *matches
+                    .subcommand_matches("send-migration")
+                    .unwrap()
+                    .get_one::<u64>("migration_timeout")
+                    .unwrap_or(&3600),
             );
             proxy.api_vm_send_migration(&send_migration_data)
         }
@@ -885,10 +905,12 @@ fn receive_migration_data(url: &str) -> String {
     serde_json::to_string(&receive_migration_data).unwrap()
 }
 
-fn send_migration_data(url: &str, local: bool) -> String {
+fn send_migration_data(url: &str, local: bool, downtime: u64, migration_timeout: u64) -> String {
     let send_migration_data = vmm::api::VmSendMigrationData {
         destination_url: url.to_owned(),
         local,
+        downtime,
+        migration_timeout,
     };
 
     serde_json::to_string(&send_migration_data).unwrap()
@@ -1050,6 +1072,22 @@ fn get_cli_commands_sorted() -> Box<[Command]> {
         Command::new("resume").about("Resume the VM"),
         Command::new("send-migration")
             .about("Initiate a VM migration")
+            .arg(
+                Arg::new("downtime")
+                    .long("downtime")
+                    .help("Set the expected maximum downtime in milliseconds")
+                    .num_args(1)
+                    .value_parser(clap::value_parser!(u64).range(1..))
+                    .default_value("500"),
+            )
+            .arg(
+                Arg::new("migration_timeout")
+                    .long("migration-timeout")
+                    .help("Set the maximum allowed migration time in seconds")
+                    .num_args(1)
+                    .value_parser(clap::value_parser!(u64))
+                    .default_value("3600"),
+            )
             .arg(
                 Arg::new("send_migration_config")
                     .index(1)
@@ -1060,6 +1098,22 @@ fn get_cli_commands_sorted() -> Box<[Command]> {
                     .long("local")
                     .num_args(0)
                     .action(ArgAction::SetTrue),
+            )
+            .arg(
+                Arg::new("downtime")
+                    .long("downtime")
+                    .help("Set the expected maximum downtime in milliseconds")
+                    .num_args(1)
+                    .value_parser(clap::value_parser!(u64).range(1..))
+                    .default_value("500"),
+            )
+            .arg(
+                Arg::new("migration_timeout")
+                    .long("migration-timeout")
+                    .help("Set the maximum allowed migration time in seconds")
+                    .num_args(1)
+                    .value_parser(clap::value_parser!(u64))
+                    .default_value("3600"),
             ),
         Command::new("shutdown").about("Shutdown the VM"),
         Command::new("shutdown-vmm").about("Shutdown the VMM"),

tests/integration.rs

@@ -520,8 +520,7 @@ fn temp_snapshot_dir_path(tmp_dir: &TempDir) -> String {
 }
 
 fn temp_vmcore_file_path(tmp_dir: &TempDir) -> String {
-    let vmcore_file = String::from(tmp_dir.as_path().join("vmcore").to_str().unwrap());
-    vmcore_file
+    String::from(tmp_dir.as_path().join("vmcore").to_str().unwrap())
 }
 
 // Creates the path for direct kernel boot and return the path.
@@ -9208,6 +9207,8 @@ mod live_migration {
         src_api_socket: &str,
         dest_api_socket: &str,
         local: bool,
+        downtime: u64,
+        timeout: u64,
     ) -> bool {
         // Start to receive migration from the destination VM
         let mut receive_migration = Command::new(clh_command("ch-remote"))
@@ -9228,6 +9229,10 @@ mod live_migration {
             format!("--api-socket={}", &src_api_socket),
             "send-migration".to_string(),
             format! {"unix:{migration_socket}"},
+            "--downtime".to_string(),
+            format!("{downtime}"),
+            "--migration-timeout".to_string(),
+            format!("{timeout}"),
         ]
         .to_vec();
 
@@ -9244,7 +9249,7 @@ mod live_migration {
 
         // The 'send-migration' command should be executed successfully within the given timeout
         let send_success = if let Some(status) = send_migration
-            .wait_timeout(std::time::Duration::from_secs(30))
+            .wait_timeout(std::time::Duration::from_secs(500))
             .unwrap()
         {
             status.success()
@@ -9266,7 +9271,7 @@ mod live_migration {
 
         // The 'receive-migration' command should be executed successfully within the given timeout
         let receive_success = if let Some(status) = receive_migration
-            .wait_timeout(std::time::Duration::from_secs(30))
+            .wait_timeout(std::time::Duration::from_secs(500))
             .unwrap()
         {
             status.success()
@@ -9447,8 +9452,18 @@ mod live_migration {
                     .unwrap(),
             );
 
+            let downtime = 100000;
+            let migration_timeout = 1000;
+
             assert!(
-                start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local),
+                start_live_migration(
+                    &migration_socket,
+                    &src_api_socket,
+                    &dest_api_socket,
+                    local,
+                    downtime,
+                    migration_timeout
+                ),
                 "Unsuccessful command: 'send-migration' or 'receive-migration'."
             );
         });
@@ -9621,8 +9636,18 @@ mod live_migration {
                     .unwrap(),
             );
 
+            let downtime = 100000;
+            let migration_timeout = 1000;
+
             assert!(
-                start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local),
+                start_live_migration(
+                    &migration_socket,
+                    &src_api_socket,
+                    &dest_api_socket,
+                    local,
+                    downtime,
+                    migration_timeout
+                ),
                 "Unsuccessful command: 'send-migration' or 'receive-migration'."
             );
         });
@@ -9839,8 +9864,18 @@ mod live_migration {
                     .unwrap(),
             );
 
+            let downtime = 100000;
+            let migration_timeout = 1000;
+
             assert!(
-                start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local),
+                start_live_migration(
+                    &migration_socket,
+                    &src_api_socket,
+                    &dest_api_socket,
+                    local,
+                    downtime,
+                    migration_timeout
+                ),
                 "Unsuccessful command: 'send-migration' or 'receive-migration'."
             );
         });
@@ -10055,8 +10090,18 @@ mod live_migration {
                     .unwrap(),
             );
 
+            let downtime = 100000;
+            let migration_timeout = 1000;
+
             assert!(
-                start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local),
+                start_live_migration(
+                    &migration_socket,
+                    &src_api_socket,
+                    &dest_api_socket,
+                    local,
+                    downtime,
+                    migration_timeout
+                ),
                 "Unsuccessful command: 'send-migration' or 'receive-migration'."
             );
         });
@@ -10165,8 +10210,18 @@ mod live_migration {
                     .unwrap(),
             );
 
+            let downtime = 100000;
+            let migration_timeout = 1000;
+
             assert!(
-                start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local),
+                start_live_migration(
+                    &migration_socket,
+                    &src_api_socket,
+                    &dest_api_socket,
+                    local,
+                    downtime,
+                    migration_timeout
+                ),
                 "Unsuccessful command: 'send-migration' or 'receive-migration'."
             );
         });
@@ -10312,8 +10367,18 @@ mod live_migration {
                     .unwrap(),
             );
 
+            let downtime = 100000;
+            let migration_timeout = 1000;
+
             assert!(
-                start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, true),
+                start_live_migration(
+                    &migration_socket,
+                    &src_api_socket,
+                    &dest_api_socket,
+                    true,
+                    downtime,
+                    migration_timeout
+                ),
                 "Unsuccessful command: 'send-migration' or 'receive-migration'."
             );
         });
@@ -10368,7 +10433,12 @@ mod live_migration {
             .port()
     }
 
-    fn start_live_migration_tcp(src_api_socket: &str, dest_api_socket: &str) -> bool {
+    fn start_live_migration_tcp(
+        src_api_socket: &str,
+        dest_api_socket: &str,
+        downtime: u64,
+        timeout: u64,
+    ) -> bool {
         // Get an available TCP port
         let migration_port = get_available_port();
         let host_ip = "127.0.0.1";
@@ -10395,6 +10465,10 @@ mod live_migration {
                 &format!("--api-socket={src_api_socket}"),
                 "send-migration",
                 &format!("tcp:{host_ip}:{migration_port}"),
+                "--downtime",
+                &format!("{downtime}"),
+                "--migration-timeout",
+                &format!("{timeout}"),
             ])
             .stdin(Stdio::null())
             .stderr(Stdio::piped())
@@ -10465,6 +10539,8 @@ mod live_migration {
             .output()
             .expect("Expect creating disk image to succeed");
         let pmem_path = String::from("/dev/pmem0");
+        let downtime = 100000;
+        let timeout = 1000;
 
         // Start the source VM
         let src_vm_path = clh_command("cloud-hypervisor");
@@ -10527,7 +10603,7 @@ mod live_migration {
             }
             // Start TCP live migration
             assert!(
-                start_live_migration_tcp(&src_api_socket, &dest_api_socket),
+                start_live_migration_tcp(&src_api_socket, &dest_api_socket, downtime, timeout),
                 "Unsuccessful command: 'send-migration' or 'receive-migration'."
             );
         });

vmm/src/api/mod.rs

@@ -260,6 +260,12 @@ pub struct VmSendMigrationData {
     /// Send memory across socket without copying
     #[serde(default)]
     pub local: bool,
+    /// Downtime
+    #[serde(default)]
+    pub downtime: u64,
+    /// Timeout for migration
+    #[serde(default)]
+    pub migration_timeout: u64,
 }
 
 pub enum ApiResponsePayload {

vmm/src/api/openapi/cloud-hypervisor.yaml

@@ -1272,6 +1272,16 @@ components:
           type: string
         local:
           type: boolean
+        downtime:
+          type: integer
+          format: int64
+          description: Maximum downtime in milliseconds during migration
+          default: 500
+        migration_timeout:
+          type: integer
+          format: int64
+          description: Total timeout for migration in milliseconds (0 = no limit)
+          default: 0
 
     VmAddUserDevice:
       required:

vmm/src/config.rs

@@ -592,7 +592,7 @@
         // ref type in the match.
         // The issue will go away once kvm_hyperv is moved under the features
         // list as it will always be checked for.
-        #[allow(unused_mut)]
+        #[allow(unused_mut, clippy::never_loop)]
         let mut features = CpuFeatures::default();
         for s in features_list.0 {
             match <std::string::String as AsRef<str>>::as_ref(&s) {
@@ -3005,7 +3005,7 @@
                 .as_ref()
                 // SAFETY: FFI call with valid FDs
                 .map(|fds| fds.iter().map(|fd| {
                     let fd_duped = unsafe { libc::dup(*fd) };
                     warn!("Cloning VM config: duping preserved FD {fd} => {fd_duped}");
                     fd_duped
                 }).collect()),