ohcldiag: Allow VTL2 accessing host files

Cargo.lock

@@ -1041,6 +1041,7 @@ dependencies = [
  "fs-err",
  "futures",
  "guid",
+ "host_file_access",
  "inspect",
  "inspect_proto",
  "mesh_rpc",
@@ -2882,6 +2883,18 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "host_file_access"
+version = "0.0.0"
+dependencies = [
+ "bitfield-struct 0.10.1",
+ "futures",
+ "open_enum",
+ "thiserror 2.0.12",
+ "tracing",
+ "zerocopy 0.8.24",
+]
+
 [[package]]
 name = "http"
 version = "1.3.1"
@@ -4757,6 +4770,7 @@ dependencies = [
  "fs-err",
  "futures",
  "futures-concurrency",
+ "host_file_access",
  "inspect",
  "kmsg",
  "mesh",
@@ -4767,6 +4781,7 @@ dependencies = [
  "thiserror 2.0.12",
  "tracing-subscriber",
  "unicycle",
+ "zerocopy 0.8.24",
 ]
 
 [[package]]
@@ -7521,6 +7536,7 @@ dependencies = [
  "guid",
  "hcl",
  "hcl_compat_uefi_nvram_storage",
+ "host_file_access",
  "hvdef",
  "hyperv_ic_guest",
  "hyperv_ic_resources",
@@ -7568,6 +7584,7 @@ dependencies = [
  "serde_helpers",
  "serde_json",
  "serial_16550_resources",
+ "sha2",
  "socket2",
  "sparse_mmap",
  "state_unit",

Cargo.toml

@@ -26,6 +26,7 @@ members = [
   # fuzzing
   "support/inspect/fuzz",
   "support/mesh/mesh_rpc/fuzz",
+  "support/host_file_access",
   "support/sparse_mmap/fuzz",
   "support/ucs2/fuzz",
   "vm/devices/chipset/fuzz",
@@ -140,6 +141,7 @@ safeatomic = { path = "support/safeatomic" }
 serde_helpers = { path = "support/serde_helpers" }
 sev_guest_device = { path = "support/sev_guest_device" }
 sparse_mmap = { path = "support/sparse_mmap" }
+host_file_access = { path = "support/host_file_access" }
 task_control = { path = "support/task_control" }
 tdx_guest_device = { path = "support/tdx_guest_device" }
 tee_call = { path = "support/tee_call" }

openhcl/diag_client/Cargo.toml

@@ -16,6 +16,7 @@ inspect.workspace = true
 mesh_rpc.workspace = true
 unix_socket.workspace = true
 pal_async.workspace = true
+host_file_access.workspace = true
 vmsocket.workspace = true
 
 anyhow.workspace = true

openhcl/diag_client/src/lib.rs

@@ -772,6 +772,27 @@ impl DiagClient {
 
         Ok(state.data)
     }
+
+    /// Connect to the synthetic data endpoint.
+    pub async fn synthetic_data(
+        &self,
+        id: &String,
+    ) -> anyhow::Result<PolledSocket<socket2::Socket>> {
+        let (conn, socket) = self.connect_data().await?;
+        self.ttrpc
+            .call()
+            .start(
+                diag_proto::UnderhillDiag::HostFile,
+                diag_proto::HostFileRequest {
+                    id: id.clone(),
+                    conn,
+                },
+            )
+            .await
+            .map_err(grpc_status)?;
+
+        Ok(socket)
+    }
 }
 
 fn grpc_status(status: Status) -> anyhow::Error {

openhcl/diag_proto/src/diag.proto

@@ -29,6 +29,7 @@ service UnderhillDiag {
     rpc ReadFile(FileRequest) returns (google.protobuf.Empty);
     rpc DumpSavedState(google.protobuf.Empty) returns (DumpSavedStateResponse);
     rpc PacketCapture(NetworkPacketCaptureRequest) returns (NetworkPacketCaptureResponse);
+    rpc HostFile(HostFileRequest) returns (google.protobuf.Empty);
 }
 
 message ExecRequest {
@@ -106,3 +107,8 @@ message NetworkPacketCaptureResponse {
 message CrashRequest {
     int32 pid = 1;
 }
+
+message HostFileRequest {
+    string id = 1;
+    uint64 conn = 2;
+}

openhcl/diag_server/src/diag_service.rs

@@ -12,6 +12,7 @@ use diag_proto::ExecRequest;
 use diag_proto::ExecResponse;
 use diag_proto::FILE_LINE_MAX;
 use diag_proto::FileRequest;
+use diag_proto::HostFileRequest;
 use diag_proto::KmsgRequest;
 use diag_proto::NetworkPacketCaptureRequest;
 use diag_proto::NetworkPacketCaptureResponse;
@@ -84,6 +85,11 @@ pub enum DiagRequest {
     Resume(FailableRpc<(), ()>),
     /// Save VTL2 state
     Save(FailableRpc<(), Vec<u8>>),
+    /// Get the `vmlinux` image from the host.
+    /// Just experimental, not used in production -- must be signed, even better come inside an IGVM.
+    VmLinux(FailableRpc<Socket, ()>),
+    /// Write the log from the host.
+    SomeLog(FailableRpc<Socket, ()>),
     /// Setup network trace
     PacketCapture(FailableRpc<PacketCaptureParams<Socket>, PacketCaptureParams<Socket>>),
     /// Profile VTL2
@@ -240,6 +246,10 @@ impl DiagServiceHandler {
             UnderhillDiag::DumpSavedState((), response) => response.send(grpc_result(
                 ctx.until_cancelled(self.handle_dump_saved_state()).await,
             )),
+            UnderhillDiag::HostFile(request, response) => response.send(grpc_result(
+                ctx.until_cancelled(self.handle_host_file_access(driver, &request))
+                    .await,
+            )),
         }
     }
 
@@ -581,6 +591,33 @@ impl DiagServiceHandler {
             .await
     }
 
+    async fn handle_host_file_access(
+        &self,
+        _driver: &(impl Driver + Spawn + Clone),
+        request: &HostFileRequest,
+    ) -> anyhow::Result<()> {
+        let params = self.take_connection(request.conn).await?;
+        match request.id.as_str() {
+            "vmlinux" => {
+                tracing::info!("Reading vmlinux from the host");
+                self.request_send
+                    .call_failable(DiagRequest::VmLinux, params.into_inner())
+                    .await?;
+            }
+            "log" => {
+                tracing::info!("Wring the log from the host");
+                self.request_send
+                    .call_failable(DiagRequest::SomeLog, params.into_inner())
+                    .await?;
+            }
+            _ => {
+                tracing::warn!("Unsupported host file access request: {}", request.id);
+            }
+        }
+
+        Ok(())
+    }
+
     async fn handle_packet_capture(
         &self,
         request: &NetworkPacketCaptureRequest,

openhcl/ohcldiag-dev/Cargo.toml

@@ -14,6 +14,7 @@ inspect.workspace = true
 mesh.workspace = true
 pal_async.workspace = true
 pal.workspace = true
+host_file_access.workspace = true
 term.workspace = true
 
 anyhow.workspace = true
@@ -27,6 +28,7 @@ socket2.workspace = true
 thiserror.workspace = true
 tracing-subscriber = { workspace = true, features = ["env-filter"] }
 unicycle.workspace = true
+zerocopy.workspace = true
 
 [lints]
 workspace = true

openhcl/ohcldiag-dev/src/main.rs

@@ -157,6 +157,24 @@ enum Command {
         #[clap(long, requires = "serial")]
         pipe_path: Option<String>,
     },
+    /// Requests data by the string ID, providing the ability to download
+    /// synthetized data such that requires seeks within the file.
+    HostFile {
+        /// The ID of the data to synthesize.
+        #[clap(short, long)]
+        id: String,
+        /// The output file path.
+        dst: PathBuf,
+        /// Maximum data size the host allows.
+        #[clap(short, long)]
+        size_limit: Option<usize>,
+        /// Allow existing file.
+        #[clap(short, long, default_value = "false")]
+        existing: bool,
+        /// Allow writing to the file.
+        #[clap(short, long, default_value = "false")]
+        write: bool,
+    },
     /// Writes the contents of the file.
     File {
         /// Keep waiting for and writing new data as its logged.
@@ -873,6 +891,41 @@ pub fn main() -> anyhow::Result<()> {
                 let mut file = create_or_stderr(&output)?;
                 file.write_all(&client.dump_saved_state().await?)?;
             }
+            Command::HostFile {
+                id,
+                dst,
+                size_limit,
+                existing,
+                write,
+            } => {
+                let medium = fs_err::OpenOptions::new()
+                    .read(true)
+                    .write(write)
+                    .create(!existing)
+                    .open(dst)
+                    .context("failed to open file")?;
+                let client = new_client(driver.clone(), &vm)?;
+                let transport = client.synthetic_data(&id).await?;
+
+                let mut data_stor =
+                    host_file_access::HostFileStorage::new(medium, size_limit.into());
+                match data_stor.run_async(transport).await {
+                    Ok(_) => {}
+                    Err(host_file_access::HostFileError::EndOfFile) => {
+                        eprintln!("Synthetic data {id} transfer complete, end of file reached.");
+                    }
+                    Err(e) => {
+                        eprintln!("Failed to run data transfer: {e}");
+                        return Err(anyhow::Error::from(e));
+                    }
+                }
+
+                let bytes_written = data_stor.bytes_written();
+                println!(
+                    "Synthetic data {id} transfer complete, {} bytes written.",
+                    bytes_written
+                );
+            }
         }
         Ok(())
     })

openhcl/underhill_core/Cargo.toml

@@ -59,6 +59,7 @@ hcl_compat_uefi_nvram_storage = { workspace = true, features = ["inspect", "save
 get_helpers.workspace = true
 get_protocol.workspace = true
 guest_emulation_transport.workspace = true
+host_file_access.workspace = true
 ide.workspace = true
 ide_resources.workspace = true
 input_core.workspace = true
@@ -162,6 +163,7 @@ parking_lot.workspace = true
 serde = { workspace = true, features = ["derive"] }
 serde_helpers.workspace = true
 serde_json.workspace = true
+sha2 = { workspace = true, features = ["std"] }
 socket2.workspace = true
 thiserror = { workspace = true, features = ["std"] }
 time = { workspace = true, features = ["macros"] }

openhcl/underhill_core/src/dispatch/mod.rs

@@ -47,10 +47,13 @@ use openhcl_dma_manager::OpenhclDmaManager;
 use pal_async::task::Spawn;
 use pal_async::task::Task;
 use parking_lot::Mutex;
+use sha2::Digest;
 use socket2::Socket;
 use state_unit::SavedStateUnit;
 use state_unit::SpawnedUnit;
 use state_unit::StateUnits;
+use std::io::Read;
+use std::io::Write;
 use std::sync::Arc;
 use std::time::Duration;
 use tracing::Instrument;
@@ -78,6 +81,8 @@ pub enum UhVmRpc {
     Pause(Rpc<(), bool>),
     Resume(Rpc<(), bool>),
     Save(FailableRpc<(), Vec<u8>>),
+    VmLinux(FailableRpc<Socket, ()>),
+    SomeLog(FailableRpc<Socket, ()>),
     ClearHalt(Rpc<(), bool>), // TODO: remove this, and use DebugRequest::Resume
     PacketCapture(FailableRpc<PacketCaptureParams<Socket>, PacketCaptureParams<Socket>>),
 }
@@ -355,6 +360,36 @@ impl LoadedVm {
                         })
                         .await
                     }
+                    UhVmRpc::VmLinux(rpc) => {
+                        tracing::info!(CVM_ALLOWED, "reading vmlinux from the host");
+
+                        pal_async::local::block_with_io(async |_| {
+                            rpc.handle_failable::<_, anyhow::Error>(async |socket| {
+                                let mut hfa = host_file_access::HostFileAccess::new(socket);
+                                let mut vmlinux = vec![];
+                                hfa.read_to_end(&mut vmlinux)?;
+
+                                let mut hasher = sha2::Sha256::new();
+                                hasher.update(&vmlinux);
+                                let result = hasher.finalize();
+                                tracing::info!(CVM_ALLOWED, "vmlinux sha256: {:x?}", result);
+                                Ok(())
+                            })
+                            .await
+                        });
+                    }
+                    UhVmRpc::SomeLog(rpc) => {
+                        tracing::info!(CVM_ALLOWED, "writing log to the host");
+
+                        pal_async::local::block_with_io(async |_| {
+                            rpc.handle_failable::<_, anyhow::Error>(async |socket| {
+                                let mut hfa = host_file_access::HostFileAccess::new(socket);
+                                hfa.write_all(b"Hello from underhill!")?;
+                                Ok(())
+                            })
+                            .await
+                        });
+                    }
                 },
                 Event::ServicingRequest(message) => {
                     // Explicitly destructure the message for easier tracking of its changes.

openhcl/underhill_core/src/lib.rs

@@ -604,6 +604,26 @@ async fn run_control(
                             })
                             .detach();
                     }
+                    diag_server::DiagRequest::VmLinux(rpc) => {
+                        tracing::info!(CVM_ALLOWED, "reading vmlinux from the host");
+                        let Some(workers) = &mut workers else {
+                            rpc.complete(Err(RemoteError::new(anyhow::anyhow!(
+                                "worker has not been started yet"
+                            ))));
+                            continue;
+                        };
+                        workers.vm_rpc.send(UhVmRpc::VmLinux(rpc));
+                    }
+                    diag_server::DiagRequest::SomeLog(rpc) => {
+                        tracing::info!(CVM_ALLOWED, "writing log to the host");
+                        let Some(workers) = &mut workers else {
+                            rpc.complete(Err(RemoteError::new(anyhow::anyhow!(
+                                "worker has not been started yet"
+                            ))));
+                            continue;
+                        };
+                        workers.vm_rpc.send(UhVmRpc::SomeLog(rpc));
+                    }
                     diag_server::DiagRequest::PacketCapture(rpc) => {
                         let Some(workers) = &mut workers else {
                             rpc.complete(Err(RemoteError::new(anyhow::anyhow!(

support/host_file_access/Cargo.toml

@@ -0,0 +1,18 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+[package]
+name = "host_file_access"
+edition.workspace = true
+rust-version.workspace = true
+
+[dependencies]
+bitfield-struct.workspace = true
+futures.workspace = true
+open_enum.workspace = true
+thiserror.workspace = true
+tracing.workspace = true
+zerocopy.workspace = true
+
+[lints]
+workspace = true