diff --git a/.github/workflows/dep_rust.yml b/.github/workflows/dep_rust.yml index ef847f2f2..ba9d3f2e6 100644 --- a/.github/workflows/dep_rust.yml +++ b/.github/workflows/dep_rust.yml @@ -124,11 +124,11 @@ jobs: run: just run-rust-examples-linux ${{ matrix.config }} ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}} - name: Run Rust Gdb tests - linux - if: runner.os == 'Linux' && matrix.hypervisor == 'kvm' + if: runner.os == 'Linux' env: CARGO_TERM_COLOR: always RUST_LOG: debug - run: just test-rust-gdb-debugging ${{ matrix.config }} + run: just test-rust-gdb-debugging ${{ matrix.config }} ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}} ### Benchmarks ### - name: Install github-cli (Linux mariner) diff --git a/Justfile b/Justfile index 2dd3fbc27..8b02d63d2 100644 --- a/Justfile +++ b/Justfile @@ -117,9 +117,9 @@ test-rust-feature-compilation-fail target=default-target: {{ if os() == "linux" { "! cargo check -p hyperlight-host --no-default-features 2> /dev/null"} else { "" } }} # Test rust gdb debugging -test-rust-gdb-debugging target=default-target: (build-rust target) - {{ set-trace-env-vars }} cargo test --profile={{ if target == "debug" { "dev" } else { target } }} --example guest-debugging --features gdb - {{ set-trace-env-vars }} cargo test --profile={{ if target == "debug" { "dev" } else { target } }} --features gdb -- test_gdb +test-rust-gdb-debugging target=default-target features="": (build-rust target) + {{ set-trace-env-vars }} cargo test --profile={{ if target == "debug" { "dev" } else { target } }} --example guest-debugging {{ if features =="" {'--features gdb'} else { "--features gdb," + features } }} + {{ set-trace-env-vars }} cargo test --profile={{ if target == "debug" { "dev" } else { target } }} {{ if features =="" {'--features gdb'} else { "--features gdb," + features } }} -- test_gdb test target=default-target: (test-rust target) diff --git a/docs/debugging-hyperlight.md b/docs/debugging-hyperlight.md index 9ff3b8229..8abb9a645 100644 --- a/docs/debugging-hyperlight.md +++ b/docs/debugging-hyperlight.md @@ -42,3 +42,7 @@ cargo test --package hyperlight-host --test integration_test --features print_de To dump the details of the memory configuration, the virtual processors register state and the contents of the VM memory set the feature `crashdump` and run a debug build. This will result in a dump file being created in the temporary directory. The name and location of the dump file will be printed to the console and logged as an error message. There are no tools at this time to analyze the dump file, but it can be useful for debugging. + +## Debugging guests + +For more information on how to debug the Hyperlight guests check the following [link](./how-to-debug-a-hyperlight-guest.md). diff --git a/docs/how-to-debug-a-hyperlight-guest.md b/docs/how-to-debug-a-hyperlight-guest.md index 7f9c14732..abf571062 100644 --- a/docs/how-to-debug-a-hyperlight-guest.md +++ b/docs/how-to-debug-a-hyperlight-guest.md @@ -1,13 +1,13 @@ -# How to debug a Hyperlight **KVM** guest using gdb +# How to debug a Hyperlight guest using gdb on Linux -Hyperlight supports gdb debugging of a **KVM** guest running inside a Hyperlight sandbox. -When Hyperlight is compiled with the `gdb` feature enabled, a Hyperlight KVM sandbox can be configured +Hyperlight supports gdb debugging of a **KVM** or **MSHV** guest running inside a Hyperlight sandbox on Linux. +When Hyperlight is compiled with the `gdb` feature enabled, a Hyperlight sandbox can be configured to start listening for a gdb connection. ## Supported features -The Hyperlight `gdb` feature enables **KVM** guest debugging: - - an entry point breakpoint is automatically set for the guest to stop +The Hyperlight `gdb` feature enables **KVM** and **MSHV** guest debugging to: + - stop at an entry point breakpoint which is automatically set by Hyperlight - add and remove HW breakpoints (maximum 4 set breakpoints at a time) - add and remove SW breakpoints - read and write registers @@ -18,7 +18,7 @@ The Hyperlight `gdb` feature enables **KVM** guest debugging: ## Expected behavior Below is a list describing some cases of expected behavior from a gdb debug -session of a guest binary running inside a KVM Hyperlight sandbox. +session of a guest binary running inside a Hyperlight sandbox on Linux. - when the `gdb` feature is enabled and a SandboxConfiguration is provided a debug port, the created sandbox will wait for a gdb client to connect on the @@ -154,7 +154,7 @@ is sent over the communication channel to the hypervisor handler for the sandbox to resolve. Below is a sequence diagram that shows the interaction between the entities -involved in the gdb debugging of a Hyperlight guest running inside a KVM sandbox. +involved in the gdb debugging of a Hyperlight guest running inside a **KVM** or **MSHV** sandbox. ``` ┌───────────────────────────────────────────────────────────────────────────────────────────────┐ diff --git a/src/hyperlight_host/build.rs b/src/hyperlight_host/build.rs index a8370725d..81649b8c0 100644 --- a/src/hyperlight_host/build.rs +++ b/src/hyperlight_host/build.rs @@ -89,7 +89,7 @@ fn main() -> Result<()> { // Essentially the kvm and mshv features are ignored on windows as long as you use #[cfg(kvm)] and not #[cfg(feature = "kvm")]. // You should never use #[cfg(feature = "kvm")] or #[cfg(feature = "mshv")] in the codebase. cfg_aliases::cfg_aliases! { - gdb: { all(feature = "gdb", debug_assertions, feature = "kvm", target_os = "linux") }, + gdb: { all(feature = "gdb", debug_assertions, any(feature = "kvm", feature = "mshv2", feature = "mshv3"), target_os = "linux") }, kvm: { all(feature = "kvm", target_os = "linux") }, mshv: { all(any(feature = "mshv2", feature = "mshv3"), target_os = "linux") }, // inprocess feature is aliased with debug_assertions to make it only available in debug-builds. diff --git a/src/hyperlight_host/examples/guest-debugging/main.rs b/src/hyperlight_host/examples/guest-debugging/main.rs index da3010b9c..9ebde14b4 100644 --- a/src/hyperlight_host/examples/guest-debugging/main.rs +++ b/src/hyperlight_host/examples/guest-debugging/main.rs @@ -107,7 +107,7 @@ mod tests { set pagination off set logging file {out_file_path} - set logging enabled on + set logging on break hyperlight_main commands @@ -118,7 +118,7 @@ mod tests { continue - set logging enabled off + set logging off quit " ) @@ -134,12 +134,17 @@ mod tests { write_cmds_file(&cmd_file_path, &out_file_path) .expect("Failed to write gdb commands to file"); + #[cfg(mshv3)] + let features = "gdb,mshv3"; + #[cfg(not(mshv3))] + let features = "gdb"; + let mut guest_child = Command::new("cargo") .arg("run") .arg("--example") .arg("guest-debugging") .arg("--features") - .arg("gdb") + .arg(features) .stdin(Stdio::piped()) .stdout(Stdio::piped()) .spawn() diff --git a/src/hyperlight_host/src/hypervisor/gdb/event_loop.rs b/src/hyperlight_host/src/hypervisor/gdb/event_loop.rs index ee009caa1..72dd97b29 100644 --- a/src/hyperlight_host/src/hypervisor/gdb/event_loop.rs +++ b/src/hyperlight_host/src/hypervisor/gdb/event_loop.rs @@ -16,14 +16,15 @@ limitations under the License. use gdbstub::common::Signal; use gdbstub::conn::ConnectionExt; -use gdbstub::stub::run_blocking::{self, WaitForStopReasonError}; -use gdbstub::stub::{BaseStopReason, DisconnectReason, GdbStub, SingleThreadStopReason}; +use gdbstub::stub::{ + run_blocking, BaseStopReason, DisconnectReason, GdbStub, SingleThreadStopReason, +}; use libc::{pthread_kill, SIGRTMIN}; use super::x86_64_target::HyperlightSandboxTarget; use super::{DebugResponse, GdbTargetError, VcpuStopReason}; -pub struct GdbBlockingEventLoop; +struct GdbBlockingEventLoop; impl run_blocking::BlockingEventLoop for GdbBlockingEventLoop { type Connection = Box>; @@ -57,13 +58,10 @@ impl run_blocking::BlockingEventLoop for GdbBlockingEventLoop { signal: Signal(SIGRTMIN() as u8), }, VcpuStopReason::Unknown => { - log::warn!("Unknown stop reason - resuming execution"); + log::warn!("Unknown stop reason received"); - target - .resume_vcpu() - .map_err(WaitForStopReasonError::Target)?; - - continue; + // Marking as a SwBreak so the gdb inspect where/why it stopped + BaseStopReason::SwBreak(()) } }; @@ -115,7 +113,7 @@ impl run_blocking::BlockingEventLoop for GdbBlockingEventLoop { } } -pub fn event_loop_thread( +pub(crate) fn event_loop_thread( debugger: GdbStub>>, target: &mut HyperlightSandboxTarget, ) { diff --git a/src/hyperlight_host/src/hypervisor/gdb/kvm_debug.rs b/src/hyperlight_host/src/hypervisor/gdb/kvm_debug.rs new file mode 100644 index 000000000..6e8f9d36f --- /dev/null +++ b/src/hyperlight_host/src/hypervisor/gdb/kvm_debug.rs @@ -0,0 +1,270 @@ +/* +Copyright 2024 The Hyperlight Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +use std::collections::HashMap; + +use kvm_bindings::{ + kvm_debug_exit_arch, kvm_guest_debug, kvm_regs, KVM_GUESTDBG_ENABLE, KVM_GUESTDBG_SINGLESTEP, + KVM_GUESTDBG_USE_HW_BP, KVM_GUESTDBG_USE_SW_BP, +}; +use kvm_ioctls::VcpuFd; + +use super::{ + GuestDebug, VcpuStopReason, X86_64Regs, DR6_BS_FLAG_MASK, DR6_HW_BP_FLAGS_MASK, + MAX_NO_OF_HW_BP, SW_BP_SIZE, +}; +use crate::{new_error, HyperlightError, Result}; + +/// Exception id for SW breakpoint +const SW_BP_ID: u32 = 3; + +/// KVM Debug struct +/// This struct is used to abstract the internal details of the kvm +/// guest debugging settings +#[derive(Default)] +pub(crate) struct KvmDebug { + /// vCPU stepping state + single_step: bool, + + /// Array of addresses for HW breakpoints + hw_breakpoints: Vec, + /// Saves the bytes modified to enable SW breakpoints + sw_breakpoints: HashMap, + + /// Sent to KVM for enabling guest debug + dbg_cfg: kvm_guest_debug, +} + +impl KvmDebug { + pub(crate) fn new() -> Self { + let dbg = kvm_guest_debug { + control: KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP, + ..Default::default() + }; + + Self { + single_step: false, + hw_breakpoints: vec![], + sw_breakpoints: HashMap::new(), + dbg_cfg: dbg, + } + } + + /// Returns the instruction pointer from the stopped vCPU + fn get_instruction_pointer(&self, vcpu_fd: &VcpuFd) -> Result { + let regs = vcpu_fd + .get_regs() + .map_err(|e| new_error!("Could not retrieve registers from vCPU: {:?}", e))?; + + Ok(regs.rip) + } + + /// This method sets the kvm debugreg fields to enable breakpoints at + /// specific addresses + /// + /// The first 4 debug registers are used to set the addresses + /// The 4th and 5th debug registers are obsolete and not used + /// The 7th debug register is used to enable the breakpoints + /// For more information see: DEBUG REGISTERS chapter in the architecture + /// manual + fn set_debug_config(&mut self, vcpu_fd: &VcpuFd, step: bool) -> Result<()> { + let addrs = &self.hw_breakpoints; + + self.dbg_cfg.arch.debugreg = [0; 8]; + for (k, addr) in addrs.iter().enumerate() { + self.dbg_cfg.arch.debugreg[k] = *addr; + self.dbg_cfg.arch.debugreg[7] |= 1 << (k * 2); + } + + if !addrs.is_empty() { + self.dbg_cfg.control |= KVM_GUESTDBG_USE_HW_BP; + } else { + self.dbg_cfg.control &= !KVM_GUESTDBG_USE_HW_BP; + } + + if step { + self.dbg_cfg.control |= KVM_GUESTDBG_SINGLESTEP; + } else { + self.dbg_cfg.control &= !KVM_GUESTDBG_SINGLESTEP; + } + + log::debug!("Setting bp: {:?} cfg: {:?}", addrs, self.dbg_cfg); + vcpu_fd + .set_guest_debug(&self.dbg_cfg) + .map_err(|e| new_error!("Could not set guest debug: {:?}", e))?; + + self.single_step = step; + + Ok(()) + } + + /// Get the reason the vCPU has stopped + pub(crate) fn get_stop_reason( + &mut self, + vcpu_fd: &VcpuFd, + debug_exit: kvm_debug_exit_arch, + entrypoint: u64, + ) -> Result { + // If the BS flag in DR6 register is set, it means a single step + // instruction triggered the exit + // Check page 19-4 Vol. 3B of Intel 64 and IA-32 + // Architectures Software Developer's Manual + if debug_exit.dr6 & DR6_BS_FLAG_MASK != 0 && self.single_step { + return Ok(VcpuStopReason::DoneStep); + } + + let ip = self.get_instruction_pointer(vcpu_fd)?; + let gpa = self.translate_gva(vcpu_fd, ip)?; + + // If any of the B0-B3 flags in DR6 register is set, it means a + // hardware breakpoint triggered the exit + // Check page 19-4 Vol. 3B of Intel 64 and IA-32 + // Architectures Software Developer's Manual + if DR6_HW_BP_FLAGS_MASK & debug_exit.dr6 != 0 && self.hw_breakpoints.contains(&gpa) { + // In case the hw breakpoint is the entry point, remove it to + // avoid hanging here as gdb does not remove breakpoints it + // has not set. + // Gdb expects the target to be stopped when connected. + if gpa == entrypoint { + self.remove_hw_breakpoint(vcpu_fd, entrypoint)?; + } + return Ok(VcpuStopReason::HwBp); + } + + // If the exception ID matches #BP (3) - it means a software breakpoint + // caused the exit + if SW_BP_ID == debug_exit.exception && self.sw_breakpoints.contains_key(&gpa) { + return Ok(VcpuStopReason::SwBp); + } + + // Log an error and provide internal debugging info for fixing + log::error!( + r"The vCPU exited because of an unknown reason: + kvm_debug_exit_arch: {:?} + single_step: {:?} + hw_breakpoints: {:?} + sw_breakpoints: {:?}", + debug_exit, + self.single_step, + self.hw_breakpoints, + self.sw_breakpoints, + ); + Ok(VcpuStopReason::Unknown) + } +} + +impl GuestDebug for KvmDebug { + type Vcpu = VcpuFd; + + fn is_hw_breakpoint(&self, addr: &u64) -> bool { + self.hw_breakpoints.contains(addr) + } + fn is_sw_breakpoint(&self, addr: &u64) -> bool { + self.sw_breakpoints.contains_key(addr) + } + fn save_hw_breakpoint(&mut self, addr: &u64) -> bool { + if self.hw_breakpoints.len() >= MAX_NO_OF_HW_BP { + false + } else { + self.hw_breakpoints.push(*addr); + + true + } + } + fn save_sw_breakpoint_data(&mut self, addr: u64, data: [u8; 1]) { + _ = self.sw_breakpoints.insert(addr, data); + } + fn delete_hw_breakpoint(&mut self, addr: &u64) { + self.hw_breakpoints.retain(|&a| a != *addr); + } + fn delete_sw_breakpoint_data(&mut self, addr: &u64) -> Option<[u8; 1]> { + self.sw_breakpoints.remove(addr) + } + + fn read_regs(&self, vcpu_fd: &Self::Vcpu, regs: &mut X86_64Regs) -> Result<()> { + log::debug!("Read registers"); + let vcpu_regs = vcpu_fd + .get_regs() + .map_err(|e| new_error!("Could not read guest registers: {:?}", e))?; + + regs.rax = vcpu_regs.rax; + regs.rbx = vcpu_regs.rbx; + regs.rcx = vcpu_regs.rcx; + regs.rdx = vcpu_regs.rdx; + regs.rsi = vcpu_regs.rsi; + regs.rdi = vcpu_regs.rdi; + regs.rbp = vcpu_regs.rbp; + regs.rsp = vcpu_regs.rsp; + regs.r8 = vcpu_regs.r8; + regs.r9 = vcpu_regs.r9; + regs.r10 = vcpu_regs.r10; + regs.r11 = vcpu_regs.r11; + regs.r12 = vcpu_regs.r12; + regs.r13 = vcpu_regs.r13; + regs.r14 = vcpu_regs.r14; + regs.r15 = vcpu_regs.r15; + + regs.rip = vcpu_regs.rip; + regs.rflags = vcpu_regs.rflags; + + Ok(()) + } + + fn set_single_step(&mut self, vcpu_fd: &Self::Vcpu, enable: bool) -> Result<()> { + self.set_debug_config(vcpu_fd, enable) + } + + fn translate_gva(&self, vcpu_fd: &Self::Vcpu, gva: u64) -> Result { + let tr = vcpu_fd + .translate_gva(gva) + .map_err(|_| HyperlightError::TranslateGuestAddress(gva))?; + + if tr.valid == 0 { + Err(HyperlightError::TranslateGuestAddress(gva)) + } else { + Ok(tr.physical_address) + } + } + + fn write_regs(&self, vcpu_fd: &Self::Vcpu, regs: &X86_64Regs) -> Result<()> { + log::debug!("Write registers"); + let new_regs = kvm_regs { + rax: regs.rax, + rbx: regs.rbx, + rcx: regs.rcx, + rdx: regs.rdx, + rsi: regs.rsi, + rdi: regs.rdi, + rbp: regs.rbp, + rsp: regs.rsp, + r8: regs.r8, + r9: regs.r9, + r10: regs.r10, + r11: regs.r11, + r12: regs.r12, + r13: regs.r13, + r14: regs.r14, + r15: regs.r15, + + rip: regs.rip, + rflags: regs.rflags, + }; + + vcpu_fd + .set_regs(&new_regs) + .map_err(|e| new_error!("Could not write guest registers: {:?}", e)) + } +} diff --git a/src/hyperlight_host/src/hypervisor/gdb/mod.rs b/src/hyperlight_host/src/hypervisor/gdb/mod.rs index b941040a3..95b81ed6a 100644 --- a/src/hyperlight_host/src/hypervisor/gdb/mod.rs +++ b/src/hyperlight_host/src/hypervisor/gdb/mod.rs @@ -15,10 +15,15 @@ limitations under the License. */ mod event_loop; -pub mod x86_64_target; +#[cfg(kvm)] +mod kvm_debug; +#[cfg(mshv)] +mod mshv_debug; +mod x86_64_target; use std::io::{self, ErrorKind}; use std::net::TcpListener; +use std::sync::{Arc, Mutex}; use std::thread; use crossbeam_channel::{Receiver, Sender, TryRecvError}; @@ -26,11 +31,42 @@ use event_loop::event_loop_thread; use gdbstub::conn::ConnectionExt; use gdbstub::stub::GdbStub; use gdbstub::target::TargetError; +use hyperlight_common::mem::PAGE_SIZE; +#[cfg(kvm)] +pub(crate) use kvm_debug::KvmDebug; +#[cfg(mshv)] +pub(crate) use mshv_debug::MshvDebug; use thiserror::Error; use x86_64_target::HyperlightSandboxTarget; +use crate::hypervisor::handlers::DbgMemAccessHandlerCaller; +use crate::mem::layout::SandboxMemoryLayout; +use crate::{new_error, HyperlightError}; + +/// Software Breakpoint size in memory +const SW_BP_SIZE: usize = 1; +/// Software Breakpoint opcode - INT3 +/// Check page 7-28 Vol. 3A of Intel 64 and IA-32 +/// Architectures Software Developer's Manual +const SW_BP_OP: u8 = 0xCC; +/// Software Breakpoint written to memory +const SW_BP: [u8; SW_BP_SIZE] = [SW_BP_OP]; +/// Maximum number of supported hardware breakpoints +const MAX_NO_OF_HW_BP: usize = 4; + +/// Check page 19-4 Vol. 3B of Intel 64 and IA-32 +/// Architectures Software Developer's Manual +/// Bit position of BS flag in DR6 debug register +const DR6_BS_FLAG_POS: usize = 14; +/// Bit mask of BS flag in DR6 debug register +const DR6_BS_FLAG_MASK: u64 = 1 << DR6_BS_FLAG_POS; +/// Bit position of HW breakpoints status in DR6 debug register +const DR6_HW_BP_FLAGS_POS: usize = 0; +/// Bit mask of HW breakpoints status in DR6 debug register +const DR6_HW_BP_FLAGS_MASK: u64 = 0x0F << DR6_HW_BP_FLAGS_POS; + #[derive(Debug, Error)] -pub enum GdbTargetError { +pub(crate) enum GdbTargetError { #[error("Error encountered while binding to address and port")] CannotBind, #[error("Error encountered while listening for connections")] @@ -68,25 +104,25 @@ impl From for TargetError { /// Struct that contains the x86_64 core registers #[derive(Debug, Default)] -pub struct X86_64Regs { - pub rax: u64, - pub rbx: u64, - pub rcx: u64, - pub rdx: u64, - pub rsi: u64, - pub rdi: u64, - pub rbp: u64, - pub rsp: u64, - pub r8: u64, - pub r9: u64, - pub r10: u64, - pub r11: u64, - pub r12: u64, - pub r13: u64, - pub r14: u64, - pub r15: u64, - pub rip: u64, - pub rflags: u64, +pub(crate) struct X86_64Regs { + pub(crate) rax: u64, + pub(crate) rbx: u64, + pub(crate) rcx: u64, + pub(crate) rdx: u64, + pub(crate) rsi: u64, + pub(crate) rdi: u64, + pub(crate) rbp: u64, + pub(crate) rsp: u64, + pub(crate) r8: u64, + pub(crate) r9: u64, + pub(crate) r10: u64, + pub(crate) r11: u64, + pub(crate) r12: u64, + pub(crate) r13: u64, + pub(crate) r14: u64, + pub(crate) r15: u64, + pub(crate) rip: u64, + pub(crate) rflags: u64, } /// Defines the possible reasons for which a vCPU can be stopped when debugging @@ -101,7 +137,7 @@ pub enum VcpuStopReason { /// Enumerates the possible actions that a debugger can ask from a Hypervisor #[derive(Debug)] -pub enum DebugMsg { +pub(crate) enum DebugMsg { AddHwBreakpoint(u64), AddSwBreakpoint(u64), Continue, @@ -118,7 +154,7 @@ pub enum DebugMsg { /// Enumerates the possible responses that a hypervisor can provide to a debugger #[derive(Debug)] -pub enum DebugResponse { +pub(crate) enum DebugResponse { AddHwBreakpoint(bool), AddSwBreakpoint(bool), Continue, @@ -135,9 +171,186 @@ pub enum DebugResponse { WriteRegisters, } +/// This trait is used to define common debugging functionality for Hypervisors +pub(crate) trait GuestDebug { + /// Type that wraps the vCPU functionality + type Vcpu; + + /// Returns true whether the provided address is a hardware breakpoint + fn is_hw_breakpoint(&self, addr: &u64) -> bool; + /// Returns true whether the provided address is a software breakpoint + fn is_sw_breakpoint(&self, addr: &u64) -> bool; + /// Stores the address of the hw breakpoint + fn save_hw_breakpoint(&mut self, addr: &u64) -> bool; + /// Stores the data that the sw breakpoint op code replaces + fn save_sw_breakpoint_data(&mut self, addr: u64, data: [u8; 1]); + /// Deletes the address of the hw breakpoint from storage + fn delete_hw_breakpoint(&mut self, addr: &u64); + /// Retrieves the saved data that the sw breakpoint op code replaces + fn delete_sw_breakpoint_data(&mut self, addr: &u64) -> Option<[u8; 1]>; + + /// Read registers + fn read_regs(&self, vcpu_fd: &Self::Vcpu, regs: &mut X86_64Regs) -> crate::Result<()>; + /// Enables or disables stepping and sets the vCPU debug configuration + fn set_single_step(&mut self, vcpu_fd: &Self::Vcpu, enable: bool) -> crate::Result<()>; + /// Translates the guest address to physical address + fn translate_gva(&self, vcpu_fd: &Self::Vcpu, gva: u64) -> crate::Result; + /// Write registers + fn write_regs(&self, vcpu_fd: &Self::Vcpu, regs: &X86_64Regs) -> crate::Result<()>; + + /// Adds hardware breakpoint + fn add_hw_breakpoint(&mut self, vcpu_fd: &Self::Vcpu, addr: u64) -> crate::Result<()> { + let addr = self.translate_gva(vcpu_fd, addr)?; + + if self.is_hw_breakpoint(&addr) { + return Ok(()); + } + + self.save_hw_breakpoint(&addr) + .then(|| self.set_single_step(vcpu_fd, false)) + .ok_or_else(|| new_error!("Failed to save hw breakpoint"))? + } + /// Overwrites the guest memory with the SW Breakpoint op code that instructs + /// the vCPU to stop when is executed and stores the overwritten data to be + /// able to restore it + fn add_sw_breakpoint( + &mut self, + vcpu_fd: &Self::Vcpu, + addr: u64, + dbg_mem_access_fn: Arc>, + ) -> crate::Result<()> { + let addr = self.translate_gva(vcpu_fd, addr)?; + + if self.is_sw_breakpoint(&addr) { + return Ok(()); + } + + // Write breakpoint OP code to write to guest memory + let mut save_data = [0; SW_BP_SIZE]; + self.read_addrs(vcpu_fd, addr, &mut save_data[..], dbg_mem_access_fn.clone())?; + self.write_addrs(vcpu_fd, addr, &SW_BP, dbg_mem_access_fn)?; + + // Save guest memory to restore when breakpoint is removed + self.save_sw_breakpoint_data(addr, save_data); + + Ok(()) + } + /// Copies the data from the guest memory address to the provided slice + /// The address is checked to be a valid guest address + fn read_addrs( + &mut self, + vcpu_fd: &Self::Vcpu, + mut gva: u64, + mut data: &mut [u8], + dbg_mem_access_fn: Arc>, + ) -> crate::Result<()> { + let data_len = data.len(); + log::debug!("Read addr: {:X} len: {:X}", gva, data_len); + + while !data.is_empty() { + let gpa = self.translate_gva(vcpu_fd, gva)?; + + let read_len = std::cmp::min( + data.len(), + (PAGE_SIZE - (gpa & (PAGE_SIZE - 1))).try_into().unwrap(), + ); + let offset = (gpa as usize) + .checked_sub(SandboxMemoryLayout::BASE_ADDRESS) + .ok_or_else(|| { + log::warn!( + "gva=0x{:#X} causes subtract with underflow: \"gpa - BASE_ADDRESS={:#X}-{:#X}\"", + gva, gpa, SandboxMemoryLayout::BASE_ADDRESS); + HyperlightError::TranslateGuestAddress(gva) + })?; + + dbg_mem_access_fn + .try_lock() + .map_err(|e| new_error!("Error locking at {}:{}: {}", file!(), line!(), e))? + .read(offset, &mut data[..read_len])?; + + data = &mut data[read_len..]; + gva += read_len as u64; + } + + Ok(()) + } + /// Removes hardware breakpoint + fn remove_hw_breakpoint(&mut self, vcpu_fd: &Self::Vcpu, addr: u64) -> crate::Result<()> { + let addr = self.translate_gva(vcpu_fd, addr)?; + + self.is_hw_breakpoint(&addr) + .then(|| { + self.delete_hw_breakpoint(&addr); + self.set_single_step(vcpu_fd, false) + }) + .ok_or_else(|| new_error!("The address: {:?} is not a hw breakpoint", addr))? + } + /// Restores the overwritten data to the guest memory + fn remove_sw_breakpoint( + &mut self, + vcpu_fd: &Self::Vcpu, + addr: u64, + dbg_mem_access_fn: Arc>, + ) -> crate::Result<()> { + let addr = self.translate_gva(vcpu_fd, addr)?; + + if self.is_sw_breakpoint(&addr) { + let save_data = self + .delete_sw_breakpoint_data(&addr) + .ok_or_else(|| new_error!("Expected to contain the sw breakpoint address"))?; + + // Restore saved data to the guest's memory + self.write_addrs(vcpu_fd, addr, &save_data, dbg_mem_access_fn)?; + + Ok(()) + } else { + Err(new_error!("The address: {:?} is not a sw breakpoint", addr)) + } + } + /// Copies the data from the provided slice to the guest memory address + /// The address is checked to be a valid guest address + fn write_addrs( + &mut self, + vcpu_fd: &Self::Vcpu, + mut gva: u64, + mut data: &[u8], + dbg_mem_access_fn: Arc>, + ) -> crate::Result<()> { + let data_len = data.len(); + log::debug!("Write addr: {:X} len: {:X}", gva, data_len); + + while !data.is_empty() { + let gpa = self.translate_gva(vcpu_fd, gva)?; + + let write_len = std::cmp::min( + data.len(), + (PAGE_SIZE - (gpa & (PAGE_SIZE - 1))).try_into().unwrap(), + ); + let offset = (gpa as usize) + .checked_sub(SandboxMemoryLayout::BASE_ADDRESS) + .ok_or_else(|| { + log::warn!( + "gva=0x{:#X} causes subtract with underflow: \"gpa - BASE_ADDRESS={:#X}-{:#X}\"", + gva, gpa, SandboxMemoryLayout::BASE_ADDRESS); + HyperlightError::TranslateGuestAddress(gva) + })?; + + dbg_mem_access_fn + .try_lock() + .map_err(|e| new_error!("Error locking at {}:{}: {}", file!(), line!(), e))? + .write(offset, data)?; + + data = &data[write_len..]; + gva += write_len as u64; + } + + Ok(()) + } +} + /// Debug communication channel that is used for sending a request type and /// receive a different response type -pub struct DebugCommChannel { +pub(crate) struct DebugCommChannel { /// Transmit channel tx: Sender, /// Receive channel @@ -145,7 +358,7 @@ pub struct DebugCommChannel { } impl DebugCommChannel { - pub fn unbounded() -> (DebugCommChannel, DebugCommChannel) { + pub(crate) fn unbounded() -> (DebugCommChannel, DebugCommChannel) { let (hyp_tx, gdb_rx): (Sender, Receiver) = crossbeam_channel::unbounded(); let (gdb_tx, hyp_rx): (Sender, Receiver) = crossbeam_channel::unbounded(); @@ -163,23 +376,23 @@ impl DebugCommChannel { } /// Sends message over the transmit channel and expects a response - pub fn send(&self, msg: T) -> Result<(), GdbTargetError> { + pub(crate) fn send(&self, msg: T) -> Result<(), GdbTargetError> { self.tx.send(msg).map_err(|_| GdbTargetError::CannotSendMsg) } /// Waits for a message over the receive channel - pub fn recv(&self) -> Result { + pub(crate) fn recv(&self) -> Result { self.rx.recv().map_err(|_| GdbTargetError::CannotReceiveMsg) } /// Checks whether there's a message waiting on the receive channel - pub fn try_recv(&self) -> Result { + pub(crate) fn try_recv(&self) -> Result { self.rx.try_recv() } } /// Creates a thread that handles gdb protocol -pub fn create_gdb_thread( +pub(crate) fn create_gdb_thread( port: u16, thread_id: u64, ) -> Result, GdbTargetError> { diff --git a/src/hyperlight_host/src/hypervisor/gdb/mshv_debug.rs b/src/hyperlight_host/src/hypervisor/gdb/mshv_debug.rs new file mode 100644 index 000000000..036aadbd3 --- /dev/null +++ b/src/hyperlight_host/src/hypervisor/gdb/mshv_debug.rs @@ -0,0 +1,283 @@ +/* +Copyright 2024 The Hyperlight Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#[cfg(mshv2)] +extern crate mshv_bindings2 as mshv_bindings; +#[cfg(mshv2)] +extern crate mshv_ioctls2 as mshv_ioctls; + +#[cfg(mshv3)] +extern crate mshv_bindings3 as mshv_bindings; +#[cfg(mshv3)] +extern crate mshv_ioctls3 as mshv_ioctls; + +use std::collections::HashMap; + +use mshv_bindings::{ + DebugRegisters, StandardRegisters, HV_TRANSLATE_GVA_VALIDATE_READ, + HV_TRANSLATE_GVA_VALIDATE_WRITE, +}; +use mshv_ioctls::VcpuFd; + +use super::{ + GuestDebug, VcpuStopReason, X86_64Regs, DR6_BS_FLAG_MASK, DR6_HW_BP_FLAGS_MASK, + MAX_NO_OF_HW_BP, SW_BP_SIZE, +}; +use crate::{new_error, HyperlightError, Result}; + +#[derive(Debug, Default)] +pub(crate) struct MshvDebug { + /// vCPU stepping state + single_step: bool, + + /// Array of addresses for HW breakpoints + hw_breakpoints: Vec, + /// Saves the bytes modified to enable SW breakpoints + sw_breakpoints: HashMap, + + /// Debug registers + dbg_cfg: DebugRegisters, +} + +impl MshvDebug { + pub(crate) fn new() -> Self { + Self { + single_step: false, + hw_breakpoints: vec![], + sw_breakpoints: HashMap::new(), + dbg_cfg: DebugRegisters::default(), + } + } + + /// Returns the instruction pointer from the stopped vCPU + fn get_instruction_pointer(&self, vcpu_fd: &VcpuFd) -> Result { + let regs = vcpu_fd + .get_regs() + .map_err(|e| new_error!("Could not retrieve registers from vCPU: {:?}", e))?; + + Ok(regs.rip) + } + + /// This method sets the vCPU debug register fields to enable breakpoints at + /// specific addresses + /// + /// The first 4 debug registers are used to set the addresses + /// The 4th and 5th debug registers are obsolete and not used + /// The 7th debug register is used to enable the breakpoints + /// For more information see: DEBUG REGISTERS chapter in the architecture + /// manual + fn set_debug_config(&mut self, vcpu_fd: &VcpuFd, step: bool) -> Result<()> { + let addrs = &self.hw_breakpoints; + + let mut dbg_cfg = DebugRegisters::default(); + for (k, addr) in addrs.iter().enumerate() { + match k { + 0 => { + dbg_cfg.dr0 = *addr; + } + 1 => { + dbg_cfg.dr1 = *addr; + } + 2 => { + dbg_cfg.dr2 = *addr; + } + 3 => { + dbg_cfg.dr3 = *addr; + } + _ => { + Err(new_error!("Tried to set more than 4 HW breakpoints"))?; + } + } + dbg_cfg.dr7 |= 1 << (k * 2); + } + + self.dbg_cfg = dbg_cfg; + vcpu_fd + .set_debug_regs(&self.dbg_cfg) + .map_err(|e| new_error!("Could not set guest debug: {:?}", e))?; + + self.single_step = step; + + let mut regs = vcpu_fd + .get_regs() + .map_err(|e| new_error!("Could not get registers: {:?}", e))?; + + // Set TF Flag to enable Traps + if self.single_step { + regs.rflags |= 1 << 8; + } else { + regs.rflags &= !(1 << 8); + } + + vcpu_fd + .set_regs(®s) + .map_err(|e| new_error!("Could not set registers: {:?}", e))?; + + Ok(()) + } + + /// Returns the vCPU stop reason + pub(crate) fn get_stop_reason( + &mut self, + vcpu_fd: &VcpuFd, + entrypoint: u64, + ) -> Result { + // MSHV does not provide info on the vCPU exits but the debug + // information can be retrieved from the DEBUG REGISTERS + let regs = vcpu_fd + .get_debug_regs() + .map_err(|e| new_error!("Cannot retrieve debug registers from vCPU: {}", e))?; + + // DR6 register contains debug state related information + let debug_status = regs.dr6; + + // If the BS flag in DR6 register is set, it means a single step + // instruction triggered the exit + // Check page 19-4 Vol. 3B of Intel 64 and IA-32 + // Architectures Software Developer's Manual + if debug_status & DR6_BS_FLAG_MASK != 0 && self.single_step { + return Ok(VcpuStopReason::DoneStep); + } + + let ip = self.get_instruction_pointer(vcpu_fd)?; + let gpa = self.translate_gva(vcpu_fd, ip)?; + + // If any of the B0-B3 flags in DR6 register is set, it means a + // hardware breakpoint triggered the exit + // Check page 19-4 Vol. 3B of Intel 64 and IA-32 + // Architectures Software Developer's Manual + if debug_status & DR6_HW_BP_FLAGS_MASK != 0 && self.hw_breakpoints.contains(&gpa) { + // In case the hw breakpoint is the entry point, remove it to + // avoid hanging here as gdb does not remove breakpoints it + // has not set. + // Gdb expects the target to be stopped when connected. + if gpa == entrypoint { + self.remove_hw_breakpoint(vcpu_fd, entrypoint)?; + } + return Ok(VcpuStopReason::HwBp); + } + + // mshv does not provide a way to specify which exception triggered the + // vCPU exit as the mshv intercepts both #DB and #BP + // We check against the SW breakpoints Hashmap to detect whether the + // vCPU exited due to a SW breakpoint + if self.sw_breakpoints.contains_key(&gpa) { + return Ok(VcpuStopReason::SwBp); + } + + Ok(VcpuStopReason::Unknown) + } +} + +impl GuestDebug for MshvDebug { + type Vcpu = VcpuFd; + + fn is_hw_breakpoint(&self, addr: &u64) -> bool { + self.hw_breakpoints.contains(addr) + } + fn is_sw_breakpoint(&self, addr: &u64) -> bool { + self.sw_breakpoints.contains_key(addr) + } + fn save_hw_breakpoint(&mut self, addr: &u64) -> bool { + if self.hw_breakpoints.len() >= MAX_NO_OF_HW_BP { + false + } else { + self.hw_breakpoints.push(*addr); + + true + } + } + fn save_sw_breakpoint_data(&mut self, addr: u64, data: [u8; 1]) { + _ = self.sw_breakpoints.insert(addr, data); + } + fn delete_hw_breakpoint(&mut self, addr: &u64) { + self.hw_breakpoints.retain(|&a| a != *addr); + } + fn delete_sw_breakpoint_data(&mut self, addr: &u64) -> Option<[u8; 1]> { + self.sw_breakpoints.remove(addr) + } + + fn read_regs(&self, vcpu_fd: &Self::Vcpu, regs: &mut X86_64Regs) -> Result<()> { + log::debug!("Read registers"); + let vcpu_regs = vcpu_fd + .get_regs() + .map_err(|e| new_error!("Could not read guest registers: {:?}", e))?; + + regs.rax = vcpu_regs.rax; + regs.rbx = vcpu_regs.rbx; + regs.rcx = vcpu_regs.rcx; + regs.rdx = vcpu_regs.rdx; + regs.rsi = vcpu_regs.rsi; + regs.rdi = vcpu_regs.rdi; + regs.rbp = vcpu_regs.rbp; + regs.rsp = vcpu_regs.rsp; + regs.r8 = vcpu_regs.r8; + regs.r9 = vcpu_regs.r9; + regs.r10 = vcpu_regs.r10; + regs.r11 = vcpu_regs.r11; + regs.r12 = vcpu_regs.r12; + regs.r13 = vcpu_regs.r13; + regs.r14 = vcpu_regs.r14; + regs.r15 = vcpu_regs.r15; + + regs.rip = vcpu_regs.rip; + regs.rflags = vcpu_regs.rflags; + + Ok(()) + } + + fn set_single_step(&mut self, vcpu_fd: &Self::Vcpu, enable: bool) -> Result<()> { + self.set_debug_config(vcpu_fd, enable) + } + + fn translate_gva(&self, vcpu_fd: &Self::Vcpu, gva: u64) -> Result { + let flags = (HV_TRANSLATE_GVA_VALIDATE_READ | HV_TRANSLATE_GVA_VALIDATE_WRITE) as u64; + let (addr, _) = vcpu_fd + .translate_gva(gva, flags) + .map_err(|_| HyperlightError::TranslateGuestAddress(gva))?; + + Ok(addr) + } + + fn write_regs(&self, vcpu_fd: &Self::Vcpu, regs: &X86_64Regs) -> Result<()> { + log::debug!("Write registers"); + let new_regs = StandardRegisters { + rax: regs.rax, + rbx: regs.rbx, + rcx: regs.rcx, + rdx: regs.rdx, + rsi: regs.rsi, + rdi: regs.rdi, + rbp: regs.rbp, + rsp: regs.rsp, + r8: regs.r8, + r9: regs.r9, + r10: regs.r10, + r11: regs.r11, + r12: regs.r12, + r13: regs.r13, + r14: regs.r14, + r15: regs.r15, + + rip: regs.rip, + rflags: regs.rflags, + }; + + vcpu_fd + .set_regs(&new_regs) + .map_err(|e| new_error!("Could not write guest registers: {:?}", e)) + } +} diff --git a/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs b/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs index 51bca3d19..a25098bf4 100644 --- a/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs +++ b/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs @@ -32,7 +32,7 @@ use gdbstub_arch::x86::X86_64_SSE as GdbTargetArch; use super::{DebugCommChannel, DebugMsg, DebugResponse, GdbTargetError, X86_64Regs}; /// Gdbstub target used by the gdbstub crate to provide GDB protocol implementation -pub struct HyperlightSandboxTarget { +pub(crate) struct HyperlightSandboxTarget { /// Hypervisor communication channels hyp_conn: DebugCommChannel, /// Thread ID @@ -40,7 +40,7 @@ pub struct HyperlightSandboxTarget { } impl HyperlightSandboxTarget { - pub fn new(hyp_conn: DebugCommChannel, thread_id: u64) -> Self { + pub(crate) fn new(hyp_conn: DebugCommChannel, thread_id: u64) -> Self { HyperlightSandboxTarget { hyp_conn, thread_id, @@ -61,23 +61,18 @@ impl HyperlightSandboxTarget { } /// Returns the thread ID - pub fn get_thread_id(&self) -> u64 { + pub(crate) fn get_thread_id(&self) -> u64 { self.thread_id } /// Waits for a response over the communication channel - pub fn recv(&self) -> Result { + pub(crate) fn recv(&self) -> Result { self.hyp_conn.recv() } - /// Non-Blocking check for a response over the communication channel - pub fn try_recv(&self) -> Result { - self.hyp_conn.try_recv() - } - /// Sends an event to the Hypervisor that tells it to resume vCPU execution /// Note: The method waits for a confirmation message - pub fn resume_vcpu(&mut self) -> Result<(), GdbTargetError> { + pub(crate) fn resume_vcpu(&mut self) -> Result<(), GdbTargetError> { log::info!("Resume vCPU execution"); match self.send_command(DebugMsg::Continue)? { @@ -89,10 +84,15 @@ impl HyperlightSandboxTarget { } } + /// Non-Blocking check for a response over the communication channel + pub(crate) fn try_recv(&self) -> Result { + self.hyp_conn.try_recv() + } + /// Sends an event to the Hypervisor that tells it to disable debugging /// and continue executing /// Note: The method waits for a confirmation message - pub fn disable_debug(&mut self) -> Result<(), GdbTargetError> { + pub(crate) fn disable_debug(&mut self) -> Result<(), GdbTargetError> { log::info!("Disable debugging and resume execution"); match self.send_command(DebugMsg::DisableDebug)? { diff --git a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs index fbbeff222..8c419c18b 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs @@ -29,6 +29,12 @@ use std::fmt::{Debug, Formatter}; use log::error; #[cfg(mshv2)] use mshv_bindings::hv_message; +#[cfg(gdb)] +use mshv_bindings::{ + hv_intercept_parameters, hv_intercept_type_HV_INTERCEPT_TYPE_EXCEPTION, + hv_message_type_HVMSG_X64_EXCEPTION_INTERCEPT, mshv_install_intercept, + HV_INTERCEPT_ACCESS_MASK_EXECUTE, +}; use mshv_bindings::{ hv_message_type, hv_message_type_HVMSG_GPA_INTERCEPT, hv_message_type_HVMSG_UNMAPPED_GPA, hv_message_type_HVMSG_X64_HALT, hv_message_type_HVMSG_X64_IO_PORT_INTERCEPT, hv_register_assoc, @@ -45,6 +51,8 @@ use tracing::{instrument, Span}; use super::fpu::{FP_CONTROL_WORD_DEFAULT, FP_TAG_WORD_DEFAULT, MXCSR_DEFAULT}; #[cfg(gdb)] +use super::gdb::{DebugCommChannel, DebugMsg, DebugResponse, GuestDebug, MshvDebug}; +#[cfg(gdb)] use super::handlers::DbgMemAccessHandlerWrapper; use super::handlers::{MemAccessHandlerWrapper, OutBHandlerWrapper}; use super::{ @@ -55,8 +63,210 @@ use crate::hypervisor::hypervisor_handler::HypervisorHandler; use crate::hypervisor::HyperlightExit; use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags}; use crate::mem::ptr::{GuestPtr, RawPtr}; +#[cfg(gdb)] +use crate::HyperlightError; use crate::{log_then_return, new_error, Result}; +#[cfg(gdb)] +mod debug { + use std::sync::{Arc, Mutex}; + + use super::{HypervLinuxDriver, *}; + use crate::hypervisor::gdb::{DebugMsg, DebugResponse, VcpuStopReason, X86_64Regs}; + use crate::hypervisor::handlers::DbgMemAccessHandlerCaller; + use crate::{new_error, Result}; + + impl HypervLinuxDriver { + /// Resets the debug information to disable debugging + fn disable_debug(&mut self) -> Result<()> { + let mut debug = MshvDebug::default(); + + debug.set_single_step(&self.vcpu_fd, false)?; + + self.debug = Some(debug); + + Ok(()) + } + + /// Get the reason the vCPU has stopped + pub(crate) fn get_stop_reason(&mut self) -> Result { + let debug = self + .debug + .as_mut() + .ok_or_else(|| new_error!("Debug is not enabled"))?; + + debug.get_stop_reason(&self.vcpu_fd, self.entrypoint) + } + + pub(crate) fn process_dbg_request( + &mut self, + req: DebugMsg, + dbg_mem_access_fn: Arc>, + ) -> Result { + if let Some(debug) = self.debug.as_mut() { + match req { + DebugMsg::AddHwBreakpoint(addr) => Ok(DebugResponse::AddHwBreakpoint( + debug + .add_hw_breakpoint(&self.vcpu_fd, addr) + .map_err(|e| { + log::error!("Failed to add hw breakpoint: {:?}", e); + + e + }) + .is_ok(), + )), + DebugMsg::AddSwBreakpoint(addr) => Ok(DebugResponse::AddSwBreakpoint( + debug + .add_sw_breakpoint(&self.vcpu_fd, addr, dbg_mem_access_fn) + .map_err(|e| { + log::error!("Failed to add sw breakpoint: {:?}", e); + + e + }) + .is_ok(), + )), + DebugMsg::Continue => { + debug.set_single_step(&self.vcpu_fd, false).map_err(|e| { + log::error!("Failed to continue execution: {:?}", e); + + e + })?; + + Ok(DebugResponse::Continue) + } + DebugMsg::DisableDebug => { + self.disable_debug().map_err(|e| { + log::error!("Failed to disable debugging: {:?}", e); + + e + })?; + + Ok(DebugResponse::DisableDebug) + } + DebugMsg::GetCodeSectionOffset => { + let offset = dbg_mem_access_fn + .try_lock() + .map_err(|e| { + new_error!("Error locking at {}:{}: {}", file!(), line!(), e) + })? + .get_code_offset() + .map_err(|e| { + log::error!("Failed to get code offset: {:?}", e); + + e + })?; + + Ok(DebugResponse::GetCodeSectionOffset(offset as u64)) + } + DebugMsg::ReadAddr(addr, len) => { + let mut data = vec![0u8; len]; + + debug + .read_addrs(&self.vcpu_fd, addr, &mut data, dbg_mem_access_fn) + .map_err(|e| { + log::error!("Failed to read from address: {:?}", e); + + e + })?; + + Ok(DebugResponse::ReadAddr(data)) + } + DebugMsg::ReadRegisters => { + let mut regs = X86_64Regs::default(); + + debug + .read_regs(&self.vcpu_fd, &mut regs) + .map_err(|e| { + log::error!("Failed to read registers: {:?}", e); + + e + }) + .map(|_| DebugResponse::ReadRegisters(regs)) + } + DebugMsg::RemoveHwBreakpoint(addr) => Ok(DebugResponse::RemoveHwBreakpoint( + debug + .remove_hw_breakpoint(&self.vcpu_fd, addr) + .map_err(|e| { + log::error!("Failed to remove hw breakpoint: {:?}", e); + + e + }) + .is_ok(), + )), + DebugMsg::RemoveSwBreakpoint(addr) => Ok(DebugResponse::RemoveSwBreakpoint( + debug + .remove_sw_breakpoint(&self.vcpu_fd, addr, dbg_mem_access_fn) + .map_err(|e| { + log::error!("Failed to remove sw breakpoint: {:?}", e); + + e + }) + .is_ok(), + )), + DebugMsg::Step => { + debug.set_single_step(&self.vcpu_fd, true).map_err(|e| { + log::error!("Failed to enable step instruction: {:?}", e); + + e + })?; + + Ok(DebugResponse::Step) + } + DebugMsg::WriteAddr(addr, data) => { + debug + .write_addrs(&self.vcpu_fd, addr, &data, dbg_mem_access_fn) + .map_err(|e| { + log::error!("Failed to write to address: {:?}", e); + + e + })?; + + Ok(DebugResponse::WriteAddr) + } + DebugMsg::WriteRegisters(regs) => debug + .write_regs(&self.vcpu_fd, ®s) + .map_err(|e| { + log::error!("Failed to write registers: {:?}", e); + + e + }) + .map(|_| DebugResponse::WriteRegisters), + } + } else { + Err(new_error!("Debugging is not enabled")) + } + } + + pub(crate) fn recv_dbg_msg(&mut self) -> Result { + let gdb_conn = self + .gdb_conn + .as_mut() + .ok_or_else(|| new_error!("Debug is not enabled"))?; + + gdb_conn.recv().map_err(|e| { + new_error!( + "Got an error while waiting to receive a + message: {:?}", + e + ) + }) + } + + pub(crate) fn send_dbg_msg(&mut self, cmd: DebugResponse) -> Result<()> { + log::debug!("Sending {:?}", cmd); + + let gdb_conn = self + .gdb_conn + .as_mut() + .ok_or_else(|| new_error!("Debug is not enabled"))?; + + gdb_conn + .send(cmd) + .map_err(|e| new_error!("Got an error while sending a response message {:?}", e)) + } + } +} + /// Determine whether the HyperV for Linux hypervisor API is present /// and functional. #[instrument(skip_all, parent = Span::current(), level = "Trace")] @@ -84,6 +294,11 @@ pub(super) struct HypervLinuxDriver { entrypoint: u64, mem_regions: Vec, orig_rsp: GuestPtr, + + #[cfg(gdb)] + debug: Option, + #[cfg(gdb)] + gdb_conn: Option>, } impl HypervLinuxDriver { @@ -101,6 +316,7 @@ impl HypervLinuxDriver { entrypoint_ptr: GuestPtr, rsp_ptr: GuestPtr, pml4_ptr: GuestPtr, + #[cfg(gdb)] gdb_conn: Option>, ) -> Result { let mshv = Mshv::new()?; let pr = Default::default(); @@ -124,6 +340,43 @@ impl HypervLinuxDriver { let mut vcpu_fd = vm_fd.create_vcpu(0)?; + #[cfg(gdb)] + let (debug, gdb_conn) = if let Some(gdb_conn) = gdb_conn { + let mut debug = MshvDebug::new(); + debug.add_hw_breakpoint(&vcpu_fd, entrypoint_ptr.absolute()?)?; + + // The bellow intercepts make the vCPU exit with the Exception Intercept exit code + // Check Table 6-1. Exceptions and Interrupts at Page 6-13 Vol. 1 + // of Intel 64 and IA-32 Architectures Software Developer's Manual + // Install intercept for #DB (1) exception + vm_fd + .install_intercept(mshv_install_intercept { + access_type_mask: HV_INTERCEPT_ACCESS_MASK_EXECUTE, + intercept_type: hv_intercept_type_HV_INTERCEPT_TYPE_EXCEPTION, + // Exception handler #DB (1) + intercept_parameter: hv_intercept_parameters { + exception_vector: 0x1, + }, + }) + .map_err(|e| new_error!("Cannot install debug exception intercept: {}", e))?; + + // Install intercept for #BP (3) exception + vm_fd + .install_intercept(mshv_install_intercept { + access_type_mask: HV_INTERCEPT_ACCESS_MASK_EXECUTE, + intercept_type: hv_intercept_type_HV_INTERCEPT_TYPE_EXCEPTION, + // Exception handler #BP (3) + intercept_parameter: hv_intercept_parameters { + exception_vector: 0x3, + }, + }) + .map_err(|e| new_error!("Cannot install breakpoint exception intercept: {}", e))?; + + (Some(debug), Some(gdb_conn)) + } else { + (None, None) + }; + mem_regions.iter().try_for_each(|region| { let mshv_region = region.to_owned().into(); vm_fd.map_user_memory(mshv_region) @@ -138,6 +391,11 @@ impl HypervLinuxDriver { mem_regions, entrypoint: entrypoint_ptr.absolute()?, orig_rsp: rsp_ptr, + + #[cfg(gdb)] + debug, + #[cfg(gdb)] + gdb_conn, }) } @@ -307,6 +565,8 @@ impl Hypervisor for HypervLinuxDriver { hv_message_type_HVMSG_X64_IO_PORT_INTERCEPT; const UNMAPPED_GPA_MESSAGE: hv_message_type = hv_message_type_HVMSG_UNMAPPED_GPA; const INVALID_GPA_ACCESS_MESSAGE: hv_message_type = hv_message_type_HVMSG_GPA_INTERCEPT; + #[cfg(gdb)] + const EXCEPTION_INTERCEPT: hv_message_type = hv_message_type_HVMSG_X64_EXCEPTION_INTERCEPT; #[cfg(mshv2)] let run_result = { @@ -364,6 +624,15 @@ impl Hypervisor for HypervLinuxDriver { None => HyperlightExit::Mmio(gpa), } } + // The only case an intercept exit is expected is when debugging is enabled + // and the intercepts are installed + #[cfg(gdb)] + EXCEPTION_INTERCEPT => match self.get_stop_reason() { + Ok(reason) => HyperlightExit::Debug(reason), + Err(e) => { + log_then_return!("Error getting stop reason: {:?}", e); + } + }, other => { crate::debug!("mshv Other Exit: Exit: {:#?} \n {:#?}", other, &self); log_then_return!("unknown Hyper-V run message type {:?}", other); @@ -391,6 +660,51 @@ impl Hypervisor for HypervLinuxDriver { fn get_memory_regions(&self) -> &[MemoryRegion] { &self.mem_regions } + + #[cfg(gdb)] + fn handle_debug( + &mut self, + dbg_mem_access_fn: std::sync::Arc< + std::sync::Mutex, + >, + stop_reason: super::gdb::VcpuStopReason, + ) -> Result<()> { + self.send_dbg_msg(DebugResponse::VcpuStopped(stop_reason)) + .map_err(|e| new_error!("Couldn't signal vCPU stopped event to GDB thread: {:?}", e))?; + + loop { + log::debug!("Debug wait for event to resume vCPU"); + + // Wait for a message from gdb + let req = self.recv_dbg_msg()?; + + let result = self.process_dbg_request(req, dbg_mem_access_fn.clone()); + + let response = match result { + Ok(response) => response, + // Treat non fatal errors separately so the guest doesn't fail + Err(HyperlightError::TranslateGuestAddress(_)) => DebugResponse::ErrorOccurred, + Err(e) => { + return Err(e); + } + }; + + // If the command was either step or continue, we need to run the vcpu + let cont = matches!( + response, + DebugResponse::Step | DebugResponse::Continue | DebugResponse::DisableDebug + ); + + self.send_dbg_msg(response) + .map_err(|e| new_error!("Couldn't send response to gdb: {:?}", e))?; + + if cont { + break; + } + } + + Ok(()) + } } impl Drop for HypervLinuxDriver { @@ -457,6 +771,14 @@ mod tests { MemoryRegionFlags::READ | MemoryRegionFlags::WRITE | MemoryRegionFlags::EXECUTE, crate::mem::memory_region::MemoryRegionType::Code, ); - super::HypervLinuxDriver::new(regions.build(), entrypoint_ptr, rsp_ptr, pml4_ptr).unwrap(); + super::HypervLinuxDriver::new( + regions.build(), + entrypoint_ptr, + rsp_ptr, + pml4_ptr, + #[cfg(gdb)] + None, + ) + .unwrap(); } } diff --git a/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs b/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs index f602b840e..4e49a66c0 100644 --- a/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs +++ b/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs @@ -943,6 +943,8 @@ fn set_up_hypervisor_partition( entrypoint_ptr, rsp_ptr, pml4_ptr, + #[cfg(gdb)] + gdb_conn, )?; Ok(Box::new(hv)) } diff --git a/src/hyperlight_host/src/hypervisor/kvm.rs b/src/hyperlight_host/src/hypervisor/kvm.rs index 5d01ec8ac..67cc8093c 100644 --- a/src/hyperlight_host/src/hypervisor/kvm.rs +++ b/src/hyperlight_host/src/hypervisor/kvm.rs @@ -26,7 +26,7 @@ use tracing::{instrument, Span}; use super::fpu::{FP_CONTROL_WORD_DEFAULT, FP_TAG_WORD_DEFAULT, MXCSR_DEFAULT}; #[cfg(gdb)] -use super::gdb::{DebugCommChannel, DebugMsg, DebugResponse, VcpuStopReason}; +use super::gdb::{DebugCommChannel, DebugMsg, DebugResponse, GuestDebug, KvmDebug, VcpuStopReason}; #[cfg(gdb)] use super::handlers::DbgMemAccessHandlerWrapper; use super::handlers::{MemAccessHandlerWrapper, OutBHandlerWrapper}; @@ -65,481 +65,182 @@ pub(crate) fn is_hypervisor_present() -> bool { #[cfg(gdb)] mod debug { - use std::collections::HashMap; use std::sync::{Arc, Mutex}; - use hyperlight_common::mem::PAGE_SIZE; - use kvm_bindings::{ - kvm_guest_debug, kvm_regs, KVM_GUESTDBG_ENABLE, KVM_GUESTDBG_SINGLESTEP, - KVM_GUESTDBG_USE_HW_BP, KVM_GUESTDBG_USE_SW_BP, - }; - use kvm_ioctls::VcpuFd; + use kvm_bindings::kvm_debug_exit_arch; use super::KVMDriver; - use crate::hypervisor::gdb::{DebugMsg, DebugResponse, VcpuStopReason, X86_64Regs}; + use crate::hypervisor::gdb::{ + DebugMsg, DebugResponse, GuestDebug, KvmDebug, VcpuStopReason, X86_64Regs, + }; use crate::hypervisor::handlers::DbgMemAccessHandlerCaller; - use crate::mem::layout::SandboxMemoryLayout; - use crate::{new_error, HyperlightError, Result}; - - /// Software Breakpoint size in memory - pub const SW_BP_SIZE: usize = 1; - /// Software Breakpoint opcode - const SW_BP_OP: u8 = 0xCC; - /// Software Breakpoint written to memory - pub const SW_BP: [u8; SW_BP_SIZE] = [SW_BP_OP]; - - /// KVM Debug struct - /// This struct is used to abstract the internal details of the kvm - /// guest debugging settings - #[derive(Default)] - pub struct KvmDebug { - /// vCPU stepping state - single_step: bool, - - /// Array of addresses for HW breakpoints - hw_breakpoints: Vec, - /// Saves the bytes modified to enable SW breakpoints - sw_breakpoints: HashMap, - - /// Sent to KVM for enabling guest debug - pub dbg_cfg: kvm_guest_debug, - } - - impl KvmDebug { - const MAX_NO_OF_HW_BP: usize = 4; - - pub fn new() -> Self { - let dbg = kvm_guest_debug { - control: KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP, - ..Default::default() - }; - - Self { - single_step: false, - hw_breakpoints: vec![], - sw_breakpoints: HashMap::new(), - dbg_cfg: dbg, - } - } - - /// This method sets the kvm debugreg fields to enable breakpoints at - /// specific addresses - /// - /// The first 4 debug registers are used to set the addresses - /// The 4th and 5th debug registers are obsolete and not used - /// The 7th debug register is used to enable the breakpoints - /// For more information see: DEBUG REGISTERS chapter in the architecture - /// manual - fn set_debug_config(&mut self, vcpu_fd: &VcpuFd, step: bool) -> Result<()> { - let addrs = &self.hw_breakpoints; - - self.dbg_cfg.arch.debugreg = [0; 8]; - for (k, addr) in addrs.iter().enumerate() { - self.dbg_cfg.arch.debugreg[k] = *addr; - self.dbg_cfg.arch.debugreg[7] |= 1 << (k * 2); - } - - if !addrs.is_empty() { - self.dbg_cfg.control |= KVM_GUESTDBG_USE_HW_BP; - } else { - self.dbg_cfg.control &= !KVM_GUESTDBG_USE_HW_BP; - } - - if step { - self.dbg_cfg.control |= KVM_GUESTDBG_SINGLESTEP; - } else { - self.dbg_cfg.control &= !KVM_GUESTDBG_SINGLESTEP; - } - - log::debug!("Setting bp: {:?} cfg: {:?}", addrs, self.dbg_cfg); - vcpu_fd - .set_guest_debug(&self.dbg_cfg) - .map_err(|e| new_error!("Could not set guest debug: {:?}", e))?; - - self.single_step = step; - - Ok(()) - } - - /// Method that adds a breakpoint - fn add_breakpoint(&mut self, vcpu_fd: &VcpuFd, addr: u64) -> Result { - if self.hw_breakpoints.len() >= Self::MAX_NO_OF_HW_BP { - Ok(false) - } else if self.hw_breakpoints.contains(&addr) { - Ok(true) - } else { - self.hw_breakpoints.push(addr); - self.set_debug_config(vcpu_fd, self.single_step)?; - - Ok(true) - } - } - - /// Method that removes a breakpoint - fn remove_breakpoint(&mut self, vcpu_fd: &VcpuFd, addr: u64) -> Result { - if self.hw_breakpoints.contains(&addr) { - self.hw_breakpoints.retain(|&a| a != addr); - self.set_debug_config(vcpu_fd, self.single_step)?; - - Ok(true) - } else { - Ok(false) - } - } - } + use crate::{new_error, Result}; impl KVMDriver { /// Resets the debug information to disable debugging fn disable_debug(&mut self) -> Result<()> { - self.debug = Some(KvmDebug::default()); + let mut debug = KvmDebug::default(); - self.set_single_step(false) - } - - /// Returns the instruction pointer from the stopped vCPU - fn get_instruction_pointer(&self) -> Result { - let regs = self - .vcpu_fd - .get_regs() - .map_err(|e| new_error!("Could not retrieve registers from vCPU: {:?}", e))?; + debug.set_single_step(&self.vcpu_fd, false)?; - Ok(regs.rip) - } - - /// Sets or clears stepping for vCPU - fn set_single_step(&mut self, enable: bool) -> Result<()> { - let debug = self - .debug - .as_mut() - .ok_or_else(|| new_error!("Debug is not enabled"))?; - - debug.set_debug_config(&self.vcpu_fd, enable) - } - - /// Translates the guest address to physical address - fn translate_gva(&self, gva: u64) -> Result { - let tr = self - .vcpu_fd - .translate_gva(gva) - .map_err(|_| HyperlightError::TranslateGuestAddress(gva))?; - - if tr.valid == 0 { - Err(HyperlightError::TranslateGuestAddress(gva)) - } else { - Ok(tr.physical_address) - } - } - - fn read_addrs( - &mut self, - mut gva: u64, - mut data: &mut [u8], - dbg_mem_access_fn: Arc>, - ) -> Result<()> { - let data_len = data.len(); - log::debug!("Read addr: {:X} len: {:X}", gva, data_len); - - while !data.is_empty() { - let gpa = self.translate_gva(gva)?; - - let read_len = std::cmp::min( - data.len(), - (PAGE_SIZE - (gpa & (PAGE_SIZE - 1))).try_into().unwrap(), - ); - let offset = gpa as usize - SandboxMemoryLayout::BASE_ADDRESS; - - dbg_mem_access_fn - .try_lock() - .map_err(|e| new_error!("Error locking at {}:{}: {}", file!(), line!(), e))? - .read(offset, &mut data[..read_len])?; - - data = &mut data[read_len..]; - gva += read_len as u64; - } - - Ok(()) - } - - fn write_addrs( - &mut self, - mut gva: u64, - mut data: &[u8], - dbg_mem_access_fn: Arc>, - ) -> Result<()> { - let data_len = data.len(); - log::debug!("Write addr: {:X} len: {:X}", gva, data_len); - - while !data.is_empty() { - let gpa = self.translate_gva(gva)?; - - let write_len = std::cmp::min( - data.len(), - (PAGE_SIZE - (gpa & (PAGE_SIZE - 1))).try_into().unwrap(), - ); - let offset = gpa as usize - SandboxMemoryLayout::BASE_ADDRESS; - - dbg_mem_access_fn - .try_lock() - .map_err(|e| new_error!("Error locking at {}:{}: {}", file!(), line!(), e))? - .write(offset, data)?; - - data = &data[write_len..]; - gva += write_len as u64; - } - - Ok(()) - } - - fn read_regs(&self, regs: &mut X86_64Regs) -> Result<()> { - log::debug!("Read registers"); - let vcpu_regs = self - .vcpu_fd - .get_regs() - .map_err(|e| new_error!("Could not read guest registers: {:?}", e))?; - - regs.rax = vcpu_regs.rax; - regs.rbx = vcpu_regs.rbx; - regs.rcx = vcpu_regs.rcx; - regs.rdx = vcpu_regs.rdx; - regs.rsi = vcpu_regs.rsi; - regs.rdi = vcpu_regs.rdi; - regs.rbp = vcpu_regs.rbp; - regs.rsp = vcpu_regs.rsp; - regs.r8 = vcpu_regs.r8; - regs.r9 = vcpu_regs.r9; - regs.r10 = vcpu_regs.r10; - regs.r11 = vcpu_regs.r11; - regs.r12 = vcpu_regs.r12; - regs.r13 = vcpu_regs.r13; - regs.r14 = vcpu_regs.r14; - regs.r15 = vcpu_regs.r15; - - regs.rip = vcpu_regs.rip; - regs.rflags = vcpu_regs.rflags; + self.debug = Some(debug); Ok(()) } - fn write_regs(&self, regs: &X86_64Regs) -> Result<()> { - log::debug!("Write registers"); - let new_regs = kvm_regs { - rax: regs.rax, - rbx: regs.rbx, - rcx: regs.rcx, - rdx: regs.rdx, - rsi: regs.rsi, - rdi: regs.rdi, - rbp: regs.rbp, - rsp: regs.rsp, - r8: regs.r8, - r9: regs.r9, - r10: regs.r10, - r11: regs.r11, - r12: regs.r12, - r13: regs.r13, - r14: regs.r14, - r15: regs.r15, - - rip: regs.rip, - rflags: regs.rflags, - }; - - self.vcpu_fd - .set_regs(&new_regs) - .map_err(|e| new_error!("Could not write guest registers: {:?}", e)) - } - - fn add_hw_breakpoint(&mut self, addr: u64) -> Result { - let addr = self.translate_gva(addr)?; - - if let Some(debug) = self.debug.as_mut() { - debug.add_breakpoint(&self.vcpu_fd, addr) - } else { - Ok(false) - } - } - - fn remove_hw_breakpoint(&mut self, addr: u64) -> Result { - let addr = self.translate_gva(addr)?; - - if let Some(debug) = self.debug.as_mut() { - debug.remove_breakpoint(&self.vcpu_fd, addr) - } else { - Ok(false) - } - } - - fn add_sw_breakpoint( - &mut self, - addr: u64, - dbg_mem_access_fn: Arc>, - ) -> Result { - let addr = { - let debug = self - .debug - .as_ref() - .ok_or_else(|| new_error!("Debug is not enabled"))?; - let addr = self.translate_gva(addr)?; - if debug.sw_breakpoints.contains_key(&addr) { - return Ok(true); - } - - addr - }; - - let mut save_data = [0; SW_BP_SIZE]; - self.read_addrs(addr, &mut save_data[..], dbg_mem_access_fn.clone())?; - self.write_addrs(addr, &SW_BP, dbg_mem_access_fn)?; - - { - let debug = self - .debug - .as_mut() - .ok_or_else(|| new_error!("Debug is not enabled"))?; - debug.sw_breakpoints.insert(addr, save_data); - } - - Ok(true) - } - - fn remove_sw_breakpoint( - &mut self, - addr: u64, - dbg_mem_access_fn: Arc>, - ) -> Result { - let (ret, data) = { - let addr = self.translate_gva(addr)?; - let debug = self - .debug - .as_mut() - .ok_or_else(|| new_error!("Debug is not enabled"))?; - - if debug.sw_breakpoints.contains_key(&addr) { - let save_data = debug - .sw_breakpoints - .remove(&addr) - .ok_or_else(|| new_error!("Expected the hashmap to contain the address"))?; - - (true, Some(save_data)) - } else { - (false, None) - } - }; - - if ret { - self.write_addrs(addr, &data.unwrap(), dbg_mem_access_fn)?; - } - - Ok(ret) - } - - /// Gdb expects the target to be stopped when connected. - /// This method provides a way to set a breakpoint at the entry point - /// it does not keep this breakpoint set after the vCPU already stopped at the address - pub fn set_entrypoint_bp(&self) -> Result<()> { - if self.debug.is_some() { - log::debug!("Setting entrypoint bp {:X}", self.entrypoint); - let mut entrypoint_debug = KvmDebug::new(); - entrypoint_debug.add_breakpoint(&self.vcpu_fd, self.entrypoint)?; - - Ok(()) - } else { - Ok(()) - } - } - /// Get the reason the vCPU has stopped - pub fn get_stop_reason(&self) -> Result { + pub(crate) fn get_stop_reason( + &mut self, + debug_exit: kvm_debug_exit_arch, + ) -> Result { let debug = self .debug - .as_ref() + .as_mut() .ok_or_else(|| new_error!("Debug is not enabled"))?; - if debug.single_step { - return Ok(VcpuStopReason::DoneStep); - } - - let ip = self.get_instruction_pointer()?; - let gpa = self.translate_gva(ip)?; - if debug.sw_breakpoints.contains_key(&gpa) { - return Ok(VcpuStopReason::SwBp); - } - - if debug.hw_breakpoints.contains(&gpa) { - return Ok(VcpuStopReason::HwBp); - } - - if ip == self.entrypoint { - return Ok(VcpuStopReason::HwBp); - } - - Ok(VcpuStopReason::Unknown) + debug.get_stop_reason(&self.vcpu_fd, debug_exit, self.entrypoint) } - pub fn process_dbg_request( + pub(crate) fn process_dbg_request( &mut self, req: DebugMsg, dbg_mem_access_fn: Arc>, ) -> Result { - match req { - DebugMsg::AddHwBreakpoint(addr) => self - .add_hw_breakpoint(addr) - .map(DebugResponse::AddHwBreakpoint), - DebugMsg::AddSwBreakpoint(addr) => self - .add_sw_breakpoint(addr, dbg_mem_access_fn) - .map(DebugResponse::AddSwBreakpoint), - DebugMsg::Continue => { - self.set_single_step(false)?; - Ok(DebugResponse::Continue) - } - DebugMsg::DisableDebug => { - self.disable_debug()?; - - Ok(DebugResponse::DisableDebug) - } - DebugMsg::GetCodeSectionOffset => { - let offset = dbg_mem_access_fn - .try_lock() - .map_err(|e| new_error!("Error locking at {}:{}: {}", file!(), line!(), e))? - .get_code_offset()?; - - Ok(DebugResponse::GetCodeSectionOffset(offset as u64)) - } - DebugMsg::ReadAddr(addr, len) => { - let mut data = vec![0u8; len]; - - self.read_addrs(addr, &mut data, dbg_mem_access_fn)?; - - Ok(DebugResponse::ReadAddr(data)) - } - DebugMsg::ReadRegisters => { - let mut regs = X86_64Regs::default(); - - self.read_regs(&mut regs) - .map(|_| DebugResponse::ReadRegisters(regs)) - } - DebugMsg::RemoveHwBreakpoint(addr) => self - .remove_hw_breakpoint(addr) - .map(DebugResponse::RemoveHwBreakpoint), - DebugMsg::RemoveSwBreakpoint(addr) => self - .remove_sw_breakpoint(addr, dbg_mem_access_fn) - .map(DebugResponse::RemoveSwBreakpoint), - DebugMsg::Step => { - self.set_single_step(true)?; - Ok(DebugResponse::Step) - } - DebugMsg::WriteAddr(addr, data) => { - self.write_addrs(addr, &data, dbg_mem_access_fn)?; - - Ok(DebugResponse::WriteAddr) + if let Some(debug) = self.debug.as_mut() { + match req { + DebugMsg::AddHwBreakpoint(addr) => Ok(DebugResponse::AddHwBreakpoint( + debug + .add_hw_breakpoint(&self.vcpu_fd, addr) + .map_err(|e| { + log::error!("Failed to add hw breakpoint: {:?}", e); + + e + }) + .is_ok(), + )), + DebugMsg::AddSwBreakpoint(addr) => Ok(DebugResponse::AddSwBreakpoint( + debug + .add_sw_breakpoint(&self.vcpu_fd, addr, dbg_mem_access_fn) + .map_err(|e| { + log::error!("Failed to add sw breakpoint: {:?}", e); + + e + }) + .is_ok(), + )), + DebugMsg::Continue => { + debug.set_single_step(&self.vcpu_fd, false).map_err(|e| { + log::error!("Failed to continue execution: {:?}", e); + + e + })?; + + Ok(DebugResponse::Continue) + } + DebugMsg::DisableDebug => { + self.disable_debug().map_err(|e| { + log::error!("Failed to disable debugging: {:?}", e); + + e + })?; + + Ok(DebugResponse::DisableDebug) + } + DebugMsg::GetCodeSectionOffset => { + let offset = dbg_mem_access_fn + .try_lock() + .map_err(|e| { + new_error!("Error locking at {}:{}: {}", file!(), line!(), e) + })? + .get_code_offset() + .map_err(|e| { + log::error!("Failed to get code offset: {:?}", e); + + e + })?; + + Ok(DebugResponse::GetCodeSectionOffset(offset as u64)) + } + DebugMsg::ReadAddr(addr, len) => { + let mut data = vec![0u8; len]; + + debug + .read_addrs(&self.vcpu_fd, addr, &mut data, dbg_mem_access_fn) + .map_err(|e| { + log::error!("Failed to read from address: {:?}", e); + + e + })?; + + Ok(DebugResponse::ReadAddr(data)) + } + DebugMsg::ReadRegisters => { + let mut regs = X86_64Regs::default(); + + debug + .read_regs(&self.vcpu_fd, &mut regs) + .map_err(|e| { + log::error!("Failed to read registers: {:?}", e); + + e + }) + .map(|_| DebugResponse::ReadRegisters(regs)) + } + DebugMsg::RemoveHwBreakpoint(addr) => Ok(DebugResponse::RemoveHwBreakpoint( + debug + .remove_hw_breakpoint(&self.vcpu_fd, addr) + .map_err(|e| { + log::error!("Failed to remove hw breakpoint: {:?}", e); + + e + }) + .is_ok(), + )), + DebugMsg::RemoveSwBreakpoint(addr) => Ok(DebugResponse::RemoveSwBreakpoint( + debug + .remove_sw_breakpoint(&self.vcpu_fd, addr, dbg_mem_access_fn) + .map_err(|e| { + log::error!("Failed to remove sw breakpoint: {:?}", e); + + e + }) + .is_ok(), + )), + DebugMsg::Step => { + debug.set_single_step(&self.vcpu_fd, true).map_err(|e| { + log::error!("Failed to enable step instruction: {:?}", e); + + e + })?; + + Ok(DebugResponse::Step) + } + DebugMsg::WriteAddr(addr, data) => { + debug + .write_addrs(&self.vcpu_fd, addr, &data, dbg_mem_access_fn) + .map_err(|e| { + log::error!("Failed to write to address: {:?}", e); + + e + })?; + + Ok(DebugResponse::WriteAddr) + } + DebugMsg::WriteRegisters(regs) => debug + .write_regs(&self.vcpu_fd, ®s) + .map_err(|e| { + log::error!("Failed to write registers: {:?}", e); + + e + }) + .map(|_| DebugResponse::WriteRegisters), } - DebugMsg::WriteRegisters(regs) => self - .write_regs(®s) - .map(|_| DebugResponse::WriteRegisters), + } else { + Err(new_error!("Debugging is not enabled")) } } - pub fn recv_dbg_msg(&mut self) -> Result { + pub(crate) fn recv_dbg_msg(&mut self) -> Result { let gdb_conn = self .gdb_conn .as_mut() @@ -553,7 +254,7 @@ mod debug { }) } - pub fn send_dbg_msg(&mut self, cmd: DebugResponse) -> Result<()> { + pub(crate) fn send_dbg_msg(&mut self, cmd: DebugResponse) -> Result<()> { log::debug!("Sending {:?}", cmd); let gdb_conn = self @@ -581,7 +282,7 @@ pub(super) struct KVMDriver { mem_regions: Vec, #[cfg(gdb)] - debug: Option, + debug: Option, #[cfg(gdb)] gdb_conn: Option>, } @@ -625,7 +326,11 @@ impl KVMDriver { #[cfg(gdb)] let (debug, gdb_conn) = if let Some(gdb_conn) = gdb_conn { - (Some(debug::KvmDebug::new()), Some(gdb_conn)) + let mut debug = KvmDebug::new(); + // Add breakpoint to the entry point address + debug.add_hw_breakpoint(&vcpu_fd, entrypoint)?; + + (Some(debug), Some(gdb_conn)) } else { (None, None) }; @@ -646,9 +351,6 @@ impl KVMDriver { gdb_conn, }; - #[cfg(gdb)] - ret.set_entrypoint_bp()?; - Ok(ret) } @@ -837,7 +539,8 @@ impl Hypervisor for KVMDriver { } } #[cfg(gdb)] - Ok(VcpuExit::Debug(_)) => match self.get_stop_reason() { + // KVM provides architecture specific information about the vCPU state when exiting + Ok(VcpuExit::Debug(debug_exit)) => match self.get_stop_reason(debug_exit) { Ok(reason) => HyperlightExit::Debug(reason), Err(e) => { log_then_return!("Error getting stop reason: {:?}", e);