diff --git a/src/hyperlight_host/src/hypervisor/gdb/arch.rs b/src/hyperlight_host/src/hypervisor/gdb/arch.rs new file mode 100644 index 000000000..5938112e3 --- /dev/null +++ b/src/hyperlight_host/src/hypervisor/gdb/arch.rs @@ -0,0 +1,111 @@ +/* +Copyright 2024 The Hyperlight Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +//! This file contains architecture specific code for the x86_64 + +use std::collections::HashMap; + +use super::VcpuStopReason; + +// Described in Table 6-1. Exceptions and Interrupts at Page 6-13 Vol. 1 +// of Intel 64 and IA-32 Architectures Software Developer's Manual +/// Exception id for #DB +const DB_EX_ID: u32 = 1; +/// Exception id for #BP - triggered by the INT3 instruction +const BP_EX_ID: u32 = 3; + +/// Software Breakpoint size in memory +pub(crate) const SW_BP_SIZE: usize = 1; +/// Software Breakpoint opcode - INT3 +/// Check page 7-28 Vol. 3A of Intel 64 and IA-32 +/// Architectures Software Developer's Manual +pub(crate) const SW_BP_OP: u8 = 0xCC; +/// Software Breakpoint written to memory +pub(crate) const SW_BP: [u8; SW_BP_SIZE] = [SW_BP_OP]; +/// Maximum number of supported hardware breakpoints +pub(crate) const MAX_NO_OF_HW_BP: usize = 4; + +/// Check page 19-4 Vol. 3B of Intel 64 and IA-32 +/// Architectures Software Developer's Manual +/// Bit position of BS flag in DR6 debug register +pub(crate) const DR6_BS_FLAG_POS: usize = 14; +/// Bit mask of BS flag in DR6 debug register +pub(crate) const DR6_BS_FLAG_MASK: u64 = 1 << DR6_BS_FLAG_POS; +/// Bit position of HW breakpoints status in DR6 debug register +pub(crate) const DR6_HW_BP_FLAGS_POS: usize = 0; +/// Bit mask of HW breakpoints status in DR6 debug register +pub(crate) const DR6_HW_BP_FLAGS_MASK: u64 = 0x0F << DR6_HW_BP_FLAGS_POS; + +/// Determine the reason the vCPU stopped +/// This is done by checking the DR6 register and the exception id +/// NOTE: Additional checks are done for the entrypoint, stored hw_breakpoints +/// and sw_breakpoints to ensure the stop reason is valid with internal state +pub(crate) fn vcpu_stop_reason( + single_step: bool, + rip: u64, + dr6: u64, + entrypoint: u64, + exception: u32, + hw_breakpoints: &[u64], + sw_breakpoints: &HashMap, +) -> VcpuStopReason { + if DB_EX_ID == exception { + // If the BS flag in DR6 register is set, it means a single step + // instruction triggered the exit + // Check page 19-4 Vol. 3B of Intel 64 and IA-32 + // Architectures Software Developer's Manual + if dr6 & DR6_BS_FLAG_MASK != 0 && single_step { + return VcpuStopReason::DoneStep; + } + + // If any of the B0-B3 flags in DR6 register is set, it means a + // hardware breakpoint triggered the exit + // Check page 19-4 Vol. 3B of Intel 64 and IA-32 + // Architectures Software Developer's Manual + if DR6_HW_BP_FLAGS_MASK & dr6 != 0 && hw_breakpoints.contains(&rip) { + if rip == entrypoint { + return VcpuStopReason::EntryPointBp; + } + return VcpuStopReason::HwBp; + } + } + + if BP_EX_ID == exception && sw_breakpoints.contains_key(&rip) { + return VcpuStopReason::SwBp; + } + + // Log an error and provide internal debugging info + log::error!( + r"The vCPU exited because of an unknown reason: + single_step: {:?} + rip: {:?} + dr6: {:?} + entrypoint: {:?} + exception: {:?} + hw_breakpoints: {:?} + sw_breakpoints: {:?} + ", + single_step, + rip, + dr6, + entrypoint, + exception, + hw_breakpoints, + sw_breakpoints, + ); + + VcpuStopReason::Unknown +} diff --git a/src/hyperlight_host/src/hypervisor/gdb/event_loop.rs b/src/hyperlight_host/src/hypervisor/gdb/event_loop.rs index 72dd97b29..c21c66042 100644 --- a/src/hyperlight_host/src/hypervisor/gdb/event_loop.rs +++ b/src/hyperlight_host/src/hypervisor/gdb/event_loop.rs @@ -49,6 +49,7 @@ impl run_blocking::BlockingEventLoop for GdbBlockingEventLoop { // Resume execution if unknown reason for stop let stop_response = match stop_reason { VcpuStopReason::DoneStep => BaseStopReason::DoneStep, + VcpuStopReason::EntryPointBp => BaseStopReason::HwBreak(()), VcpuStopReason::SwBp => BaseStopReason::SwBreak(()), VcpuStopReason::HwBp => BaseStopReason::HwBreak(()), // This is a consequence of the GDB client sending an interrupt signal diff --git a/src/hyperlight_host/src/hypervisor/gdb/kvm_debug.rs b/src/hyperlight_host/src/hypervisor/gdb/kvm_debug.rs index 6e8f9d36f..354b385b3 100644 --- a/src/hyperlight_host/src/hypervisor/gdb/kvm_debug.rs +++ b/src/hyperlight_host/src/hypervisor/gdb/kvm_debug.rs @@ -22,15 +22,10 @@ use kvm_bindings::{ }; use kvm_ioctls::VcpuFd; -use super::{ - GuestDebug, VcpuStopReason, X86_64Regs, DR6_BS_FLAG_MASK, DR6_HW_BP_FLAGS_MASK, - MAX_NO_OF_HW_BP, SW_BP_SIZE, -}; +use super::arch::{vcpu_stop_reason, MAX_NO_OF_HW_BP, SW_BP_SIZE}; +use super::{GuestDebug, VcpuStopReason, X86_64Regs}; use crate::{new_error, HyperlightError, Result}; -/// Exception id for SW breakpoint -const SW_BP_ID: u32 = 3; - /// KVM Debug struct /// This struct is used to abstract the internal details of the kvm /// guest debugging settings @@ -118,51 +113,29 @@ impl KvmDebug { debug_exit: kvm_debug_exit_arch, entrypoint: u64, ) -> Result { - // If the BS flag in DR6 register is set, it means a single step - // instruction triggered the exit - // Check page 19-4 Vol. 3B of Intel 64 and IA-32 - // Architectures Software Developer's Manual - if debug_exit.dr6 & DR6_BS_FLAG_MASK != 0 && self.single_step { - return Ok(VcpuStopReason::DoneStep); - } + let rip = self.get_instruction_pointer(vcpu_fd)?; + let rip = self.translate_gva(vcpu_fd, rip)?; - let ip = self.get_instruction_pointer(vcpu_fd)?; - let gpa = self.translate_gva(vcpu_fd, ip)?; + // Check if the vCPU stopped because of a hardware breakpoint + let reason = vcpu_stop_reason( + self.single_step, + rip, + debug_exit.dr6, + entrypoint, + debug_exit.exception, + &self.hw_breakpoints, + &self.sw_breakpoints, + ); - // If any of the B0-B3 flags in DR6 register is set, it means a - // hardware breakpoint triggered the exit - // Check page 19-4 Vol. 3B of Intel 64 and IA-32 - // Architectures Software Developer's Manual - if DR6_HW_BP_FLAGS_MASK & debug_exit.dr6 != 0 && self.hw_breakpoints.contains(&gpa) { + if let VcpuStopReason::EntryPointBp = reason { // In case the hw breakpoint is the entry point, remove it to // avoid hanging here as gdb does not remove breakpoints it // has not set. // Gdb expects the target to be stopped when connected. - if gpa == entrypoint { - self.remove_hw_breakpoint(vcpu_fd, entrypoint)?; - } - return Ok(VcpuStopReason::HwBp); + self.remove_hw_breakpoint(vcpu_fd, entrypoint)?; } - // If the exception ID matches #BP (3) - it means a software breakpoint - // caused the exit - if SW_BP_ID == debug_exit.exception && self.sw_breakpoints.contains_key(&gpa) { - return Ok(VcpuStopReason::SwBp); - } - - // Log an error and provide internal debugging info for fixing - log::error!( - r"The vCPU exited because of an unknown reason: - kvm_debug_exit_arch: {:?} - single_step: {:?} - hw_breakpoints: {:?} - sw_breakpoints: {:?}", - debug_exit, - self.single_step, - self.hw_breakpoints, - self.sw_breakpoints, - ); - Ok(VcpuStopReason::Unknown) + Ok(reason) } } diff --git a/src/hyperlight_host/src/hypervisor/gdb/mod.rs b/src/hyperlight_host/src/hypervisor/gdb/mod.rs index 95b81ed6a..46c26d1dc 100644 --- a/src/hyperlight_host/src/hypervisor/gdb/mod.rs +++ b/src/hyperlight_host/src/hypervisor/gdb/mod.rs @@ -14,6 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ +mod arch; mod event_loop; #[cfg(kvm)] mod kvm_debug; @@ -26,6 +27,7 @@ use std::net::TcpListener; use std::sync::{Arc, Mutex}; use std::thread; +use arch::{SW_BP, SW_BP_SIZE}; use crossbeam_channel::{Receiver, Sender, TryRecvError}; use event_loop::event_loop_thread; use gdbstub::conn::ConnectionExt; @@ -43,28 +45,6 @@ use crate::hypervisor::handlers::DbgMemAccessHandlerCaller; use crate::mem::layout::SandboxMemoryLayout; use crate::{new_error, HyperlightError}; -/// Software Breakpoint size in memory -const SW_BP_SIZE: usize = 1; -/// Software Breakpoint opcode - INT3 -/// Check page 7-28 Vol. 3A of Intel 64 and IA-32 -/// Architectures Software Developer's Manual -const SW_BP_OP: u8 = 0xCC; -/// Software Breakpoint written to memory -const SW_BP: [u8; SW_BP_SIZE] = [SW_BP_OP]; -/// Maximum number of supported hardware breakpoints -const MAX_NO_OF_HW_BP: usize = 4; - -/// Check page 19-4 Vol. 3B of Intel 64 and IA-32 -/// Architectures Software Developer's Manual -/// Bit position of BS flag in DR6 debug register -const DR6_BS_FLAG_POS: usize = 14; -/// Bit mask of BS flag in DR6 debug register -const DR6_BS_FLAG_MASK: u64 = 1 << DR6_BS_FLAG_POS; -/// Bit position of HW breakpoints status in DR6 debug register -const DR6_HW_BP_FLAGS_POS: usize = 0; -/// Bit mask of HW breakpoints status in DR6 debug register -const DR6_HW_BP_FLAGS_MASK: u64 = 0x0F << DR6_HW_BP_FLAGS_POS; - #[derive(Debug, Error)] pub(crate) enum GdbTargetError { #[error("Error encountered while binding to address and port")] @@ -129,6 +109,10 @@ pub(crate) struct X86_64Regs { #[derive(Debug)] pub enum VcpuStopReason { DoneStep, + /// Hardware breakpoint inserted by the hypervisor so the guest can be stopped + /// at the entry point. This is used to avoid the guest from executing + /// the entry point code before the debugger is connected + EntryPointBp, HwBp, SwBp, Interrupt, diff --git a/src/hyperlight_host/src/hypervisor/gdb/mshv_debug.rs b/src/hyperlight_host/src/hypervisor/gdb/mshv_debug.rs index 036aadbd3..7d48fcae0 100644 --- a/src/hyperlight_host/src/hypervisor/gdb/mshv_debug.rs +++ b/src/hyperlight_host/src/hypervisor/gdb/mshv_debug.rs @@ -32,10 +32,8 @@ use mshv_bindings::{ }; use mshv_ioctls::VcpuFd; -use super::{ - GuestDebug, VcpuStopReason, X86_64Regs, DR6_BS_FLAG_MASK, DR6_HW_BP_FLAGS_MASK, - MAX_NO_OF_HW_BP, SW_BP_SIZE, -}; +use super::arch::{vcpu_stop_reason, MAX_NO_OF_HW_BP, SW_BP_SIZE}; +use super::{GuestDebug, VcpuStopReason, X86_64Regs}; use crate::{new_error, HyperlightError, Result}; #[derive(Debug, Default)] @@ -133,10 +131,9 @@ impl MshvDebug { pub(crate) fn get_stop_reason( &mut self, vcpu_fd: &VcpuFd, + exception: u16, entrypoint: u64, ) -> Result { - // MSHV does not provide info on the vCPU exits but the debug - // information can be retrieved from the DEBUG REGISTERS let regs = vcpu_fd .get_debug_regs() .map_err(|e| new_error!("Cannot retrieve debug registers from vCPU: {}", e))?; @@ -144,41 +141,28 @@ impl MshvDebug { // DR6 register contains debug state related information let debug_status = regs.dr6; - // If the BS flag in DR6 register is set, it means a single step - // instruction triggered the exit - // Check page 19-4 Vol. 3B of Intel 64 and IA-32 - // Architectures Software Developer's Manual - if debug_status & DR6_BS_FLAG_MASK != 0 && self.single_step { - return Ok(VcpuStopReason::DoneStep); - } + let rip = self.get_instruction_pointer(vcpu_fd)?; + let rip = self.translate_gva(vcpu_fd, rip)?; - let ip = self.get_instruction_pointer(vcpu_fd)?; - let gpa = self.translate_gva(vcpu_fd, ip)?; + let reason = vcpu_stop_reason( + self.single_step, + rip, + debug_status, + entrypoint, + exception as u32, + &self.hw_breakpoints, + &self.sw_breakpoints, + ); - // If any of the B0-B3 flags in DR6 register is set, it means a - // hardware breakpoint triggered the exit - // Check page 19-4 Vol. 3B of Intel 64 and IA-32 - // Architectures Software Developer's Manual - if debug_status & DR6_HW_BP_FLAGS_MASK != 0 && self.hw_breakpoints.contains(&gpa) { + if let VcpuStopReason::EntryPointBp = reason { // In case the hw breakpoint is the entry point, remove it to // avoid hanging here as gdb does not remove breakpoints it // has not set. // Gdb expects the target to be stopped when connected. - if gpa == entrypoint { - self.remove_hw_breakpoint(vcpu_fd, entrypoint)?; - } - return Ok(VcpuStopReason::HwBp); - } - - // mshv does not provide a way to specify which exception triggered the - // vCPU exit as the mshv intercepts both #DB and #BP - // We check against the SW breakpoints Hashmap to detect whether the - // vCPU exited due to a SW breakpoint - if self.sw_breakpoints.contains_key(&gpa) { - return Ok(VcpuStopReason::SwBp); + self.remove_hw_breakpoint(vcpu_fd, entrypoint)?; } - Ok(VcpuStopReason::Unknown) + Ok(reason) } } diff --git a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs index 8419cb073..3c0ff59d0 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs @@ -71,6 +71,7 @@ use crate::{log_then_return, new_error, Result}; mod debug { use std::sync::{Arc, Mutex}; + use super::mshv_bindings::hv_x64_exception_intercept_message; use super::{HypervLinuxDriver, *}; use crate::hypervisor::gdb::{DebugMsg, DebugResponse, VcpuStopReason, X86_64Regs}; use crate::hypervisor::handlers::DbgMemAccessHandlerCaller; @@ -89,13 +90,16 @@ mod debug { } /// Get the reason the vCPU has stopped - pub(crate) fn get_stop_reason(&mut self) -> Result { + pub(crate) fn get_stop_reason( + &mut self, + ex_info: hv_x64_exception_intercept_message, + ) -> Result { let debug = self .debug .as_mut() .ok_or_else(|| new_error!("Debug is not enabled"))?; - debug.get_stop_reason(&self.vcpu_fd, self.entrypoint) + debug.get_stop_reason(&self.vcpu_fd, ex_info.exception_vector, self.entrypoint) } pub(crate) fn process_dbg_request( @@ -626,14 +630,27 @@ impl Hypervisor for HypervLinuxDriver { } } // The only case an intercept exit is expected is when debugging is enabled - // and the intercepts are installed + // and the intercepts are installed. + // Provide the extra information about the exception to accurately determine + // the stop reason #[cfg(gdb)] - EXCEPTION_INTERCEPT => match self.get_stop_reason() { - Ok(reason) => HyperlightExit::Debug(reason), - Err(e) => { - log_then_return!("Error getting stop reason: {:?}", e); + EXCEPTION_INTERCEPT => { + // Extract exception info from the message so we can figure out + // more information about the vCPU state + let ex_info = match m.to_exception_info() { + Ok(info) => info, + Err(e) => { + log_then_return!("Error converting to exception info: {:?}", e); + } + }; + + match self.get_stop_reason(ex_info) { + Ok(reason) => HyperlightExit::Debug(reason), + Err(e) => { + log_then_return!("Error getting stop reason: {:?}", e); + } } - }, + } other => { crate::debug!("mshv Other Exit: Exit: {:#?} \n {:#?}", other, &self); log_then_return!("unknown Hyper-V run message type {:?}", other);