Gdb debug improvements

src/hyperlight_host/src/hypervisor/gdb/arch.rs

@@ -0,0 +1,111 @@
+/*
+Copyright 2024 The Hyperlight Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+//! This file contains architecture specific code for the x86_64
+
+use std::collections::HashMap;
+
+use super::VcpuStopReason;
+
+// Described in Table 6-1. Exceptions and Interrupts at Page 6-13 Vol. 1
+// of Intel 64 and IA-32 Architectures Software Developer's Manual
+/// Exception id for #DB
+const DB_EX_ID: u32 = 1;
+/// Exception id for #BP - triggered by the INT3 instruction
+const BP_EX_ID: u32 = 3;
+
+/// Software Breakpoint size in memory
+pub(crate) const SW_BP_SIZE: usize = 1;
+/// Software Breakpoint opcode - INT3
+/// Check page 7-28 Vol. 3A of Intel 64 and IA-32
+/// Architectures Software Developer's Manual
+pub(crate) const SW_BP_OP: u8 = 0xCC;
+/// Software Breakpoint written to memory
+pub(crate) const SW_BP: [u8; SW_BP_SIZE] = [SW_BP_OP];
+/// Maximum number of supported hardware breakpoints
+pub(crate) const MAX_NO_OF_HW_BP: usize = 4;
+
+/// Check page 19-4 Vol. 3B of Intel 64 and IA-32
+/// Architectures Software Developer's Manual
+/// Bit position of BS flag in DR6 debug register
+pub(crate) const DR6_BS_FLAG_POS: usize = 14;
+/// Bit mask of BS flag in DR6 debug register
+pub(crate) const DR6_BS_FLAG_MASK: u64 = 1 << DR6_BS_FLAG_POS;
+/// Bit position of HW breakpoints status in DR6 debug register
+pub(crate) const DR6_HW_BP_FLAGS_POS: usize = 0;
+/// Bit mask of HW breakpoints status in DR6 debug register
+pub(crate) const DR6_HW_BP_FLAGS_MASK: u64 = 0x0F << DR6_HW_BP_FLAGS_POS;
+
+/// Determine the reason the vCPU stopped
+/// This is done by checking the DR6 register and the exception id
+/// NOTE: Additional checks are done for the entrypoint, stored hw_breakpoints
+/// and sw_breakpoints to ensure the stop reason is valid with internal state
+pub(crate) fn vcpu_stop_reason(
+    single_step: bool,
+    rip: u64,
+    dr6: u64,
+    entrypoint: u64,
+    exception: u32,
+    hw_breakpoints: &[u64],
+    sw_breakpoints: &HashMap<u64, [u8; SW_BP_SIZE]>,
+) -> VcpuStopReason {
+    if DB_EX_ID == exception {
+        // If the BS flag in DR6 register is set, it means a single step
+        // instruction triggered the exit
+        // Check page 19-4 Vol. 3B of Intel 64 and IA-32
+        // Architectures Software Developer's Manual
+        if dr6 & DR6_BS_FLAG_MASK != 0 && single_step {
+            return VcpuStopReason::DoneStep;
+        }
+
+        // If any of the B0-B3 flags in DR6 register is set, it means a
+        // hardware breakpoint triggered the exit
+        // Check page 19-4 Vol. 3B of Intel 64 and IA-32
+        // Architectures Software Developer's Manual
+        if DR6_HW_BP_FLAGS_MASK & dr6 != 0 && hw_breakpoints.contains(&rip) {
+            if rip == entrypoint {
+                return VcpuStopReason::EntryPointBp;
+            }
+            return VcpuStopReason::HwBp;
+        }
+    }
+
+    if BP_EX_ID == exception && sw_breakpoints.contains_key(&rip) {
+        return VcpuStopReason::SwBp;
+    }
+
+    // Log an error and provide internal debugging info
+    log::error!(
+        r"The vCPU exited because of an unknown reason:
+        single_step: {:?}
+        rip: {:?}
+        dr6: {:?}
+        entrypoint: {:?}
+        exception: {:?}
+        hw_breakpoints: {:?}
+        sw_breakpoints: {:?}
+        ",
+        single_step,
+        rip,
+        dr6,
+        entrypoint,
+        exception,
+        hw_breakpoints,
+        sw_breakpoints,
+    );
+
+    VcpuStopReason::Unknown
+}

src/hyperlight_host/src/hypervisor/gdb/event_loop.rs

@@ -49,6 +49,7 @@ impl run_blocking::BlockingEventLoop for GdbBlockingEventLoop {
                     // Resume execution if unknown reason for stop
                     let stop_response = match stop_reason {
                         VcpuStopReason::DoneStep => BaseStopReason::DoneStep,
+                        VcpuStopReason::EntryPointBp => BaseStopReason::HwBreak(()),
                         VcpuStopReason::SwBp => BaseStopReason::SwBreak(()),
                         VcpuStopReason::HwBp => BaseStopReason::HwBreak(()),
                         // This is a consequence of the GDB client sending an interrupt signal

src/hyperlight_host/src/hypervisor/gdb/kvm_debug.rs

@@ -22,15 +22,10 @@ use kvm_bindings::{
 };
 use kvm_ioctls::VcpuFd;
 
-use super::{
-    GuestDebug, VcpuStopReason, X86_64Regs, DR6_BS_FLAG_MASK, DR6_HW_BP_FLAGS_MASK,
-    MAX_NO_OF_HW_BP, SW_BP_SIZE,
-};
+use super::arch::{vcpu_stop_reason, MAX_NO_OF_HW_BP, SW_BP_SIZE};
+use super::{GuestDebug, VcpuStopReason, X86_64Regs};
 use crate::{new_error, HyperlightError, Result};
 
-/// Exception id for SW breakpoint
-const SW_BP_ID: u32 = 3;
-
 /// KVM Debug struct
 /// This struct is used to abstract the internal details of the kvm
 /// guest debugging settings
@@ -118,51 +113,29 @@ impl KvmDebug {
         debug_exit: kvm_debug_exit_arch,
         entrypoint: u64,
     ) -> Result<VcpuStopReason> {
-        // If the BS flag in DR6 register is set, it means a single step
-        // instruction triggered the exit
-        // Check page 19-4 Vol. 3B of Intel 64 and IA-32
-        // Architectures Software Developer's Manual
-        if debug_exit.dr6 & DR6_BS_FLAG_MASK != 0 && self.single_step {
-            return Ok(VcpuStopReason::DoneStep);
-        }
+        let rip = self.get_instruction_pointer(vcpu_fd)?;
+        let rip = self.translate_gva(vcpu_fd, rip)?;
 
-        let ip = self.get_instruction_pointer(vcpu_fd)?;
-        let gpa = self.translate_gva(vcpu_fd, ip)?;
+        // Check if the vCPU stopped because of a hardware breakpoint
+        let reason = vcpu_stop_reason(
+            self.single_step,
+            rip,
+            debug_exit.dr6,
+            entrypoint,
+            debug_exit.exception,
+            &self.hw_breakpoints,
+            &self.sw_breakpoints,
+        );
 
-        // If any of the B0-B3 flags in DR6 register is set, it means a
-        // hardware breakpoint triggered the exit
-        // Check page 19-4 Vol. 3B of Intel 64 and IA-32
-        // Architectures Software Developer's Manual
-        if DR6_HW_BP_FLAGS_MASK & debug_exit.dr6 != 0 && self.hw_breakpoints.contains(&gpa) {
+        if let VcpuStopReason::EntryPointBp = reason {
             // In case the hw breakpoint is the entry point, remove it to
             // avoid hanging here as gdb does not remove breakpoints it
             // has not set.
             // Gdb expects the target to be stopped when connected.
-            if gpa == entrypoint {
-                self.remove_hw_breakpoint(vcpu_fd, entrypoint)?;
-            }
-            return Ok(VcpuStopReason::HwBp);
+            self.remove_hw_breakpoint(vcpu_fd, entrypoint)?;
         }
 
-        // If the exception ID matches #BP (3) - it means a software breakpoint
-        // caused the exit
-        if SW_BP_ID == debug_exit.exception && self.sw_breakpoints.contains_key(&gpa) {
-            return Ok(VcpuStopReason::SwBp);
-        }
-
-        // Log an error and provide internal debugging info for fixing
-        log::error!(
-            r"The vCPU exited because of an unknown reason:
-            kvm_debug_exit_arch: {:?}
-            single_step: {:?}
-            hw_breakpoints: {:?}
-            sw_breakpoints: {:?}",
-            debug_exit,
-            self.single_step,
-            self.hw_breakpoints,
-            self.sw_breakpoints,
-        );
-        Ok(VcpuStopReason::Unknown)
+        Ok(reason)
     }
 }
 

src/hyperlight_host/src/hypervisor/gdb/mod.rs

@@ -14,6 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+mod arch;
 mod event_loop;
 #[cfg(kvm)]
 mod kvm_debug;
@@ -26,6 +27,7 @@ use std::net::TcpListener;
 use std::sync::{Arc, Mutex};
 use std::thread;
 
+use arch::{SW_BP, SW_BP_SIZE};
 use crossbeam_channel::{Receiver, Sender, TryRecvError};
 use event_loop::event_loop_thread;
 use gdbstub::conn::ConnectionExt;
@@ -43,28 +45,6 @@ use crate::hypervisor::handlers::DbgMemAccessHandlerCaller;
 use crate::mem::layout::SandboxMemoryLayout;
 use crate::{new_error, HyperlightError};
 
-/// Software Breakpoint size in memory
-const SW_BP_SIZE: usize = 1;
-/// Software Breakpoint opcode - INT3
-/// Check page 7-28 Vol. 3A of Intel 64 and IA-32
-/// Architectures Software Developer's Manual
-const SW_BP_OP: u8 = 0xCC;
-/// Software Breakpoint written to memory
-const SW_BP: [u8; SW_BP_SIZE] = [SW_BP_OP];
-/// Maximum number of supported hardware breakpoints
-const MAX_NO_OF_HW_BP: usize = 4;
-
-/// Check page 19-4 Vol. 3B of Intel 64 and IA-32
-/// Architectures Software Developer's Manual
-/// Bit position of BS flag in DR6 debug register
-const DR6_BS_FLAG_POS: usize = 14;
-/// Bit mask of BS flag in DR6 debug register
-const DR6_BS_FLAG_MASK: u64 = 1 << DR6_BS_FLAG_POS;
-/// Bit position of HW breakpoints status in DR6 debug register
-const DR6_HW_BP_FLAGS_POS: usize = 0;
-/// Bit mask of HW breakpoints status in DR6 debug register
-const DR6_HW_BP_FLAGS_MASK: u64 = 0x0F << DR6_HW_BP_FLAGS_POS;
-
 #[derive(Debug, Error)]
 pub(crate) enum GdbTargetError {
     #[error("Error encountered while binding to address and port")]
@@ -129,6 +109,10 @@ pub(crate) struct X86_64Regs {
 #[derive(Debug)]
 pub enum VcpuStopReason {
     DoneStep,
+    /// Hardware breakpoint inserted by the hypervisor so the guest can be stopped
+    /// at the entry point. This is used to avoid the guest from executing
+    /// the entry point code before the debugger is connected
+    EntryPointBp,
     HwBp,
     SwBp,
     Interrupt,

src/hyperlight_host/src/hypervisor/gdb/mshv_debug.rs

@@ -32,10 +32,8 @@ use mshv_bindings::{
 };
 use mshv_ioctls::VcpuFd;
 
-use super::{
-    GuestDebug, VcpuStopReason, X86_64Regs, DR6_BS_FLAG_MASK, DR6_HW_BP_FLAGS_MASK,
-    MAX_NO_OF_HW_BP, SW_BP_SIZE,
-};
+use super::arch::{vcpu_stop_reason, MAX_NO_OF_HW_BP, SW_BP_SIZE};
+use super::{GuestDebug, VcpuStopReason, X86_64Regs};
 use crate::{new_error, HyperlightError, Result};
 
 #[derive(Debug, Default)]
@@ -133,52 +131,38 @@ impl MshvDebug {
     pub(crate) fn get_stop_reason(
         &mut self,
         vcpu_fd: &VcpuFd,
+        exception: u16,
         entrypoint: u64,
     ) -> Result<VcpuStopReason> {
-        // MSHV does not provide info on the vCPU exits but the debug
-        // information can be retrieved from the DEBUG REGISTERS
         let regs = vcpu_fd
             .get_debug_regs()
             .map_err(|e| new_error!("Cannot retrieve debug registers from vCPU: {}", e))?;
 
         // DR6 register contains debug state related information
         let debug_status = regs.dr6;
 
-        // If the BS flag in DR6 register is set, it means a single step
-        // instruction triggered the exit
-        // Check page 19-4 Vol. 3B of Intel 64 and IA-32
-        // Architectures Software Developer's Manual
-        if debug_status & DR6_BS_FLAG_MASK != 0 && self.single_step {
-            return Ok(VcpuStopReason::DoneStep);
-        }
+        let rip = self.get_instruction_pointer(vcpu_fd)?;
+        let rip = self.translate_gva(vcpu_fd, rip)?;
 
-        let ip = self.get_instruction_pointer(vcpu_fd)?;
-        let gpa = self.translate_gva(vcpu_fd, ip)?;
+        let reason = vcpu_stop_reason(
+            self.single_step,
+            rip,
+            debug_status,
+            entrypoint,
+            exception as u32,
+            &self.hw_breakpoints,
+            &self.sw_breakpoints,
+        );
 
-        // If any of the B0-B3 flags in DR6 register is set, it means a
-        // hardware breakpoint triggered the exit
-        // Check page 19-4 Vol. 3B of Intel 64 and IA-32
-        // Architectures Software Developer's Manual
-        if debug_status & DR6_HW_BP_FLAGS_MASK != 0 && self.hw_breakpoints.contains(&gpa) {
+        if let VcpuStopReason::EntryPointBp = reason {
             // In case the hw breakpoint is the entry point, remove it to
             // avoid hanging here as gdb does not remove breakpoints it
             // has not set.
             // Gdb expects the target to be stopped when connected.
-            if gpa == entrypoint {
-                self.remove_hw_breakpoint(vcpu_fd, entrypoint)?;
-            }
-            return Ok(VcpuStopReason::HwBp);
-        }
-
-        // mshv does not provide a way to specify which exception triggered the
-        // vCPU exit as the mshv intercepts both #DB and #BP
-        // We check against the SW breakpoints Hashmap to detect whether the
-        // vCPU exited due to a SW breakpoint
-        if self.sw_breakpoints.contains_key(&gpa) {
-            return Ok(VcpuStopReason::SwBp);
+            self.remove_hw_breakpoint(vcpu_fd, entrypoint)?;
         }
 
-        Ok(VcpuStopReason::Unknown)
+        Ok(reason)
     }
 }