From 72a28bdbf016478800385b93884a0215618f3006 Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Sat, 11 Feb 2023 14:58:41 +0100 Subject: [PATCH 01/11] add `flush_broadcast` --- src/instructions/tlb.rs | 61 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/src/instructions/tlb.rs b/src/instructions/tlb.rs index b300a2a74..1b0247f1d 100644 --- a/src/instructions/tlb.rs +++ b/src/instructions/tlb.rs @@ -1,6 +1,11 @@ //! Functions to flush the translation lookaside buffer (TLB). -use crate::VirtAddr; +use bit_field::BitField; + +use crate::{ + structures::paging::{page::NotGiantPageSize, Page, PageSize, Size2MiB}, + VirtAddr, +}; use core::arch::asm; /// Invalidate the given address in the TLB using the `invlpg` instruction. @@ -97,3 +102,57 @@ pub unsafe fn flush_pcid(command: InvPicdCommand) { asm!("invpcid {0}, [{1}]", in(reg) kind, in(reg) &desc, options(nostack, preserves_flags)); } } + +/// Invalidates TLB entry(s) with Broadcast. +/// +/// # Safety +/// +/// This function is unsafe as it requires CPUID.(EAX=8000_0008H, ECX=0H):EBX.INVLPGB +/// to be 1 and count to be less than or equal to CPUID.(EAX=8000_0008H, ECX=0H):EDX[0..=15]. +#[inline] +pub unsafe fn flush_broadcast( + va_and_count: Option<(Page, u16)>, + pcid: Option, + asid: Option, + include_global: bool, + final_translation_only: bool, + include_nested_translations: bool, +) where + S: NotGiantPageSize, +{ + let mut rax = 0; + let mut ecx = 0; + let mut edx = 0; + + if let Some((va, count)) = va_and_count { + rax.set_bit(0, true); + rax.set_bits(12.., va.start_address().as_u64().get_bits(12..)); + + ecx.set_bits(0..=15, u32::from(count)); + ecx.set_bit(31, S::SIZE == Size2MiB::SIZE); + } + + if let Some(pcid) = pcid { + rax.set_bit(1, true); + edx.set_bits(16..=27, u32::from(pcid.value())); + } + + if let Some(asid) = asid { + rax.set_bit(2, true); + edx.set_bits(0..=15, u32::from(asid)); + } + + rax.set_bit(3, include_global); + rax.set_bit(4, final_translation_only); + rax.set_bit(5, include_nested_translations); + + unsafe { + asm!( + "invlpgb", + in("rax") rax, + in("ecx") ecx, + in("edx") edx, + options(nomem, preserves_flags), + ); + } +} From 54190d52828e4c8fc43cd70007816c14c0cc8f49 Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Sat, 11 Feb 2023 14:58:52 +0100 Subject: [PATCH 02/11] add `tlbsync` --- src/instructions/tlb.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/instructions/tlb.rs b/src/instructions/tlb.rs index 1b0247f1d..dea133223 100644 --- a/src/instructions/tlb.rs +++ b/src/instructions/tlb.rs @@ -156,3 +156,15 @@ pub unsafe fn flush_broadcast( ); } } + +/// Synchronize broadcasted TLB Invalidations. +/// +/// # Safety +/// +/// This function is unsafe as it requires CPUID.(EAX=8000_0008H, ECX=0H):EBX.INVLPGB to be 1. +#[inline] +pub unsafe fn tlbsync() { + unsafe { + asm!("tlbsync", options(nomem, preserves_flags)); + } +} From c23357fdb2d0f718c54ffcc103a29f3ce971b804 Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Tue, 14 Feb 2023 09:58:16 +0100 Subject: [PATCH 03/11] use `nostack` instead of `nomem` --- src/instructions/tlb.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/instructions/tlb.rs b/src/instructions/tlb.rs index dea133223..53361e7c1 100644 --- a/src/instructions/tlb.rs +++ b/src/instructions/tlb.rs @@ -152,7 +152,7 @@ pub unsafe fn flush_broadcast( in("rax") rax, in("ecx") ecx, in("edx") edx, - options(nomem, preserves_flags), + options(nostack, preserves_flags), ); } } From 83be8d715d590ee13515680e28baf265d56b651f Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Tue, 14 Feb 2023 12:01:27 +0100 Subject: [PATCH 04/11] move parts of the `Step` impl into normal functions This allows us to these functions internally without requiring the nightly `step_trait` feature. --- src/addr.rs | 68 +++++++++++++++++++---------------- src/structures/paging/page.rs | 26 +++++++++----- 2 files changed, 56 insertions(+), 38 deletions(-) diff --git a/src/addr.rs b/src/addr.rs index d3bec6a8b..0d60ffefc 100644 --- a/src/addr.rs +++ b/src/addr.rs @@ -1,6 +1,5 @@ //! Physical and virtual addresses manipulation -#[cfg(feature = "step_trait")] use core::convert::TryFrom; use core::fmt; #[cfg(feature = "step_trait")] @@ -11,7 +10,6 @@ use crate::structures::paging::page_table::PageTableLevel; use crate::structures::paging::{PageOffset, PageTableIndex}; use bit_field::BitField; -#[cfg(feature = "step_trait")] const ADDRESS_SPACE_SIZE: u64 = 0x1_0000_0000_0000; /// A canonical 64-bit virtual memory address. @@ -226,6 +224,42 @@ impl VirtAddr { pub const fn page_table_index(self, level: PageTableLevel) -> PageTableIndex { PageTableIndex::new_truncate((self.0 >> 12 >> ((level as u8 - 1) * 9)) as u16) } + + // FIXME: Move this into the `Step` impl, once `Step` is stabilized. + pub(crate) fn steps_between(start: &Self, end: &Self) -> Option { + let mut steps = end.0.checked_sub(start.0)?; + + // Check if we jumped the gap. + if end.0.get_bit(47) && !start.0.get_bit(47) { + steps = steps.checked_sub(0xffff_0000_0000_0000).unwrap(); + } + + usize::try_from(steps).ok() + } + + // FIXME: Move this into the `Step` impl, once `Step` is stabilized. + pub(crate) fn forward_checked(start: Self, count: usize) -> Option { + let offset = u64::try_from(count).ok()?; + if offset > ADDRESS_SPACE_SIZE { + return None; + } + + let mut addr = start.0.checked_add(offset)?; + + match addr.get_bits(47..) { + 0x1 => { + // Jump the gap by sign extending the 47th bit. + addr.set_bits(47.., 0x1ffff); + } + 0x2 => { + // Address overflow + return None; + } + _ => {} + } + + Some(Self::new(addr)) + } } impl fmt::Debug for VirtAddr { @@ -346,37 +380,11 @@ impl Sub for VirtAddr { #[cfg(feature = "step_trait")] impl Step for VirtAddr { fn steps_between(start: &Self, end: &Self) -> Option { - let mut steps = end.0.checked_sub(start.0)?; - - // Check if we jumped the gap. - if end.0.get_bit(47) && !start.0.get_bit(47) { - steps = steps.checked_sub(0xffff_0000_0000_0000).unwrap(); - } - - usize::try_from(steps).ok() + Self::steps_between(start, end) } fn forward_checked(start: Self, count: usize) -> Option { - let offset = u64::try_from(count).ok()?; - if offset > ADDRESS_SPACE_SIZE { - return None; - } - - let mut addr = start.0.checked_add(offset)?; - - match addr.get_bits(47..) { - 0x1 => { - // Jump the gap by sign extending the 47th bit. - addr.set_bits(47.., 0x1ffff); - } - 0x2 => { - // Address overflow - return None; - } - _ => {} - } - - Some(Self::new(addr)) + Self::forward_checked(start, count) } fn backward_checked(start: Self, count: usize) -> Option { diff --git a/src/structures/paging/page.rs b/src/structures/paging/page.rs index 3e2878bfe..f58d66c0d 100644 --- a/src/structures/paging/page.rs +++ b/src/structures/paging/page.rs @@ -148,6 +148,22 @@ impl Page { pub fn range_inclusive(start: Self, end: Self) -> PageRangeInclusive { PageRangeInclusive { start, end } } + + // FIXME: Move this into the `Step` impl, once `Step` is stabilized. + pub(crate) fn steps_between(start: &Self, end: &Self) -> Option { + VirtAddr::steps_between(&start.start_address, &end.start_address) + .map(|steps| steps / S::SIZE as usize) + } + + // FIXME: Move this into the `Step` impl, once `Step` is stabilized. + pub(crate) fn forward_checked(start: Self, count: usize) -> Option { + let count = count.checked_mul(S::SIZE as usize)?; + let start_address = VirtAddr::forward_checked(start.start_address, count)?; + Some(Self { + start_address, + size: PhantomData, + }) + } } impl Page { @@ -270,17 +286,11 @@ impl Sub for Page { #[cfg(feature = "step_trait")] impl Step for Page { fn steps_between(start: &Self, end: &Self) -> Option { - Step::steps_between(&start.start_address, &end.start_address) - .map(|steps| steps / S::SIZE as usize) + Self::steps_between(start, end) } fn forward_checked(start: Self, count: usize) -> Option { - let count = count.checked_mul(S::SIZE as usize)?; - let start_address = Step::forward_checked(start.start_address, count)?; - Some(Self { - start_address, - size: PhantomData, - }) + Self::forward_checked(start, count) } fn backward_checked(start: Self, count: usize) -> Option { From bae6119533f19cd5c3557d5a302524f1d72456c5 Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Tue, 14 Feb 2023 12:02:03 +0100 Subject: [PATCH 05/11] move to a builder pattern --- src/instructions/tlb.rs | 273 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 253 insertions(+), 20 deletions(-) diff --git a/src/instructions/tlb.rs b/src/instructions/tlb.rs index 53361e7c1..c33671970 100644 --- a/src/instructions/tlb.rs +++ b/src/instructions/tlb.rs @@ -3,10 +3,14 @@ use bit_field::BitField; use crate::{ - structures::paging::{page::NotGiantPageSize, Page, PageSize, Size2MiB}, - VirtAddr, + instructions::segmentation::{Segment, CS}, + structures::paging::{ + page::{NotGiantPageSize, PageRange}, + Page, PageSize, Size2MiB, Size4KiB, + }, + PrivilegeLevel, VirtAddr, }; -use core::arch::asm; +use core::{arch::asm, cmp, convert::TryFrom}; /// Invalidate the given address in the TLB using the `invlpg` instruction. #[inline] @@ -103,14 +107,255 @@ pub unsafe fn flush_pcid(command: InvPicdCommand) { } } -/// Invalidates TLB entry(s) with Broadcast. +/// Used to broadcast flushes to all logical processors. /// -/// # Safety +/// ```no_run +/// use x86_64::VirtAddr; +/// use x86_64::structures::paging::Page; +/// use x86_64::instructions::tlb::Invlpgb; /// -/// This function is unsafe as it requires CPUID.(EAX=8000_0008H, ECX=0H):EBX.INVLPGB -/// to be 1 and count to be less than or equal to CPUID.(EAX=8000_0008H, ECX=0H):EDX[0..=15]. +/// // Check that `invlpgb` and `tlbsync` are supported. +/// let invlpgb = Invlpgb::new().unwrap(); +/// +/// // Broadcast flushing some pages to all logical processors. +/// let start: Page = Page::from_start_address(VirtAddr::new(0xf000_0000)).unwrap(); +/// let pages = Page::range(start, start + 3); +/// invlpgb.build().pages(pages).include_global().flush(); +/// +/// // Wait for all logical processors to respond. +/// invlpgb.tlbsync(); +/// ``` +#[derive(Debug, Clone, Copy)] +pub struct Invlpgb { + invlpgb_count_max: u16, + tlb_flush_nested: bool, + nasid: u32, +} + +impl Invlpgb { + /// Check that `invlpgb` and `tlbsync` are supported and query limits. + /// + /// # Panics + /// + /// Panics if the CPL is not 0. + pub fn new() -> Option { + let cs = CS::get_reg(); + assert_eq!(cs.rpl(), PrivilegeLevel::Ring0); + + // Check if the `INVLPGB` and `TLBSYNC` instruction are supported. + let cpuid = unsafe { core::arch::x86_64::__cpuid(0x8000_0008) }; + if !cpuid.ebx.get_bit(3) { + return None; + } + + let tlb_flush_nested = cpuid.ebx.get_bit(21); + let invlpgb_count_max = cpuid.edx.get_bits(0..=15) as u16; + + // Figure out the number of supported ASIDs. + let cpuid = unsafe { core::arch::x86_64::__cpuid(0x8000_000a) }; + let nasid = cpuid.ebx; + + Some(Self { + tlb_flush_nested, + invlpgb_count_max, + nasid, + }) + } + + /// Returns the maximum count of pages to be flushed supported by the processor. + #[inline] + pub fn invlpgb_count_max(&self) -> u16 { + self.invlpgb_count_max + } + + /// Returns whether the processor supports flushing translations used for guest translation. + #[inline] + pub fn tlb_flush_nested(&self) -> bool { + self.tlb_flush_nested + } + + /// Returns the number of available address space identifiers. + #[inline] + pub fn nasid(&self) -> u32 { + self.nasid + } + + /// Create a `InvlpgbFlushBuilder`. + pub fn build(&self) -> InvlpgbFlushBuilder<'_> { + InvlpgbFlushBuilder { + invlpgb: self, + page_range: None, + pcid: None, + asid: None, + include_global: false, + final_translation_only: false, + include_nested_translations: false, + } + } + + /// Wait for all previous `invlpgb` instruction executed on the current + /// logical processor to be acknowledged by all other logical processors. + #[inline] + pub fn tlbsync(&self) { + unsafe { + asm!("tlbsync", options(nomem, preserves_flags)); + } + } +} + +/// A builder struct to construct the parameters for the `invlpgb` instruction. +#[derive(Debug, Clone)] +#[must_use] +pub struct InvlpgbFlushBuilder<'a, S = Size4KiB> +where + S: NotGiantPageSize, +{ + invlpgb: &'a Invlpgb, + page_range: Option>, + pcid: Option, + asid: Option, + include_global: bool, + final_translation_only: bool, + include_nested_translations: bool, +} + +impl<'a, S> InvlpgbFlushBuilder<'a, S> +where + S: NotGiantPageSize, +{ + /// Flush a range of pages. + /// + /// If the range doesn't fit within `invlpgb_count_max`, `invlpgb` is + /// executed multiple times. + pub fn pages(self, page_range: PageRange) -> InvlpgbFlushBuilder<'a, T> + where + T: NotGiantPageSize, + { + InvlpgbFlushBuilder { + invlpgb: self.invlpgb, + page_range: Some(page_range), + pcid: self.pcid, + asid: self.asid, + include_global: self.include_global, + final_translation_only: self.final_translation_only, + include_nested_translations: self.include_nested_translations, + } + } + + /// Only flush TLB entries with the given PCID. + /// + /// # Safety + /// + /// The caller has to ensure that PCID is enabled in CR4 when the flush is executed. + pub unsafe fn pcid(mut self, pcid: Pcid) -> Self { + self.pcid = Some(pcid); + self + } + + /// Only flush TLB entries with the given ASID. + /// + /// # Safety + /// + /// The caller has to ensure that SVM is enabled in EFER when the flush is executed. + // FIXME: Make ASID a type and remove error type. + pub unsafe fn asid(mut self, asid: u16) -> Result { + if u32::from(asid) > self.invlpgb.nasid { + return Err(AsidOutOfRangeError { + asid, + nasid: self.invlpgb.nasid, + }); + } + + self.asid = Some(asid); + Ok(self) + } + + /// Also flush global pages. + pub fn include_global(mut self) -> Self { + self.include_global = true; + self + } + + /// Only flush the final translation and not the cached upper level TLB entries. + pub fn final_translation_only(mut self) -> Self { + self.final_translation_only = true; + self + } + + /// Also flush nestred translations that could be used for guest translation. + pub fn include_nested_translations(mut self) -> Self { + assert!( + self.invlpgb.tlb_flush_nested, + "flushing all nested translations is not supported" + ); + + self.include_nested_translations = true; + self + } + + /// Execute the flush. + pub fn flush(self) { + if let Some(mut pages) = self.page_range { + while !pages.is_empty() { + // Calculate out how many pages we still need to flush. + let count = Page::::steps_between(&pages.start, &pages.end).unwrap(); + + // Make sure that we never jump the gap in the address space when flushing. + let second_half_start = + Page::::containing_address(VirtAddr::new(0xffff_8000_0000_0000)); + let count = if pages.start < second_half_start { + let count_to_second_half = + Page::steps_between(&pages.start, &second_half_start).unwrap(); + cmp::min(count, count_to_second_half) + } else { + count + }; + + // We can flush at most u16::MAX pages at once. + let count = u16::try_from(count).unwrap_or(u16::MAX); + + // Cap the count by the maximum supported count of the processor. + let count = cmp::min(count, self.invlpgb.invlpgb_count_max); + + unsafe { + flush_broadcast( + Some((pages.start, count)), + self.pcid, + self.asid, + self.include_global, + self.final_translation_only, + self.include_nested_translations, + ); + } + + // Even if the count is zero, one page is still flushed and so + // we need to advance by at least one. + let inc_count = cmp::max(count, 1); + pages.start = Page::forward_checked(pages.start, usize::from(inc_count)).unwrap(); + } + } else { + unsafe { + flush_broadcast::( + None, + self.pcid, + self.asid, + self.include_global, + self.final_translation_only, + self.include_nested_translations, + ); + } + } + } +} + +#[derive(Debug)] +pub struct AsidOutOfRangeError { + asid: u16, + nasid: u32, +} + #[inline] -pub unsafe fn flush_broadcast( +unsafe fn flush_broadcast( va_and_count: Option<(Page, u16)>, pcid: Option, asid: Option, @@ -156,15 +401,3 @@ pub unsafe fn flush_broadcast( ); } } - -/// Synchronize broadcasted TLB Invalidations. -/// -/// # Safety -/// -/// This function is unsafe as it requires CPUID.(EAX=8000_0008H, ECX=0H):EBX.INVLPGB to be 1. -#[inline] -pub unsafe fn tlbsync() { - unsafe { - asm!("tlbsync", options(nomem, preserves_flags)); - } -} From ee1075352c9588f853881c310654c88c00d7e3ae Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Tue, 14 Mar 2023 12:49:55 +0100 Subject: [PATCH 06/11] rename the inherent step methods This avoids any confusion between the trait methods and inherent methods. --- src/addr.rs | 8 ++++---- src/instructions/tlb.rs | 7 ++++--- src/structures/paging/page.rs | 12 ++++++------ 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/addr.rs b/src/addr.rs index 0d60ffefc..b442541c9 100644 --- a/src/addr.rs +++ b/src/addr.rs @@ -226,7 +226,7 @@ impl VirtAddr { } // FIXME: Move this into the `Step` impl, once `Step` is stabilized. - pub(crate) fn steps_between(start: &Self, end: &Self) -> Option { + pub(crate) fn steps_between_impl(start: &Self, end: &Self) -> Option { let mut steps = end.0.checked_sub(start.0)?; // Check if we jumped the gap. @@ -238,7 +238,7 @@ impl VirtAddr { } // FIXME: Move this into the `Step` impl, once `Step` is stabilized. - pub(crate) fn forward_checked(start: Self, count: usize) -> Option { + pub(crate) fn forward_checked_impl(start: Self, count: usize) -> Option { let offset = u64::try_from(count).ok()?; if offset > ADDRESS_SPACE_SIZE { return None; @@ -380,11 +380,11 @@ impl Sub for VirtAddr { #[cfg(feature = "step_trait")] impl Step for VirtAddr { fn steps_between(start: &Self, end: &Self) -> Option { - Self::steps_between(start, end) + Self::steps_between_impl(start, end) } fn forward_checked(start: Self, count: usize) -> Option { - Self::forward_checked(start, count) + Self::forward_checked_impl(start, count) } fn backward_checked(start: Self, count: usize) -> Option { diff --git a/src/instructions/tlb.rs b/src/instructions/tlb.rs index c33671970..ff0ff5ae0 100644 --- a/src/instructions/tlb.rs +++ b/src/instructions/tlb.rs @@ -298,14 +298,14 @@ where if let Some(mut pages) = self.page_range { while !pages.is_empty() { // Calculate out how many pages we still need to flush. - let count = Page::::steps_between(&pages.start, &pages.end).unwrap(); + let count = Page::::steps_between_impl(&pages.start, &pages.end).unwrap(); // Make sure that we never jump the gap in the address space when flushing. let second_half_start = Page::::containing_address(VirtAddr::new(0xffff_8000_0000_0000)); let count = if pages.start < second_half_start { let count_to_second_half = - Page::steps_between(&pages.start, &second_half_start).unwrap(); + Page::steps_between_impl(&pages.start, &second_half_start).unwrap(); cmp::min(count, count_to_second_half) } else { count @@ -331,7 +331,8 @@ where // Even if the count is zero, one page is still flushed and so // we need to advance by at least one. let inc_count = cmp::max(count, 1); - pages.start = Page::forward_checked(pages.start, usize::from(inc_count)).unwrap(); + pages.start = + Page::forward_checked_impl(pages.start, usize::from(inc_count)).unwrap(); } } else { unsafe { diff --git a/src/structures/paging/page.rs b/src/structures/paging/page.rs index f58d66c0d..230fbf0ba 100644 --- a/src/structures/paging/page.rs +++ b/src/structures/paging/page.rs @@ -150,15 +150,15 @@ impl Page { } // FIXME: Move this into the `Step` impl, once `Step` is stabilized. - pub(crate) fn steps_between(start: &Self, end: &Self) -> Option { - VirtAddr::steps_between(&start.start_address, &end.start_address) + pub(crate) fn steps_between_impl(start: &Self, end: &Self) -> Option { + VirtAddr::steps_between_impl(&start.start_address, &end.start_address) .map(|steps| steps / S::SIZE as usize) } // FIXME: Move this into the `Step` impl, once `Step` is stabilized. - pub(crate) fn forward_checked(start: Self, count: usize) -> Option { + pub(crate) fn forward_checked_impl(start: Self, count: usize) -> Option { let count = count.checked_mul(S::SIZE as usize)?; - let start_address = VirtAddr::forward_checked(start.start_address, count)?; + let start_address = VirtAddr::forward_checked_impl(start.start_address, count)?; Some(Self { start_address, size: PhantomData, @@ -286,11 +286,11 @@ impl Sub for Page { #[cfg(feature = "step_trait")] impl Step for Page { fn steps_between(start: &Self, end: &Self) -> Option { - Self::steps_between(start, end) + Self::steps_between_impl(start, end) } fn forward_checked(start: Self, count: usize) -> Option { - Self::forward_checked(start, count) + Self::forward_checked_impl(start, count) } fn backward_checked(start: Self, count: usize) -> Option { From 28f95118918847648907aab0e2fc48dfef0a1d52 Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Tue, 14 Mar 2023 13:09:58 +0100 Subject: [PATCH 07/11] use mutable references in `InvlpgbFlushBuilder` This works for all methods except `pages` which changes the type. --- src/instructions/tlb.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/instructions/tlb.rs b/src/instructions/tlb.rs index ff0ff5ae0..afb7d8309 100644 --- a/src/instructions/tlb.rs +++ b/src/instructions/tlb.rs @@ -247,7 +247,7 @@ where /// # Safety /// /// The caller has to ensure that PCID is enabled in CR4 when the flush is executed. - pub unsafe fn pcid(mut self, pcid: Pcid) -> Self { + pub unsafe fn pcid(&mut self, pcid: Pcid) -> &mut Self { self.pcid = Some(pcid); self } @@ -258,7 +258,7 @@ where /// /// The caller has to ensure that SVM is enabled in EFER when the flush is executed. // FIXME: Make ASID a type and remove error type. - pub unsafe fn asid(mut self, asid: u16) -> Result { + pub unsafe fn asid(&mut self, asid: u16) -> Result<&mut Self, AsidOutOfRangeError> { if u32::from(asid) > self.invlpgb.nasid { return Err(AsidOutOfRangeError { asid, @@ -271,13 +271,13 @@ where } /// Also flush global pages. - pub fn include_global(mut self) -> Self { + pub fn include_global(&mut self) -> &mut Self { self.include_global = true; self } /// Only flush the final translation and not the cached upper level TLB entries. - pub fn final_translation_only(mut self) -> Self { + pub fn final_translation_only(&mut self) -> &mut Self { self.final_translation_only = true; self } @@ -294,7 +294,7 @@ where } /// Execute the flush. - pub fn flush(self) { + pub fn flush(&self) { if let Some(mut pages) = self.page_range { while !pages.is_empty() { // Calculate out how many pages we still need to flush. From 861bbbff162955b5fda576e9ace85283342964de Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Tue, 14 Mar 2023 13:17:29 +0100 Subject: [PATCH 08/11] fix off-by-one for ASID check --- src/instructions/tlb.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/instructions/tlb.rs b/src/instructions/tlb.rs index afb7d8309..31b86c402 100644 --- a/src/instructions/tlb.rs +++ b/src/instructions/tlb.rs @@ -259,7 +259,7 @@ where /// The caller has to ensure that SVM is enabled in EFER when the flush is executed. // FIXME: Make ASID a type and remove error type. pub unsafe fn asid(&mut self, asid: u16) -> Result<&mut Self, AsidOutOfRangeError> { - if u32::from(asid) > self.invlpgb.nasid { + if u32::from(asid) >= self.invlpgb.nasid { return Err(AsidOutOfRangeError { asid, nasid: self.invlpgb.nasid, From 3ecd11c5b1b34f47394adc2f53112c11b3ba0c80 Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Tue, 14 Mar 2023 13:27:26 +0100 Subject: [PATCH 09/11] add `Display` impl for `AsidOutOfRangeError` --- src/instructions/tlb.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/instructions/tlb.rs b/src/instructions/tlb.rs index 31b86c402..059a515f8 100644 --- a/src/instructions/tlb.rs +++ b/src/instructions/tlb.rs @@ -10,7 +10,7 @@ use crate::{ }, PrivilegeLevel, VirtAddr, }; -use core::{arch::asm, cmp, convert::TryFrom}; +use core::{arch::asm, cmp, convert::TryFrom, fmt}; /// Invalidate the given address in the TLB using the `invlpg` instruction. #[inline] @@ -355,6 +355,16 @@ pub struct AsidOutOfRangeError { nasid: u32, } +impl fmt::Display for AsidOutOfRangeError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{} is out of the range of available ASIDS ({})", + self.asid, self.nasid + ) + } +} + #[inline] unsafe fn flush_broadcast( va_and_count: Option<(Page, u16)>, From 0d55f015efc4fb297ba11cdd60fd00413af9307f Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Tue, 14 Mar 2023 13:27:43 +0100 Subject: [PATCH 10/11] make fields public and add documentation --- src/instructions/tlb.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/instructions/tlb.rs b/src/instructions/tlb.rs index 059a515f8..289acd233 100644 --- a/src/instructions/tlb.rs +++ b/src/instructions/tlb.rs @@ -349,10 +349,13 @@ where } } +/// An error returned when trying to use an invalid ASID. #[derive(Debug)] pub struct AsidOutOfRangeError { - asid: u16, - nasid: u32, + /// The requested ASID. + pub asid: u16, + /// The number of valid ASIDS. + pub nasid: u32, } impl fmt::Display for AsidOutOfRangeError { From 5cf5f86ab094451e593e5009c49a7523193210c6 Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Tue, 14 Mar 2023 13:30:13 +0100 Subject: [PATCH 11/11] add reference to `flush_broadcast` --- src/instructions/tlb.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/instructions/tlb.rs b/src/instructions/tlb.rs index 289acd233..7f5fd48a1 100644 --- a/src/instructions/tlb.rs +++ b/src/instructions/tlb.rs @@ -368,6 +368,7 @@ impl fmt::Display for AsidOutOfRangeError { } } +/// See `INVLPGB` in AMD64 Architecture Programmer's Manual Volume 3 #[inline] unsafe fn flush_broadcast( va_and_count: Option<(Page, u16)>,