diff --git a/src/external_trait_impls/rayon/set.rs b/src/external_trait_impls/rayon/set.rs
index bbdfa7799d..ee4f6e6693 100644
--- a/src/external_trait_impls/rayon/set.rs
+++ b/src/external_trait_impls/rayon/set.rs
@@ -198,9 +198,16 @@ where
     where
         C: UnindexedConsumer<Self::Item>,
     {
-        self.a
+        // We'll iterate one set in full, and only the remaining difference from the other.
+        // Use the smaller set for the difference in order to reduce hash lookups.
+        let (smaller, larger) = if self.a.len() <= self.b.len() {
+            (self.a, self.b)
+        } else {
+            (self.b, self.a)
+        };
+        larger
             .into_par_iter()
-            .chain(self.b.par_difference(self.a))
+            .chain(smaller.par_difference(larger))
             .drive_unindexed(consumer)
     }
 }
diff --git a/src/set.rs b/src/set.rs
index c09876ee9e..a451bcc771 100644
--- a/src/set.rs
+++ b/src/set.rs
@@ -736,7 +736,9 @@ where
     /// ```
     #[cfg_attr(feature = "inline-more", inline)]
     pub fn union<'a>(&'a self, other: &'a Self) -> Union<'a, T, S, A> {
-        let (smaller, larger) = if self.len() >= other.len() {
+        // We'll iterate one set in full, and only the remaining difference from the other.
+        // Use the smaller set for the difference in order to reduce hash lookups.
+        let (smaller, larger) = if self.len() <= other.len() {
             (self, other)
         } else {
             (other, self)