[WIP] Impl a more strict transmute check

src/liballoc/sync.rs

@@ -517,6 +517,13 @@ impl<T: ?Sized> Arc<T> {
         unsafe { self.ptr.as_ref() }
     }
 
+    #[inline]
+    pub(crate) unsafe fn into_ptr<U>(self) -> NonNull<U> {
+        let ptr = self.ptr;
+        mem::forget(self);
+        ptr.cast()
+    }
+
     // Non-inlined part of `drop`.
     #[inline(never)]
     unsafe fn drop_slow(&mut self) {

src/liballoc/task.rs

@@ -19,9 +19,8 @@ pub use self::if_arc::*;
 mod if_arc {
     use super::*;
     use core::marker::PhantomData;
-    use core::mem;
-    use core::ptr::{self, NonNull};
     use sync::Arc;
+    use core::ptr;
 
     /// A way of waking up a specific task.
     ///
@@ -83,9 +82,9 @@ mod if_arc {
     {
         fn from(rc: Arc<T>) -> Self {
             unsafe {
-                let ptr = mem::transmute::<Arc<T>, NonNull<ArcWrapped<T>>>(rc);
-                Waker::new(ptr)
+                Waker::new(rc.into_ptr::<ArcWrapped<T>>())
             }
+
         }
     }
 
@@ -96,8 +95,7 @@ mod if_arc {
     /// will call `wake.wake()` if awoken after being converted to a `Waker`.
     #[inline]
     pub unsafe fn local_waker<W: Wake + 'static>(wake: Arc<W>) -> LocalWaker {
-        let ptr = mem::transmute::<Arc<W>, NonNull<ArcWrapped<W>>>(wake);
-        LocalWaker::new(ptr)
+        LocalWaker::new(wake.into_ptr::<ArcWrapped<W>>())
     }
 
     struct NonLocalAsLocal<T>(ArcWrapped<T>);
@@ -133,8 +131,7 @@ mod if_arc {
     #[inline]
     pub fn local_waker_from_nonlocal<W: Wake + 'static>(wake: Arc<W>) -> LocalWaker {
         unsafe {
-            let ptr = mem::transmute::<Arc<W>, NonNull<NonLocalAsLocal<W>>>(wake);
-            LocalWaker::new(ptr)
+            LocalWaker::new(wake.into_ptr::<NonLocalAsLocal<W>>())
         }
     }
 }

src/libcore/str/lossy.rs

@@ -12,7 +12,6 @@ use char;
 use str as core_str;
 use fmt;
 use fmt::Write;
-use mem;
 
 /// Lossy UTF-8 string.
 #[unstable(feature = "str_internals", issue = "0")]
@@ -26,7 +25,7 @@ impl Utf8Lossy {
     }
 
     pub fn from_bytes(bytes: &[u8]) -> &Utf8Lossy {
-        unsafe { mem::transmute(bytes) }
+        unsafe { &*(bytes as *const [u8] as *const Utf8Lossy) }
     }
 
     pub fn chunks(&self) -> Utf8LossyChunksIter {

src/libpanic_unwind/lib.rs

@@ -51,7 +51,7 @@ extern crate unwind;
 
 use alloc::boxed::Box;
 use core::intrinsics;
-use core::mem;
+use core::any::Any;
 use core::raw;
 use core::panic::BoxMeUp;
 
@@ -105,7 +105,8 @@ pub unsafe extern "C" fn __rust_maybe_catch_panic(f: fn(*mut u8),
     if intrinsics::try(f, data, &mut payload as *mut _ as *mut _) == 0 {
         0
     } else {
-        let obj = mem::transmute::<_, raw::TraitObject>(imp::cleanup(payload));
+        let raw = &Box::into_raw(imp::cleanup(payload));
+        let obj = *(raw as *const *mut (Any + Send) as *const raw::TraitObject);
         *data_ptr = obj.data as usize;
         *vtable_ptr = obj.vtable as usize;
         1

src/librustc/diagnostics.rs

@@ -2133,8 +2133,8 @@ register_diagnostics! {
     E0688, // in-band lifetimes cannot be mixed with explicit lifetime binders
 
     E0697, // closures cannot be static
-
     E0707, // multiple elided lifetimes used in arguments of `async fn`
     E0708, // `async` non-`move` closures with arguments are not currently supported
     E0709, // multiple different lifetimes used in arguments of `async fn`
+    E0912, // transmutation between types of unspecified layout
 }

src/librustc/middle/intrinsicck.rs

@@ -8,15 +8,17 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
+use hir;
 use hir::def::Def;
 use hir::def_id::DefId;
-use ty::{self, Ty, TyCtxt};
+use hir::intravisit::{self, Visitor, NestedVisitorMap};
 use ty::layout::{LayoutError, Pointer, SizeSkeleton};
+use ty::{self, Ty, AdtKind, TyCtxt, TypeFoldable};
+use ty::subst::Substs;
 
 use rustc_target::spec::abi::Abi::RustIntrinsic;
+use syntax_pos::DUMMY_SP;
 use syntax_pos::Span;
-use hir::intravisit::{self, Visitor, NestedVisitorMap};
-use hir;
 
 pub fn check_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>) {
     let mut visitor = ItemVisitor {
@@ -64,17 +66,171 @@ fn unpack_option_like<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
     ty
 }
 
+/// Check if this enum can be safely exported based on the
+/// "nullable pointer optimization". Currently restricted
+/// to function pointers and references, but could be
+/// expanded to cover NonZero raw pointers and newtypes.
+/// FIXME: This duplicates code in codegen.
+pub fn is_repr_nullable_ptr<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
+                                  def: &'tcx ty::AdtDef,
+                                  substs: &Substs<'tcx>)
+                                  -> bool {
+    if def.variants.len() == 2 {
+        let data_idx;
+
+        if def.variants[0].fields.is_empty() {
+            data_idx = 1;
+        } else if def.variants[1].fields.is_empty() {
+            data_idx = 0;
+        } else {
+            return false;
+        }
+
+        if def.variants[data_idx].fields.len() == 1 {
+            match def.variants[data_idx].fields[0].ty(tcx, substs).sty {
+                ty::TyFnPtr(_) => {
+                    return true;
+                }
+                ty::TyRef(..) => {
+                    return true;
+                }
+                _ => {}
+            }
+        }
+    }
+    false
+}
+
 impl<'a, 'tcx> ExprVisitor<'a, 'tcx> {
     fn def_id_is_transmute(&self, def_id: DefId) -> bool {
         self.tcx.fn_sig(def_id).abi() == RustIntrinsic &&
         self.tcx.item_name(def_id) == "transmute"
     }
 
+    /// Calculate whether a type has a specified layout.
+    ///
+    /// The function returns `None` in indeterminate cases (such as `TyError`).
+    fn is_layout_specified(&self, ty: Ty<'tcx>) -> Option<bool> {
+        match ty.sty {
+            // These types have a specified layout
+            // Reference: Primitive type layout
+            ty::TyBool |
+            ty::TyChar |
+            ty::TyInt(_) |
+            ty::TyUint(_) |
+            ty::TyFloat(_) |
+            // Reference: Pointers and references layout
+            ty::TyFnPtr(_) => Some(true),
+            // Reference: Array layout (depends on the contained type)
+            ty::TyArray(ty, _) => self.is_layout_specified(ty),
+            // Reference: Tuple layout (only specified if empty)
+            ty::TyTuple(ref tys) => Some(tys.is_empty()),
+
+            // Cases with definitely unspecified layouts
+            ty::TyClosure(_, _) |
+            ty::TyGenerator(_, _, _) |
+            ty::TyGeneratorWitness(_) => Some(false),
+            // Currently ZST, but does not seem to be guaranteed
+            ty::TyFnDef(_, _) => Some(false),
+            // Unsized types
+            ty::TyForeign(_) |
+            ty::TyNever |
+            ty::TyStr |
+            ty::TySlice(_) |
+            ty::TyDynamic(_, _) => Some(false),
+
+            // Indeterminate cases
+            ty::TyInfer(_) |
+            // should we report `Some(false)` for TyParam(_)? It this possible to reach this branch?
+            ty::TyParam(_) |
+            ty::TyError => None,
+
+            // The “it’s complicated™” cases
+            // Reference: Pointers and references layout
+            ty::TyRawPtr(ty::TypeAndMut { ty: pointee, .. }) |
+            ty::TyRef(_, pointee, _) => {
+                let pointee = self.tcx.normalize_erasing_regions(self.param_env, pointee);
+                // Pointers to unsized types have no specified layout.
+                Some(pointee.is_sized(self.tcx.at(DUMMY_SP), self.param_env))
+            }
+            ty::TyProjection(_) | ty::TyAnon(_, _) => {
+                let normalized = self.tcx.normalize_erasing_regions(self.param_env, ty);
+                if ty == normalized {
+                    None
+                } else {
+                    self.is_layout_specified(normalized)
+                }
+            }
+            ty::TyAdt(def, substs) => {
+                // Documentation guarantees 0-size.
+                if def.is_phantom_data() {
+                    return Some(true);
+                }
+                match def.adt_kind() {
+                    AdtKind::Struct | AdtKind::Union => {
+                        if !def.repr.c() && !def.repr.transparent() && !def.repr.simd() {
+                            return Some(false);
+                        }
+                        // FIXME: do we guarantee 0-sizedness for structs with 0 fields?
+                        // If not, they should cause Some(false) here.
+                        let mut seen_none = false;
+                        for field in &def.non_enum_variant().fields {
+                            let field_ty = field.ty(self.tcx, substs);
+                            match self.is_layout_specified(field_ty) {
+                                Some(true) => continue,
+                                None => {
+                                    seen_none = true;
+                                    continue;
+                                }
+                                x => return x,
+                            }
+                        }
+                        return if seen_none { None } else { Some(true) };
+                    }
+                    AdtKind::Enum => {
+                        if !def.repr.c() && def.repr.int.is_none() {
+                            if !is_repr_nullable_ptr(self.tcx, def, substs) {
+                                return Some(false);
+                            }
+                        }
+                        return Some(true);
+                    }
+                }
+            }
+        }
+    }
+
     fn check_transmute(&self, span: Span, from: Ty<'tcx>, to: Ty<'tcx>) {
+        // Check for unspecified types before checking for same size.
+        assert!(!from.has_infer_types());
+        assert!(!to.has_infer_types());
+
+        let unspecified_layout = |msg, ty| {
+            if ::std::env::var("RUSTC_BOOTSTRAP").is_ok() {
+                struct_span_warn!(self.tcx.sess, span, E0912, "{}", msg)
+                    .note(&format!("{} has an unspecified layout", ty))
+                    .note("this will become a hard error in the future")
+                    .emit();
+            } else {
+                struct_span_err!(self.tcx.sess, span, E0912, "{}", msg)
+                    .note(&format!("{} has an unspecified layout", ty))
+                    .note("this will become a hard error in the future")
+                    .emit();
+            }
+        };
+
+        if self.is_layout_specified(from) == Some(false) {
+            unspecified_layout("transmutation from a type with an unspecified layout", from);
+        }
+
+        if self.is_layout_specified(to) == Some(false) {
+            unspecified_layout("transmutation to a type with an unspecified layout", to);
+        }
+
+        // Check for same size using the skeletons.
         let sk_from = SizeSkeleton::compute(from, self.tcx, self.param_env);
         let sk_to = SizeSkeleton::compute(to, self.tcx, self.param_env);
 
-        // Check for same size using the skeletons.
         if let (Ok(sk_from), Ok(sk_to)) = (sk_from, sk_to) {
             if sk_from.same_size(sk_to) {
                 return;

src/librustc_lint/types.rs

@@ -11,9 +11,9 @@
 #![allow(non_snake_case)]
 
 use rustc::hir::map as hir_map;
-use rustc::ty::subst::Substs;
-use rustc::ty::{self, AdtKind, ParamEnv, Ty, TyCtxt};
+use rustc::ty::{self, AdtKind, ParamEnv, Ty};
 use rustc::ty::layout::{self, IntegerExt, LayoutOf};
+use rustc::middle::intrinsicck::is_repr_nullable_ptr;
 use util::nodemap::FxHashSet;
 use lint::{LateContext, LintContext, LintArray};
 use lint::{LintPass, LateLintPass};
@@ -428,40 +428,6 @@ enum FfiResult<'tcx> {
     },
 }
 
-/// Check if this enum can be safely exported based on the
-/// "nullable pointer optimization". Currently restricted
-/// to function pointers and references, but could be
-/// expanded to cover NonZero raw pointers and newtypes.
-/// FIXME: This duplicates code in codegen.
-fn is_repr_nullable_ptr<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
-                                  def: &'tcx ty::AdtDef,
-                                  substs: &Substs<'tcx>)
-                                  -> bool {
-    if def.variants.len() == 2 {
-        let data_idx;
-
-        if def.variants[0].fields.is_empty() {
-            data_idx = 1;
-        } else if def.variants[1].fields.is_empty() {
-            data_idx = 0;
-        } else {
-            return false;
-        }
-
-        if def.variants[data_idx].fields.len() == 1 {
-            match def.variants[data_idx].fields[0].ty(tcx, substs).sty {
-                ty::TyFnPtr(_) => {
-                    return true;
-                }
-                ty::TyRef(..) => {
-                    return true;
-                }
-                _ => {}
-            }
-        }
-    }
-    false
-}
 
 impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> {
     /// Check if the given type is "ffi-safe" (has a stable, well-defined

src/libstd/error.rs

@@ -30,7 +30,6 @@ use cell;
 use char;
 use core::array;
 use fmt::{self, Debug, Display};
-use mem::transmute;
 use num;
 use str;
 use string;
@@ -483,7 +482,7 @@ impl Error + Send {
         let err: Box<Error> = self;
         <Error>::downcast(err).map_err(|s| unsafe {
             // reapply the Send marker
-            transmute::<Box<Error>, Box<Error + Send>>(s)
+            Box::from_raw(Box::into_raw(s) as *mut (Error + Send))
         })
     }
 }
@@ -497,7 +496,7 @@ impl Error + Send + Sync {
         let err: Box<Error> = self;
         <Error>::downcast(err).map_err(|s| unsafe {
             // reapply the Send+Sync marker
-            transmute::<Box<Error>, Box<Error + Send + Sync>>(s)
+            Box::from_raw(Box::into_raw(s) as *mut (Error + Send + Sync))
         })
     }
 }

src/libstd/panicking.rs

@@ -297,10 +297,10 @@ pub unsafe fn try<R, F: FnOnce() -> R>(f: F) -> Result<R, Box<Any + Send>> {
     } else {
         update_panic_count(-1);
         debug_assert!(update_panic_count(0) == 0);
-        Err(mem::transmute(raw::TraitObject {
+        Err(Box::from_raw(*(&raw::TraitObject {
             data: any_data as *mut _,
             vtable: any_vtable as *mut _,
-        }))
+        } as *const _ as *const *mut _)))
     };
 
     fn do_call<F: FnOnce() -> R, R>(data: *mut u8) {