Implement likely/unlikely intrinsics

src/libcore/intrinsics.rs

@@ -602,4 +602,24 @@ extern "rust-intrinsic" {
     /// Returns the value of the discriminant for the variant in 'v',
     /// cast to a `u64`; if `T` has no discriminant, returns 0.
     pub fn discriminant_value<T>(v: &T) -> u64;
+
+    /// Hints to the compiler that a branch is likely to be taken. Returns the value
+    /// passed to it. In order for the hint to take effect, it should be used as follows:
+    ///
+    /// ```
+    /// unsafe fn foo(a: i32, b: i32) -> i32 {
+    ///     if likely(a == b) {
+    ///         1
+    ///     } else {
+    ///         2
+    ///     }
+    /// }
+    /// ```
+    #[cfg(not(stage0))]
+    pub fn likely(v: bool) -> bool;
+
+    /// Hints to the compiler that a branch is not likely to be taken. See `likely` for
+    /// more details
+    #[cfg(not(stage0))]
+    pub fn unlikely(v: bool) -> bool;
 }

src/librustc_trans/trans/base.rs

@@ -1672,8 +1672,7 @@ pub fn trans_named_tuple_constructor<'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>,
                                                  disr: ty::Disr,
                                                  args: callee::CallArgs,
                                                  dest: expr::Dest,
-                                                 debug_loc: DebugLoc)
-                                                 -> Result<'blk, 'tcx> {
+                                                 debug_loc: DebugLoc) -> Result<'blk, 'tcx> {
 
     let ccx = bcx.fcx.ccx;
     let tcx = ccx.tcx();
@@ -1717,6 +1716,12 @@ pub fn trans_named_tuple_constructor<'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>,
         }
     }
 
+    let val = if type_is_immediate(ccx, result_ty) && !type_is_zero_size(ccx, result_ty) {
+        load_ty(bcx, llresult, result_ty)
+    } else {
+        common::C_nil(ccx)
+    };
+
     // If the caller doesn't care about the result
     // drop the temporary we made
     let bcx = match dest {
@@ -1730,7 +1735,7 @@ pub fn trans_named_tuple_constructor<'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>,
         }
     };
 
-    Result::new(bcx, llresult)
+    Result::new(bcx, val)
 }
 
 pub fn trans_tuple_struct<'a, 'tcx>(ccx: &CrateContext<'a, 'tcx>,

src/librustc_trans/trans/build.rs

@@ -31,7 +31,14 @@ pub fn terminate(cx: Block, _: &str) {
 
 pub fn check_not_terminated(cx: Block) {
     if cx.terminated.get() {
-        panic!("already terminated!");
+        let fcx = cx.fcx;
+        if let Some(span) = fcx.span {
+            cx.tcx().sess.span_bug(
+                span,
+                "already terminated!");
+        } else {
+            cx.tcx().sess.bug("already terminated!");
+        }
     }
 }
 

src/librustc_trans/trans/builder.rs

@@ -718,6 +718,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     pub fn icmp(&self, op: IntPredicate, lhs: ValueRef, rhs: ValueRef) -> ValueRef {
         self.count_insn("icmp");
         unsafe {
+            assert!(val_ty(lhs) == val_ty(rhs), "Cannot compare {} and {}",
+                    self.ccx.tn().val_to_string(lhs),
+                    self.ccx.tn().val_to_string(rhs));
             llvm::LLVMBuildICmp(self.llbuilder, op as c_uint, lhs, rhs, noname())
         }
     }

src/librustc_trans/trans/callee.rs

@@ -87,8 +87,8 @@ pub struct Callee<'blk, 'tcx: 'blk> {
     pub data: CalleeData<'tcx>,
 }
 
-fn trans<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, expr: &ast::Expr)
-                     -> Callee<'blk, 'tcx> {
+pub fn trans<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, expr: &ast::Expr)
+                         -> Callee<'blk, 'tcx> {
     let _icx = push_ctxt("trans_callee");
     debug!("callee::trans(expr={})", expr.repr(bcx.tcx()));
 
@@ -860,13 +860,25 @@ pub fn trans_call_inner<'a, 'blk, 'tcx, F>(bcx: Block<'blk, 'tcx>,
                          abi);
         fcx.scopes.borrow_mut().last_mut().unwrap().drop_non_lifetime_clean();
 
-        bcx = foreign::trans_native_call(bcx,
+        let (llret, b) = foreign::trans_native_call(bcx,
                                          callee_ty,
                                          llfn,
                                          opt_llretslot.unwrap(),
                                          &llargs[..],
                                          arg_tys,
                                          debug_loc);
+
+        bcx = b;
+        match (opt_llretslot, ret_ty) {
+            (Some(_), ty::FnConverging(ret_ty)) => {
+                if !type_of::return_uses_outptr(bcx.ccx(), ret_ty) &&
+                    !common::type_is_zero_size(bcx.ccx(), ret_ty)
+                {
+                    llresult = llret;
+                }
+            }
+            (_, _) => {}
+        }
     }
 
     fcx.pop_and_trans_custom_cleanup_scope(bcx, arg_cleanup_scope);

src/librustc_trans/trans/controlflow.rs

@@ -19,6 +19,7 @@ use trans::cleanup::CleanupMethods;
 use trans::cleanup;
 use trans::common::*;
 use trans::consts;
+use trans::datum;
 use trans::debuginfo;
 use trans::debuginfo::{DebugLoc, ToDebugLoc};
 use trans::expr;
@@ -144,6 +145,48 @@ pub fn trans_block<'blk, 'tcx>(bcx: Block<'blk, 'tcx>,
     return bcx;
 }
 
+/// Same as `trans_block` except it returns a `DatumBlock` instead of storing the
+/// the result to a destination. This avoids going through a temporary when it's
+/// not needed, primarily to ensure that `unsafe { likely(cond) }` and similar patterns
+/// work.
+pub fn trans_block_datum<'blk, 'tcx>(bcx: Block<'blk, 'tcx>,
+                                     b: &ast::Block)
+                                     -> datum::DatumBlock<'blk, 'tcx, datum::Expr> {
+    let _icx = push_ctxt("trans_block_datum");
+
+    if bcx.unreachable.get() {
+        let llval = C_nil(bcx.ccx());
+        let datum = datum::immediate_rvalue(llval, ty::mk_nil(bcx.tcx()));
+        return datum::DatumBlock {bcx: bcx, datum: datum.to_expr_datum()};
+    }
+
+    let fcx = bcx.fcx;
+    let mut bcx = bcx;
+
+    let cleanup_debug_loc =
+        debuginfo::get_cleanup_debug_loc_for_ast_node(bcx.ccx(), b.id, b.span, true);
+    fcx.push_ast_cleanup_scope(cleanup_debug_loc);
+
+    for s in &b.stmts {
+        bcx = trans_stmt(bcx, &**s);
+    }
+
+    let datum = match b.expr {
+        Some(ref e) if !bcx.unreachable.get() => {
+            unpack_datum!(bcx, expr::trans(bcx, &**e))
+        }
+        _ => {
+            let llval = C_nil(bcx.ccx());
+            datum::immediate_rvalue(llval, ty::mk_nil(bcx.tcx())).to_expr_datum()
+        }
+    };
+
+    bcx = fcx.pop_and_trans_ast_cleanup_scope(bcx, b.id);
+
+    datum::DatumBlock {bcx: bcx, datum: datum}
+}
+
+
 pub fn trans_if<'blk, 'tcx>(bcx: Block<'blk, 'tcx>,
                             if_id: ast::NodeId,
                             cond: &ast::Expr,

src/librustc_trans/trans/expr.rs

@@ -571,16 +571,17 @@ pub fn trans_to_lvalue<'blk, 'tcx>(bcx: Block<'blk, 'tcx>,
 /// A version of `trans` that ignores adjustments. You almost certainly do not want to call this
 /// directly.
 fn trans_unadjusted<'blk, 'tcx>(bcx: Block<'blk, 'tcx>,
-                                expr: &ast::Expr)
-                                -> DatumBlock<'blk, 'tcx, Expr> {
+                                expr: &ast::Expr) -> DatumBlock<'blk, 'tcx, Expr> {
     let mut bcx = bcx;
 
     debug!("trans_unadjusted(expr={})", bcx.expr_to_string(expr));
     let _indenter = indenter();
 
     debuginfo::set_source_location(bcx.fcx, expr.id, expr.span);
 
-    return match ty::expr_kind(bcx.tcx(), expr) {
+    let ty = expr_ty(bcx, expr);
+
+    return match expr_kind(bcx, expr) {
         ty::LvalueExpr | ty::RvalueDatumExpr => {
             let datum = unpack_datum!(bcx, {
                 trans_datum_unadjusted(bcx, expr)
@@ -591,11 +592,10 @@ fn trans_unadjusted<'blk, 'tcx>(bcx: Block<'blk, 'tcx>,
 
         ty::RvalueStmtExpr => {
             bcx = trans_rvalue_stmt_unadjusted(bcx, expr);
-            nil(bcx, expr_ty(bcx, expr))
+            nil(bcx, ty)
         }
 
         ty::RvalueDpsExpr => {
-            let ty = expr_ty(bcx, expr);
             if type_is_zero_size(bcx.ccx(), ty) {
                 bcx = trans_rvalue_dps_unadjusted(bcx, expr, Ignore);
                 nil(bcx, ty)
@@ -629,6 +629,55 @@ fn trans_unadjusted<'blk, 'tcx>(bcx: Block<'blk, 'tcx>,
     }
 }
 
+// Get the appropriate expression kind for the expression. Most of the time this just uses
+// ty::expr_kind, but `ExprCall`s can be treated as `RvalueDatumExpr`s in some cases.
+fn expr_kind<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, expr: &ast::Expr) -> ty::ExprKind {
+    let ty = expr_ty(bcx, expr);
+    match expr.node {
+        ast::ExprCall(ref f, _) if !bcx.tcx().is_method_call(expr.id) => {
+            if let ast::ExprPath(..) = f.node {
+                let fn_ty = expr_ty_adjusted(bcx, f);
+
+                let ret_ty = match fn_ty.sty {
+                    ty::TyBareFn(_, ref fty) =>
+                        ty::erase_late_bound_regions(bcx.tcx(), &fty.sig.output()),
+                    _ => bcx.tcx().sess.bug("Not calling a function?")
+                };
+
+                let is_datum = if let ty::FnConverging(output) = ret_ty {
+                    !type_of::return_uses_outptr(bcx.ccx(), output) &&
+                        !bcx.fcx.type_needs_drop(output)
+                } else {
+                    true
+                };
+
+
+                if is_datum {
+                    return ty::RvalueDatumExpr;
+                }
+            }
+        }
+        ast::ExprBlock(ref b) if !bcx.fcx.type_needs_drop(ty) => {
+            // Only consider the final expression if it's reachable
+            let reachable = if let Some(ref cfg) = bcx.fcx.cfg {
+                cfg.node_is_reachable(expr.id)
+            } else {
+                true
+            };
+            // Use the kind of the last expression in the block, since
+            // it's the only one that actually matters
+            if let Some(ref expr) = b.expr {
+                if reachable {
+                    return expr_kind(bcx, expr);
+                }
+            }
+        }
+        _ => ()
+    }
+
+    return ty::expr_kind(bcx.tcx(), expr);
+}
+
 fn trans_datum_unadjusted<'blk, 'tcx>(bcx: Block<'blk, 'tcx>,
                                       expr: &ast::Expr)
                                       -> DatumBlock<'blk, 'tcx, Expr> {
@@ -696,6 +745,38 @@ fn trans_datum_unadjusted<'blk, 'tcx>(bcx: Block<'blk, 'tcx>,
             // Datum output mode means this is a scalar cast:
             trans_imm_cast(bcx, &**val, expr.id)
         }
+        ast::ExprCall(ref f, ref args) => {
+            let fn_ty = expr_ty_adjusted(bcx, f);
+
+            let ret_ty = match fn_ty.sty {
+                ty::TyBareFn(_, ref fty) => {
+                    ty::erase_late_bound_regions(bcx.tcx(), &fty.sig.output())
+                }
+                _ => panic!("Not calling a function?!")
+            };
+
+            let args = callee::ArgExprs(&args[..]);
+            let result = callee::trans_call_inner(bcx,
+                                                  expr.debug_loc(),
+                                                  fn_ty,
+                                                  |cx, _| callee::trans(cx, f),
+                                                  args, Some(Ignore));
+
+            if let ty::FnConverging(ret_ty) = ret_ty {
+                immediate_rvalue_bcx(result.bcx, result.val, ret_ty)
+                    .to_expr_datumblock()
+            } else {
+                // We called a diverging function, generate an undef value of the appropriate
+                // type.
+                let ty = expr_ty(bcx, expr);
+                let llval = C_undef(type_of::arg_type_of(bcx.ccx(), ty));
+                let datum = immediate_rvalue(llval, ty);
+                DatumBlock::new(bcx, datum.to_expr_datum())
+            }
+        }
+        ast::ExprBlock(ref b) => {
+            controlflow::trans_block_datum(bcx, b)
+        }
         _ => {
             bcx.tcx().sess.span_bug(
                 expr.span,
@@ -1018,6 +1099,9 @@ fn trans_rvalue_stmt_unadjusted<'blk, 'tcx>(bcx: Block<'blk, 'tcx>,
         ast::ExprInlineAsm(ref a) => {
             asm::trans_inline_asm(bcx, a)
         }
+        ast::ExprBlock(ref b) => {
+            controlflow::trans_block(bcx, b, Ignore)
+        }
         _ => {
             bcx.tcx().sess.span_bug(
                 expr.span,
@@ -1036,6 +1120,10 @@ fn trans_rvalue_dps_unadjusted<'blk, 'tcx>(bcx: Block<'blk, 'tcx>,
     let mut bcx = bcx;
     let tcx = bcx.tcx();
 
+    if bcx.unreachable.get() {
+        return bcx;
+    }
+
     debuginfo::set_source_location(bcx.fcx, expr.id, expr.span);
 
     match expr.node {
@@ -2418,13 +2506,10 @@ impl OverflowOpViaIntrinsic {
 
         let val = Call(bcx, llfn, &[lhs, rhs], None, binop_debug_loc);
         let result = ExtractValue(bcx, val, 0); // iN operation result
-        let overflow = ExtractValue(bcx, val, 1); // i1 "did it overflow?"
-
-        let cond = ICmp(bcx, llvm::IntEQ, overflow, C_integral(Type::i1(bcx.ccx()), 1, false),
-                        binop_debug_loc);
+        let cond = ExtractValue(bcx, val, 1); // i1 "did it overflow?"
 
         let expect = bcx.ccx().get_intrinsic(&"llvm.expect.i1");
-        Call(bcx, expect, &[cond, C_integral(Type::i1(bcx.ccx()), 0, false)],
+        let cond = Call(bcx, expect, &[cond, C_integral(Type::i1(bcx.ccx()), 0, false)],
              None, binop_debug_loc);
 
         let bcx =

src/librustc_trans/trans/foreign.rs

@@ -230,7 +230,7 @@ pub fn trans_native_call<'blk, 'tcx>(bcx: Block<'blk, 'tcx>,
                                      llargs_rust: &[ValueRef],
                                      passed_arg_tys: Vec<Ty<'tcx>>,
                                      call_debug_loc: DebugLoc)
-                                     -> Block<'blk, 'tcx>
+                                     -> (ValueRef, Block<'blk, 'tcx>)
 {
     let ccx = bcx.ccx();
     let tcx = bcx.tcx();
@@ -389,6 +389,7 @@ pub fn trans_native_call<'blk, 'tcx>(bcx: Block<'blk, 'tcx>,
     // type to match because some ABIs will use a different type than
     // the Rust type. e.g., a {u32,u32} struct could be returned as
     // u64.
+    let llretval = llforeign_retval;
     if llsig.ret_def && !fn_type.ret_ty.is_indirect() {
         let llrust_ret_ty = llsig.llret_ty;
         let llforeign_ret_ty = match fn_type.ret_ty.cast {
@@ -436,7 +437,7 @@ pub fn trans_native_call<'blk, 'tcx>(bcx: Block<'blk, 'tcx>,
         }
     }
 
-    return bcx;
+    return (llretval, bcx);
 }
 
 // feature gate SIMD types in FFI, since I (huonw) am not sure the