Add asm goto support to asm!

compiler/rustc_ast/src/ast.rs

@@ -2294,6 +2294,9 @@ pub enum InlineAsmOperand {
     Sym {
         sym: InlineAsmSym,
     },
+    Label {
+        block: P<Block>,
+    },
 }
 
 impl InlineAsmOperand {
@@ -2303,7 +2306,7 @@ impl InlineAsmOperand {
             | Self::Out { reg, .. }
             | Self::InOut { reg, .. }
             | Self::SplitInOut { reg, .. } => Some(reg),
-            Self::Const { .. } | Self::Sym { .. } => None,
+            Self::Const { .. } | Self::Sym { .. } | Self::Label { .. } => None,
         }
     }
 }

compiler/rustc_ast/src/mut_visit.rs

@@ -1330,6 +1330,7 @@ pub fn noop_visit_inline_asm<T: MutVisitor>(asm: &mut InlineAsm, vis: &mut T) {
             }
             InlineAsmOperand::Const { anon_const } => vis.visit_anon_const(anon_const),
             InlineAsmOperand::Sym { sym } => vis.visit_inline_asm_sym(sym),
+            InlineAsmOperand::Label { block } => vis.visit_block(block),
         }
     }
 }

compiler/rustc_ast/src/visit.rs

@@ -885,6 +885,7 @@ pub fn walk_inline_asm<'a, V: Visitor<'a>>(visitor: &mut V, asm: &'a InlineAsm)
                 try_visit!(visitor.visit_anon_const(anon_const))
             }
             InlineAsmOperand::Sym { sym } => try_visit!(visitor.visit_inline_asm_sym(sym)),
+            InlineAsmOperand::Label { block } => try_visit!(visitor.visit_block(block)),
         }
     }
     V::Result::output()

compiler/rustc_ast_lowering/messages.ftl

@@ -88,6 +88,9 @@ ast_lowering_invalid_abi_suggestion = did you mean
 ast_lowering_invalid_asm_template_modifier_const =
     asm template modifiers are not allowed for `const` arguments
 
+ast_lowering_invalid_asm_template_modifier_label =
+    asm template modifiers are not allowed for `label` arguments
+
 ast_lowering_invalid_asm_template_modifier_reg_class =
     invalid asm template modifier for this register class
 

compiler/rustc_ast_lowering/src/asm.rs

@@ -3,9 +3,9 @@ use crate::{ImplTraitContext, ImplTraitPosition, ParamMode, ResolverAstLoweringE
 use super::errors::{
     AbiSpecifiedMultipleTimes, AttSyntaxOnlyX86, ClobberAbiNotSupported,
     InlineAsmUnsupportedTarget, InvalidAbiClobberAbi, InvalidAsmTemplateModifierConst,
-    InvalidAsmTemplateModifierRegClass, InvalidAsmTemplateModifierRegClassSub,
-    InvalidAsmTemplateModifierSym, InvalidRegister, InvalidRegisterClass, RegisterClassOnlyClobber,
-    RegisterConflict,
+    InvalidAsmTemplateModifierLabel, InvalidAsmTemplateModifierRegClass,
+    InvalidAsmTemplateModifierRegClassSub, InvalidAsmTemplateModifierSym, InvalidRegister,
+    InvalidRegisterClass, RegisterClassOnlyClobber, RegisterConflict,
 };
 use super::LoweringContext;
 
@@ -236,6 +236,18 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                             }
                         }
                     }
+                    InlineAsmOperand::Label { block } => {
+                        if !self.tcx.features().asm_goto {
+                            feature_err(
+                                sess,
+                                sym::asm_goto,
+                                *op_sp,
+                                "label operands for inline assembly are unstable",
+                            )
+                            .emit();
+                        }
+                        hir::InlineAsmOperand::Label { block: self.lower_block(block, false) }
+                    }
                 };
                 (op, self.lower_span(*op_sp))
             })
@@ -295,6 +307,12 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                             op_span: op_sp,
                         });
                     }
+                    hir::InlineAsmOperand::Label { .. } => {
+                        self.dcx().emit_err(InvalidAsmTemplateModifierLabel {
+                            placeholder_span,
+                            op_span: op_sp,
+                        });
+                    }
                 }
             }
         }
@@ -334,7 +352,8 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
 
                         hir::InlineAsmOperand::Const { .. }
                         | hir::InlineAsmOperand::SymFn { .. }
-                        | hir::InlineAsmOperand::SymStatic { .. } => {
+                        | hir::InlineAsmOperand::SymStatic { .. }
+                        | hir::InlineAsmOperand::Label { .. } => {
                             unreachable!("{op:?} is not a register operand");
                         }
                     };

compiler/rustc_ast_lowering/src/errors.rs

@@ -262,6 +262,16 @@ pub struct InvalidAsmTemplateModifierSym {
     pub op_span: Span,
 }
 
+#[derive(Diagnostic, Clone, Copy)]
+#[diag(ast_lowering_invalid_asm_template_modifier_label)]
+pub struct InvalidAsmTemplateModifierLabel {
+    #[primary_span]
+    #[label(ast_lowering_template_modifier)]
+    pub placeholder_span: Span,
+    #[label(ast_lowering_argument)]
+    pub op_span: Span,
+}
+
 #[derive(Diagnostic, Clone, Copy)]
 #[diag(ast_lowering_register_class_only_clobber)]
 pub struct RegisterClassOnlyClobber {

compiler/rustc_ast_pretty/src/pprust/state.rs

@@ -1331,6 +1331,10 @@ impl<'a> State<'a> {
                             s.print_path(&sym.path, true, 0);
                         }
                     }
+                    InlineAsmOperand::Label { block } => {
+                        s.head("label");
+                        s.print_block(block);
+                    }
                 }
             }
             AsmArg::ClobberAbi(abi) => {

compiler/rustc_borrowck/src/lib.rs

@@ -723,7 +723,7 @@ impl<'cx, 'tcx, R> rustc_mir_dataflow::ResultsVisitor<'cx, 'tcx, R> for MirBorro
                 operands,
                 options: _,
                 line_spans: _,
-                destination: _,
+                targets: _,
                 unwind: _,
             } => {
                 for op in operands {
@@ -749,7 +749,8 @@ impl<'cx, 'tcx, R> rustc_mir_dataflow::ResultsVisitor<'cx, 'tcx, R> for MirBorro
                         }
                         InlineAsmOperand::Const { value: _ }
                         | InlineAsmOperand::SymFn { value: _ }
-                        | InlineAsmOperand::SymStatic { def_id: _ } => {}
+                        | InlineAsmOperand::SymStatic { def_id: _ }
+                        | InlineAsmOperand::Label { target_index: _ } => {}
                     }
                 }
             }

compiler/rustc_borrowck/src/polonius/loan_invalidations.rs

@@ -161,7 +161,7 @@ impl<'cx, 'tcx> Visitor<'tcx> for LoanInvalidationsGenerator<'cx, 'tcx> {
                 operands,
                 options: _,
                 line_spans: _,
-                destination: _,
+                targets: _,
                 unwind: _,
             } => {
                 for op in operands {
@@ -182,7 +182,8 @@ impl<'cx, 'tcx> Visitor<'tcx> for LoanInvalidationsGenerator<'cx, 'tcx> {
                         }
                         InlineAsmOperand::Const { value: _ }
                         | InlineAsmOperand::SymFn { value: _ }
-                        | InlineAsmOperand::SymStatic { def_id: _ } => {}
+                        | InlineAsmOperand::SymStatic { def_id: _ }
+                        | InlineAsmOperand::Label { target_index: _ } => {}
                     }
                 }
             }

compiler/rustc_borrowck/src/type_check/mod.rs

@@ -1771,8 +1771,8 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                 self.assert_iscleanup(body, block_data, real_target, is_cleanup);
                 self.assert_iscleanup_unwind(body, block_data, unwind, is_cleanup);
             }
-            TerminatorKind::InlineAsm { destination, unwind, .. } => {
-                if let Some(target) = destination {
+            TerminatorKind::InlineAsm { ref targets, unwind, .. } => {
+                for &target in targets {
                     self.assert_iscleanup(body, block_data, target, is_cleanup);
                 }
                 self.assert_iscleanup_unwind(body, block_data, unwind, is_cleanup);

compiler/rustc_builtin_macros/messages.ftl

@@ -19,6 +19,8 @@ builtin_macros_asm_expected_other = expected operand, {$is_global_asm ->
 
 builtin_macros_asm_explicit_register_name = explicit register arguments cannot have names
 
+builtin_macros_asm_mayunwind = asm labels are not allowed with the `may_unwind` option
+
 builtin_macros_asm_modifier_invalid = asm template modifier must be a single character
 
 builtin_macros_asm_mutually_exclusive = the `{$opt1}` and `{$opt2}` options are mutually exclusive

compiler/rustc_builtin_macros/src/asm.rs

@@ -166,6 +166,9 @@ pub fn parse_asm_args<'a>(
                 path: path.clone(),
             };
             ast::InlineAsmOperand::Sym { sym }
+        } else if !is_global_asm && p.eat_keyword(sym::label) {
+            let block = p.parse_block()?;
+            ast::InlineAsmOperand::Label { block }
         } else if allow_templates {
             let template = p.parse_expr()?;
             // If it can't possibly expand to a string, provide diagnostics here to include other
@@ -242,6 +245,7 @@ pub fn parse_asm_args<'a>(
     let mut have_real_output = false;
     let mut outputs_sp = vec![];
     let mut regclass_outputs = vec![];
+    let mut labels_sp = vec![];
     for (op, op_sp) in &args.operands {
         match op {
             ast::InlineAsmOperand::Out { reg, expr, .. }
@@ -259,6 +263,9 @@ pub fn parse_asm_args<'a>(
                     regclass_outputs.push(*op_sp);
                 }
             }
+            ast::InlineAsmOperand::Label { .. } => {
+                labels_sp.push(*op_sp);
+            }
             _ => {}
         }
     }
@@ -270,6 +277,9 @@ pub fn parse_asm_args<'a>(
         // Bail out now since this is likely to confuse MIR
         return Err(err);
     }
+    if args.options.contains(ast::InlineAsmOptions::MAY_UNWIND) && !labels_sp.is_empty() {
+        dcx.emit_err(errors::AsmMayUnwind { labels_sp });
+    }
 
     if args.clobber_abis.len() > 0 {
         if is_global_asm {

compiler/rustc_builtin_macros/src/errors.rs

@@ -787,6 +787,13 @@ pub(crate) struct AsmNoReturn {
     pub(crate) outputs_sp: Vec<Span>,
 }
 
+#[derive(Diagnostic)]
+#[diag(builtin_macros_asm_mayunwind)]
+pub(crate) struct AsmMayUnwind {
+    #[primary_span]
+    pub(crate) labels_sp: Vec<Span>,
+}
+
 #[derive(Diagnostic)]
 #[diag(builtin_macros_global_asm_clobber_abi)]
 pub(crate) struct GlobalAsmClobberAbi {

compiler/rustc_codegen_cranelift/src/base.rs

@@ -445,7 +445,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                 template,
                 operands,
                 options,
-                destination,
+                targets,
                 line_spans: _,
                 unwind: _,
             } => {
@@ -456,13 +456,25 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                     );
                 }
 
+                let have_labels = if options.contains(InlineAsmOptions::NORETURN) {
+                    !targets.is_empty()
+                } else {
+                    targets.len() > 1
+                };
+                if have_labels {
+                    fx.tcx.dcx().span_fatal(
+                        source_info.span,
+                        "cranelift doesn't support labels in inline assembly.",
+                    );
+                }
+
                 crate::inline_asm::codegen_inline_asm_terminator(
                     fx,
                     source_info.span,
                     template,
                     operands,
                     *options,
-                    *destination,
+                    targets.get(0).copied(),
                 );
             }
             TerminatorKind::UnwindTerminate(reason) => {

compiler/rustc_codegen_cranelift/src/global_asm.rs

@@ -78,7 +78,8 @@ pub(crate) fn codegen_global_asm_item(tcx: TyCtxt<'_>, global_asm: &mut String,
                         InlineAsmOperand::In { .. }
                         | InlineAsmOperand::Out { .. }
                         | InlineAsmOperand::InOut { .. }
-                        | InlineAsmOperand::SplitInOut { .. } => {
+                        | InlineAsmOperand::SplitInOut { .. }
+                        | InlineAsmOperand::Label { .. } => {
                             span_bug!(op_sp, "invalid operand type for global_asm!")
                         }
                     }

compiler/rustc_codegen_cranelift/src/inline_asm.rs

@@ -129,6 +129,9 @@ pub(crate) fn codegen_inline_asm_terminator<'tcx>(
                 let instance = Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx);
                 CInlineAsmOperand::Symbol { symbol: fx.tcx.symbol_name(instance).name.to_owned() }
             }
+            InlineAsmOperand::Label { .. } => {
+                span_bug!(span, "asm! label operands are not yet supported");
+            }
         })
         .collect::<Vec<_>>();
 

compiler/rustc_codegen_gcc/src/asm.rs

@@ -107,7 +107,7 @@ enum ConstraintOrRegister {
 
 
 impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
-    fn codegen_inline_asm(&mut self, template: &[InlineAsmTemplatePiece], rust_operands: &[InlineAsmOperandRef<'tcx, Self>], options: InlineAsmOptions, span: &[Span], instance: Instance<'_>, _dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>) {
+    fn codegen_inline_asm(&mut self, template: &[InlineAsmTemplatePiece], rust_operands: &[InlineAsmOperandRef<'tcx, Self>], options: InlineAsmOptions, span: &[Span], instance: Instance<'_>, dest: Option<Self::BasicBlock>, _catch_funclet: Option<(Self::BasicBlock, Option<&Self::Funclet>)>) {
         if options.contains(InlineAsmOptions::MAY_UNWIND) {
             self.sess().dcx()
                 .create_err(UnwindingInlineAsm { span: span[0] })
@@ -126,6 +126,10 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
         // added to `outputs.len()`
         let mut inputs = vec![];
 
+        // GCC index of a label equals its position in the array added to
+        // `outputs.len() + inputs.len()`.
+        let mut labels = vec![];
+
         // Clobbers collected from `out("explicit register") _` and `inout("expl_reg") var => _`
         let mut clobbers = vec![];
 
@@ -269,6 +273,10 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                     // some targets to add a leading underscore (Mach-O).
                     constants_len += self.tcx.symbol_name(Instance::mono(self.tcx, def_id)).name.len();
                 }
+
+                InlineAsmOperandRef::Label { label } => {
+                    labels.push(label);
+                }
             }
         }
 
@@ -368,6 +376,10 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                 InlineAsmOperandRef::Const { .. } => {
                     // processed in the previous pass
                 }
+
+                InlineAsmOperandRef::Label { .. } => {
+                    // processed in the previous pass
+                }
             }
         }
 
@@ -454,6 +466,14 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                         InlineAsmOperandRef::Const { ref string } => {
                             template_str.push_str(string);
                         }
+
+                        InlineAsmOperandRef::Label { label } => {
+                            let label_gcc_index = labels.iter()
+                                .position(|&l| l == label)
+                                .expect("wrong rust index");
+                            let gcc_index = label_gcc_index + outputs.len() + inputs.len();
+                            push_to_template(Some('l'), gcc_index);
+                        }
                     }
                 }
             }
@@ -466,7 +486,12 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
         // 4. Generate Extended Asm block
 
         let block = self.llbb();
-        let extended_asm = block.add_extended_asm(None, &template_str);
+        let extended_asm = if let Some(dest) = dest {
+            assert!(!labels.is_empty());
+            block.end_with_extended_asm_goto(None, &template_str, &labels, Some(dest))
+        } else {
+            block.add_extended_asm(None, &template_str)
+        };
 
         for op in &outputs {
             extended_asm.add_output_operand(None, &op.to_constraint(), op.tmp_var);
@@ -494,7 +519,7 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
         if !options.contains(InlineAsmOptions::NOSTACK) {
             // TODO(@Commeownist): figure out how to align stack
         }
-        if options.contains(InlineAsmOptions::NORETURN) {
+        if dest.is_none() && options.contains(InlineAsmOptions::NORETURN) {
             let builtin_unreachable = self.context.get_builtin_function("__builtin_unreachable");
             let builtin_unreachable: RValue<'gcc> = unsafe { std::mem::transmute(builtin_unreachable) };
             self.call(self.type_void(), None, None, builtin_unreachable, &[], None);