Call to Rust extern "fastcall" function does not follow GNU/Clang fastcall convention

[rprichard]

For a call to an extern "fastcall" function on 32-bit Linux, rustc passes arguments via the stack instead of via registers, and it doesn't pop the arguments from the stack after the call. I was expecting rustc to follow the GNU/Clang "fastcall" convention (http://en.wikipedia.org/wiki/X86_calling_conventions#fastcall). Instead, it seems to be following something like the "stdcall" convention.

Test case:

cat > fastcall_callee.c << EOF
#include <stdio.h>
void callee(int x, int y) __attribute__((fastcall));
void callee(int x, int y) {
    printf("%d %d\n", x, y);
}
EOF

cat > fastcall_good_caller.c << EOF
void callee(int x, int y) __attribute__((fastcall));
void good_caller() {
    callee(7, 11);
    callee(17, 19);
}
EOF

cat > fastcall_caller.rs << EOF
extern crate libc;
#[link(name = "fastcall_callee")]
extern "fastcall" {
    fn callee(x: libc::c_int, y: libc::c_int);
}
pub fn main() {
    unsafe {
        callee(7, 11);
        callee(17, 19);
    }
}
EOF

clang -m32 fastcall_callee.c -c -O2
rm -f libfastcall_callee.a
ar rf libfastcall_callee.a fastcall_callee.o

export LD_LIBRARY_PATH=$PWD/rust-nightly-i686-unknown-linux-gnu/lib
rust-nightly-i686-unknown-linux-gnu/bin/rustc fastcall_caller.rs -L . -O

./fastcall_caller

Output:

-159272960 -7321680
0 -7323132
Segmentation fault (core dumped)

Here's the assembly output from main (with some stuff elided):

_ZN4main20h75001e02b331b561raaE:
    cmpl    %gs:48, %esp
    ja  .LBB0_2
    pushl   $0
    pushl   $12
    calll   __morestack
    retl
.LBB0_2:
    pushl   %ebx
    subl    $8, %esp
    calll   .L0$pb
.L0$pb:
    popl    %ebx
.Ltmp3:
    addl    $_GLOBAL_OFFSET_TABLE_+(.Ltmp3-.L0$pb), %ebx
    movl    $11, 4(%esp)
    movl    $7, (%esp)
    calll   callee@PLT
    subl    $8, %esp
    movl    $19, 4(%esp)
    movl    $17, (%esp)
    calll   callee@PLT
    popl    %ebx
    retl

I compiled both fastcall_good_caller.c and fastcall_caller.rs to LLVM IR, and the difference seems to be an inreg keyword. Clang outputs it, but Rust doesn't.

fastcall_good_caller.ll:

; Function Attrs: nounwind
define void @good_caller() #0 {
  tail call x86_fastcallcc void @callee(i32 inreg 7, i32 inreg 11) #2
  tail call x86_fastcallcc void @callee(i32 inreg 17, i32 inreg 19) #2
  ret void
}

attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
attributes #2 = { nounwind }

fastcall_caller.ll:

; Function Attrs: uwtable
define internal void @_ZN4main20h75001e02b331b561raaE() unnamed_addr #1 {
entry-block:
  tail call x86_fastcallcc void @callee(i32 7, i32 11)
  tail call x86_fastcallcc void @callee(i32 17, i32 19)
  ret void
}

attributes #1 = { uwtable "split-stack" }

[rprichard]

This llvm-dev thread looks relevant: http://lists.cs.uiuc.edu/pipermail/llvmdev/2012-October/054629.html

[XMPPwocky]

This is currently blocking a lot of things I'd like to do in Rust, and is requiring an awful C wrapper library just for calling conventions.

[steveklabnik]

Traige: trying to reproduce, adding #![feature(libc)] to fastcall_caller.rs so as not to bother with extern crates, I now get

note: /usr/bin/ld.gold.real: warning: skipping incompatible ./libfastcall_callee.a while searching for fastcall_callee
/usr/bin/ld.gold.real: error: cannot find -lfastcall_callee
fastcall_caller.0.o:fastcall_caller.0.rs:function main::h11bb15aa3118a7afoaa: error: undefined reference to 'callee'
fastcall_caller.0.o:fastcall_caller.0.rs:function main::h11bb15aa3118a7afoaa: error: undefined reference to 'callee'
collect2: error: ld returned 1 exit status

This seems better than a segfault, but still not good?

[pravic]

Same here. It seems, "fastcall" is not correct still: testcase.

extern "fastcall" fn inner_call(a: u32, b: u8, c: u16) -> u32 {
    a + b as u32 + c as u32
}

#[link(name="outer")]
extern "fastcall"
{
    fn outer_call(a: u32, b: u8, c: u16) -> u32;
}

fn main() {
    let r1 = inner_call(1,2,3);
    let r2 = unsafe { outer_call(1,2,3) };
    println!("fcall: {} and {}", r1, r2);
}

unsigned __fastcall outer_call(unsigned int a, unsigned char b, unsigned short c) {
    return a + b + c;
}

Meta

rustc 1.9.0-nightly (526f2bf 2016-04-09)
binary: rustc
commit-hash: 526f2bf
commit-date: 2016-04-09
host: i686-pc-windows-msvc
release: 1.9.0-nightly

[eddyb]

From @pravic's examples, clang generates:

declare x86_fastcallcc i32 @"\01@outer_call@12"(i32 inreg, i8 inreg zeroext, i16 zeroext)

While rustc only has:

declare x86_fastcallcc i32 @outer_call(i32, i8 zeroext, i16 zeroext)

Naming aside (it ends up linking just fine), we're missing inreg attributes. cc @dotdash

[Ericson2314]

@whitequark @edef1c we could use this for libfringe.