Linking CUDA: errors

[David-OConnor]

Problem: CUDA is unable to link using cc or with manually nvcc-compiled libraries, on two independent hardware setups. rustc version: rustc 1.73.0 (cc66ad468 2023-10-03) cargo clean between each build. main.rs and cuda.cu code is the same for both setups:

main.rs:

#[link(name = "cuda", kind = "static")]
extern "C" {
    fn ffi_test();
}

fn main() {
    unsafe {
        ffi_test();
    }
}

cuda.cu:

extern "C" void ffi_test() {
    std::cout << "FFI TEST" << std::endl;
}

System 1: Windows 11. Environment description:

• CUDA 12.2 installed
• MSVC 2022 Community installed, with v143 C/C++ build tools
• LLVM17.4 installed
• CUDA_PATH env variable set to C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.2 (also added to PATH)
• LLVM_PATH env variable set to C:\Program Files\LLVM\bin (also added to PATH, and LIBCLANG_PATH)
• CXX env var set to C:\\Program Files\\Microsoft Visual Studio\\2022\\Community\\VC\\Tools\\MSVC\\14.37.32822\\bin\\Hostx64\\x64

build.rs main:

fn main() {
    // Tell Cargo that if the given file changes, to rerun this build script.
    println!("cargo:rerun-if-changed=src/cuda.cu");

    cc::Build::new()
        .cuda(true)
        .cudart("shared")  // Defaults to `static`; have tried both
        .flag("-gencode").flag("arch=compute_75,code=sm_75")
        .file("src/cuda.cu")
        .compile("cuda");

    println!("cargo:rustc-link-lib=msvc");
    println!("cargo:rustc-link-lib=cudart");
    println!("cargo:rustc-link-search=native=C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v12.2/lib/x64");
}

Error:

  running: "nvcc" "-ccbin=C:\\Program Files\\Microsoft Visual Studio\\2022\\Community\\VC\\Tools\\MSVC\\14.37.32822\\bin\\Hostx64\\x64" "-Xcompiler" "-O0" "-Xcompiler" "-ffunction-sections" "-Xcompiler" "-fdata-sections" "-G" "-Xcompiler" "-g" "-Xcompiler" "-fno-omit-frame-pointer" "-m64" "-Xcompiler" "-Wall" "-Xcompiler" "-Wextra" "-t0" "-o" "C:\\Users\\the_a\\code\\wf-lab\\target\\debug\\build\\wf_lab-c50242c302aa4eb9\\out\\src/cuda.o" "-c" "src/cuda.cu"
  cuda.cu
  cargo:warning=cl : Command line error D8021 : invalid numeric argument '/Wextra'

  cargo:warning=cl : Command line error D8021 : invalid numeric argument '/Wextra'

  exit code: 0xffffffff

  --- stderr

If removing the cc build and compiling manually with nvcc, using the same commands, but without the Wextra flag:

hains\\stable-x86_64-pc-windows-msvc\\lib\\rustlib\\etc\\libstd.natvis"  = note: LINK : warning LNK4098: defaultlib 'LIBCMT' conflicts with use of other libs; use /NODEFAULTLIB:library
        cuda.lib : error LNK2019: unresolved external symbol __cudaRegisterFatBinary referenced in function "void __cdecl __nv_cudaEntityRegisterCallback(void * *)" (?__nv_cudaEntityRegisterCallback@@YAXPEAPEAX@Z)
        cuda.lib : error LNK2019: unresolved external symbol __cudaRegisterFatBinaryEnd referenced in function "void __cdecl __nv_cudaEntityRegisterCallback(void * *)" (?__nv_cudaEntityRegisterCallback@@YAXPEAPEAX@Z)
        cuda.lib : error LNK2019: unresolved external symbol __cudaUnregisterFatBinary referenced in function "void __cdecl __cudaUnregisterBinaryUtil(void)" (?__cudaUnregisterBinaryUtil@@YAXXZ)
        C:\Users\the_a\code\wf-lab\target\debug\deps\wf_lab.exe : fatal error LNK1120: 3 unresolved externals

(Note that a simplified bare minimum version may be nvcc src/cuda.cu -gencode "arch=compute_75,code=sm_75" -c -o cuda.lib; the one generated by cc has more flags.)

System 2: Ubuntu 20.04 LTS Clean install. Environment description:

• CUDA 12.3 installed, via the .run file from nvidia.
• sudo apt install build-essential run
• sudo apt update run
• CUDA_PATH env variable set to C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.2 (also added to PATH)
• /usr/local/cuda-12.3/bin added to PATH environment variable.

build.rs main:

fn main() {
    // Tell Cargo that if the given file changes, to rerun this build script.
    println!("cargo:rerun-if-changed=src/cuda.cu");

    cc::Build::new()
        .cuda(true)
        .cudart("shared")  // Defaults to `static`; have tried both
        .flag("-gencode").flag("arch=compute_86,code=sm_86")
        .file("src/cuda.cu")
        .compile("libcuda.a");

    println!("cargo:rustc-link-search=native=/usr/local/cuda/lib64");
    println!("cargo:rustc-link-lib=cudart");
    // println!("cargo:rustc-link-lib=dylib=cudart");  // Have also tried these lines.
    // println!("cargo:rustc-link-lib=dylib=cublas");
}

Error:

"-Wl,--gc-sections" "-pie" "-Wl,-z,relro,-z,now" "-nodefaultlibs"
  = note: /usr/bin/ld: /home/david/code/wf-lab/target/debug/build/wf_lab-022d4f9dfab3e335/out/libcuda.a(cuda.o): in function `ffi_test':
          /home/david/code/wf-lab/src/cuda.cu:46: undefined reference to `std::cout'
          /usr/bin/ld: /home/david/code/wf-lab/src/cuda.cu:46: undefined reference to `std::basic_ostream<char, std::char_traits<char> >& std::operator<< <std::char_traits<char> >(std::basic_ostream<char, std::char_traits<char> >&, char const*)'
          /usr/bin/ld: /home/david/code/wf-lab/src/cuda.cu:46: undefined reference to `std::basic_ostream<char, std::char_traits<char> >& std::endl<char, std::char_traits<char> >(std::basic_ostream<char, std::char_traits<char> >&)'
          /usr/bin/ld: /home/david/code/wf-lab/src/cuda.cu:46: undefined reference to `std::ostream::operator<<(std::ostream& (*)(std::ostream&))'
          /usr/bin/ld: /home/david/code/wf-lab/target/debug/build/wf_lab-022d4f9dfab3e335/out/libcuda.a(cuda.o): in function `__cudaUnregisterBinaryUtil()':
          /usr/include/crt/host_runtime.h:259: undefined reference to `__cudaUnregisterFatBinary'
          /usr/bin/ld: /home/david/code/wf-lab/target/debug/build/wf_lab-022d4f9dfab3e335/out/libcuda.a(cuda.o): in function `__sti____cudaRegisterAll()':
          /tmp/tmpxft_000060fa_00000000-6_cuda.cudafe1.stub.c:13: undefined reference to `__cudaRegisterFatBinary'
          /usr/bin/ld: /tmp/tmpxft_000060fa_00000000-6_cuda.cudafe1.stub.c:13: undefined reference to `__cudaRegisterFatBinaryEnd'
          /usr/bin/ld: /home/david/code/wf-lab/target/debug/build/wf_lab-022d4f9dfab3e335/out/libcuda.a(cuda.o): in function `__static_initialization_and_destruction_0(int, int)':
          /usr/include/c++/9/iostream:74: undefined reference to `std::ios_base::Init::Init()'
          /usr/bin/ld: /usr/include/c++/9/iostream:74: undefined reference to `std::ios_base::Init::~Init()'
          collect2: error: ld returned 1 exit status

A clue : both systems produce errors about linking to this: __cudaUnregisterFatBinary.

I would love to get a reproducible CUDA + FFI on rust example repository set up. Cuda is awesome. Rust is awesome. Linking is stressful. Let's do this!