Cargo double build is necessary to compile bin project with its own dynamic lib inside

Hi, I'm trying to have a bin project that has its own dynamic lib (as .so, aka crate-type = ["cdylib"]) built and used in the same project, but what I notice is that, after a cargo clean, I have to run cargo build twice in order for compilation to succeed, the first build will fail to link. Even after, it only runs through cargo run (rpath issue).

I start like this:

cargo new all_in_one
     Created binary (application) `all_in_one` package

Contents of src/lib2.rs (as to why not src/lib.rs, because I get a warning about special_module_name) are as as follows:

#[no_mangle]
pub extern "C" fn add(left: usize, right: usize) -> usize {
    left + right
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn it_works() {
        let result = add(2, 2);
        assert_eq!(result, 4);
    }
}

Contents of src/main.rs are:

extern "C" {
    fn add(left: usize, right: usize) -> usize;
}

#[link(name = "all_in_one")] //FIXME: how to use package.name here from Cargo.toml?
extern {}

fn main() {
    println!("Hello, world!{}",
             unsafe { add(1,2) }
             );
}

[package]
name = "all_in_one"
version = "0.1.0"
edition = "2021"

# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[lib]
crate-type = ["cdylib"]
path = "src/lib2.rs"

[profile.dev]
opt-level = 0
incremental=false #true/false has no effect on double building with cargo being needed to succeed

[dependencies]

Now I do:
cargo clean
then the first cargo build shows me this:

$ cargo build
   Compiling all_in_one v0.1.0 (/tmp/1/all_in_one)
error: linking with `x86_64-pc-linux-gnu-gcc` failed: exit status: 1
  |
  = note: LC_ALL="C" PATH="/usr/lib/rust/1.76.0/lib/rustlib/x86_64-unknown-linux-gnu/bin:/swcode/bin.prepend:/home/user/bin/binprio:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/bin:/usr/lib/llvm/18/bin:/usr/lib/llvm/17/bin:/home/user/bin:/home/user/bin/oldbin:/opt/depot_tools/:/swcode/bin.append:/home/user/.cargo/bin" VSLANG="1033" "x86_64-pc-linux-gnu-gcc" "-m64" "/tmp/rustcob3FDD/symbols.o" "/tmp/1/all_in_one/target/debug/deps/all_in_one-b047c1515e2e1edf.all_in_one.c48bf73702433d06-cgu.0.rcgu.o" "/tmp/1/all_in_one/target/debug/deps/all_in_one-b047c1515e2e1edf.5g6kiq4hwdv7xc7l.rcgu.o" "-Wl,--as-needed" "-L" "/tmp/1/all_in_one/target/debug/deps" "-L" "/usr/lib/rust/1.76.0/lib/rustlib/x86_64-unknown-linux-gnu/lib" "-Wl,-Bdynamic" "-lall_in_one" "-Wl,-Bstatic" "/usr/lib/rust/1.76.0/lib/rustlib/x86_64-unknown-linux-gnu/lib/libstd-a65e73eb992a68d8.rlib" "/usr/lib/rust/1.76.0/lib/rustlib/x86_64-unknown-linux-gnu/lib/libpanic_unwind-9f6656651c7fe4e3.rlib" "/usr/lib/rust/1.76.0/lib/rustlib/x86_64-unknown-linux-gnu/lib/libobject-4db84c099590338a.rlib" "/usr/lib/rust/1.76.0/lib/rustlib/x86_64-unknown-linux-gnu/lib/libmemchr-6335b5a1c2e7e5a0.rlib" "/usr/lib/rust/1.76.0/lib/rustlib/x86_64-unknown-linux-gnu/lib/libaddr2line-9b651b731f1ef76e.rlib" "/usr/lib/rust/1.76.0/lib/rustlib/x86_64-unknown-linux-gnu/lib/libgimli-c8bd0479a2e9f8bb.rlib" "/usr/lib/rust/1.76.0/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc_demangle-732731b4def56af0.rlib" "/usr/lib/rust/1.76.0/lib/rustlib/x86_64-unknown-linux-gnu/lib/libstd_detect-cb193f06c1f8da78.rlib" "/usr/lib/rust/1.76.0/lib/rustlib/x86_64-unknown-linux-gnu/lib/libhashbrown-5b1537a272cafac1.rlib" "/usr/lib/rust/1.76.0/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc_std_workspace_alloc-011ceab966df1728.rlib" "/usr/lib/rust/1.76.0/lib/rustlib/x86_64-unknown-linux-gnu/lib/libminiz_oxide-7938c4c950efaf6a.rlib" "/usr/lib/rust/1.76.0/lib/rustlib/x86_64-unknown-linux-gnu/lib/libadler-5da751c7f8a6d18b.rlib" "/usr/lib/rust/1.76.0/lib/rustlib/x86_64-unknown-linux-gnu/lib/libunwind-725b8301e9288129.rlib" "/usr/lib/rust/1.76.0/lib/rustlib/x86_64-unknown-linux-gnu/lib/libcfg_if-b317e4c5a886ad10.rlib" "/usr/lib/rust/1.76.0/lib/rustlib/x86_64-unknown-linux-gnu/lib/liblibc-a5c4e38572432451.rlib" "/usr/lib/rust/1.76.0/lib/rustlib/x86_64-unknown-linux-gnu/lib/liballoc-72d34c71513008c1.rlib" "/usr/lib/rust/1.76.0/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc_std_workspace_core-29fe682da95c746c.rlib" "/usr/lib/rust/1.76.0/lib/rustlib/x86_64-unknown-linux-gnu/lib/libcore-09ae7bfdddacb91b.rlib" "/usr/lib/rust/1.76.0/lib/rustlib/x86_64-unknown-linux-gnu/lib/libcompiler_builtins-3654f674d9afce6c.rlib" "-Wl,-Bdynamic" "-lgcc_s" "-lutil" "-lrt" "-lpthread" "-lm" "-ldl" "-lc" "-Wl,--eh-frame-hdr" "-Wl,-z,noexecstack" "-L" "/usr/lib/rust/1.76.0/lib/rustlib/x86_64-unknown-linux-gnu/lib" "-o" "/tmp/1/all_in_one/target/debug/deps/all_in_one-b047c1515e2e1edf" "-Wl,--gc-sections" "-pie" "-Wl,-z,relro,-z,now" "-nodefaultlibs"
  = note: /usr/lib/gcc/x86_64-pc-linux-gnu/13/../../../../x86_64-pc-linux-gnu/bin/ld: /tmp/1/all_in_one/target/debug/deps/all_in_one-b047c1515e2e1edf.all_in_one.c48bf73702433d06-cgu.0.rcgu.o: in function `all_in_one::main':
          /tmp/1/all_in_one/src/main.rs:10:(.text._ZN10all_in_one4main17h62df9d360bb29cc3E+0x7): undefined reference to `add'
          collect2: error: ld returned 1 exit status

  = note: some `extern` functions couldn't be found; some native libraries may need to be installed or have their path specified
  = note: use the `-l` flag to specify native libraries to link
  = note: use the `cargo:rustc-link-lib` directive to specify the native libraries to link with Cargo (see https://doc.rust-lang.org/cargo/reference/build-scripts.html#cargorustc-link-libkindname)

error: could not compile `all_in_one` (bin "all_in_one") due to 1 previous error

now the second cargo build shows me this:

$ cargo build
   Compiling all_in_one v0.1.0 (/tmp/1/all_in_one)
    Finished dev [unoptimized + debuginfo] target(s) in 0.13s

What's the difference in target/ dir:

$ diff -upr /tmp/target{1,2} |& sort
Only in /tmp/target1/debug/deps: all_in_one-b047c1515e2e1edf.5g6kiq4hwdv7xc7l.rcgu.o
Only in /tmp/target1/debug/deps: all_in_one-b047c1515e2e1edf.all_in_one.c48bf73702433d06-cgu.0.rcgu.o
Only in /tmp/target1/debug/.fingerprint/all_in_one-b047c1515e2e1edf: output-bin-all_in_one
Only in /tmp/target2/debug: all_in_one
Only in /tmp/target2/debug: all_in_one.d
Only in /tmp/target2/debug/deps: all_in_one-b047c1515e2e1edf
Only in /tmp/target2/debug/.fingerprint/all_in_one-b047c1515e2e1edf: bin-all_in_one
Only in /tmp/target2/debug/.fingerprint/all_in_one-b047c1515e2e1edf: bin-all_in_one.json
Only in /tmp/target2/debug/.fingerprint/all_in_one-b047c1515e2e1edf: dep-bin-all_in_one
Only in /tmp/target2/debug: liball_in_one.d

cargo run:

$ cargo run
    Finished dev [unoptimized + debuginfo] target(s) in 0.01s
     Running `target/debug/all_in_one`
Hello, world!3

Attempting to run it by itself:

$ ./target/debug/all_in_one
./target/debug/all_in_one: error while loading shared libraries: liball_in_one.so: cannot open shared object file: No such file or directory

forcing:

$ LD_LIBRARY_PATH=./target/debug ./target/debug/all_in_one
Hello, world!3

Summary:

  1. Double cargo build runs are needed to succeed, why?
  2. rpath isn't set(or something) in the executable (and doing rpath=true in Cargo.toml under [profile.dev] has no effect), how to do?

Notes:

  • Using a separate bin and lib project does work well as a "workaround" for the above failings - the lib is still dynamically linked .so and has rpath set correctly(due to rpath=true). But my goal of having them both in one project instead of two isn't accomplished this way.
  • Not making the lib dynamic does work but then it's a rlib(i guess) and it's statically linked into the exe.
  • I've tried some build.rs shenanigans but I still get the first error on cargo build and still failed to set rpath from in there. Most likely I don't know how to do it properly.

Any ideas? I can try things and post results. Will thank you later :slight_smile:

I think it's just a race condition. can you try to build with parallel building disabled, i.e.:

$ cargo build --jobs 1
1 Like

An excellent suggestion! Thank you!

The results are unpredictable:

  • sometimes it works, both cargo build compile;
  • sometimes both don't compile; (well, even a third cargo run doesn't, so it's 3 times in a row.
  • sometimes the first doesn't compile, but the second does;

I've made a batch for it, the above hold when running it:

#!/usr/bin/env bash
export RUST_BACKTRACE=1
cargo clean
export CARGO_INCREMENTAL=0
cargo build -vv --jobs 1 || cargo build -vv --jobs 1

cargo run -vv --jobs 1
ec="$?"
set +vx
pushd target/debug
#FIXME: need to set lib path, make it not necessary!
LD_LIBRARY_PATH=./ ./all_in_one
ec2="$?"
./all_in_one
ec3="$?"
popd
echo "exit code: $ec $ec2 $ec3"

EDIT: easier to reproduce when not idle, it seems, like $ openssl speed running in another terminal.

version details

$ cargo --version
cargo 1.76.0-nightly
$ rustc -vV
rustc 1.76.0-nightly (07dca489a 2024-02-04) (gentoo)
binary: rustc
commit-hash: 07dca489ac2d933c78d3c5158e3f43beefeb02ce
commit-date: 2024-02-04
host: x86_64-unknown-linux-gnu
release: 1.76.0-nightly
LLVM version: 17.0.6
They're from Gentoo, but slightly modified

it might be not accurate, but my guess is like this: cargo only knows the dependency graph of the "rust part", e.g. it will start building the binary after the library is compiled, but before it finishes linking (since it is a cdylib, the library artifact needs a linking step). this means the building of the binary is running concurrently with the linking of the library, thus the race condition.

I don't think you can tell cargo that the binary has a dependency on the final .so artifact of the library (not just the rlib or rmeta files) when they are different crates of the same package.

it won't be a problem if they are separate packages, as cargo won't concurrently build a package before all the dependencies finishes building.

I thought reduce the concurrent job number to 1 should work, but on a second thought, if the dependency graph is incomplete, the build order might still be incorrect.

I think there's a rfc about artifact dependencies (or binary dependencies, I can't remember exactly the name), but I don't know if applies when it is the same package, you may try to search for it.

but for now, I think the options you have is either:

a) put the binary and library into different cargo packages, and maybe use a workspace to organize your project, or:

b) accept the fact you have to build the package in two steps:

$ cargo build --lib
$ cargo build --bins

as for the rpath problem, I have absolute no idea why it doesn't work.

1 Like

This is a very good workaround! Much appreciated!
I will mark that as the solution until something better comes along, if ever.

I did want to mention that I've another, uglier, workaround(shown below, but also link) that tells cargo there's no lib(for it to compile), and instead build.rs is used to build the lib and tell cargo how to link it to the bin, which also happens to solve the rpath problem. But it's kinda ugly as it manually invokes rustc, and I'd prefer cargo to do it, that's why I like your workaround better(but still have to find a fix for the rpath then).

Here it is:
remove these from Cargo.toml, so cargo's unaware of any lib.

-[lib]
-crate-type = ["cdylib"]
-path = "src/lib2.rs"

Then create ./build.rs:

fn main() {
    let out_dir = std::env::var("OUT_DIR").unwrap();
    let lib_name = "liball_in_one.so";
    let status = std::process::Command::new("rustc")
        .args(&["src/lib2.rs", "--crate-type=cdylib", "-o"])
        .arg(&format!("{}/{}", out_dir, lib_name))
        .status()
        .expect("failed to compile custom abort library");

    if !status.success() {
        panic!("Failed to compile custom abort library");
    }

    // Output the path to the shared library
    println!("cargo:rustc-link-search=native={}", out_dir); // needed
    println!("cargo:rustc-link-arg=-Wl,--no-as-needed,-rpath={}",out_dir);
    //--no-as-needed has no effect here,
}

This topic was automatically closed 90 days after the last reply. We invite you to open a new topic if you have further questions or comments.