It looks like just the act of intercepting readlink hangs when using on "cargo build" as a test
I am writing an open source LD_PRELOAD library that tracks file system activities.
While testing it I ran into hang.
I have simplified it as far as I can to get it to pretty much bare bones
- intercept readlink,
- Call the original readlink.
There is nothing else going.
The test I am doing is to invoke
LD_PRELOAD=/path/libreadlink.so cargo build
I initialized a new project "cargo new"
Cargo.toml is as follows
[package]
name = "readlink"
version = "0.1.0"
authors = ["Saravanan Shanmugham <sarvi@cisco.com>"]
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[lib]
name = "readlink"
crate_type = ["dylib"]
[dependencies]
libc = "0.2"
The code looks like this. If this looks familiar, its because it a fork of the redhook crate that helps write other ld_preload program. Infact, redhook, does have its own readlink interception example that works just fine by itself. But fails when running the LD_PRELOAD on "cargo build"
The example below is simpler and does away with all the macros and is simple enough to be cut and pare and executed.
extern crate core;
extern crate libc;
use libc::{c_void,c_char,c_int,size_t,ssize_t};
use std::sync::atomic;
#[cfg(any(target_os = "macos", target_os = "ios"))]
pub mod dyld_insert_libraries;
/* Some Rust library functionality (e.g., jemalloc) initializes
* lazily, after the hooking library has inserted itself into the call
* path. If the initialization uses any hooked functions, this will lead
* to an infinite loop. Work around this by running some initialization
* code in a static constructor, and bypassing all hooks until it has
* completed. */
static INIT_STATE: atomic::AtomicBool = atomic::AtomicBool::new(false);
pub fn initialized() -> bool {
INIT_STATE.load(atomic::Ordering::SeqCst)
}
extern "C" fn initialize() {
Box::new(0u8);
INIT_STATE.store(true, atomic::Ordering::SeqCst);
}
#[link(name = "dl")]
extern "C" {
fn dlsym(handle: *const c_void, symbol: *const c_char) -> *const c_void;
}
const RTLD_NEXT: *const c_void = -1isize as *const c_void;
pub unsafe fn dlsym_next(symbol: &'static str) -> *const u8 {
let ptr = dlsym(RTLD_NEXT, symbol.as_ptr() as *const c_char);
if ptr.is_null() {
panic!("redhook: Unable to find underlying function for {}", symbol);
}
ptr as *const u8
}
/* Rust doesn't directly expose __attribute__((constructor)), but this
* is how GNU implements it. */
#[link_section = ".init_array"]
pub static INITIALIZE_CTOR: extern "C" fn() = ::initialize;
#[allow(non_camel_case_types)]
pub struct readlink {__private_field: ()}
#[allow(non_upper_case_globals)]
static readlink: readlink = readlink {__private_field: ()};
impl readlink {
fn get(&self) -> unsafe extern fn (path: *const c_char, buf: *mut c_char, bufsiz: size_t) -> ssize_t {
use ::std::sync::Once;
static mut REAL: *const u8 = 0 as *const u8;
static mut ONCE: Once = Once::new();
unsafe {
ONCE.call_once(|| {
REAL = dlsym_next(concat!("readlink", "\0"));
});
::std::mem::transmute(REAL)
}
}
#[no_mangle]
pub unsafe extern "C" fn readlink(path: *const c_char, buf: *mut c_char, bufsiz: size_t) -> ssize_t {
if initialized() {
::std::panic::catch_unwind(|| my_readlink ( path, buf, bufsiz )).ok()
} else {
None
}.unwrap_or_else(|| readlink.get() ( path, buf, bufsiz ))
}
}
pub unsafe fn my_readlink(path: *const c_char, buf: *mut c_char, bufsiz: size_t) -> ssize_t {
readlink.get()(path, buf, bufsiz)
}
Once setup I do the following
cargo clean
cargo build
#Copy the libreadlink.so to local, so as to not cleaup when doing the test
cp target/debug/libreadlink.so .
cargo clean; LD_PRELOAD=`pwd`/libreadlink.so cargo build
This hangs.
Ran an strace
brk(NULL) = 0x5644c60b7000
read(3, "U5qfeCTiRShFAh6d8WWQUe7UREN3+v9X"..., 4096) = 4096
read(3, "LLCo4MBANzX2hFxc469CeP6nyQ1Q6g2E"..., 4096) = 4096
read(3, "\nQ2l0eTEkMCIGA1UECgwbVHJ1c3RDb3I"..., 4096) = 4096
read(3, "P8nm9rZ/+I8U6laUpSNwXqxhaN0sSZ0Y"..., 4096) = 4096
read(3, "dGlmaWNhdGlvbiBBdXRob3JpdHkwHhcN"..., 4096) = 4096
read(3, "PVQT60nKWVSFJuUrjxuf6/WhkcIz\nSdh"..., 4096) = 4096
read(3, "TxzhT5yvDwyd93gN2PQ1VoDat20Xj50e"..., 4096) = 4096
read(3, "gMCGgUAMAcGBWcqAwAABBRFsMLH\nClZ8"..., 4096) = 4096
brk(NULL) = 0x5644c60b7000
brk(0x5644c60d8000) = 0x5644c60d8000
read(3, "GcxCzAJBgNVBAYTAklO\nMRMwEQYDVQQL"..., 4096) = 4096
read(3, "9yaXpl\nZCB1c2Ugb25seTEkMCIGA1UEA"..., 4096) = 964
read(3, "", 4096) = 0
close(3) = 0
getuid() = 19375
geteuid() = 19375
getgid() = 25
getegid() = 25
futex(0x5644c553fcb0, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x5644c553f634, FUTEX_WAKE_PRIVATE, 2147483647) = 0
access("/users/sarvi/.gitconfig", F_OK) = 0
stat("/users/sarvi/.gitconfig", {st_mode=S_IFREG|0644, st_size=367, ...}) = 0
access("/users/sarvi/.gitconfig", F_OK) = 0
access("/users/sarvi/.gitconfig", R_OK) = 0
stat("/users/sarvi/.gitconfig", {st_mode=S_IFREG|0644, st_size=367, ...}) = 0
stat("/users/sarvi/.gitconfig", {st_mode=S_IFREG|0644, st_size=367, ...}) = 0
openat(AT_FDCWD, "/users/sarvi/.gitconfig", O_RDONLY) = 3
read(3, "[user]\n\tname = Saravanan Shanmug"..., 367) = 367
close(3) = 0
access("/users/sarvi/.config/git/config", F_OK) = -1 ENOENT (No such file or directory)
access("/etc/gitconfig", F_OK) = -1 ENOENT (No such file or directory)
statx(AT_FDCWD, "/ws/sarvi-sjc/redhook/examples/readlink/Cargo.toml", AT_STATX_SYNC_AS_STAT, STATX_ALL, {stx_mask=STATX_BASIC_STATS, stx_attributes=0, stx_mode=S_IFREG|0644, stx_size=274, ...}) = 0
openat(AT_FDCWD, "/ws/sarvi-sjc/redhook/examples/readlink/Cargo.toml", O_RDONLY|O_CLOEXEC) = 3
statx(3, "", AT_STATX_SYNC_AS_STAT|AT_EMPTY_PATH, STATX_ALL, {stx_mask=STATX_BASIC_STATS, stx_attributes=0, stx_mode=S_IFREG|0644, stx_size=274, ...}) = 0
read(3, "[package]\nname = \"readlink\"\nvers"..., 275) = 274
read(3, "", 1) = 0
close(3) = 0
statx(AT_FDCWD, "/ws/sarvi-sjc/redhook/examples/readlink/src/lib.rs", AT_STATX_SYNC_AS_STAT, STATX_ALL, {stx_mask=STATX_BASIC_STATS, stx_attributes=0, stx_mode=S_IFREG|0644, stx_size=2537, ...}) = 0
statx(AT_FDCWD, "/ws/sarvi-sjc/redhook/examples/readlink/src/main.rs", AT_STATX_SYNC_AS_STAT, STATX_ALL, {stx_mask=STATX_BASIC_STATS, stx_attributes=0, stx_mode=S_IFREG|0644, stx_size=45, ...}) = 0
openat(AT_FDCWD, "/ws/sarvi-sjc/redhook/examples/readlink/src/bin", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/ws/sarvi-sjc/redhook/examples/readlink/examples", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/ws/sarvi-sjc/redhook/examples/readlink/tests", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/ws/sarvi-sjc/redhook/examples/readlink/benches", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = -1 ENOENT (No such file or directory)
statx(AT_FDCWD, "/ws/sarvi-sjc/redhook/examples/readlink/build.rs", AT_STATX_SYNC_AS_STAT, STATX_ALL, 0x7fffdec88230) = -1 ENOENT (No such file or directory)
statx(AT_FDCWD, "/ws/sarvi-sjc/redhook/examples/readlink/README.md", AT_STATX_SYNC_AS_STAT, STATX_ALL, 0x7fffdec88610) = -1 ENOENT (No such file or directory)
statx(AT_FDCWD, "/ws/sarvi-sjc/redhook/examples/readlink/README.txt", AT_STATX_SYNC_AS_STAT, STATX_ALL, 0x7fffdec88610) = -1 ENOENT (No such file or directory)
statx(AT_FDCWD, "/ws/sarvi-sjc/redhook/examples/readlink/README", AT_STATX_SYNC_AS_STAT, STATX_ALL, 0x7fffdec88610) = -1 ENOENT (No such file or directory)
statx(AT_FDCWD, "/ws/sarvi-sjc/redhook/examples/Cargo.toml", AT_STATX_SYNC_AS_STAT, STATX_ALL, 0x7fffdec8b550) = -1 ENOENT (No such file or directory)
statx(AT_FDCWD, "/ws/sarvi-sjc/redhook/Cargo.toml", AT_STATX_SYNC_AS_STAT, STATX_ALL, {stx_mask=STATX_BASIC_STATS, stx_attributes=0, stx_mode=S_IFREG|0644, stx_size=507, ...}) = 0
openat(AT_FDCWD, "/ws/sarvi-sjc/redhook/Cargo.toml", O_RDONLY|O_CLOEXEC) = 3
statx(3, "", AT_STATX_SYNC_AS_STAT|AT_EMPTY_PATH, STATX_ALL, {stx_mask=STATX_BASIC_STATS, stx_attributes=0, stx_mode=S_IFREG|0644, stx_size=507, ...}) = 0
read(3, "[package]\nname = \"redhook\"\nversi"..., 508) = 507
read(3, "", 1) = 0
close(3) = 0
statx(AT_FDCWD, "/ws/sarvi-sjc/redhook/src/lib.rs", AT_STATX_SYNC_AS_STAT, STATX_ALL, {stx_mask=STATX_BASIC_STATS, stx_attributes=0, stx_mode=S_IFREG|0644, stx_size=930, ...}) = 0
statx(AT_FDCWD, "/ws/sarvi-sjc/redhook/src/main.rs", AT_STATX_SYNC_AS_STAT, STATX_ALL, 0x7fffdec88230) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/ws/sarvi-sjc/redhook/src/bin", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/ws/sarvi-sjc/redhook/examples", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 3
fstat(3, {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
getdents64(3, /* 7 entries */, 32768) = 208
statx(AT_FDCWD, "/ws/sarvi-sjc/redhook/examples/fakeroot/main.rs", AT_STATX_SYNC_AS_STAT, STATX_ALL, 0x7fffdec879e0) = -1 ENOENT (No such file or directory)
statx(AT_FDCWD, "/ws/sarvi-sjc/redhook/examples/neverfree/main.rs", AT_STATX_SYNC_AS_STAT, STATX_ALL, 0x7fffdec879e0) = -1 ENOENT (No such file or directory)
statx(AT_FDCWD, "/ws/sarvi-sjc/redhook/examples/readlinkspy/main.rs", AT_STATX_SYNC_AS_STAT, STATX_ALL, 0x7fffdec879e0) = -1 ENOENT (No such file or directory)
statx(AT_FDCWD, "/ws/sarvi-sjc/redhook/examples/readlink/main.rs", AT_STATX_SYNC_AS_STAT, STATX_ALL, 0x7fffdec879e0) = -1 ENOENT (No such file or directory)
statx(AT_FDCWD, "/ws/sarvi-sjc/redhook/examples/varprintspy/main.rs", AT_STATX_SYNC_AS_STAT, STATX_ALL, 0x7fffdec879e0) = -1 ENOENT (No such file or directory)
getdents64(3, /* 0 entries */, 32768) = 0
close(3) = 0
openat(AT_FDCWD, "/ws/sarvi-sjc/redhook/tests", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/ws/sarvi-sjc/redhook/benches", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = -1 ENOENT (No such file or directory)
statx(AT_FDCWD, "/ws/sarvi-sjc/redhook/build.rs", AT_STATX_SYNC_AS_STAT, STATX_ALL, 0x7fffdec88230) = -1 ENOENT (No such file or directory)
statx(AT_FDCWD, "/ws/sarvi-sjc/Cargo.toml", AT_STATX_SYNC_AS_STAT, STATX_ALL, 0x7fffdec8b550) = -1 ENOENT (No such file or directory)
statx(AT_FDCWD, "/ws/Cargo.toml", AT_STATX_SYNC_AS_STAT, STATX_ALL, 0x7fffdec8b550) = -1 ENOENT (No such file or directory)
statx(AT_FDCWD, "/Cargo.toml", AT_STATX_SYNC_AS_STAT, STATX_ALL, 0x7fffdec8b550) = -1 ENOENT (No such file or directory)
sched_getaffinity(0, 128, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) = 16
statx(AT_FDCWD, "/users/sarvi/.cargo/bin/rustc", AT_STATX_SYNC_AS_STAT, STATX_ALL, {stx_mask=STATX_BASIC_STATS, stx_attributes=0, stx_mode=S_IFREG|0755, stx_size=12391632, ...}) = 0
lstat("/users", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
lstat("/users/sarvi", {st_mode=S_IFDIR|0755, st_size=36864, ...}) = 0
lstat("/users/sarvi/.cargo", {st_mode=S_IFLNK|0777, st_size=19, ...}) = 0
readlink("/users/sarvi/.cargo", "/ws/sarvi-sjc/cargo", 4095) = 19
lstat("/ws", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
lstat("/ws/sarvi-sjc", {st_mode=S_IFDIR|0755, st_size=16384, ...}) = 0
lstat("/ws/sarvi-sjc/cargo", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
lstat("/ws/sarvi-sjc/cargo/bin", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
lstat("/ws/sarvi-sjc/cargo/bin/rustc", {st_mode=S_IFREG|0755, st_size=12391632, ...}) = 0
statx(AT_FDCWD, "/ws/sarvi-sjc/cargo/bin/rustc", AT_STATX_SYNC_AS_STAT, STATX_ALL, {stx_mask=STATX_BASIC_STATS, stx_attributes=0, stx_mode=S_IFREG|0755, stx_size=12391632, ...}) = 0
statx(AT_FDCWD, "/users/sarvi/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu/bin/rustc", AT_STATX_SYNC_AS_STAT, STATX_ALL, {stx_mask=STATX_BASIC_STATS, stx_attributes=0, stx_mode=S_IFREG|0755, stx_size=2707888, ...}) = 0
openat(AT_FDCWD, "/ws/sarvi-sjc/redhook/examples/readlink/target/.rustc_info.json", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/dev/null", O_RDONLY|O_CLOEXEC) = 3
pipe2([4, 5], O_CLOEXEC) = 0
pipe2([6, 7], O_CLOEXEC) = 0
futex(0x7f005d94d0e8, FUTEX_WAKE_PRIVATE, 2147483647) = 0
prlimit64(0, RLIMIT_NOFILE, NULL, {rlim_cur=4*1024, rlim_max=4*1024}) = 0
prlimit64(0, RLIMIT_NOFILE, NULL, {rlim_cur=4*1024, rlim_max=4*1024}) = 0
prlimit64(0, RLIMIT_NOFILE, NULL, {rlim_cur=4*1024, rlim_max=4*1024}) = 0
prlimit64(0, RLIMIT_NOFILE, NULL, {rlim_cur=4*1024, rlim_max=4*1024}) = 0
prlimit64(0, RLIMIT_NOFILE, NULL, {rlim_cur=4*1024, rlim_max=4*1024}) = 0
prlimit64(0, RLIMIT_NOFILE, NULL, {rlim_cur=4*1024, rlim_max=4*1024}) = 0
mmap(NULL, 36864, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0x7f005e6a9000
rt_sigprocmask(SIG_BLOCK, ~[], [], 8) = 0
clone(child_stack=0x7f005e6b1ff0, flags=CLONE_VM|CLONE_VFORK|SIGCHLD) = 20534
munmap(0x7f005e6a9000, 36864) = 0
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
close(3) = 0
close(5) = 0
close(7) = 0
ioctl(4, FIONBIO, [1]) = 0
ioctl(6, FIONBIO, [1]) = 0
poll([{fd=4, events=POLLIN}, {fd=6, events=POLLIN}], 2, -1) = 1 ([{fd=4, revents=POLLIN}])
read(4, "readlink(\"/proc/self/exe\")\n", 32) = 27
read(4, 0x5644c601543b, 5) = -1 EAGAIN (Resource temporarily unavailable)
poll([{fd=4, events=POLLIN}, {fd=6, events=POLLIN}], 2, -1
Can anyone explain why this might be hanging? Cosidering the intercept code is pretty much a No-Op, I am not sure why it is hanging. And only on "cargo build"
this for example works fine, Added an extrace "println!()" in the following case to make sure it indeed works.
LD_PRELOAD=/ws/sarvi-sjc/redhook/examples/readlinkspy/target/debug/libreadlinkspy.so ls -al ./test.link
readlink("./test.link")
lrwxrwxrwx 1 sarvi eng 10 Aug 29 22:30 ./test.link -> Cargo.toml