Docker alpine:3 Segmentation Fault

nicram · September 1, 2021, 12:05pm

Hi,
I have strange problem. I've started write simple app. It runs on debian every time, also in docker containter with debian, but in alpine I get segmentation fault errors and "memory allocation of .... bytes failed". Not always this happens, let's say that for 10 times it starts 3-4 times.
cargo version 1.54
This is strace from segmetation fault:

/app # strace ./gurita-core 
execve("./gurita-core", ["./gurita-core"], 0x7fff4afa89a0 /* 6 vars */) = 0
arch_prctl(ARCH_SET_FS, 0x7f8171d9c7b8) = 0
set_tid_address(0x7f8171d9c8f0)         = 41
brk(NULL)                               = 0x55cd7f02e000
brk(0x55cd7f02f000)                     = 0x55cd7f02f000
readlink("/proc/self/exe", "/app/gurita-core", 4096) = 16
execve("/lib/ld-musl-x86_64.so.1", ["ld-linux-x86-64.so.2", "--argv0", "./gurita-core", "--preload", "/lib/libgcompat.so.0 ", "--", "/app/gurita-core"], 0x7fff09074568 /* 6 vars */) = 0
arch_prctl(ARCH_SET_FS, 0x7fab2e4d9b48) = 0
set_tid_address(0x7fab2e4d9f90)         = 41
open("/app/gurita-core", O_RDONLY|O_LARGEFILE) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`\245\1\0\0\0\0\0"..., 960) = 960
mmap(NULL, 1531904, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fab2e2cd000
mmap(0x7fab2e2de000, 1126400, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED, 3, 0x11000) = 0x7fab2e2de000
mmap(0x7fab2e3f1000, 286720, PROT_READ, MAP_PRIVATE|MAP_FIXED, 3, 0x124000) = 0x7fab2e3f1000
mmap(0x7fab2e438000, 45056, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 3, 0x16a000) = 0x7fab2e438000
close(3)                                = 0
brk(NULL)                               = 0x5555566eb000
brk(0x5555566ed000)                     = 0x5555566ed000
mmap(0x5555566eb000, 4096, PROT_NONE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x5555566eb000
open("/lib/libgcompat.so.0", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = 3
fcntl(3, F_SETFD, FD_CLOEXEC)           = 0
fstat(3, {st_mode=S_IFREG|0755, st_size=63088, ...}) = 0
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0m\0\0\0\0\0\0"..., 960) = 960
mmap(NULL, 73728, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fab2e2bb000
mmap(0x7fab2e2c1000, 20480, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED, 3, 0x6000) = 0x7fab2e2c1000
mmap(0x7fab2e2c6000, 12288, PROT_READ, MAP_PRIVATE|MAP_FIXED, 3, 0xb000) = 0x7fab2e2c6000
mmap(0x7fab2e2ca000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 3, 0xe000) = 0x7fab2e2ca000
mmap(0x7fab2e2cc000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fab2e2cc000
close(3)                                = 0
open("/etc/ld-musl-x86_64.path", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = -1 ENOENT (No such file or directory)
open("/lib/libgcc_s.so.1", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = -1 ENOENT (No such file or directory)
open("/usr/local/lib/libgcc_s.so.1", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = -1 ENOENT (No such file or directory)
open("/usr/lib/libgcc_s.so.1", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = 3
fcntl(3, F_SETFD, FD_CLOEXEC)           = 0
fstat(3, {st_mode=S_IFREG|0644, st_size=100096, ...}) = 0
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\3602\0\0\0\0\0\0"..., 960) = 960
mmap(NULL, 106496, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fab2e2a1000
mmap(0x7fab2e2a4000, 69632, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED, 3, 0x3000) = 0x7fab2e2a4000
mmap(0x7fab2e2b5000, 12288, PROT_READ, MAP_PRIVATE|MAP_FIXED, 3, 0x14000) = 0x7fab2e2b5000
mmap(0x7fab2e2b9000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 3, 0x17000) = 0x7fab2e2b9000
close(3)                                = 0
open("/lib/libucontext.so.1", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = 3
fcntl(3, F_SETFD, FD_CLOEXEC)           = 0
fstat(3, {st_mode=S_IFREG|0755, st_size=13912, ...}) = 0
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0p\20\0\0\0\0\0\0"..., 960) = 960
mmap(NULL, 20480, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fab2e29c000
mmap(0x7fab2e29d000, 4096, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED, 3, 0x1000) = 0x7fab2e29d000
mmap(0x7fab2e29e000, 4096, PROT_READ, MAP_PRIVATE|MAP_FIXED, 3, 0x2000) = 0x7fab2e29e000
mmap(0x7fab2e29f000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 3, 0x2000) = 0x7fab2e29f000
close(3)                                = 0
open("/lib/libobstack.so.1", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = -1 ENOENT (No such file or directory)
open("/usr/local/lib/libobstack.so.1", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = -1 ENOENT (No such file or directory)
open("/usr/lib/libobstack.so.1", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = 3
fcntl(3, F_SETFD, FD_CLOEXEC)           = 0
fstat(3, {st_mode=S_IFREG|0755, st_size=13920, ...}) = 0
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240\20\0\0\0\0\0\0"..., 960) = 960
mmap(NULL, 20480, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fab2e297000
mmap(0x7fab2e298000, 4096, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED, 3, 0x1000) = 0x7fab2e298000
mmap(0x7fab2e299000, 4096, PROT_READ, MAP_PRIVATE|MAP_FIXED, 3, 0x2000) = 0x7fab2e299000
mmap(0x7fab2e29a000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 3, 0x2000) = 0x7fab2e29a000
close(3)                                = 0
mprotect(0x7fab2e2ca000, 4096, PROT_READ) = 0
mprotect(0x7fab2e4d6000, 4096, PROT_READ) = 0
mprotect(0x7fab2e2b9000, 4096, PROT_READ) = 0
mprotect(0x7fab2e29f000, 4096, PROT_READ) = 0
mprotect(0x7fab2e29a000, 4096, PROT_READ) = 0
mprotect(0x7fab2e438000, 40960, PROT_READ) = 0
arch_prctl(ARCH_SET_FS, 0x7fab2e2ccc20) = 0
set_tid_address(0x7fab2e4d9f90)         = 41
poll([{fd=0, events=0}, {fd=1, events=0}, {fd=2, events=0}], 3, 0) = 0 (Timeout)
rt_sigaction(SIGPIPE, {sa_handler=SIG_IGN, sa_mask=[], sa_flags=SA_RESTORER|SA_RESTART, sa_restorer=0x7fab2e489c8b}, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8) = 0
rt_sigaction(SIGSEGV, NULL, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RT_1 RT_2], NULL, 8) = 0
rt_sigaction(SIGSEGV, {sa_handler=0x7fab2e3ce3e0, sa_mask=[], sa_flags=SA_RESTORER|SA_ONSTACK|SA_SIGINFO, sa_restorer=0x7fab2e489c8b}, NULL, 8) = 0
rt_sigaction(SIGBUS, NULL, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8) = 0
rt_sigaction(SIGBUS, {sa_handler=0x7fab2e3ce3e0, sa_mask=[], sa_flags=SA_RESTORER|SA_ONSTACK|SA_SIGINFO, sa_restorer=0x7fab2e489c8b}, NULL, 8) = 0
sigaltstack(NULL, {ss_sp=NULL, ss_flags=SS_DISABLE, ss_size=0}) = 0
mmap(NULL, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fab2e294000
mprotect(0x7fab2e294000, 4096, PROT_NONE) = 0
sigaltstack({ss_sp=0x7fab2e295000, ss_flags=0, ss_size=8192}, NULL) = 0
mremap(0x7ffc3777c000, 4096, 8192, 0)   = -1 ENOMEM (Out of memory)
mremap(0x7ffc3777b000, 4096, 8192, 0)   = -1 ENOMEM (Out of memory)
mremap(0x7ffc3777a000, 4096, 8192, 0)   = -1 ENOMEM (Out of memory)
mremap(0x7ffc37779000, 4096, 8192, 0)   = -1 ENOMEM (Out of memory)
mremap(0x7ffc37778000, 4096, 8192, 0)   = -1 ENOMEM (Out of memory)
mremap(0x7ffc37777000, 4096, 8192, 0)   = -1 ENOMEM (Out of memory)
mremap(0x7ffc37776000, 4096, 8192, 0)   = -1 ENOMEM (Out of memory)
mremap(0x7ffc37775000, 4096, 8192, 0)   = -1 ENOMEM (Out of memory)
mremap(0x7ffc37774000, 4096, 8192, 0)   = -1 ENOMEM (Out of memory)
mremap(0x7ffc37773000, 4096, 8192, 0)   = -1 ENOMEM (Out of memory)
mremap(0x7ffc37772000, 4096, 8192, 0)   = -1 ENOMEM (Out of memory)
mremap(0x7ffc37771000, 4096, 8192, 0)   = -1 ENOMEM (Out of memory)
mremap(0x7ffc37770000, 4096, 8192, 0)   = -1 ENOMEM (Out of memory)
mremap(0x7ffc3776f000, 4096, 8192, 0)   = -1 ENOMEM (Out of memory)
mremap(0x7ffc3776e000, 4096, 8192, 0)   = -1 ENOMEM (Out of memory)
mremap(0x7ffc3776d000, 4096, 8192, 0)   = -1 ENOMEM (Out of memory)
mremap(0x7ffc3776c000, 4096, 8192, 0)   = -1 ENOMEM (Out of memory)
mremap(0x7ffc3776b000, 4096, 8192, 0)   = -1 ENOMEM (Out of memory)
mremap(0x7ffc3776a000, 4096, 8192, 0)   = -1 ENOMEM (Out of memory)
mremap(0x7ffc37769000, 4096, 8192, 0)   = -1 ENOMEM (Out of memory)
mremap(0x7ffc37768000, 4096, 8192, 0)   = -1 ENOMEM (Out of memory)
mremap(0x7ffc37767000, 4096, 8192, 0)   = -1 ENOMEM (Out of memory)
mremap(0x7ffc37766000, 4096, 8192, 0)   = -1 ENOMEM (Out of memory)
mremap(0x7ffc37765000, 4096, 8192, 0)   = -1 ENOMEM (Out of memory)
mremap(0x7ffc37764000, 4096, 8192, 0)   = -1 ENOMEM (Out of memory)
mremap(0x7ffc37763000, 4096, 8192, 0)   = -1 ENOMEM (Out of memory)
mremap(0x7ffc37762000, 4096, 8192, 0)   = -1 ENOMEM (Out of memory)
mremap(0x7ffc37761000, 4096, 8192, 0)   = -1 ENOMEM (Out of memory)
mremap(0x7ffc37760000, 4096, 8192, 0)   = -1 ENOMEM (Out of memory)
mremap(0x7ffc3775f000, 4096, 8192, 0)   = -1 ENOMEM (Out of memory)
mremap(0x7ffc3775e000, 4096, 8192, 0)   = -1 ENOMEM (Out of memory)
mremap(0x7ffc3775d000, 4096, 8192, 0)   = -1 EFAULT (Bad address)
mmap(NULL, 18644094068, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fa6d6e2f000
--- SIGSEGV {si_signo=SIGSEGV, si_code=SEGV_MAPERR, si_addr=0x80} ---
rt_sigaction(SIGSEGV, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=SA_RESTORER, sa_restorer=0x7fab2e489c8b}, NULL, 8) = 0
rt_sigreturn({mask=[]})                 = 128
--- SIGSEGV {si_signo=SIGSEGV, si_code=SEGV_MAPERR, si_addr=0x80} ---
+++ killed by SIGSEGV (core dumped) +++
Segmentation fault (core dumped)

linked libraries on alpine:

/app # ldd ./gurita-core 
	/lib64/ld-linux-x86-64.so.2 (0x7f21c028a000)
	libc.so.6 => /lib64/ld-linux-x86-64.so.2 (0x7f21c028a000)
	ld-linux-x86-64.so.2 => /lib/ld-linux-x86-64.so.2 (0x7f21c010e000)
	libm.so.6 => /lib64/ld-linux-x86-64.so.2 (0x7f21c028a000)
	libpthread.so.0 => /lib64/ld-linux-x86-64.so.2 (0x7f21c028a000)
	libgcc_s.so.1 => /usr/lib/libgcc_s.so.1 (0x7f21c00f4000)
	libdl.so.2 => /lib64/ld-linux-x86-64.so.2 (0x7f21c028a000)
/app #

and the same on debian:10-slim

root@7cd9bcf943b1:/app# ldd ./gurita-core 
	linux-vdso.so.1 (0x00007fff274a1000)
	libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007ff23104c000)
	/lib64/ld-linux-x86-64.so.2 (0x00007ff231387000)
	libm.so.6 => /lib/x86_64-linux-gnu/libm.so.6 (0x00007ff230ec9000)
	libpthread.so.0 => /lib/x86_64-linux-gnu/libpthread.so.0 (0x00007ff230ea8000)
	libgcc_s.so.1 => /lib/x86_64-linux-gnu/libgcc_s.so.1 (0x00007ff230e8e000)
	libdl.so.2 => /lib/x86_64-linux-gnu/libdl.so.2 (0x00007ff230e89000)
root@7cd9bcf943b1:/app#

Why there is problem on alpine?

Hyeonu · September 1, 2021, 12:58pm

My wild guess is to blame musl libc's malloc impl. Try jemallocator and see what happens?

nicram · September 1, 2021, 1:37pm

Thaks for you answer. I added jemallocator but problem remains

lwojtow · September 1, 2021, 5:50pm

Do you compile it on alpian or is it docker image build on debian failing on alpian? Is alpian running on different processor? I had this problem when something compiled on ryzen (linux mint) with -native was running on intel (fedora).

nicram · September 1, 2021, 6:13pm

I compile this with Docker using rust:1.54 image and later copy it into alpine container. Everything is on the same phisical machine.
The same behaviour is in gitlab usig CI/CD where docker image is build on docker:20.10-dind next rust:1.54 and finally alpine:3. In alpine I have to add libgcc and gcompat packages

this is my Dockerfile:

FROM rust:1.54.0 as build
WORKDIR /app
COPY . .
RUN rustc --version \
    && cargo --version \
    && cargo test --workspace --verbose --release \
    && cargo build --release

#FROM debian:10-slim
FROM alpine:3
ENV TZ=Europe/Warsaw
ENV LANG pl_PL.utf8
WORKDIR /app
RUN apk --no-cache add libgcc gcompat

COPY --from=build /app/target/release/gurita-core .
COPY config/gurita.toml /etc/gurita/gurita.toml

CMD ["/app/gurita-core"]

I've created simple hello app, just cargo new hello and build it from the same Dockerfile with alpine, and cannot reproduce this error

For now in my project I use these crates:

clap = "2.33.3"
config = "0.11.0"
serde = "1.0.130"
serde_derive = "1.0.130"
ip_network = "0.4.0"
google-authenticator = {version = "0.2.1", features = ["with-qrcode"]}
argon2 = "0.3.0"
rand_core = { version = "0.6.3", features = ["std"]}

nicram · September 1, 2021, 6:39pm

// cut

I have created repo with this issue. https://github.com/nicraMarcin/alpine_segfault

// edit
I've made a test. I compiled this directly on alpine, not rust:1.54 image which is based on debian 10, and now works

FROM alpine:3 as build
WORKDIR /app
COPY . .
RUN apk add --no-cache gcc curl \
    && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs -o rst.sh && chmod 777 rst.sh && ./rst.sh -y \
    && source $HOME/.cargo/env \
    && rustc --version \
    && cargo --version \
    && cargo test --workspace --verbose --release \
    && cargo build --release

FROM alpine:3
ENV TZ=Europe/Warsaw
WORKDIR /app
RUN apk --no-cache add libgcc gcompat
COPY --from=build /app/target/release/gcore .
COPY config/gurita.toml /etc/gurita/gurita.toml

CMD ["/app/gcore"]

Now build image and run container:

$ docker build -t newtest .
// ...
Successfully built d6173481ea72
Successfully tagged newtest:latest

$ docker run --rm -it newtest:latest sh
/app # ./gcore -h
gurita 0.1.0
GURITA

USAGE:
    gcore [FLAGS] [OPTIONS]

FLAGS:
    -d               Run as daemon
    -h, --help       Prints help information
    -v               Sets the level of verbosity (-v -vv -vvv)
    -V, --version    Prints version information

OPTIONS:
    -c, --config <FILE>    Sets custom conifg file. [default: /etc/gurita/gurita.toml]
    -p, --port <PORT>      Set listen port
/app # ./gcore 
Value of config set: /etc/gurita/gurita.toml
/app # ./gcore 
Value of config set: /etc/gurita/gurita.toml
/app # ./gcore 
Value of config set: /etc/gurita/gurita.toml
/app # ./gcore 
Value of config set: /etc/gurita/gurita.toml
/app # ./gcore 
Value of config set: /etc/gurita/gurita.toml
/app # ./gcore 
Value of config set: /etc/gurita/gurita.toml
/app # ./gcore 
Value of config set: /etc/gurita/gurita.tom 
/app # ./gcore 
Value of config set: /etc/gurita/gurita.toml

and now no more segmentation fault

riking · September 2, 2021, 9:08am

Also consider telling cargo to compile for a -linux-musl target - Debian is glibc, Alpine is musl

nicram · September 2, 2021, 9:23am

Thank you for support.

system · December 1, 2021, 9:24am

This topic was automatically closed 90 days after the last reply. We invite you to open a new topic if you have further questions or comments.

Topic		Replies	Views
Segmentation fault (opencv + tesseract + tensorflow) help	22	1815	October 20, 2022
SIGSEGV with program linked against OpenSSL in an Alpine container help	5	4070	March 2, 2021
SegFault when linking against shared libraries on Alpine help	10	627	November 18, 2023
Segfaulting, don't know where! help	66	2132	December 3, 2023
Binary won't run - FFI, Alpine, musl and C help	14	302	March 17, 2025

Docker alpine:3 Segmentation Fault

Related topics