Centos 5.3 tar and rpm give segmentation fault always - can you tell why from the strace?

Question

(Update: Today, tar works without segfault without known changes on the system. The "rm" command is now segfaulting though. Argh! Maybe this is being caused by ESXi? Or, maybe the hardware.)

I have a Centos 5.3 system running on VMWare ESXi. It's been running for quite a while without much problem. Recently, however, I noticed that the tar command and the rpm command gives me a segmentation fault when I run them.

So, I tried using strace (found someone suggested this online) and below is what I get when using strace. I'd like to know how to repair my system and how I can prevent this from happening in the future.

Thanks all in advance!

The output:

[root@wagon init.d]# strace -f rpm
execve("/bin/rpm", ["rpm"], [/* 20 vars */]) = 0
brk(0)                                  = 0x9e98000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY)      = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=51424, ...}) = 0
mmap2(NULL, 51424, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb7f79000
close(3)                                = 0
open("/usr/lib/librpm-4.4.so", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360\257\0\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=378528, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7f78000
mmap2(NULL, 588416, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xd43000
mmap2(0xd9d000, 16384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x59) = 0xd9d000
mmap2(0xda1000, 203392, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xda1000
close(3)                                = 0
open("/usr/lib/librpmdb-4.4.so", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0P\307\1\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=1134540, ...}) = 0
mmap2(NULL, 1139296, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x627000
mmap2(0x739000, 16384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x111) = 0x739000
mmap2(0x73d000, 608, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x73d000
close(3)                                = 0
open("/lib/libselinux.so.1", O_RDONLY)  = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\0005\0\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=91892, ...}) = 0
mmap2(NULL, 97112, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xc25000
mmap2(0xc3b000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x15) = 0xc3b000
close(3)                                = 0
open("/usr/lib/librpmio-4.4.so", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\276\0\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=396276, ...}) = 0
mmap2(NULL, 540192, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb64000
mmap2(0xbc3000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x5e) = 0xbc3000
mmap2(0xbc6000, 138784, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xbc6000
close(3)                                = 0
open("/usr/lib/libpopt.so.0", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\260\20\0\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=29008, ...}) = 0
mmap2(NULL, 31856, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7c5000
mmap2(0x7cc000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6) = 0x7cc000
close(3)                                = 0
open("/usr/lib/libsqlite3.so.0", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\340=\327\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=385180, ...}) = 0
mmap2(0xd6a000, 387072, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x110000
mmap2(0x16d000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x5c) = 0x16d000
close(3)                                = 0
open("/usr/lib/libelf.so.1", O_RDONLY)  = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0000\35\0\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=87508, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7f77000
mmap2(NULL, 90296, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x9cf000
mmap2(0x9e4000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14) = 0x9e4000
close(3)                                = 0
open("/lib/libm.so.6", O_RDONLY)        = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\0204\0\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=206380, ...}) = 0
mmap2(NULL, 155760, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x16f000
mmap2(0x194000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x24) = 0x194000
close(3)                                = 0
open("/usr/lib/libz.so.1", O_RDONLY)    = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\300\245\303\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=75028, ...}) = 0
mmap2(0xc39000, 76400, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x196000
mmap2(0x1a8000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x11) = 0x1a8000
close(3)                                = 0
open("/usr/lib/libnss3.so", O_RDONLY)   = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\0mr\0054\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=1306252, ...}) = 0
mmap2(0x5714000, 1310584, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x5714000
mmap2(0x584e000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x139) = 0x584e000
close(3)                                = 0
open("/usr/lib/libplds4.so", O_RDONLY)  = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0000\312\205\0054\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=11460, ...}) = 0
mmap2(0x585c000, 8672, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x585c000
mmap2(0x585e000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2) = 0x585e000
close(3)                                = 0
open("/usr/lib/libplc4.so", O_RDONLY)   = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360m\205\0054\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=15556, ...}) = 0
mmap2(0x5856000, 12744, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x5856000
mmap2(0x5859000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3) = 0x5859000
close(3)                                = 0
open("/usr/lib/libnspr4.so", O_RDONLY)  = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0P\227\206\0054\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=228028, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7f76000
mmap2(0x5861000, 232928, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x5861000
mmap2(0x5897000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x36) = 0x5897000
mmap2(0x5898000, 7648, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x5898000
close(3)                                = 0
open("/lib/libdl.so.2", O_RDONLY)       = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0P\n\0\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=14644, ...}) = 0
mmap2(NULL, 12408, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x2a7000
mmap2(0x2a9000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1) = 0x2a9000
close(3)                                = 0
open("/lib/librt.so.1", O_RDONLY)       = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\200\30\0\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=42048, ...}) = 0
mmap2(NULL, 33324, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7d0000
mmap2(0x7d7000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6) = 0x7d7000
close(3)                                = 0
open("/lib/libpthread.so.0", O_RDONLY)  = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0000H\0\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=123596, ...}) = 0
mmap2(NULL, 90592, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x1a9000
mmap2(0x1bc000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x12) = 0x1bc000
mmap2(0x1be000, 4576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x1be000
close(3)                                = 0
open("/usr/lib/libbz2.so.1", O_RDONLY)  = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\240\17\364\0034\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=71852, ...}) = 0
mmap2(0x3f40000, 69128, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x3f40000
mmap2(0x3f50000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x10) = 0x3f50000
close(3)                                = 0
open("/lib/libc.so.6", O_RDONLY)        = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\320_\1\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=1603392, ...}) = 0
mmap2(NULL, 1324452, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x2ab000
mmap2(0x3e9000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x13e) = 0x3e9000
mmap2(0x3ec000, 9636, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x3ec000
close(3)                                = 0
open("/lib/libsepol.so.1", O_RDONLY)    = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\340\256\306\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=242880, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7f75000
mmap2(0xc68000, 286624, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xc68000
mmap2(0xca3000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3a) = 0xca3000
mmap2(0xca4000, 40864, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xca4000
close(3)                                = 0
open("/lib/libgcc_s.so.1", O_RDONLY)    = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\26\0\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=44992, ...}) = 0
mmap2(NULL, 48036, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x435000
mmap2(0x440000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xa) = 0x440000
close(3)                                = 0
open("/usr/lib/libnssutil3.so", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\20\350o\0054\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=96764, ...}) = 0
mmap2(0x56fa000, 98028, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x56fa000
mmap2(0x570f000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14) = 0x570f000
close(3)                                = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7f74000
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7f73000
set_thread_area({entry_number:-1 -> 6, base_addr:0xb7f736d0, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}) = 0
mprotect(0x3e9000, 8192, PROT_READ)     = 0
mprotect(0x1bc000, 4096, PROT_READ)     = 0
mprotect(0x7d7000, 4096, PROT_READ)     = 0
mprotect(0x2a9000, 4096, PROT_READ)     = 0
mprotect(0x194000, 4096, PROT_READ)     = 0
mprotect(0xff9000, 4096, PROT_READ)     = 0
munmap(0xb7f79000, 51424)               = 0
set_tid_address(0xb7f73718)             = 5604
set_robust_list(0xb7f73720, 0xc)        = 0
futex(0xbf9763c4, FUTEX_WAKE_PRIVATE, 1) = 0
rt_sigaction(SIGRTMIN, {0x1ad3d0, [], SA_SIGINFO}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {0x1ad2e0, [], SA_RESTART|SA_SIGINFO}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
getrlimit(RLIMIT_STACK, {rlim_cur=10240*1024, rlim_max=RLIM_INFINITY}) = 0
uname({sys="Linux", node="wagon.localdomain", ...}) = 0
access("/etc/selinux/", F_OK)           = 0
brk(0)                                  = 0x9e98000
brk(0x9eb9000)                          = 0x9eb9000
open("/etc/selinux/config", O_RDONLY|O_LARGEFILE) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=511, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7f85000
read(3, "# This file controls the state o"..., 4096) = 511
read(3, "", 4096)                       = 0
close(3)                                = 0
munmap(0xb7f85000, 4096)                = 0
open("/proc/mounts", O_RDONLY|O_LARGEFILE) = 3
fstat64(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7f85000
read(3, "rootfs / rootfs rw 0 0\n/dev/root"..., 4096) = 587
read(3, "", 4096)                       = 0
close(3)                                = 0
munmap(0xb7f85000, 4096)                = 0
--- SIGSEGV (Segmentation fault) @ 0 (0) ---
+++ killed by SIGSEGV +++
[root@wagon init.d]# 

Answer 1

Even without much investigation, from your description that a few programs started to segfault, then they began to work correctly while others started failing, you have either bad memory modules, or a broken prelink.

First, stop all your virtual machines, reboot the host and run a memory test. You have to do this from outside ESXi. If you find any memory defects, that is your problem. Replace the hardware.

If no memory defects were found, check that you are running the latest available kernel from CentOS for your architecture.

Boot the host and the server again, the server in single-user mode (pass "single" to grub kernel parameters) and run:

prelink -avf

When prelink finishes, you should reboot the server. You could also do telinit u && init 3 to resume booting, but it is better to reboot in order to assure that all binaries will be reloaded with their new memory mappings.

Answer 2

This is a guess, but, try turning off se-linix and see if it works.

It shouldn't segfault, but, there may be a file that can't be opened because of se-linux that tar never thought would happen.

Else you'll have to install the source rpm for tar, and, build it with debugging, and then run it in the debugger to see why.

Answer 3

Ok, so not as complex as I thought. The server was compromised and the genius hacker screwed up the root kit install. So, the affect was binaries segfaulting. Another affect was unexpected network traffic from the server. Thanks all who responded!

Answer 4

Alright, sounds like great fun.

First, do you have two identically up-to-date boxes (one of which is not segfaulting)?

If so, check the md5sum on the binaries that are segfaulting, make sure they're the same.

Next, run ldd on the binaries that are failing, then run md5sum on the libraries to see if they're different.

Now, assuming the libraries are the same, something else is wrong, probably machine environment or configuration, but lets see if the strace sheds any light.

To quote your strace, near the end, you have:

 open("/proc/mounts", O_RDONLY|O_LARGEFILE) = 3
 fstat64(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
 mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7f85000
 read(3, "rootfs / rootfs rw 0 0\n/dev/root"..., 4096) = 587
 read(3, "", 4096)                       = 0
 close(3)                                = 0
 munmap(0xb7f85000, 4096)                = 0

The next line is the segfault message.

On my CentOS 5.3 machine (rpm version 4.4.2.3), my rpm immediately does this:

open("/usr/lib/locale/locale-archive", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=56458752, ...}) = 0
mmap(NULL, 56458752, PROT_READ, MAP_PRIVATE, 3, 0) = 0x2b36f9d01000
close(3)                                = 0

I would thus venture that the problem is either closing the memory out after reading /proc/mounts or in opening /usr/lib/locale/locale-archive. Since munmap() returned 0, I'd bark up the locale tree.

Incidentally, if your configuration excludes the locale, the next block mine executes is:

 open("/usr/lib/rpm/rpmpopt-4.4.2.3", O_RDONLY) = 3
 lseek(3, 0, SEEK_END)                   = 23100
 lseek(3, 0, SEEK_SET)                   = 0
 read(3, "#/*! \\page config_rpmpopt Defaul"..., 23100) = 23100
 close(3)        

Good luck and let us know what you find.