12

df fails to return any output, and just hangs. There have been no changes to fstab in over a year. There are no network drives.

OS: CentOS 7.0

Output of strace df:

execve("/usr/bin/df", ["df"], [/* 22 vars */]) = 0
brk(NULL)                               = 0x1499000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f628668f000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=45250, ...}) = 0
mmap(NULL, 45250, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f6286683000
close(3)                                = 0
open("/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P%\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=2173512, ...}) = 0
mmap(NULL, 3981792, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f62860a2000
mprotect(0x7f6286265000, 2093056, PROT_NONE) = 0
mmap(0x7f6286464000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1c2000) = 0x7f6286464000
mmap(0x7f628646a000, 16864, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f628646a000
close(3)                                = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f6286682000
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f6286680000
arch_prctl(ARCH_SET_FS, 0x7f6286680740) = 0
mprotect(0x7f6286464000, 16384, PROT_READ) = 0
mprotect(0x616000, 4096, PROT_READ)     = 0
mprotect(0x7f6286690000, 4096, PROT_READ) = 0
munmap(0x7f6286683000, 45250)           = 0
brk(NULL)                               = 0x1499000
brk(0x14ba000)                          = 0x14ba000
brk(NULL)                               = 0x14ba000
open("/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=106070960, ...}) = 0
mmap(NULL, 106070960, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f627fb79000
close(3)                                = 0
open("/usr/share/locale/locale.alias", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=2502, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f628668e000
read(3, "# Locale name alias data base.\n#"..., 4096) = 2502
read(3, "", 4096)                       = 0
close(3)                                = 0
munmap(0x7f628668e000, 4096)            = 0
open("/usr/share/locale/en_US.UTF-8/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en_US.utf8/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en_US/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en.UTF-8/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en.utf8/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/etc/mtab", O_RDONLY|O_CLOEXEC)   = 3
fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f628668e000
read(3, "rootfs / rootfs rw 0 0\nsysfs /sy"..., 1024) = 1024
read(3, "oexec,relatime,memory 0 0\ncgroup"..., 1024) = 1024
read(3, "rw,nosuid,nodev,relatime,size=14"..., 1024) = 167
read(3, "", 1024)                       = 0
close(3)                                = 0
munmap(0x7f628668e000, 4096)            = 0
stat("/", {st_mode=S_IFDIR|0555, st_size=4096, ...}) = 0
stat("/sys", {st_mode=S_IFDIR|0555, st_size=0, ...}) = 0
stat("/proc", {st_mode=S_IFDIR|0555, st_size=0, ...}) = 0
stat("/dev", {st_mode=S_IFDIR|0755, st_size=2780, ...}) = 0
stat("/sys/kernel/security", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
stat("/dev/shm", {st_mode=S_IFDIR|S_ISVTX|0777, st_size=40, ...}) = 0
stat("/dev/pts", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
stat("/run", {st_mode=S_IFDIR|0755, st_size=1100, ...}) = 0
stat("/sys/fs/cgroup", {st_mode=S_IFDIR|0755, st_size=340, ...}) = 0
stat("/sys/fs/cgroup/systemd", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
stat("/sys/fs/pstore", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
stat("/sys/fs/cgroup/devices", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
stat("/sys/fs/cgroup/pids", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
stat("/sys/fs/cgroup/cpu,cpuacct", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
stat("/sys/fs/cgroup/memory", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
stat("/sys/fs/cgroup/hugetlb", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
stat("/sys/fs/cgroup/freezer", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
stat("/sys/fs/cgroup/blkio", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
stat("/sys/fs/cgroup/perf_event", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
stat("/sys/fs/cgroup/cpuset", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
stat("/sys/fs/cgroup/net_cls,net_prio", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
stat("/sys/kernel/config", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
stat("/", {st_mode=S_IFDIR|0555, st_size=4096, ...}) = 0
stat("/proc/sys/fs/binfmt_misc",

fstab:

/dev/xvda1              /                       ext4     defaults,noatime 1 1
/swapfile   swap    swap    sw  0   0
  • 1
    Does everything else on this node work fine? – Spooler May 11 '18 at 19:12
  • 1
    What happens when you run stat /proc/sys/fs/binfmt_misc from a shell? – Mark Wagner May 11 '18 at 19:15
  • Also what does the mount command show? – Mikhail Khirgiy May 12 '18 at 4:13
  • Have you considered installing updates? – Michael Hampton May 13 '18 at 7:50
19

Started having this issue on few servers after 7.5 upgrade. A quick reboot will fix the issue but I couldn't afford rebooting the production hosts, after hours of troubleshooting following fixed it for me.

systemctl restart proc-sys-fs-binfmt_misc.mount
| improve this answer | |
2

i have the same issue on lots of servers

quick & dirty fix is

yum -y reinstall systemd
| improve this answer | |
  • This worked for me and made me avoid a restart. I got into this situation after a yum update. – tavi Jun 21 '18 at 0:12
1

The solution turned out to be an odd one. A new issue manifested itself in the form of certain network routes failing to resolve. In research, it was dicovered that the underlying host force updated routing tables, updating kernel. Effectively, the issue was solved with a reboot, and re-entry of correct routing tables.

| improve this answer | |

Not the answer you're looking for? Browse other questions tagged or ask your own question.