CentOS 7: df started to hang

Question

df fails to return any output, and just hangs. There have been no changes to fstab in over a year. There are no network drives.

OS: CentOS 7.0

Output of strace df:

execve("/usr/bin/df", ["df"], [/* 22 vars */]) = 0
brk(NULL)                               = 0x1499000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f628668f000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=45250, ...}) = 0
mmap(NULL, 45250, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f6286683000
close(3)                                = 0
open("/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P%\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=2173512, ...}) = 0
mmap(NULL, 3981792, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f62860a2000
mprotect(0x7f6286265000, 2093056, PROT_NONE) = 0
mmap(0x7f6286464000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1c2000) = 0x7f6286464000
mmap(0x7f628646a000, 16864, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f628646a000
close(3)                                = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f6286682000
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f6286680000
arch_prctl(ARCH_SET_FS, 0x7f6286680740) = 0
mprotect(0x7f6286464000, 16384, PROT_READ) = 0
mprotect(0x616000, 4096, PROT_READ)     = 0
mprotect(0x7f6286690000, 4096, PROT_READ) = 0
munmap(0x7f6286683000, 45250)           = 0
brk(NULL)                               = 0x1499000
brk(0x14ba000)                          = 0x14ba000
brk(NULL)                               = 0x14ba000
open("/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=106070960, ...}) = 0
mmap(NULL, 106070960, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f627fb79000
close(3)                                = 0
open("/usr/share/locale/locale.alias", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=2502, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f628668e000
read(3, "# Locale name alias data base.\n#"..., 4096) = 2502
read(3, "", 4096)                       = 0
close(3)                                = 0
munmap(0x7f628668e000, 4096)            = 0
open("/usr/share/locale/en_US.UTF-8/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en_US.utf8/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en_US/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en.UTF-8/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en.utf8/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/etc/mtab", O_RDONLY|O_CLOEXEC)   = 3
fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f628668e000
read(3, "rootfs / rootfs rw 0 0\nsysfs /sy"..., 1024) = 1024
read(3, "oexec,relatime,memory 0 0\ncgroup"..., 1024) = 1024
read(3, "rw,nosuid,nodev,relatime,size=14"..., 1024) = 167
read(3, "", 1024)                       = 0
close(3)                                = 0
munmap(0x7f628668e000, 4096)            = 0
stat("/", {st_mode=S_IFDIR|0555, st_size=4096, ...}) = 0
stat("/sys", {st_mode=S_IFDIR|0555, st_size=0, ...}) = 0
stat("/proc", {st_mode=S_IFDIR|0555, st_size=0, ...}) = 0
stat("/dev", {st_mode=S_IFDIR|0755, st_size=2780, ...}) = 0
stat("/sys/kernel/security", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
stat("/dev/shm", {st_mode=S_IFDIR|S_ISVTX|0777, st_size=40, ...}) = 0
stat("/dev/pts", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
stat("/run", {st_mode=S_IFDIR|0755, st_size=1100, ...}) = 0
stat("/sys/fs/cgroup", {st_mode=S_IFDIR|0755, st_size=340, ...}) = 0
stat("/sys/fs/cgroup/systemd", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
stat("/sys/fs/pstore", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
stat("/sys/fs/cgroup/devices", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
stat("/sys/fs/cgroup/pids", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
stat("/sys/fs/cgroup/cpu,cpuacct", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
stat("/sys/fs/cgroup/memory", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
stat("/sys/fs/cgroup/hugetlb", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
stat("/sys/fs/cgroup/freezer", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
stat("/sys/fs/cgroup/blkio", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
stat("/sys/fs/cgroup/perf_event", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
stat("/sys/fs/cgroup/cpuset", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
stat("/sys/fs/cgroup/net_cls,net_prio", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
stat("/sys/kernel/config", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
stat("/", {st_mode=S_IFDIR|0555, st_size=4096, ...}) = 0
stat("/proc/sys/fs/binfmt_misc",

fstab:

/dev/xvda1              /                       ext4     defaults,noatime 1 1
/swapfile   swap    swap    sw  0   0

Answer 1

Started having this issue on few servers after 7.5 upgrade. A quick reboot will fix the issue but I couldn't afford rebooting the production hosts, after hours of troubleshooting following fixed it for me.

systemctl restart proc-sys-fs-binfmt_misc.mount

Answer 2

i have the same issue on lots of servers

quick & dirty fix is

yum -y reinstall systemd

Answer 3

The solution turned out to be an odd one. A new issue manifested itself in the form of certain network routes failing to resolve. In research, it was dicovered that the underlying host force updated routing tables, updating kernel. Effectively, the issue was solved with a reboot, and re-entry of correct routing tables.