Take the 2-minute tour ×
Server Fault is a question and answer site for professional system and network administrators. It's 100% free, no registration required.

All httpd child processes Segmentation fault near MaxRequestsPerChild, seemingly when it is time for the process to recycle. MaxRequestsPerChild currently set to 10000. I am also not able to reproduce this on a test system. Child process exits clean when attached with gdb. Child process also exits clean on stops/restarts.

[notice] child pid 14054 exit signal Segmentation fault (11)

Info:

RHEL 6.4

Package: httpd.x86_64 2.2.15-29.el6_4

Server version: Apache/2.2.15 (Unix)
Server built:   Aug  2 2013 08:02:15
Server's Module Magic Number: 20051115:25
Server loaded:  APR 1.3.9, APR-Util 1.3.9
Compiled using: APR 1.3.9, APR-Util 1.3.9
Architecture:   64-bit
Server MPM:     Prefork
  threaded:     no
    forked:     yes (variable process count)
Server compiled with....
 -D APACHE_MPM_DIR="server/mpm/prefork"
 -D APR_HAS_SENDFILE
 -D APR_HAS_MMAP
 -D APR_HAVE_IPV6 (IPv4-mapped addresses enabled)
 -D APR_USE_SYSVSEM_SERIALIZE
 -D APR_USE_PTHREAD_SERIALIZE
 -D SINGLE_LISTEN_UNSERIALIZED_ACCEPT
 -D APR_HAS_OTHER_CHILD
 -D AP_HAVE_RELIABLE_PIPED_LOGS
 -D DYNAMIC_MODULE_LIMIT=128
 -D HTTPD_ROOT="/etc/httpd"
 -D SUEXEC_BIN="/usr/sbin/suexec"
 -D DEFAULT_PIDLOG="run/httpd.pid"
 -D DEFAULT_SCOREBOARD="logs/apache_runtime_status"
 -D DEFAULT_LOCKFILE="logs/accept.lock"
 -D DEFAULT_ERRORLOG="logs/error_log"
 -D AP_TYPES_CONFIG_FILE="conf/mime.types"
 -D SERVER_CONFIG_FILE="conf/httpd.conf"

#

[debug] prefork.c(1018): AcceptMutex: sysvsem (default: sysvsem)

bt full

#0  __pthread_mutex_lock (mutex=0x0) at pthread_mutex_lock.c:50
        type = <value optimized out>
        id = <value optimized out>
#1  0x00007f6595fd30b9 in PR_Lock (lock=0x0) at ../../../mozilla/nsprpub/pr/src/pthreads/ptsynch.c:174
No locals.
#2  0x00007f659686a038 in PK11_GetAttributes (arena=0x7f659d795560, slot=0x7f659d4e9310, obj=13, 
    attr=0x7fffbed0ef50, count=2) at pk11obj.c:190
        i = <value optimized out>
        mark = 0x0
        crv = <value optimized out>
#3  0x00007f659686a39c in PK11_MatchItem (slot=0x7f659d4e9310, searchID=13, matchclass=3) at pk11obj.c:1657
        theTemplate = {{type = 258, pValue = 0x0, ulValueLen = 0}, {type = 0, pValue = 0x0, ulValueLen = 0}}
        keyclass = 0x7fffbed0ef68
        peerID = <value optimized out>
        parent = <value optimized out>
        arena = 0x7f659d795560
        crv = <value optimized out>
#4  0x00007f6596892106 in nssToken_IsPrivateKeyAvailable (token=0x7f659d787e20, c=<value optimized out>, 
    instance=0x7f659cf99450) at devtoken.c:1579
        theClass = <value optimized out>
#5  0x00007f659688998c in NSSCertificate_IsPrivateKeyAvailable (c=0x7f659d805508, uhh=<value optimized out>, 
    statusOpt=<value optimized out>) at certificate.c:730
        instance = <value optimized out>
        isUser = <value optimized out>
        ip = 0x7f659d7dcb40
        instances = 0x7f659d7dcb40
#6  0x00007f65968908ea in nssTrust_GetCERTCertTrustForCert (c=0x7f659d805508, cc=<value optimized out>)
    at pki3hack.c:634
        rvTrust = 0x7f659d7f73a0
        td = <value optimized out>
        t = 0x0
#7  0x00007f659689148e in fill_CERTCertificateFields (c=0x7f659d805508, forceUpdate=<value optimized out>)
    at pki3hack.c:826
        nssTrust = <value optimized out>
        context = 0x0
        trust = 0x0
        instance = 0x7f659d7dce50
        stanNick = <value optimized out>
#8  stan_GetCERTCertificate (c=0x7f659d805508, forceUpdate=<value optimized out>) at pki3hack.c:894
        dc = <value optimized out>
        cc = 0x7f659d7f66b0
        certTrust = {sslFlags = 2642323008, emailFlags = 32613, objectSigningFlags = 2525549026}
#9  0x00007f659688d2d0 in nssTrustDomain_RemoveTokenCertsFromCache (td=<value optimized out>, 
    token=<value optimized out>) at tdcache.c:452
        certs = <value optimized out>
        i = <value optimized out>
        dtor = {token = 0x7f659cebee70, cache = 0x7f659cea7240, certs = 0x7f659d806c70, numCerts = 2, 
          arrSize = 10}
#10 0x00007f659689198d in STAN_RemoveModuleFromDefaultTrustDomain (module=0x7f659ceae130) at pki3hack.c:194
---Type <return> to continue, or q <return> to quit--- 
        token = 0x7f659cebee70
        td = 0x7f659cea6980
        i = <value optimized out>
#11 0x00007f659686a628 in SECMOD_UnloadUserModule (mod=0x7f659ceae130) at pk11pars.c:1127
        rv = SECSuccess
        atype = 0
        moduleLock = 0x7f659cfa9f20
#12 0x00007f659741fcda in tlsm_nss_shutdown_cb (appData=<value optimized out>, nssData=<value optimized out>)
    at ../../../libraries/libldap/tls_m.c:1227
No locals.
#13 0x00007f659683a9b2 in nss_ShutdownShutdownList () at nssinit.c:1052
        funcPair = <value optimized out>
        rv = <value optimized out>
        i = <value optimized out>
#14 nss_Shutdown () at nssinit.c:1080
        shutdownRV = SECSuccess
        rv = <value optimized out>
        status = <value optimized out>
        temp = <value optimized out>
#15 0x00007f659683aba3 in NSS_ShutdownContext (context=<value optimized out>) at nssinit.c:1215
        rv = SECSuccess
#16 0x00007f6597420e72 in tlsm_ctx_free (ctx=0x7f659cfae1a0) at ../../../libraries/libldap/tls_m.c:2178
        c = 0x7f659cfae1a0
        refcount = <value optimized out>
#17 0x00007f659741cc92 in ldap_int_tls_destroy (lo=0x7f6597635f60) at ../../../libraries/libldap/tls2.c:105
No locals.
#18 0x00007f65973f87cf in __do_global_dtors_aux () from /lib64/libldap-2.4.so.2.5.6
No symbol table info available.
#19 0x0000000000000000 in ?? ()
No symbol table info available.

strace sample:

*Filtered

    open("/etc/pki/nssdb/pkcs11.txt", O_RDONLY) = 23
    close(23)                               = 0
    open("/etc/ld.so.cache", O_RDONLY)      = 23
    close(23)                               = 0
    open("/usr/lib64/libnsssysinit.so", O_RDONLY) = 23
    close(23)                               = 0
    ...
    open("/etc/pki/nssdb/cert9.db", O_RDONLY) = 23
    fcntl(23, F_GETFD)                      = 0
    fcntl(23, F_SETFD, FD_CLOEXEC)          = 0
    fstat(23, {st_dev=makedev(8, 3), st_ino=1047848, st_mode=S_IFREG|0644, st_nlink=1, st_uid=0, st_gid=0, 
    st_blksize=4096, st_blocks=24, st_size=9216, st_atime=2010/01/12-15:18:17, st_mtime=2010/01/12-15:18:17
    , st_ctime=2013/08/20-09:59:34}) = 0
    lseek(23, 0, SEEK_SET)                  = 0
    read(23, "SQLite format 3\0\4\0\1\1\0@  \0\0\0\1\0\0\0\0"..., 100) = 100
    fcntl(23, F_SETLK, {type=F_RDLCK, whence=SEEK_SET, start=1073741824, len=1}) = 0
    fcntl(23, F_SETLK, {type=F_RDLCK, whence=SEEK_SET, start=1073741826, len=510}) = 0
    fcntl(23, F_SETLK, {type=F_UNLCK, whence=SEEK_SET, start=1073741824, len=1}) = 0
    fstat(23, {st_dev=makedev(8, 3), st_ino=1047848, st_mode=S_IFREG|0644, st_nlink=1, st_uid=0, st_gid=0, 
    st_blksize=4096, st_blocks=24, st_size=9216, st_atime=2010/01/12-15:18:17, st_mtime=2010/01/12-15:18:17
    , st_ctime=2013/08/20-09:59:34}) = 0
    lseek(23, 0, SEEK_SET)                  = 0
    read(23, "SQLite format 3\0\4\0\1\1\0@  \0\0\0\1\0\0\0\0"..., 1024) = 1024
    lseek(23, 3072, SEEK_SET)               = 3072
    read(23, "\r\0\0\0\1\0J\0\0J\3\315\0\0\0\0\0\0\0\0\0\0\0\2073\1\7\27\37\37\1\216"..., 1024) = 1024
    lseek(23, 5120, SEEK_SET)               = 5120
    read(23, "\r\0\0\0\5\2\302\0\3\212\3\275\3D\3\4\2\302\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 1024) = 1024
    fcntl(23, F_SETLK, {type=F_UNLCK, whence=SEEK_SET, start=0, len=0}) = 0
    fcntl(23, F_SETLK, {type=F_RDLCK, whence=SEEK_SET, start=1073741824, len=1}) = 0
    fcntl(23, F_SETLK, {type=F_RDLCK, whence=SEEK_SET, start=1073741826, len=510}) = 0
    fcntl(23, F_SETLK, {type=F_UNLCK, whence=SEEK_SET, start=1073741824, len=1}) = 0
    fstat(23, {st_dev=makedev(8, 3), st_ino=1047848, st_mode=S_IFREG|0644, st_nlink=1, st_uid=0, st_gid=0, 
    st_blksize=4096, st_blocks=24, st_size=9216, st_atime=2010/01/12-15:18:17, st_mtime=2010/01/12-15:18:17
    , st_ctime=2013/08/20-09:59:34}) = 0

    ...

rt_sigprocmask(SIG_BLOCK, [INT], NULL, 8)      = 0
rt_sigaction(SIGINT, {0x7f8dc4ec0e50, [], SA_RESTORER, 0x7f8dc33f0920}, {0x7f8db3507f0c, ~[ILL ABRT BUS FPE KILL SEGV USR2 STOP XCPU XFSZ SYS RTMIN RT_1], SA_RESTORER|SA_SIGINFO, 0x7f8dc33f0920}, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [INT], NULL, 8) = 0
munmap(0x7f8dae20f000, 2178616) = 0
munmap(0x7f8daeced000, 2129488)         = 0
munmap(0x7f8daead9000, 2176864)         = 0
munmap(0x7f8dae89c000, 2343352)         = 0
munmap(0x7f8dae627000, 2574912)         = 0
munmap(0x7f8dae423000, 2109528)         = 0
munmap(0x7f8daeef5000, 2142192)         = 0
munmap(0x7f8daf100000, 2125616)         = 0
munmap(0x7f8daf307000, 2128512) = 0
munmap(0x7f8daf50f000, 2107408) = 0
munmap(0x7f8daf712000, 2103968) = 0
munmap(0x7f8daf914000, 2111904) = 0
munmap(0x7f8dafb18000, 2162656) = 0
munmap(0x7f8dafd28000, 2140264)         = 0
munmap(0x7f8db0911000, 2116672) = 0
munmap(0x7f8db044f000, 2099376)            = 0
munmap(0x7f8db0650000, 2885912)                      = 0
munmap(0x7f8db0b16000, 2123736)                      = 0
munmap(0x7f8db0d1d000, 2357704) = 0
munmap(0x7f8db11ec000, 2119816) = 0
munmap(0x7f8db13f2000, 2125296)         = 0
munmap(0x7f8db15f9000, 2198440)         = 0
munmap(0x7f8db467b000, 2289928)         = 0
munmap(0x7f8db48ab000, 2232272) = 0
munmap(0x7f8db4e50000, 2149768) = 0
munmap(0x7f8db4acc000, 3684744) = 0
munmap(0x7f8db5290000, 2142072) = 0
munmap(0x7f8db505d000, 2303840)                              = 0
munmap(0x7f8db549b000, 4278992)     = 0
munmap(0x7f8db58b0000, 2148288)         = 0
munmap(0x7f8db5abd000, 2131392) = 0
munmap(0x7f8db6d48000, 2440032)         = 0
munmap(0x7f8db6aab000, 2737840)         = 0
munmap(0x7f8db655d000, 2165496) = 0
munmap(0x7f8db676e000, 3394936) = 0
munmap(0x7f8db6337000, 2250552)         = 0
munmap(0x7f8db60e7000, 2423648)         = 0
munmap(0x7f8db5ec9000, 2217576)         = 0
munmap(0x7f8db5cc6000, 2106112)         = 0
munmap(0x7f8db6f9c000, 3939304)         = 0
munmap(0x7f8db735e000, 2268048)         = 0
fcntl(23, F_SETLK, {type=F_RDLCK, whence=SEEK_SET, start=1073741824, len=1}) = 0
fcntl(23, F_SETLK, {type=F_RDLCK, whence=SEEK_SET, start=1073741826, len=510}) = 0
fcntl(23, F_SETLK, {type=F_UNLCK, whence=SEEK_SET, start=1073741824, len=1}) = 0
access("/etc/pki/nssdb/cert9.db-journal", F_OK)                              = -1 ENOENT (No such file or directory)
fstat(23, {st_dev=makedev(8, 3), st_ino=1047848, st_mode=S_IFREG|0644, st_nlink=1, st_uid=0, st_gid=0, st_blksize=4096, st_blocks=24, st_size=9216, st_atime=2010/01/12-15:18:17, st_mtime=2010/01/12-15:18:17, st_ctime=2013/08/20-09:59:34}) = 0
lseek(23, 24, SEEK_SET) = 24
read(23, "\0\0\0\1\0\0\0\0\0\0\0\0\0\0\0\0", 16) = 16
fcntl(23, F_SETLK, {type=F_UNLCK, whence=SEEK_SET, start=0, len=0}) = 0
fcntl(23, F_SETLK, {type=F_RDLCK, whence=SEEK_SET, start=1073741824, len=1})                              = 0
fcntl(23, F_SETLK, {type=F_RDLCK, whence=SEEK_SET, start=1073741826, len=510}) = 0
fcntl(23, F_SETLK, {type=F_UNLCK, whence=SEEK_SET, start=1073741824, len=1}) = 0
access("/etc/pki/nssdb/cert9.db-journal", F_OK) = -1 ENOENT (No such file or directory)
fstat(23, {st_dev=makedev(8, 3), st_ino=1047848, st_mode=S_IFREG|0644, st_nlink=1, st_uid=0, st_gid=0, st_blksize=4096, st_blocks=24, st_size=9216, st_atime=2010/01/12-15:18:17, st_mtime=2010/01/12-15:18:17, st_ctime=2013/08/20-09:59:34}) = 0
lseek(23, 24, SEEK_SET)                 = 24
read(23, "\0\0\0\1\0\0\0\0\0\0\0\0\0\0\0\0", 16) = 16
fcntl(23, F_SETLK, {type=F_UNLCK, whence=SEEK_SET, start=0, len=0}) = 0
brk(0x7f8dc6191000)                     = 0x7f8dc6191000
munmap(0x7f8db7e3e000, 2161288)         = 0
munmap(0x7f8db7bea000, 2438600)         = 0
munmap(0x7f8db79b8000, 2301736)         = 0
munmap(0x7f8db7588000, 2257152)         = 0
close(8)                                = 0
close(7)                                = 0
fcntl(23, F_SETLK, {type=F_RDLCK, whence=SEEK_SET, start=1073741824, len=1}) = 0
fcntl(23, F_SETLK, {type=F_RDLCK, whence=SEEK_SET, start=1073741826, len=510}) = 0
fcntl(23, F_SETLK, {type=F_UNLCK, whence=SEEK_SET, start=1073741824, len=1}) = 0
access("/etc/pki/nssdb/cert9.db-journal", F_OK) = -1 ENOENT (No such file or directory)
fstat(23, {st_dev=makedev(8, 3), st_ino=1047848, st_mode=S_IFREG|0644, st_nlink=1, st_uid=0, st_gid=0, st_blksize=4096, st_blocks=24, st_size=9216, st_atime=2010/01/12-15:18:17, st_mtime=2010/01/12-15:18:17, st_ctime=2013/08/20-09:59:34}) = 0
lseek(23, 24, SEEK_SET) = 24
read(23, "\0\0\0\1\0\0\0\0\0\0\0\0\0\0\0\0", 16) = 16
fcntl(23, F_SETLK, {type=F_UNLCK, whence=SEEK_SET, start=0, len=0}) = 0
--- SIGSEGV (Segmentation fault) @ 0 (0) ---
chdir("/apache2/apache-dumps") = 0
rt_sigaction(SIGSEGV, {SIG_DFL, [], SA_RESTORER|SA_INTERRUPT, 0x7f8dc3760500}, {SIG_DFL, [], SA_RESTORER|SA_RESETHAND, 0x7f8dc3760500}, 8) = 0
kill(26317, SIGSEGV) = 0
rt_sigreturn(0x66cd) = 140246846382624
--- SIGSEGV (Segmentation fault) @ 0 (0) ---
share|improve this question
    
Now the great question: what file is fd 21? –  ott-- Aug 28 '13 at 20:59
    
I'm not especially experienced at this level of debugging, so this is where I've hit my wall. This is a little over me, but here are some additional observations. The backtrace references pthread mutex, which is not the configured AcceptMutex. However, mod_ssl SSLMutex is configured as "default", though I cannot find what value is being taken. I could assume that it is taking pthread and mod_ssl is producing the pthread reference in backtrace. This might also bridge a correlation between the info in backtrace and nss file accesses in the strace? –  riggs685 Aug 28 '13 at 22:08
    
If fd 21 is created when the process starts, how might I get a strace on a new httpd pid from start? –  riggs685 Aug 28 '13 at 22:15
    
Can you add the part where fd 21 is opened, or use strace -fv ... to include PIDs? –  ott-- Aug 28 '13 at 22:47
    
ott I have added strace sample 2 which has more information about its respective fd. –  riggs685 Aug 29 '13 at 14:59
show 5 more comments

Your Answer

 
discard

By posting your answer, you agree to the privacy policy and terms of service.

Browse other questions tagged or ask your own question.