0

We have a number of Android app works on different APIs from our main domain.They all are images apps. Recently after spike in app traffic, our APIs are going down frequently, I though it was my dedicated server issue.

But after thorough discussion with my Web host, i figured that server is fine and the problem lies in API which is causing those crashes.

Our API urls usually looks like

http://*****.com/scripts/api.php?limit=48&action=previews&page=1&type=ico_v&category=10

So tried to see a trace from random requests and the following commands are happening for about 20-30 times in a single process.

stat("../images/pages/ico_v/40741.jpg", {st_mode=S_IFREG|0644, st_size=60090, ...}) = 0
getcwd("/home/hd****/public_html/****.com/scripts", 4096) = 50
lstat("/home/****/public_html/*****.com/scripts/../images/pages/ico_v/40741.jpg", {st_mode=S_IFREG|0644, st_size=60090, ...}) = 0
open("/home/****/public_html/****.com/images/pages/ico_v/40741.jpg", O_RDONLY) = 10
fstat(10, {st_mode=S_IFREG|0644, st_size=60090, ...}) = 0
lseek(10, 0, SEEK_CUR)                  = 0
read(10, "\377\330\377\340\0\20JFIF\0\1\1\1\0`\0`\0\0\377\376\0;CREATOR:"..., 8192) = 8192
close(10)                               = 0
open("../images/pages/ico_v", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 10
getdents(10, /* 1024 entries */, 32768) = 32752
getdents(10, /* 1024 entries */, 32768) = 32768
getdents(10, /* 1024 entries */, 32768) = 32768
getdents(10, /* 1024 entries */, 32768) = 32768
getdents(10, /* 1024 entries */, 32768) = 32768
getdents(10, /* 1024 entries */, 32768) = 32768
getdents(10, /* 1024 entries */, 32768) = 32768
getdents(10, /* 1024 entries */, 32768) = 32768
getdents(10, /* 1024 entries */, 32768) = 32768
getdents(10, /* 1024 entries */, 32768) = 32768
getdents(10, /* 1024 entries */, 32768) = 32768
getdents(10, /* 1024 entries */, 32768) = 32768
getdents(10, /* 1024 entries */, 32768) = 32768
getdents(10, /* 1024 entries */, 32768) = 32768
getdents(10, /* 1024 entries */, 32768) = 32768
getdents(10, /* 1024 entries */, 32768) = 32768
getdents(10, /* 1024 entries */, 32768) = 32768
getdents(10, /* 1024 entries */, 32768) = 32768
getdents(10, /* 1024 entries */, 32768) = 32768
getdents(10, /* 1024 entries */, 32768) = 32768
getdents(10, /* 1024 entries */, 32768) = 32768
getdents(10, /* 1024 entries */, 32768) = 32768
getdents(10, /* 1024 entries */, 32768) = 32768
getdents(10, /* 1024 entries */, 32768) = 32768
getdents(10, /* 1024 entries */, 32768) = 32768
getdents(10, /* 1024 entries */, 32768) = 32768
getdents(10, /* 1024 entries */, 32768) = 32768
getdents(10, /* 1024 entries */, 32768) = 32768
getdents(10, /* 1024 entries */, 32768) = 32768
getdents(10, /* 1024 entries */, 32768) = 32768
getdents(10, /* 1024 entries */, 32768) = 32768
getdents(10, /* 1024 entries */, 32768) = 32768
getdents(10, /* 1024 entries */, 32768) = 32768
getdents(10, /* 1024 entries */, 32768) = 32768
getdents(10, /* 1024 entries */, 32768) = 32768
getdents(10, /* 1024 entries */, 32768) = 32768
getdents(10, /* 1024 entries */, 32768) = 32768
getdents(10, /* 1024 entries */, 32768) = 32768
getdents(10, /* 763 entries */, 32768)  = 24416
getdents(10, /* 0 entries */, 32768)    = 0
close(10)                               = 0

After 20-30 sets of above commands trace ends with following code.....

fstat(10, {st_mode=S_IFREG|0644, st_size=87069, ...}) = 0
lseek(10, 0, SEEK_CUR)                  = 0
read(10, "\211PNG\r\n\32\n\0\0\0\rIHDR\0\0\1h\0\0\2\200\10\2\0\0\0>xu"..., 8192) = 8192
close(10)                               = 0
chdir("/opt/cpanel/ea-php72/root/usr/bin") = 0
setitimer(ITIMER_PROF, {it_interval={0, 0}, it_value={0, 0}}, NULL) = 0
pwrite(8, "", 0, 0)                     = 0
close(8)                                = 0
sendto(9, "\1\0\0\0\1", 5, MSG_DONTWAIT, NULL, 0) = -1 EPIPE (Broken pipe)
--- SIGPIPE {si_signo=SIGPIPE, si_code=SI_USER, si_pid=13472, si_uid=503} ---
close(9)                                = 0
setitimer(ITIMER_PROF, {it_interval={0, 0}, it_value={0, 0}}, NULL) = 0
setitimer(ITIMER_PROF, {it_interval={0, 0}, it_value={30, 0}}, NULL) = 0
fcntl(3, F_SETLK, {type=F_UNLCK, whence=SEEK_SET, start=0, len=0}) = 0
setitimer(ITIMER_PROF, {it_interval={0, 0}, it_value={0, 0}}, NULL) = 0
writev(6, [{"LS\3\0X\1\0\0\7\0\0\0\310\0\0\0t\0'\0003\0\21\0.\0\27\0\26\0", 30}, {"Set-Cookie: sid=5560bf357e132065"..., 314}, {"LS\4\0009\2\0\0", 8}, {"\37\213\10\0\0\0\0\0\0\3\275\230\317\216\2330\30\304_e\345CN(\370\17`@\212\366A\252"..., 561}, {"LS\5\0\10\0\0\0", 8}], 5) = 921
select(7, [6], NULL, NULL, {1, 0})      = 1 (in [6], left {0, 999044})
read(6, "LS\1\0\256\6\0\0\316\1\0\0&\0\0\0\272\2\0\0k\3\0\0\5\3\0\0\312\3\0\0"..., 8192) = 1748
write(6, "LS\7\0\10\0\0\0", 8)          = 8
setitimer(ITIMER_PROF, {it_interval={0, 0}, it_value={0, 0}}, NULL) = 0
setitimer(ITIMER_PROF, {it_interval={0, 0}, it_value={60, 0}}, NULL) = 0
rt_sigaction(SIGPROF, NULL, {0x63b780, ~[ILL TRAP ABRT BUS FPE KILL SEGV CONT STOP TSTP TTIN TTOU SYS RTMIN RT_1], SA_RESTORER|SA_SIGINFO, 0x2b522e85f570}, 8) = 0
rt_sigaction(SIGHUP, NULL, {0x63b780, ~[ILL TRAP ABRT BUS FPE KILL SEGV CONT STOP TSTP TTIN TTOU SYS RTMIN RT_1], SA_RESTORER|SA_SIGINFO, 0x2b522e85f570}, 8) = 0
rt_sigaction(SIGINT, NULL, {0x63b780, ~[ILL TRAP ABRT BUS FPE KILL SEGV CONT STOP TSTP TTIN TTOU SYS RTMIN RT_1], SA_RESTORER|SA_SIGINFO, 0x2b522e85f570}, 8) = 0
rt_sigaction(SIGQUIT, NULL, {0x63b780, ~[ILL TRAP ABRT BUS FPE KILL SEGV CONT STOP TSTP TTIN TTOU SYS RTMIN RT_1], SA_RESTORER|SA_SIGINFO, 0x2b522e85f570}, 8) = 0
rt_sigaction(SIGTERM, NULL, {0x63b780, ~[ILL TRAP ABRT BUS FPE KILL SEGV CONT STOP TSTP TTIN TTOU SYS RTMIN RT_1], SA_RESTORER|SA_SIGINFO, 0x2b522e85f570}, 8) = 0
rt_sigaction(SIGUSR1, NULL, {0x63b780, ~[ILL TRAP ABRT BUS FPE KILL SEGV CONT STOP TSTP TTIN TTOU SYS RTMIN RT_1], SA_RESTORER|SA_SIGINFO, 0x2b522e85f570}, 8) = 0
rt_sigaction(SIGUSR2, NULL, {0x63b780, ~[ILL TRAP ABRT BUS FPE KILL SEGV CONT STOP TSTP TTIN TTOU SYS RTMIN RT_1], SA_RESTORER|SA_SIGINFO, 0x2b522e85f570}, 8) = 0
setitimer(ITIMER_PROF, {it_interval={0, 0}, it_value={120, 0}}, NULL) = 0
rt_sigaction(SIGPROF, {0x63b780, ~[ILL TRAP ABRT BUS FPE KILL SEGV CONT STOP TSTP TTIN TTOU SYS RTMIN RT_1], SA_RESTORER|SA_SIGINFO, 0x2b522e85f570}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [PROF], NULL, 8) = 0
getcwd("/opt/cpanel/ea-php72/root/usr/bin", 4095) = 34
chdir("/home/*****/public_html/*****.com/scripts") = 0
setitimer(ITIMER_PROF, {it_interval={0, 0}, it_value={60, 0}}, NULL) = 0
fcntl(3, F_SETLK, {type=F_RDLCK, whence=SEEK_SET, start=1, len=1}) = 0
access("/home/*****/public_html/*****.com/scripts/api.php", R_OK) = 0
getcwd("/home/*****/public_html/*****.com/scripts", 4096) = 50
access("/home/*****/public_html/*****.com/config.php", R_OK) = 0
read(7, "u\373\312\332\303\v\266\217\17\352\373jjc\345v\335w\275\23\320\177J\356\252\243\367\232\34\17d`"..., 92) = 92
stat("/var/cpanel/php/sessions/ea-php72/sess_57bfacad3cb06bf8f0aebfa6a6365e67", 0x7ffc7b381fc0) = -1 ENOENT (No such file or directory)
open("/var/cpanel/php/sessions/ea-php72/sess_57bfacad3cb06bf8f0aebfa6a6365e67", O_RDWR|O_CREAT|O_NOFOLLOW, 0600) = 8
fstat(8, {st_mode=S_IFREG|0600, st_size=0, ...}) = 0
getuid()                                = 503
flock(8, LOCK_EX)                       = 0
fcntl(8, F_SETFD, FD_CLOEXEC)           = 0
fstat(8, {st_mode=S_IFREG|0600, st_size=0, ...}) = 0
access("/home/*****/public_html/*****.com/classes.php", R_OK) = 0
access("/home/*****/public_html/*****.com/functions.php", R_OK) = 0
socket(PF_LOCAL, SOCK_STREAM, 0)        = 9
fcntl(9, F_GETFL)                       = 0x2 (flags O_RDWR)
fcntl(9, F_SETFL, O_RDWR|O_NONBLOCK)    = 0
connect(9, {sa_family=AF_LOCAL, sun_path="/var/lib/mysql/mysql.sock"}, 27) = 0
fcntl(9, F_SETFL, O_RDWR)               = 0
poll([{fd=9, events=POLLIN|POLLERR|POLLHUP}], 1, 86400000) = 1 ([{fd=9, revents=POLLIN}])
recvfrom(9, "Y\0\0\0", 4, MSG_DONTWAIT, NULL, NULL) = 4
poll([{fd=9, events=POLLIN|POLLERR|POLLHUP}], 1, 86400000) = 1 ([{fd=9, revents=POLLIN}])
recvfrom(9, "\n5.5.5-10.1.37-MariaDB\0\223\376\30\2m:vf}"..., 93, MSG_DONTWAIT, NULL, NULL) = 89
sendto(9, "o\0\0\1\205\242\n\0\0\0\0\300\10\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 115, MSG_DONTWAIT, NULL, 0) = 115
poll([{fd=9, events=POLLIN|POLLERR|POLLHUP}], 1, 86400000) = 1 ([{fd=9, revents=POLLIN}])
recvfrom(9, ",\0\0\2", 4, MSG_DONTWAIT, NULL, NULL) = 4
poll([{fd=9, events=POLLIN|POLLERR|POLLHUP}], 1, 86400000) = 1 ([{fd=9, revents=POLLIN}])
recvfrom(9, "\376mysql_native_password\0m:vf}`I,="..., 93, MSG_DONTWAIT, NULL, NULL) = 44
sendto(9, "\24\0\0\3EJ-N?\377\354\266\25x\326\t\262Z\352\324\261\212\6\354", 24, MSG_DONTWAIT, NULL, 0) = 24
poll([{fd=9, events=POLLIN|POLLERR|POLLHUP}], 1, 86400000) = 1 ([{fd=9, revents=POLLIN}])
recvfrom(9, "\7\0\0\4\0\0\0\2\0\0\0", 49, MSG_DONTWAIT, NULL, NULL) = 11
sendto(9, "\r\0\0\0\2*****_app", 17, MSG_DONTWAIT, NULL, 0) = 17
poll([{fd=9, events=POLLIN|POLLERR|POLLHUP}], 1, 86400000) = 1 ([{fd=9, revents=POLLIN}])
recvfrom(9, "\7\0\0\1\0\0\0\2\0\0\0", 38, MSG_DONTWAIT, NULL, NULL) = 11
sendto(9, "\17\0\0\0\3SET NAMES UTF8", 19, MSG_DONTWAIT, NULL, 0) = 19
poll([{fd=9, events=POLLIN|POLLERR|POLLHUP}], 1, 86400000) = 1 ([{fd=9, revents=POLLIN}])
recvfrom(9, "\7\0\0\1\0\0\0\2\0\0\0", 27, MSG_DONTWAIT, NULL, NULL) = 11
sendto(9, "8\0\0\0\3UPDATE galleries SET loads "..., 60, MSG_DONTWAIT, NULL, 0) = 60
poll([{fd=9, events=POLLIN|POLLERR|POLLHUP}], 1, 86400000) = 1 ([{fd=9, revents=POLLIN}])
recvfrom(9, "0\0\0\1\0\1\0\2\0\0\0(Rows", 16, MSG_DONTWAIT, NULL, NULL) = 16
poll([{fd=9, events=POLLIN|POLLERR|POLLHUP}], 1, 86400000) = 1 ([{fd=9, revents=POLLIN}])
recvfrom(9, " matched: 1  Changed: 1  Warning"..., 93, MSG_DONTWAIT, NULL, NULL) = 36
sendto(9, ">\0\0\0\3SELECT id FROM loads WHERE "..., 66, MSG_DONTWAIT, NULL, 0) = 66
poll([{fd=9, events=POLLIN|POLLERR|POLLHUP}], 1, 86400000) = 1 ([{fd=9, revents=POLLIN}])
recvfrom(9, "\1\0\0\1\0010\0\0\2\3def\f*****_app\5loads"..., 57, MSG_DONTWAIT, NULL, NULL) = 57
poll([{fd=9, events=POLLIN|POLLERR|POLLHUP}], 1, 86400000) = 1 ([{fd=9, revents=POLLIN}])
recvfrom(9, "\5\0\0\3\376\0\0\2\0\10\0\0\4\0072607367\5\0\0\5\376\0\0\2\0", 93, MSG_DONTWAIT, NULL, NULL) = 30
sendto(9, "2\0\0\0\3UPDATE loads SET `times`=ti"..., 54, MSG_DONTWAIT, NULL, 0) = 54
poll([{fd=9, events=POLLIN|POLLERR|POLLHUP}], 1, 86400000) = 1 ([{fd=9, revents=POLLIN}])
recvfrom(9, "0\0\0\1\0\1\0\2\0\0\0(Rows matched: 1  Cha"..., 63, MSG_DONTWAIT, NULL, NULL) = 52
chdir("/opt/cpanel/ea-php72/root/usr/bin") = 0
sendto(9, "\1\0\0\0\1", 5, MSG_DONTWAIT, NULL, 0) = 5
close(9)                                = 0
setitimer(ITIMER_PROF, {it_interval={0, 0}, it_value={0, 0}}, NULL) = 0
pwrite(8, "", 0, 0)                     = 0
close(8)                                = 0
setitimer(ITIMER_PROF, {it_interval={0, 0}, it_value={0, 0}}, NULL) = 0
setitimer(ITIMER_PROF, {it_interval={0, 0}, it_value={30, 0}}, NULL) = 0
fcntl(3, F_SETLK, {type=F_UNLCK, whence=SEEK_SET, start=0, len=0}) = 0
setitimer(ITIMER_PROF, {it_interval={0, 0}, it_value={0, 0}}, NULL) = 0
writev(6, [{"LS\3\0'\1\0\0\5\0\0\0\310\0\0\0t\0'\0003\0\21\0.\0", 26}, {"Set-Cookie: sid=57bfacad3cb06bf8"..., 269}, {"LS\4\0009\0\0\0", 8}, {"{\"status\":\"success\",\"message\":\"R"..., 49}, {"LS\5\0\10\0\0\0", 8}], 5) = 360
select(7, [6], NULL, NULL, {1, 0})      = 1 (in [6], left {0, 666236})
read(6, "", 8192)                       = 0
close(6)                                = 0
select(5, [4], NULL, NULL, {1, 0})      = 1 (in [4], left {0, 628531})
accept(4, 0x7ffc7b385010, [128])        = -1 EAGAIN (Resource temporarily unavailable)
select(5, [4], NULL, NULL, {1, 0})      = 0 (Timeout)
munmap(0x2b5238e76000, 134217728)       = 0
close(3)                                = 0
munmap(0x2b5238aee000, 2125696)         = 0
munmap(0x2b52388e7000, 2124512)         = 0
munmap(0x2b52386e1000, 2120968)         = 0
munmap(0x2b52384d7000, 2136488)         = 0
munmap(0x2b52382d0000, 2123440)         = 0
munmap(0x2b52380b0000, 2226448)         = 0
munmap(0x2b5237e90000, 2226424)         = 0
munmap(0x2b5237a08000, 2124936)         = 0
munmap(0x3770800000, 2176864)           = 0
munmap(0x3771400000, 2343352)           = 0
munmap(0x2b5237c1b000, 2574912)         = 0
munmap(0x3776800000, 2109528)           = 0
munmap(0x2b52377fd000, 2139752)         = 0
munmap(0x2b52375f2000, 2142168)         = 0
munmap(0x2b52373ee000, 2112856)         = 0
munmap(0x2b52371e2000, 2144096)         = 0
munmap(0x3772400000, 2679408)           = 0
munmap(0x2b5236fcd000, 2183088)         = 0
munmap(0x2b5236d82000, 2403880)         = 0
munmap(0x2b5236b76000, 2144584)         = 0
munmap(0x2b523696e000, 2126488)         = 0
munmap(0x2b523672b000, 2370648)         = 0
munmap(0x2b5234e30000, 2222560)         = 0
munmap(0x2b523505b000, 2258168)         = 0
munmap(0x2b5235283000, 2465000)         = 0
munmap(0x2b52354dd000, 2155816)         = 0
munmap(0x3773400000, 2201520)           = 0
munmap(0x2b52356ec000, 2434568)         = 0
munmap(0x2b523593f000, 2280672)         = 0
munmap(0x2b5235b6c000, 3438440)         = 0
munmap(0x2b5235eb4000, 2288592)         = 0
munmap(0x2b52360e3000, 2109704)         = 0
munmap(0x2b52362e7000, 2113912)         = 0
munmap(0x2b52364ec000, 2354368)         = 0
munmap(0x2b5234c16000, 2201760)         = 0
munmap(0x2b52349d6000, 2356640)         = 0
munmap(0x2b5232165000, 2178096)         = 0
munmap(0x2b5234434000, 2136176)         = 0
munmap(0x2b5233acf000, 2182984)         = 0
munmap(0x2b5233ce4000, 3218408)         = 0
munmap(0x2b5234002000, 2150944)         = 0
munmap(0x2b5234210000, 2241080)         = 0
munmap(0x2b52338c5000, 2137048)         = 0
munmap(0x2b52336c2000, 2107704)         = 0
munmap(0x2b5232c56000, 2493216)         = 0
munmap(0x3776000000, 2165496)           = 0
munmap(0x2b5232ec3000, 3394952)         = 0
munmap(0x3776400000, 2250552)           = 0
munmap(0x3773000000, 2423648)           = 0
munmap(0x2b5233200000, 2739312)         = 0
munmap(0x2b523349d000, 2246440)         = 0
munmap(0x3775000000, 2106112)           = 0
munmap(0x2b5232a48000, 2152768)         = 0
munmap(0x2b5232379000, 7138824)         = 0
munmap(0x2b523463e000, 3765696)         = 0
munmap(0x2b5231a3a000, 2116512)         = 0
munmap(0x377a200000, 2139552)           = 0
munmap(0x2b5231c4b000, 2107224)         = 0
munmap(0x2b5231e4e000, 3237912)         = 0
munmap(0x2b523180f000, 2272632)         = 0
munmap(0x2b5230488000, 2185392)         = 0
munmap(0x2b523069e000, 2645320)         = 0
munmap(0x2b5230924000, 2246552)         = 0
munmap(0x2b5230b55000, 2257376)         = 0
munmap(0x2b5230d7d000, 2235440)         = 0
munmap(0x2b5230f9f000, 2538792)         = 0
munmap(0x2b523120b000, 4083672)         = 0
munmap(0x2b52315f0000, 2222432)         = 0
munmap(0x2b5230285000, 2108952)         = 0
munmap(0x2b523007d000, 2126208)         = 0
munmap(0x2b522fe78000, 2117016)         = 0
munmap(0x3777000000, 2162768)           = 0
munmap(0x2b522fc70000, 2128320)         = 0
close(7)                                = 0
munmap(0x2b522c7e6000, 65536)           = 0
brk(0x1544000)                          = 0x1544000
munmap(0x2b522fa00000, 2555064)         = 0
munmap(0x2b5238d1a000, 299008)          = 0
munmap(0x2b522f800000, 2097152)         = 0
munmap(0x2b522f62c000, 323584)          = 0
exit_group(0)                           = ?
+++ exited with 0 +++

I really need to find what causing APIs crashing.

Your Answer

By clicking "Post Your Answer", you acknowledge that you have read our updated terms of service, privacy policy and cookie policy, and that your continued use of the website is subject to these policies.

Browse other questions tagged or ask your own question.