We are having an occasional (ie non-reproducible) issue where the Oracle client process (EnterpriseOne kernel process) and the Oracle shadow process seem to get out of sync. I perform a stack dump of the two processes, and they are both stuck waiting on a read from the other. Both processes are running on same instance of AIX 5.3.
I have spent some time with Oracle support when it was happening fairly regularly (once a month or so), but unable to make any progress with them, getting bounced between the application and database team. In the last few days it has happened a couple of times, so looking for some ideas about how to get further.
I have some ideas are that it might somehow be network related, however the fact that it is always the same point in the application code leads me to believe it may be some kind of bug in the application code maybe causing corruption. EnterpriseOne comes with source at the application level, but obviously I don't have it for the low-level stuff or the Oracle internal stuff.
Any ideas on how I can progress with this? If I connected to the hung process via debugger (dbx) to the kernel process, would I be able to guess somehow if corruption had occurred, and maybe why?
#Incident 1
44158994: jdenet_k 6209
0xd0363a54 read(??, ??, ??) + 0x1a8
0x202f3d18 ntusread(??, ??, ??, ??) + 0xf0
0x202e0664 nsprecv(??, ??, ??) + 0x934
0x202dc95c nsrdr(??, ??) + 0x1a8
0x202d8c94 nsdo(0x2310e1d0, 0x55, 0x24026938, 0x2310e2a4, 0x2ff19a80, 0x0, 0x3) + 0x16a0
0x2076d964 nioqrc(??, ??, ??, ??, ??) + 0x54c
0x20761864 ttcdrv(??, ??) + 0x780
0x20770c20 nioqwa(??, ??, ??, ??, ??, ??) + 0x4c
0x2063beac upirtrc(??, ??, ??, ??, ??, ??, ??, ??) + 0x63c
0x2066b8c4 kpurcsc(??, ??, ??, ??, ??, ??, ??, ??) + 0x68
0x207483fc kpuexecv8(??, ??, ??, ??, ??, ??) + 0xcb0
0x207498ac kpuexec(??, ??, ??, ??, ??, ??, ??, ??) + 0x1028
0x2012705c OCIStmtExecute(??, ??, ??, ??, ??, ??, ??, ??) + 0x18
0x208e9218 BFOCIStmtExecute(0x24023878, 0x2748fdfc, 0x27ec426c, 0x1, 0x0, 0x0, 0x0, 0x20) + 0x4c
0x208ff288 performRequestInternal(0x21984478, 0x1) + 0x110
0x208ff840 dballPerformRequest(0x21984478) + 0xfc
0x208ff89c DBPerformRequest(0x21984478) + 0x14
0xd1fbd8b8 JDB_DBPerformRequest(0x2566d318, 0x21984478, 0x254b6448) + 0x40
0xd2245b54 TM_DBPerformRequest(0x20c51ce8, 0x20c158f8, 0x254b6448, 0x254b6338) + 0x290
0xd1f9d078 DeleteTable(0x254b6338, 0x2ff2062c, 0x0, 0x1, 0x2ff2064c, 0x20002, 0x1, 0x0) + 0x184c
0xd1f9e340 JDB_DeleteTable(0x254b6338, 0x2ff2062c, 0x0, 0x1, 0x2ff2064c, 0x20002) + 0xb0
0xd42af274 BatchReviseOnExit() + 0x7f4
0xd0e2a88c jdeCallObject(0x20c2cbd8, 0x0, 0x20ebee08, 0x20e50ab8, 0x2129ec58, 0x0, 0x0, 0x20c2e648) + 0x23d0
0xd20349f4 JDEK_ProcessCallRequest(0x13ece, 0xa54010b, 0x0, 0x2005c4f8, 0x21fe5078, 0x20c158f8) + 0xcb4
0xd203543c JDEK_StartCallRequest(0x13ece, 0xa54010b, 0x0, 0x2005c4f8) + 0x46c
0xd201a4a8 JDEK_DispatchCallObjectMessage(0x13ece, 0xa54010b, 0x0, 0x2005c4f8, 0x0, 0x3850385, 0x0) + 0x4c4
0xd0a51b0c XMLCallObjectDispatch() + 0xd0
0xd0ce3a14 callDispatchFunction(0x5, 0x13ece, 0xa54010b, 0x0, 0x2005c4f8, 0x0, 0x385, 0x8000) + 0x514
0xd0ce3c58 kernelMsgThread(0x20bb1d78) + 0x1c0
0xd0ce5624 processKernelQueueMsg(0x20bb1d78) + 0x14
0xd0cd4204 processKernelQueue() + 0x49c
0xd0cc9430 JDENET_RunKernel(0x2ff22599) + 0x180
0x10001f70 main(0x2, 0x2ff224d4) + 0x290
0x100001c0 __start() + 0x98
40972518: oracleJDE (LOCAL=NO)
0x0900000000067c78 read(??, ??, ??) + 0x1c4
0x00000001002344c0 ntusread(??, ??, ??, ??) + 0x110
0x000000010021a030 nsprecv(??, ??, ??) + 0x9e0
0x0000000100215e84 nsrdr(??, ??) + 0x1d0
0x0000000100211da8 nsdo(0x110271a38, 0x5500000055, 0x110292420, 0x110271ba0, 0xfffffffffffb7c0, 0x0, 0x300000003) + 0x1818
0x000000010081eca4 nioqrc(??, ??, ??, ??, ??) + 0x5c4
0x00000001022caeac opikndf2(??, ??, ??, ??) + 0x6a8
0x000000010215b66c opitsk(??, ??) + 0x5f8
0x00000001022cbde0 opiino(??, ??, ??) + 0x758
0x000000010096a1c0 opiodr(??, ??, ??, ??) + 0x8cc
0x0000000102306690 opidrv(??, ??, ??) + 0x32c
0x0000000102580bd4 sou2o(??, ??, ??, ??) + 0x28
0x0000000100000648 main(??, ??) + 0x138
0x0000000100000338 __start() + 0x98
#Incident 2
44859576: jdenet_k 6209
0xd0363a54 read(??, ??, ??) + 0x1a8
0x202f3d18 ntusread(??, ??, ??, ??) + 0xf0
0x202e0664 nsprecv(??, ??, ??) + 0x934
0x202dc95c nsrdr(??, ??) + 0x1a8
0x202d8c94 nsdo(0x228ab680, 0x55, 0x228e6258, 0x228ab754, 0x2ff19a80, 0x0, 0x3) + 0x16a0
0x2076d964 nioqrc(??, ??, ??, ??, ??) + 0x54c
0x20761864 ttcdrv(??, ??) + 0x780
0x20770c20 nioqwa(??, ??, ??, ??, ??, ??) + 0x4c 0x2063beac upirtrc(??, ??, ??, ??, ??, ??, ??, ??) + 0x63c
0x2066b8c4 kpurcsc(??, ??, ??, ??, ??, ??, ??, ??) + 0x68 0x207483fc kpuexecv8(??, ??, ??, ??, ??, ??) + 0xcb0 0x207498ac kpuexec(??, ??, ??, ??, ??, ??, ??, ??) + 0x1028 0x2012705c OCIStmtExecute(??, ??, ??, ??, ??, ??, ??, ??) + 0x18
0x208e9218 BFOCIStmtExecute(0x23c69e18, 0x284e35ec, 0x2708d9e0, 0x1, 0x0, 0x0, 0x0, 0x20) + 0x4c
0x208ff288 performRequestInternal(0x23a94e18, 0x1) + 0x110
0x208ff840 dballPerformRequest(0x23a94e18) + 0xfc 0x208ff89c DBPerformRequest(0x23a94e18) + 0x14
0xd1fbd8b8 JDB_DBPerformRequest(0x2414aeb8, 0x23a94e18, 0x26af8d08) + 0x40
0xd2245b54 TM_DBPerformRequest(0x21287208, 0x21285608, 0x26af8d08, 0x26a348c8) + 0x290
0xd1f9d078 DeleteTable(0x26a348c8, 0x2ff2062c, 0x0, 0x1, 0x2ff2064c, 0x20002, 0x1, 0x0) + 0x184c
0xd1f9e340 JDB_DeleteTable(0x26a348c8, 0x2ff2062c, 0x0, 0x1, 0x2ff2064c, 0x20002) + 0xb0
0xd42af274 BatchReviseOnExit() + 0x7f4
0xd0e2a88c jdeCallObject(0x20c2cca8, 0x0, 0x21f82708, 0x21f829e8, 0x22800498, 0x0, 0x0, 0x20c3bb38) + 0x23d0
0xd20349f4 JDEK_ProcessCallRequest(0xca24, 0xa54010b, 0x0, 0x2002a148, 0x228bbfc8, 0x21285608) + 0xcb4 0xd203543c JDEK_StartCallRequest(0xca24, 0xa54010b, 0x0, 0x2002a148) + 0x46c
0xd201a4a8 JDEK_DispatchCallObjectMessage(0xca24, 0xa54010b, 0x0, 0x2002a148, 0x0, 0x3850385, 0x0) + 0x4c4 0xd0a51b0c XMLCallObjectDispatch() + 0xd0
0xd0ce3a14 callDispatchFunction(0x5, 0xca24, 0xa54010b, 0x0, 0x2002a148, 0x0, 0x385, 0x8000) + 0x514
0xd0ce3c58 kernelMsgThread(0x2002a648) + 0x1c0
0xd0ce5624 processKernelQueueMsg(0x2002a648) + 0x14
0xd0cd4204 processKernelQueue() + 0x49c
0xd0cc9430 JDENET_RunKernel(0x2ff22599) + 0x180
0x10001f70 main(0x2, 0x2ff224d4) + 0x290
0x100001c0 __start() + 0x98
root@jdelogichk:/home/root>
root@jdelogichk:/home/root>procstack 42033402
42033402: oracleJDE (LOCAL=NO)
0x0900000000067c78 read(??, ??, ??) + 0x1c4
0x00000001002344c0 ntusread(??, ??, ??, ??) + 0x110
0x000000010021a030 nsprecv(??, ??, ??) + 0x9e0
0x0000000100215e84 nsrdr(??, ??) + 0x1d0
0x0000000100211da8 nsdo(0x110271a38, 0x5500000055, 0x11028fce0, 0x110271ba0, 0xfffffffffffb7c0, 0x0, 0x300000003) + 0x1818
0x000000010081eca4 nioqrc(??, ??, ??, ??, ??) + 0x5c4 0x00000001022caeac opikndf2(??, ??, ??, ??) + 0x6a8 0x000000010215b66c opitsk(??, ??) + 0x5f8
0x00000001022cbde0 opiino(??, ??, ??) + 0x758
0x000000010096a1c0 opiodr(??, ??, ??, ??) + 0x8cc
0x0000000102306690 opidrv(??, ??, ??) + 0x32c
0x0000000102580bd4 sou2o(??, ??, ??, ??) + 0x28
0x0000000100000648 main(??, ??) + 0x138
0x0000000100000338 __start() + 0x98
