I have noticed in our thread dumps of Weblogic 8.1 managed servers, the following lock:
"ExecuteThread: '2' for queue: 'weblogic.admin.RMI'" id=186 idx=0x182 tid=5292 prio=5 alive, in native, native_blocked, daemon
at jrockit/vm/Allocator.allocLargeArray(JIZ)Ljava/lang/Object;(Native Method)
at jrockit/vm/Allocator.allocArray(JIZ)Ljava/lang/Object;(Unknown Source)
at java/util/HashMap.resize(I)V(Unknown Source)
at java/util/HashMap.addEntry(ILjava/lang/Object;Ljava/lang/Object;I)V(Unknown Source)
at java/util/HashMap.put(Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;(Unknown Source)
at java/util/HashSet.add(Ljava/lang/Object;)Z(HashSet.java:192)
at com/sun/management/jmx/RepositorySupport.query(Ljavax/management/ObjectName;Ljavax/management/QueryExp;)Ljava/util/Set;(RepositorySupport.java:241)
at com/sun/management/jmx/MBeanServerImpl.queryMBeans(Ljavax/management/ObjectName;Ljavax/management/QueryExp;)Ljava/util/Set;(MBeanServerImpl.java:1042)
^-- Holding lock: weblogic/management/internal/RemoteMBeanServerImpl@0x2000000008e78fc0[fat lock]
at weblogic/management/internal/RemoteMBeanServerImpl.queryMBeans(Ljavax/management/ObjectName;Ljavax/management/QueryExp;)Ljava/util/Set;(RemoteMBeanServerImpl.java:911)
at com/sun/management/jmx/MBeanServerImpl.queryNames(Ljavax/management/ObjectName;Ljavax/management/QueryExp;)Ljava/util/Set;(MBeanServerImpl.java:1075)
at weblogic/management/internal/MBeanHomeImpl.mbeanServerQuery(Ljavax/management/ObjectName;Ljavax/management/QueryExp;)Ljava/util/Set;(MBeanHomeImpl.java:487)
at weblogic/management/internal/MBeanHomeImpl.getMBeansByClass(Ljava/lang/Class;Ljava/lang/String;)Ljava/util/Set;(MBeanHomeImpl.java:311)
at weblogic/management/internal/MBeanHomeImpl_WLSkel.invoke(ILweblogic/rmi/spi/InboundRequest;Lweblogic/rmi/spi/OutboundResponse;Ljava/lang/Object;)Lweblogic/rmi/spi/OutboundResponse;(Unknown Source)
at weblogic/rmi/internal/BasicServerRef.invoke(Lweblogic/rmi/extensions/server/RuntimeMethodDescriptor;Lweblogic/rmi/spi/InboundRequest;Lweblogic/rmi/spi/OutboundResponse;)V(BasicServerRef.java:492)
at weblogic/rmi/internal/BasicServerRef$1.run()Ljava/lang/Object;(BasicServerRef.java:435)
at weblogic/security/acl/internal/AuthenticatedSubject.doAs(Lweblogic/security/subject/AbstractSubject;Ljava/security/PrivilegedExceptionAction;)Ljava/lang/Object;(AuthenticatedSubject.java:363)
at weblogic/security/service/SecurityManager.runAs(Lweblogic/security/acl/internal/AuthenticatedSubject;Lweblogic/security/acl/internal/AuthenticatedSubject;Ljava/security/PrivilegedExceptionAction;)Ljava/lang/Object;(SecurityManager.java:147)
at weblogic/rmi/internal/BasicServerRef.handleRequest(Lweblogic/rmi/spi/InboundRequest;)V(BasicServerRef.java:430)
at weblogic/rmi/internal/BasicExecuteRequest.execute(Lweblogic/kernel/ExecuteThread;)V(BasicExecuteRequest.java:35)
at weblogic/kernel/ExecuteThread.execute(Lweblogic/kernel/ExecuteRequest;)V(ExecuteThread.java:224)
at weblogic/kernel/ExecuteThread.run()V(ExecuteThread.java:183)
at jrockit/vm/RNI.c2java(JJJJ)V(Native Method)
-- end of trace
Then a big number of other threads are blocked by this lock. An example is:
"ExecuteThread: '1' for queue: 'HZOZMsgEjbExecuteThreadQueue'" id=17 idx=0x30 tid=5123 prio=5 alive, in native, blocked, daemon
-- Blocked trying to get lock: weblogic/management/internal/RemoteMBeanServerImpl@0x2000000008e78fc0[fat lock]
at jrockit/vm/Threads.waitForSignal(J)Z(Native Method)
at jrockit/vm/Locks.fatLockBlockOrSpin(JLjrockit/vm/ObjectMonitor;II)V(Unknown Source)
at jrockit/vm/Locks.lockFat(Ljava/lang/Object;JLjrockit/vm/ObjectMonitor;Z)Ljava/lang/Object;(Unknown Source)
at jrockit/vm/Locks.monitorEnterSecondStage(Ljava/lang/Object;I)Ljava/lang/Object;(Unknown Source)
at jrockit/vm/Locks.monitorEnter(Ljava/lang/Object;)Ljava/lang/Object;(Unknown Source)
at com/sun/management/jmx/MBeanServerImpl.internal_addObject(Ljava/lang/Object;Ljavax/management/ObjectName;)V(MBeanServerImpl.java:2379)
at com/sun/management/jmx/MBeanServerImpl.registerMBean(Ljava/lang/Object;Ljavax/management/ObjectName;)Ljavax/management/ObjectInstance;(MBeanServerImpl.java:876)
at weblogic/management/internal/RemoteMBeanServerImpl.private_registerMBean(Ljava/lang/Object;Ljavax/management/ObjectName;)Ljavax/management/ObjectInstance;(RemoteMBeanServerImpl.java:614)
at weblogic/management/internal/RemoteMBeanServerImpl.registerMBean(Ljava/lang/Object;Ljavax/management/ObjectName;)Ljavax/management/ObjectInstance;(RemoteMBeanServerImpl.java:554)
at weblogic/management/runtime/RuntimeMBeanDelegate.register()V(RuntimeMBeanDelegate.java:166)
at weblogic/management/runtime/RuntimeMBeanDelegate.<init>(Ljava/lang/String;Lweblogic/management/runtime/RuntimeMBean;Z)V(RuntimeMBeanDelegate.java:122)
at weblogic/management/runtime/RuntimeMBeanDelegate.<init>(Ljava/lang/String;Lweblogic/management/runtime/RuntimeMBean;)V(RuntimeMBeanDelegate.java:85)
at weblogic/jms/frontend/FEConnection.<init>(Lweblogic/jms/frontend/FEConnectionFactory;Ljava/lang/String;Lweblogic/jms/common/JMSID;Lweblogic/jms/dispatcher/Dispatcher;IIJJJJLjava/lang/String;JZZIIIZZ)V(FEConnection.java:183)
at weblogic/jms/frontend/FEConnectionFactory$1.run()Ljava/lang/Object;(FEConnectionFactory.java:434)
at weblogic/security/acl/internal/AuthenticatedSubject.doAs(Lweblogic/security/subject/AbstractSubject;Ljava/security/PrivilegedExceptionAction;)Ljava/lang/Object;(AuthenticatedSubject.java:363)
at weblogic/security/service/SecurityManager.runAs(Lweblogic/security/acl/internal/AuthenticatedSubject;Lweblogic/security/acl/internal/AuthenticatedSubject;Ljava/security/PrivilegedExceptionAction;)Ljava/lang/Object;(SecurityManager.java:147)
at weblogic/jms/frontend/FEConnectionFactory.connectionCreateInternal(Lweblogic/jms/dispatcher/DispatcherWrapper;Z)Lweblogic/jms/client/JMSConnection;(FEConnectionFactory.java:430)
at weblogic/jms/frontend/FEConnectionFactory.connectionCreate(Lweblogic/jms/dispatcher/DispatcherWrapper;)Lweblogic/jms/client/JMSConnection;(FEConnectionFactory.java:371)
at weblogic/jms/client/JMSConnectionFactory.createConnectionInternal(Ljava/lang/String;Ljava/lang/String;Z)Lweblogic/jms/client/JMSConnection;(JMSConnectionFactory.java:148)
at weblogic/jms/client/JMSConnectionFactory.createQueueConnection()Ljavax/jms/QueueConnection;(JMSConnectionFactory.java:75)
at gr/cosmote/common/util/UtilLib.putObjToQueue(IILjava/io/Serializable;JLgr/cosmote/common/bus/LoginInfo;)V(UtilLib.java:664)
at gr/cosmote/common/util/UtilLib.putObjToQueue(IILjava/io/Serializable;Lgr/cosmote/common/bus/LoginInfo;)V(UtilLib.java:636)
at gr/cosmote/bus/session/HZOZQueueBean.handleMozartCallbackVO(Lgr/cosmote/common/bus/mozartVOs/MozartCallbackVO;)V(HZOZQueueBean.java:418)
at gr/cosmote/bus/session/HZOZQueueBean.onMessage(Ljavax/jms/Message;)V(HZOZQueueBean.java:191)
at weblogic/ejb20/internal/MDListener.execute(Lweblogic/kernel/ExecuteThread;)V(MDListener.java:400)
at weblogic/ejb20/internal/MDListener.transactionalOnMessage(Ljavax/jms/Message;)V(MDListener.java:333)
at weblogic/ejb20/internal/MDListener.onMessage(Ljavax/jms/Message;)V(MDListener.java:298)
at weblogic/jms/client/JMSSession.onMessage(Ljavax/jms/MessageListener;Lweblogic/jms/common/MessageImpl;)V(JMSSession.java:2686)
at weblogic/jms/client/JMSSession.execute(Lweblogic/kernel/ExecuteThread;)V(JMSSession.java:2598)
at weblogic/kernel/ExecuteThread.execute(Lweblogic/kernel/ExecuteRequest;)V(ExecuteThread.java:224)
at weblogic/kernel/ExecuteThread.run()V(ExecuteThread.java:183)
at jrockit/vm/RNI.c2java(JJJJ)V(Native Method)
-- end of trace
What is this 'weblogic.admin.RMI' queue? What I understand so far is that the object that appears as locked (RemoteMBeanServerImpl) is used to initiate and instantiate ejb/rmi calls from the managed servers. Does it attempt some sort of connection to the admin? Something else quite interesting that I read somewhere, that when this call (queryMBeans) is performed, the result is handled by a non thread-safe collection object which could (as a result of a weblogic bug) result in an infinite loop.
By the way, our set up is 1 cluster, 4 managed servers in 4 different machines. The admin is installed along with one of the managed. All servers are configured to start/stop/run independently.
Anybody any ideas?
