WebLogic Version:
WebLogic Server 10.3.3.0 Fri Apr 9 00:05:28 PDT 2010 1321401
Java Version:
1.6.0_17
OS Name:
Linux
OS Version:
2.6.18-238.el5

<span style="font-size:14px;"><Jul 31, 201312:21:05 AM PDT> <Error> <Cluster> <BEA-000170> <Server ms1 did not receive the multicast packets that were sent by itself>
<Jul 31, 201312:21:05 AM PDT> <Critical> <Health> <BEA-310006> <Critical Subsystem Cluster has failed. Setting server state to FAILED.
Reason: Unable to receive self generated multicast messages>
<Jul 31, 201312:21:05 AM PDT> <Critical> <WebLogicServer> <BEA-000385> <Server health failed. Reason: health of critical service 'Cluster' failed>
<Jul 31, 201312:21:05 AM PDT> <Notice> <WebLogicServer> <BEA-000365> <Server state changed to FAILED>
<Jul 31, 201312:21:06 AM PDT> <Error> <> <BEA-000000> <
===== FULL THREAD DUMP ===============
Wed Jul 3100:21:052013
Oracle JRockit(R) R28.0.0-679-130297-1.6.0_17-20100312-2128-linux-ia32
"Main Thread" id=1 idx=0x4 tid=10345 prio=5 alive, waiting, native_blocked
-- Waiting for notification on: weblogic/t3/srvr/T3Srvr@0xa85524b0[fat lock]
at jrockit/vm/Threads.waitForNotifySignal(JLjava/lang/Object;)Z(Native Method)
at java/lang/Object.wait(J)V(Native Method)
at java/lang/Object.wait(Object.java:485)
at weblogic/t3/srvr/T3Srvr.waitForDeath(T3Srvr.java:902)
^-- Lock released while waiting: weblogic/t3/srvr/T3Srvr@0xa85524b0[fat lock]
at weblogic/t3/srvr/T3Srvr.run(T3Srvr.java:465)
at weblogic/Server.main(Server.java:67)
at jrockit/vm/RNI.c2java(IIIII)V(Native Method)
-- end of trace
"(Signal Handler)" id=2 idx=0x8 tid=10346 prio=5 alive, native_blocked, daemon
"(OC Main Thread)" id=3 idx=0xc tid=10347 prio=5 alive, native_waiting, daemon
"(Code Generation Thread 1)" id=4 idx=0x10 tid=10348 prio=5 alive, native_waiting, daemon
"(Code Optimization Thread 1)" id=5 idx=0x14 tid=10349 prio=5 alive, native_waiting, daemon
"(VM Periodic Task)" id=6 idx=0x18 tid=10350 prio=10 alive, native_blocked, daemon
"Finalizer" id=7 idx=0x1c tid=10351 prio=8 alive, native_waiting, daemon
at jrockit/memory/Finalizer.waitForFinalizees(J[Ljava/lang/Object;)I(Native Method)
at jrockit/memory/Finalizer.access$700(Finalizer.java:12)
at jrockit/memory/Finalizer$4.run(Finalizer.java:189)
at java/lang/Thread.run(Thread.java:619)
at jrockit/vm/RNI.c2java(IIIII)V(Native Method)
-- end of trace
"Reference Handler" id=8 idx=0x20 tid=10352 prio=10 alive, native_waiting, daemon
at java/lang/ref/Reference.waitForActivatedQueue(J)Ljava/lang/ref/Reference;(Native Method)
at java/lang/ref/Reference.access$100(Reference.java:11)
at java/lang/ref/Reference$ReferenceHandler.run(Reference.java:82)
at jrockit/vm/RNI.c2java(IIIII)V(Native Method)
-- end of trace
"(Sensor Event Thread)" id=9 idx=0x24 tid=10353 prio=5 alive, native_blocked, daemon
"VM JFR Buffer Thread" id=10 idx=0x28 tid=10354 prio=5 alive, in native, daemon
"Timer-0" id=13 idx=0x2c tid=10369 prio=5 alive, waiting, native_blocked, daemon
-- Waiting for notification on: java/util/TaskQueue@0xa8558c48[fat lock]
at jrockit/vm/Threads.waitForNotifySignal(JLjava/lang/Object;)Z(Native Method)
at java/lang/Object.wait(J)V(Native Method)
at java/lang/Object.wait(Object.java:485)
at java/util/TimerThread.mainLoop(Timer.java:483)
^-- Lock released while waiting: java/util/TaskQueue@0xa8558c48[fat lock]
at java/util/TimerThread.run(Timer.java:462)
at jrockit/vm/RNI.c2java(IIIII)V(Native Method)
-- end of trace
"Timer-1" id=14 idx=0x30 tid=10381 prio=5 alive, waiting, native_blocked, daemon
-- Waiting for notification on: java/util/TaskQueue@0xa8558cb0[fat lock]
at jrockit/vm/Threads.waitForNotifySignal(JLjava/lang/Object;)Z(Native Method)
at java/lang/Object.wait(J)V(Native Method)
at java/util/TimerThread.mainLoop(Timer.java:509)
^-- Lock released while waiting: java/util/TaskQueue@0xa8558cb0[fat lock]
at java/util/TimerThread.run(Timer.java:462)
at jrockit/vm/RNI.c2java(IIIII)V(Native Method)
-- end of trace
"[ACTIVE] ExecuteThread: '0' for queue: 'weblogic.kernel.Default (self-tuning)'" id=15 idx=0x34 tid=10382 prio=5 alive, waiting, native_blocked, daemon
-- Waiting for notification on: weblogic/work/ExecuteThread@0xa7ecf450[fat lock]
at jrockit/vm/Threads.waitForNotifySignal(JLjava/lang/Object;)Z(Native Method)
at java/lang/Object.wait(J)V(Native Method)
at java/lang/Object.wait(Object.java:485)
at weblogic/work/ExecuteThread.waitForRequest(ExecuteThread.java:157)
^-- Lock released while waiting: weblogic/work/ExecuteThread@0xa7ecf450[fat lock]
at weblogic/work/ExecuteThread.run(ExecuteThread.java:178)
at jrockit/vm/RNI.c2java(IIIII)V(Native Method)
-- end of trace
"JFR request timer" id=16 idx=0x38 tid=10383 prio=5 alive, waiting, native_blocked, daemon
-- Waiting for notification on: java/util/TaskQueue@0xa8558d30[fat lock]
at jrockit/vm/Threads.waitForNotifySignal(JLjava/lang/Object;)Z(Native Method)
at java/lang/Object.wait(J)V(Native Method)
at java/lang/Object.wait(Object.java:485)
at java/util/TimerThread.mainLoop(Timer.java:483)
^-- Lock released while waiting: java/util/TaskQueue@0xa8558d30[fat lock]
at java/util/TimerThread.run(Timer.java:462)
at jrockit/vm/RNI.c2java(IIIII)V(Native Method)
-- end of trace
"weblogic.time.TimeEventGenerator" id=18 idx=0x3c tid=10396 prio=9 alive, waiting, native_blocked, daemon
-- Waiting for notification on: weblogic/time/common/internal/TimeTable@0xa8557be0[fat lock]
at jrockit/vm/Threads.waitForNotifySignal(JLjava/lang/Object;)Z(Native Method)
at java/lang/Object.wait(J)V(Native Method)
at weblogic/time/common/internal/TimeTable.snooze(TimeTable.java:286)
^-- Lock released while waiting: weblogic/time/common/internal/TimeTable@0xa8557be0[fat lock]
at weblogic/time/common/internal/TimeEventGenerator.run(TimeEventGenerator.java:117)
at java/lang/Thread.run(Thread.java:619)
at jrockit/vm/RNI.c2java(IIIII)V(Native Method)
-- end of trace
"JMAPI event thread" id=19 idx=0x40 tid=10397 prio=5 alive, in native, daemon
"weblogic.timers.TimerThread" id=20 idx=0x44 tid=10398 prio=9 alive, waiting, native_blocked, daemon
-- Waiting for notification on: weblogic/timers/internal/TimerThread@0xa8558e30[fat lock]
at jrockit/vm/Threads.waitForNotifySignal(JLjava/lang/Object;)Z(Native Method)
at java/lang/Object.wait(J)V(Native Method)
at weblogic/timers/internal/TimerThread$Thread.run(TimerThread.java:262)
^-- Lock released while waiting: weblogic/timers/internal/TimerThread@0xa8558e30[fat lock]
at jrockit/vm/RNI.c2java(IIIII)V(Native Method)
-- end of trace
"[STANDBY] ExecuteThread: '1' for queue: 'weblogic.kernel.Default (self-tuning)'" id=23 idx=0x48 tid=10400 prio=5 alive, daemon
at bea/jmapi/DiagnosticCommandImpl.execute(Ljava/lang/String;Ljava/io/FileDescriptor;)V(Native Method)
at bea/jmapi/DiagnosticCommandImpl.execute(DiagnosticCommandImpl.java:53)
at com/bea/jvm/DiagnosticCommand.execute(DiagnosticCommand.java:245)
at com/bea/jvm/DiagnosticCommand$Command.execute(DiagnosticCommand.java:404)
at bea/jmapi/ThreadSystemImpl.getThreadStackDump(ThreadSystemImpl.java:102)
at weblogic/platform/JRockitVM.threadDump(JRockitVM.java:61)
at weblogic/t3/srvr/T3Srvr.logThreadDump(T3Srvr.java:280)
at weblogic/t3/srvr/T3Srvr.failed(T3Srvr.java:207)
^-- Holding lock: weblogic/t3/srvr/T3Srvr@0xa85524b0[fat lock]
at weblogic/health/HealthMonitorService$2.run(HealthMonitorService.java:188)
at weblogic/work/SelfTuningWorkManagerImpl$WorkAdapterImpl.run(SelfTuningWorkManagerImpl.java:528)
at weblogic/work/ExecuteThread.execute(ExecuteThread.java:201)
at weblogic/work/ExecuteThread.run(ExecuteThread.java:173)
at jrockit/vm/RNI.c2java(IIIII)V(Native Method)
-- end of trace
"ExecuteThread: '0' for queue: 'weblogic.socket.Muxer'" id=24 idx=0x4c tid=10409 prio=5 alive, in native, daemon
at jrockit/ext/epoll/EPoll.epollWait0(ILjava/nio/ByteBuffer;II)I(Native Method)
at jrockit/ext/epoll/EPoll.epollWait(EPoll.java:115)
at weblogic/socket/EPollSocketMuxer.processSockets(EPollSocketMuxer.java:156)
^-- Holding lock: java/lang/String@0xa8103cc0[fat lock]
at weblogic/socket/SocketReaderRequest.run(SocketReaderRequest.java:29)
at weblogic/socket/SocketReaderRequest.execute(SocketReaderRequest.java:42)
at weblogic/kernel/ExecuteThread.execute(ExecuteThread.java:145)
at weblogic/kernel/ExecuteThread.run(ExecuteThread.java:117)
at jrockit/vm/RNI.c2java(IIIII)V(Native Method)
-- end of trace
"ExecuteThread: '1' for queue: 'weblogic.socket.Muxer'" id=25 idx=0x50 tid=10410 prio=5 alive, blocked, native_blocked, daemon
-- Blocked trying to get lock: java/lang/String@0xa8103cc0[fat lock]
at jrockit/vm/Threads.waitForUnblockSignal()V(Native Method)
at jrockit/vm/Locks.fatLockBlockOrSpin(Locks.java:1402)
at jrockit/vm/Locks.fatLockReacquire(Locks.java:1640)
at jrockit/vm/Locks.lockFat(Locks.java:1508)
at jrockit/vm/Locks.monitorEnterSecondStageHard(Locks.java:1045)
at jrockit/vm/Locks.monitorEnterSecondStage(Locks.java:996)
at jrockit/vm/Locks.monitorEnter(Locks.java:2170)
at weblogic/socket/EPollSocketMuxer.processSockets(EPollSocketMuxer.java:153)
at weblogic/socket/SocketReaderRequest.run(SocketReaderRequest.java:29)
at weblogic/socket/SocketReaderRequest.execute(SocketReaderRequest.java:42)
at weblogic/kernel/ExecuteThread.execute(ExecuteThread.java:145)
at weblogic/kernel/ExecuteThread.run(ExecuteThread.java:117)
at jrockit/vm/RNI.c2java(IIIII)V(Native Method)
-- end of trace
"[ACTIVE] ExecuteThread: '2' for queue: 'weblogic.kernel.Default (self-tuning)'" id=26 idx=0x54 tid=10411 prio=5 alive, waiting, native_blocked, daemon
-- Waiting for notification on: weblogic/work/ExecuteThread@0xa7ecf600[fat lock]
at jrockit/vm/Threads.waitForNotifySignal(JLjava/lang/Object;)Z(Native Method)
at java/lang/Object.wait(J)V(Native Method)
at java/lang/Object.wait(Object.java:485)
at weblogic/work/ExecuteThread.waitForRequest(ExecuteThread.java:157)
^-- Lock released while waiting: weblogic/work/ExecuteThread@0xa7ecf600[fat lock]
at weblogic/work/ExecuteThread.run(ExecuteThread.java:178)
at jrockit/vm/RNI.c2java(IIIII)V(Native Method)
-- end of trace
"VDE Transaction Processor Thread" id=29 idx=0x58 tid=10459 prio=2 alive, waiting, native_blocked, daemon
-- Waiting for notification on: com/octetstring/vde/backend/standard/TransactionProcessor@0xa81d2668[fat lock]
at jrockit/vm/Threads.waitForNotifySignal(JLjava/lang/Object;)Z(Native Method)
at java/lang/Object.wait(J)V(Native Method)
at java/lang/Object.wait(Object.java:485)
at com/octetstring/vde/backend/standard/TransactionProcessor.waitChange(TransactionProcessor.java:367)
^-- Lock released while waiting: com/octetstring/vde/backend/standard/TransactionProcessor@0xa81d2668[fat lock]
at com/octetstring/vde/backend/standard/TransactionProcessor.run(TransactionProcessor.java:212)
at jrockit/vm/RNI.c2java(IIIII)V(Native Method)
-- end of trace
"DoSManager" id=32 idx=0x60 tid=10461 prio=6 alive, sleeping, native_waiting, daemon
at java/lang/Thread.sleep(J)V(Native Method)
at com/octetstring/vde/DoSManager.run(DoSManager.java:433)
at jrockit/vm/RNI.c2java(IIIII)V(Native Method)
-- end of trace
"Thread-12" id=33 idx=0x64 tid=10475 prio=5 alive, parked, native_blocked, daemon
-- Parking to wait for: java/util/concurrent/locks/AbstractQueuedSynchronizer$ConditionObject@0xa82f1980
at jrockit/vm/Locks.park0(J)V(Native Method)
at jrockit/vm/Locks.park(Locks.java:2221)
at sun/misc/Unsafe.park(ZJ)V(Native Method)
at java/util/concurrent/locks/LockSupport.park(LockSupport.java:158)
at java/util/concurrent/locks/AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
at java/util/concurrent/LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
at weblogic/utils/concurrent/JDK15ConcurrentBlockingQueue.take(JDK15ConcurrentBlockingQueue.java:89)
at weblogic/store/internal/PersistentStoreImpl.getOutstandingWork(PersistentStoreImpl.java:650)
at weblogic/store/internal/PersistentStoreImpl.run(PersistentStoreImpl.java:707)
at weblogic/store/internal/PersistentStoreImpl$2.run(PersistentStoreImpl.java:464)
at jrockit/vm/RNI.c2java(IIIII)V(Native Method)
-- end of trace
"[ACTIVE] ExecuteThread: '3' for queue: 'weblogic.kernel.Default (self-tuning)'" id=34 idx=0x68 tid=10508 prio=5 alive, waiting, native_blocked, daemon
at jrockit/vm/Threads.waitForNotifySignal(JLjava/lang/Object;)Z(Native Method)
at java/lang/Object.wait(J)V(Native Method)
at java/lang/Object.wait(Object.java:485)
at weblogic/work/ExecuteThread.waitForRequest(ExecuteThread.java:157)
^-- Lock released while waiting: weblogic/work/ExecuteThread@0xa294b8b0[fat lock]
at weblogic/work/ExecuteThread.run(ExecuteThread.java:178)
at jrockit/vm/RNI.c2java(IIIII)V(Native Method)
-- end of trace
"Thread-13" id=35 idx=0x6c tid=10513 prio=5 alive, parked, native_blocked, daemon
-- Parking to wait for: java/util/concurrent/locks/AbstractQueuedSynchronizer$ConditionObject@0xa55be778
at jrockit/vm/Locks.park0(J)V(Native Method)
at jrockit/vm/Locks.park(Locks.java:2221)
at sun/misc/Unsafe.park(ZJ)V(Native Method)
at java/util/concurrent/locks/LockSupport.park(LockSupport.java:158)
at java/util/concurrent/locks/AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
at java/util/concurrent/LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
at weblogic/utils/concurrent/JDK15ConcurrentBlockingQueue.take(JDK15ConcurrentBlockingQueue.java:89)
at weblogic/store/internal/PersistentStoreImpl.getOutstandingWork(PersistentStoreImpl.java:650)
at weblogic/store/internal/PersistentStoreImpl.run(PersistentStoreImpl.java:707)
at weblogic/store/internal/PersistentStoreImpl$2.run(PersistentStoreImpl.java:464)
at jrockit/vm/RNI.c2java(IIIII)V(Native Method)
-- end of trace
"weblogic.cluster.MessageReceiver" id=36 idx=0x70 tid=10514 prio=5 alive, in native, daemon
at java/net/PlainDatagramSocketImpl.receive0(Ljava/net/DatagramPacket;)V(Native Method)
^-- Holding lock: java/net/PlainDatagramSocketImpl@0xa295d588[recursive]
at java/net/PlainDatagramSocketImpl.receive(PlainDatagramSocketImpl.java:136)
^-- Holding lock: java/net/PlainDatagramSocketImpl@0xa295d588[biased lock]
at java/net/DatagramSocket.receive(DatagramSocket.java:712)
^-- Holding lock: java/net/DatagramPacket@0xa295d438[biased lock]
^-- Holding lock: java/net/MulticastSocket@0xa295d458[biased lock]
at weblogic/cluster/MulticastFragmentSocket.receive(MulticastFragmentSocket.java:239)
at weblogic/cluster/FragmentSocketWrapper.receive(FragmentSocketWrapper.java:98)
at weblogic/cluster/MulticastManager.run(MulticastManager.java:460)
at java/lang/Thread.run(Thread.java:619)
at jrockit/vm/RNI.c2java(IIIII)V(Native Method)
-- end of trace
"DynamicListenThread[Default]" id=37 idx=0x74 tid=12242 prio=9 alive, in native, daemon
at java/net/PlainSocketImpl.socketAccept(Ljava/net/SocketImpl;)V(Native Method)
at java/net/PlainSocketImpl.accept(PlainSocketImpl.java:390)
^-- Holding lock: java/net/SocksSocketImpl@0xa549f658[biased lock]
at java/net/ServerSocket.implAccept(ServerSocket.java:453)
at java/net/ServerSocket.accept(ServerSocket.java:421)
at weblogic/socket/WeblogicServerSocket.accept(WeblogicServerSocket.java:38)
at weblogic/server/channels/DynamicListenThread$SocketAccepter.accept(DynamicListenThread.java:535)
at weblogic/server/channels/DynamicListenThread$SocketAccepter.access$200(DynamicListenThread.java:420)
at weblogic/server/channels/DynamicListenThread.run(DynamicListenThread.java:171)
at java/lang/Thread.run(Thread.java:619)
at jrockit/vm/RNI.c2java(IIIII)V(Native Method)
-- end of trace
"[STANDBY] ExecuteThread: '4' for queue: 'weblogic.kernel.Default (self-tuning)'" id=38 idx=0x78 tid=12517 prio=5 alive, waiting, native_blocked, daemon
-- Waiting for notification on: weblogic/work/ExecuteThread@0x971b5898[fat lock]
at jrockit/vm/Threads.waitForNotifySignal(JLjava/lang/Object;)Z(Native Method)
at java/lang/Object.wait(J)V(Native Method)
at java/lang/Object.wait(Object.java:485)
at weblogic/work/ExecuteThread.waitForRequest(ExecuteThread.java:157)
^-- Lock released while waiting: weblogic/work/ExecuteThread@0x971b5898[fat lock]
at weblogic/work/ExecuteThread.run(ExecuteThread.java:178)
at jrockit/vm/RNI.c2java(IIIII)V(Native Method)
-- end of trace
Open lock chains
================
Chain 1:
"ExecuteThread: '1' for queue: 'weblogic.socket.Muxer'" id=25 idx=0x50 tid=10410 waiting for java/lang/String@0xa8103cc0 held by:
"ExecuteThread: '0' for queue: 'weblogic.socket.Muxer'" id=24 idx=0x4c tid=10409 (active)
===== END OF THREAD DUMP ===============
>
<Jul 31, 201312:21:15 AM PDT> <Error> <Cluster> <BEA-000170> <Server ms1 did not receive the multicast packets that were sent by itself>
<Jul 31, 201312:21:15 AM PDT> <Critical> <Health> <BEA-310006> <Critical Subsystem Cluster has failed. Setting server state to FAILED.
Reason: Unable to receive self generated multicast messages>
<Jul 31, 201312:21:25 AM PDT> <Error> <Cluster> <BEA-000170> <Server ms1 did not receive the multicast packets that were sent by itself>
<Jul 31, 201312:21:25 AM PDT> <Critical> <Health> <BEA-310006> <Critical Subsystem Cluster has failed. Setting server state to FAILED.
Reason: Unable to receive self generated multicast messages>
<Jul 31, 201312:21:35 AM PDT> <Error> <Cluster> <BEA-000170> <Server ms1 did not receive the multicast packets that were sent by itself>
<Jul 31, 201312:21:35 AM PDT> <Critical> <Health> <BEA-310006> <Critical Subsystem Cluster has failed. Setting server state to FAILED.
Reason: Unable to receive self generated multicast messages>
<Jul 31, 201312:21:45 AM PDT> <Error> <Cluster> <BEA-000170> <Server ms1 did not receive the multicast packets that were sent by itself>
<Jul 31, 201312:21:45 AM PDT> <Critical> <Health> <BEA-310006> <Critical Subsystem Cluster has failed. Setting server state to FAILED.
Reason: Unable to receive self generated multicast messages>
<Jul 31, 201312:21:55 AM PDT> <Error> <Cluster> <BEA-000170> <Server ms1 did not receive the multicast packets that were sent by itself>
<Jul 31, 201312:21:55 AM PDT> <Critical> <Health> <BEA-310006> <Critical Subsystem Cluster has failed. Setting server state to FAILED.
Reason: Unable to receive self generated multicast messages>
<Jul 31, 201312:22:05 AM PDT> <Error> <Cluster> <BEA-000170> <Server ms1 did not receive the multicast packets that were sent by itself>
<Jul 31, 201312:22:05 AM PDT> <Critical> <Health> <BEA-310006> <Critical Subsystem Cluster has failed. Setting server state to FAILED.
Reason: Unable to receive self generated multicast messages> </span>

#service iptables status
#发现iptables，处于启动状态