users@glassfish.java.net

glassfish pauses for certain periods

From: Cam Bazz <cambazz_at_gmail.com>
Date: Mon, 1 Feb 2010 00:20:45 +0200

Hello,

I have this problem of glassfish v3, pausing in certain intervals for
a certain time. It has been going on for a while. I tried increased
both the number of worker-threads thread and http-threads, and also
lowered the idle thread timeout to 120 seconds. This has helped to
reduce the time of the lockups, but the problem did not go away. I
have been monitoring my glassfish with jvisualvm, and taking thread
dumps when this happens. Below is the thread dump copied and pasted.

Any ideas/recomendations and help greatly appreciated.

Best,
-C.B.


2010-01-31 23:47:00
Full thread dump Java HotSpot(TM) 64-Bit Server VM (16.0-b13 mixed mode):

"RMI TCP Connection(421)-88.250.217.202" - Thread t_at_240
   java.lang.Thread.State: RUNNABLE
        at java.net.SocketInputStream.socketRead0(Native Method)
        at java.net.SocketInputStream.read(SocketInputStream.java:129)
        at java.io.BufferedInputStream.fill(BufferedInputStream.java:218)
        at java.io.BufferedInputStream.read(BufferedInputStream.java:237)
        - locked java.io.BufferedInputStream_at_2d1e1c19
        at java.io.FilterInputStream.read(FilterInputStream.java:66)
        at sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:517)
        at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:790)
        at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:649)
        at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - locked java.util.concurrent.locks.ReentrantLock$NonfairSync_at_13c6c473

"RMI TCP Connection(idle)" - Thread t_at_239
   java.lang.Thread.State: TIMED_WAITING on
java.util.concurrent.SynchronousQueue$TransferStack_at_26f69ea3
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:424)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:323)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:874)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:945)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"RMI TCP Connection(418)-88.250.217.202" - Thread t_at_238
   java.lang.Thread.State: RUNNABLE
        at java.net.SocketOutputStream.socketWrite0(Native Method)
        at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:92)
        at java.net.SocketOutputStream.write(SocketOutputStream.java:136)
        at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:65)
        at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:123)
        - locked java.io.BufferedOutputStream_at_115cf03d
        at java.io.ObjectOutputStream$BlockDataOutputStream.flush(ObjectOutputStream.java:1784)
        at java.io.ObjectOutputStream.flush(ObjectOutputStream.java:691)
        at sun.rmi.transport.StreamRemoteCall.releaseOutputStream(StreamRemoteCall.java:96)
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:346)
        at sun.rmi.transport.Transport$1.run(Transport.java:159)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:155)
        at sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:535)
        at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:790)
        at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:649)
        at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - locked java.util.concurrent.locks.ReentrantLock$NonfairSync_at_6ca03db2

"RMI TCP Connection(420)-88.250.217.202" - Thread t_at_237
   java.lang.Thread.State: RUNNABLE
        at java.net.SocketInputStream.socketRead0(Native Method)
        at java.net.SocketInputStream.read(SocketInputStream.java:129)
        at java.io.BufferedInputStream.fill(BufferedInputStream.java:218)
        at java.io.BufferedInputStream.read(BufferedInputStream.java:237)
        - locked java.io.BufferedInputStream_at_51ea3e34
        at java.io.FilterInputStream.read(FilterInputStream.java:66)
        at sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:517)
        at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:790)
        at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:649)
        at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - locked java.util.concurrent.locks.ReentrantLock$NonfairSync_at_6999f1ef

"RMI TCP Connection(404)-88.250.217.202" - Thread t_at_235
   java.lang.Thread.State: RUNNABLE
        at sun.management.ThreadImpl.dumpThreads0(Native Method)
        at sun.management.ThreadImpl.dumpAllThreads(ThreadImpl.java:374)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
        at java.lang.reflect.Method.invoke(Method.java:597)
        at com.sun.jmx.mbeanserver.ConvertingMethod.invokeWithOpenReturn(ConvertingMethod.java:167)
        at com.sun.jmx.mbeanserver.MXBeanIntrospector.invokeM2(MXBeanIntrospector.java:96)
        at com.sun.jmx.mbeanserver.MXBeanIntrospector.invokeM2(MXBeanIntrospector.java:33)
        at com.sun.jmx.mbeanserver.MBeanIntrospector.invokeM(MBeanIntrospector.java:208)
        at com.sun.jmx.mbeanserver.PerInterface.invoke(PerInterface.java:120)
        at com.sun.jmx.mbeanserver.MBeanSupport.invoke(MBeanSupport.java:262)
        at javax.management.StandardMBean.invoke(StandardMBean.java:391)
        at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.invoke(DefaultMBeanServerInterceptor.java:836)
        at com.sun.jmx.mbeanserver.JmxMBeanServer.invoke(JmxMBeanServer.java:761)
        at javax.management.remote.rmi.RMIConnectionImpl.doOperation(RMIConnectionImpl.java:1426)
        at javax.management.remote.rmi.RMIConnectionImpl.access$200(RMIConnectionImpl.java:72)
        at javax.management.remote.rmi.RMIConnectionImpl$PrivilegedOperation.run(RMIConnectionImpl.java:1264)
        at javax.management.remote.rmi.RMIConnectionImpl.doPrivilegedOperation(RMIConnectionImpl.java:1359)
        at javax.management.remote.rmi.RMIConnectionImpl.invoke(RMIConnectionImpl.java:788)
        at sun.reflect.GeneratedMethodAccessor187.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
        at java.lang.reflect.Method.invoke(Method.java:597)
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:305)
        at sun.rmi.transport.Transport$1.run(Transport.java:159)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:155)
        at sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:535)
        at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:790)
        at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:649)
        at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - locked java.util.concurrent.locks.ReentrantLock$NonfairSync_at_79dab812

"RMI TCP Connection(410)-88.250.217.202" - Thread t_at_225
   java.lang.Thread.State: RUNNABLE
        at java.net.SocketInputStream.socketRead0(Native Method)
        at java.net.SocketInputStream.read(SocketInputStream.java:129)
        at java.io.BufferedInputStream.fill(BufferedInputStream.java:218)
        at java.io.BufferedInputStream.read(BufferedInputStream.java:237)
        - locked java.io.BufferedInputStream_at_53aea5bc
        at java.io.FilterInputStream.read(FilterInputStream.java:66)
        at sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:517)
        at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:790)
        at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:649)
        at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - locked java.util.concurrent.locks.ReentrantLock$NonfairSync_at_6f03048f

"Timer-2" - Thread t_at_186
   java.lang.Thread.State: TIMED_WAITING on java.util.TaskQueue_at_20f563ec
        at java.lang.Object.wait(Native Method)
        at java.util.TimerThread.mainLoop(Timer.java:509)
        at java.util.TimerThread.run(Timer.java:462)

   Locked ownable synchronizers:
        - None

"pool-9-thread-1" - Thread t_at_168
   java.lang.Thread.State: WAITING on
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject_at_b0abb2
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"http-thread-pool-4848-(16)" - Thread t_at_167
   java.lang.Thread.State: TIMED_WAITING on java.lang.Object_at_1be34f
        at java.lang.Object.wait(Native Method)
        at com.sun.grizzly.util.SyncThreadPool$SyncThreadWorker.getTask(SyncThreadPool.java:445)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:323)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.run(AbstractThreadPool.java:309)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"http-thread-pool-4848-(15)" - Thread t_at_166
   java.lang.Thread.State: TIMED_WAITING on java.lang.Object_at_1be34f
        at java.lang.Object.wait(Native Method)
        at com.sun.grizzly.util.SyncThreadPool$SyncThreadWorker.getTask(SyncThreadPool.java:445)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:323)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.run(AbstractThreadPool.java:309)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"http-thread-pool-4848-(14)" - Thread t_at_165
   java.lang.Thread.State: TIMED_WAITING on java.lang.Object_at_1be34f
        at java.lang.Object.wait(Native Method)
        at com.sun.grizzly.util.SyncThreadPool$SyncThreadWorker.getTask(SyncThreadPool.java:445)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:323)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.run(AbstractThreadPool.java:309)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"http-thread-pool-4848-(13)" - Thread t_at_164
   java.lang.Thread.State: TIMED_WAITING on java.lang.Object_at_1be34f
        at java.lang.Object.wait(Native Method)
        at com.sun.grizzly.util.SyncThreadPool$SyncThreadWorker.getTask(SyncThreadPool.java:445)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:323)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.run(AbstractThreadPool.java:309)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"http-thread-pool-4848-(12)" - Thread t_at_163
   java.lang.Thread.State: TIMED_WAITING on java.lang.Object_at_1be34f
        at java.lang.Object.wait(Native Method)
        at com.sun.grizzly.util.SyncThreadPool$SyncThreadWorker.getTask(SyncThreadPool.java:445)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:323)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.run(AbstractThreadPool.java:309)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"http-thread-pool-4848-(11)" - Thread t_at_162
   java.lang.Thread.State: TIMED_WAITING on java.lang.Object_at_1be34f
        at java.lang.Object.wait(Native Method)
        at com.sun.grizzly.util.SyncThreadPool$SyncThreadWorker.getTask(SyncThreadPool.java:445)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:323)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.run(AbstractThreadPool.java:309)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"http-thread-pool-4848-(10)" - Thread t_at_161
   java.lang.Thread.State: TIMED_WAITING on java.lang.Object_at_1be34f
        at java.lang.Object.wait(Native Method)
        at com.sun.grizzly.util.SyncThreadPool$SyncThreadWorker.getTask(SyncThreadPool.java:445)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:323)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.run(AbstractThreadPool.java:309)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"http-thread-pool-4848-(9)" - Thread t_at_160
   java.lang.Thread.State: TIMED_WAITING on java.lang.Object_at_1be34f
        at java.lang.Object.wait(Native Method)
        at com.sun.grizzly.util.SyncThreadPool$SyncThreadWorker.getTask(SyncThreadPool.java:445)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:323)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.run(AbstractThreadPool.java:309)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"http-thread-pool-4848-(8)" - Thread t_at_159
   java.lang.Thread.State: TIMED_WAITING on java.lang.Object_at_1be34f
        at java.lang.Object.wait(Native Method)
        at com.sun.grizzly.util.SyncThreadPool$SyncThreadWorker.getTask(SyncThreadPool.java:445)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:323)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.run(AbstractThreadPool.java:309)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"http-thread-pool-4848-(7)" - Thread t_at_158
   java.lang.Thread.State: TIMED_WAITING on java.lang.Object_at_1be34f
        at java.lang.Object.wait(Native Method)
        at com.sun.grizzly.util.SyncThreadPool$SyncThreadWorker.getTask(SyncThreadPool.java:445)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:323)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.run(AbstractThreadPool.java:309)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"http-thread-pool-4848-(6)" - Thread t_at_157
   java.lang.Thread.State: TIMED_WAITING on java.lang.Object_at_1be34f
        at java.lang.Object.wait(Native Method)
        at com.sun.grizzly.util.SyncThreadPool$SyncThreadWorker.getTask(SyncThreadPool.java:445)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:323)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.run(AbstractThreadPool.java:309)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"http-thread-pool-4848-(5)" - Thread t_at_156
   java.lang.Thread.State: TIMED_WAITING on java.lang.Object_at_1be34f
        at java.lang.Object.wait(Native Method)
        at com.sun.grizzly.util.SyncThreadPool$SyncThreadWorker.getTask(SyncThreadPool.java:445)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:323)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.run(AbstractThreadPool.java:309)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"http-thread-pool-4848-(4)" - Thread t_at_155
   java.lang.Thread.State: TIMED_WAITING on java.lang.Object_at_1be34f
        at java.lang.Object.wait(Native Method)
        at com.sun.grizzly.util.SyncThreadPool$SyncThreadWorker.getTask(SyncThreadPool.java:445)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:323)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.run(AbstractThreadPool.java:309)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"ContainerBackgroundProcessor[StandardEngine[com.sun.appserv].StandardHost[__asadmin].StandardContext[]]"
- Thread t_at_151
   java.lang.Thread.State: TIMED_WAITING
        at java.lang.Thread.sleep(Native Method)
        at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.run(ContainerBase.java:1774)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"http-thread-pool-4848-(3)" - Thread t_at_150
   java.lang.Thread.State: TIMED_WAITING on java.lang.Object_at_1be34f
        at java.lang.Object.wait(Native Method)
        at com.sun.grizzly.util.SyncThreadPool$SyncThreadWorker.getTask(SyncThreadPool.java:445)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:323)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.run(AbstractThreadPool.java:309)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"http-thread-pool-4848-(2)" - Thread t_at_149
   java.lang.Thread.State: TIMED_WAITING on java.lang.Object_at_1be34f
        at java.lang.Object.wait(Native Method)
        at com.sun.grizzly.util.SyncThreadPool$SyncThreadWorker.getTask(SyncThreadPool.java:445)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:323)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.run(AbstractThreadPool.java:309)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"Grizzly" - Thread t_at_141
   java.lang.Thread.State: TIMED_WAITING on
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject_at_26e857
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963)
        at java.util.concurrent.DelayQueue.take(DelayQueue.java:164)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:583)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:576)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"http-thread-pool-4848-(1)" - Thread t_at_140
   java.lang.Thread.State: TIMED_WAITING on java.lang.Object_at_1be34f
        at java.lang.Object.wait(Native Method)
        at com.sun.grizzly.util.SyncThreadPool$SyncThreadWorker.getTask(SyncThreadPool.java:445)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:323)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.run(AbstractThreadPool.java:309)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"JMX server connection timeout 130" - Thread t_at_130
   java.lang.Thread.State: TIMED_WAITING on [I_at_46a1eaf7
        at java.lang.Object.wait(Native Method)
        at com.sun.jmx.remote.internal.ServerCommunicatorAdmin$Timeout.run(ServerCommunicatorAdmin.java:150)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"JMX server connection timeout 129" - Thread t_at_129
   java.lang.Thread.State: TIMED_WAITING on [I_at_3049225b
        at java.lang.Object.wait(Native Method)
        at com.sun.jmx.remote.internal.ServerCommunicatorAdmin$Timeout.run(ServerCommunicatorAdmin.java:150)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"pool-29-thread-1" - Thread t_at_126
   java.lang.Thread.State: WAITING on
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject_at_116e9d
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
        at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:317)
        at org.jvnet.hk2.config.Transactions$Notifier$1.run(Transactions.java:157)
        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:441)
        at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
        at java.util.concurrent.FutureTask.run(FutureTask.java:138)
        at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - locked java.util.concurrent.locks.ReentrantLock$NonfairSync_at_42d75a8a

"AMXConfigLoader.AMXConfigLoaderThread" - Thread t_at_125
   java.lang.Thread.State: WAITING on
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject_at_750dea
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
        at org.glassfish.admin.mbeanserver.PendingConfigBeans.take(PendingConfigBeans.java:96)
        at org.glassfish.admin.amx.impl.config.AMXConfigLoader$AMXConfigLoaderThread.doRun(AMXConfigLoader.java:582)
        at org.glassfish.admin.amx.impl.config.AMXConfigLoader$AMXConfigLoaderThread.run(AMXConfigLoader.java:533)

   Locked ownable synchronizers:
        - None

"ComplianceMonitor.ValidatorThread" - Thread t_at_120
   java.lang.Thread.State: WAITING on
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject_at_649c66
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
        at org.glassfish.admin.amx.impl.mbean.ComplianceMonitor$ValidatorThread.doRun(ComplianceMonitor.java:240)
        at org.glassfish.admin.amx.impl.mbean.ComplianceMonitor$ValidatorThread.run(ComplianceMonitor.java:227)

   Locked ownable synchronizers:
        - None

"JMX server connection timeout 119" - Thread t_at_119
   java.lang.Thread.State: TIMED_WAITING on [I_at_58185680
        at java.lang.Object.wait(Native Method)
        at com.sun.jmx.remote.internal.ServerCommunicatorAdmin$Timeout.run(ServerCommunicatorAdmin.java:150)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"http-thread-pool-8080-(16)" - Thread t_at_118
   java.lang.Thread.State: TIMED_WAITING on java.lang.Object_at_70729a2b
        at java.lang.Object.wait(Native Method)
        at com.sun.grizzly.util.SyncThreadPool$SyncThreadWorker.getTask(SyncThreadPool.java:445)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:323)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.run(AbstractThreadPool.java:309)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"http-thread-pool-8080-(15)" - Thread t_at_117
   java.lang.Thread.State: TIMED_WAITING on java.lang.Object_at_70729a2b
        at java.lang.Object.wait(Native Method)
        at com.sun.grizzly.util.SyncThreadPool$SyncThreadWorker.getTask(SyncThreadPool.java:445)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:323)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.run(AbstractThreadPool.java:309)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"http-thread-pool-8080-(14)" - Thread t_at_116
   java.lang.Thread.State: TIMED_WAITING on java.lang.Object_at_70729a2b
        at java.lang.Object.wait(Native Method)
        at com.sun.grizzly.util.SyncThreadPool$SyncThreadWorker.getTask(SyncThreadPool.java:445)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:323)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.run(AbstractThreadPool.java:309)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"http-thread-pool-8080-(13)" - Thread t_at_115
   java.lang.Thread.State: TIMED_WAITING on java.lang.Object_at_70729a2b
        at java.lang.Object.wait(Native Method)
        at com.sun.grizzly.util.SyncThreadPool$SyncThreadWorker.getTask(SyncThreadPool.java:445)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:323)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.run(AbstractThreadPool.java:309)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"http-thread-pool-8080-(12)" - Thread t_at_114
   java.lang.Thread.State: TIMED_WAITING on java.lang.Object_at_70729a2b
        at java.lang.Object.wait(Native Method)
        at com.sun.grizzly.util.SyncThreadPool$SyncThreadWorker.getTask(SyncThreadPool.java:445)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:323)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.run(AbstractThreadPool.java:309)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"http-thread-pool-8080-(11)" - Thread t_at_113
   java.lang.Thread.State: TIMED_WAITING on java.lang.Object_at_70729a2b
        at java.lang.Object.wait(Native Method)
        at com.sun.grizzly.util.SyncThreadPool$SyncThreadWorker.getTask(SyncThreadPool.java:445)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:323)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.run(AbstractThreadPool.java:309)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"http-thread-pool-8080-(10)" - Thread t_at_112
   java.lang.Thread.State: TIMED_WAITING on java.lang.Object_at_70729a2b
        at java.lang.Object.wait(Native Method)
        at com.sun.grizzly.util.SyncThreadPool$SyncThreadWorker.getTask(SyncThreadPool.java:445)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:323)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.run(AbstractThreadPool.java:309)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"http-thread-pool-8080-(9)" - Thread t_at_111
   java.lang.Thread.State: TIMED_WAITING on java.lang.Object_at_70729a2b
        at java.lang.Object.wait(Native Method)
        at com.sun.grizzly.util.SyncThreadPool$SyncThreadWorker.getTask(SyncThreadPool.java:445)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:323)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.run(AbstractThreadPool.java:309)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"http-thread-pool-8080-(8)" - Thread t_at_110
   java.lang.Thread.State: TIMED_WAITING on java.lang.Object_at_70729a2b
        at java.lang.Object.wait(Native Method)
        at com.sun.grizzly.util.SyncThreadPool$SyncThreadWorker.getTask(SyncThreadPool.java:445)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:323)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.run(AbstractThreadPool.java:309)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"http-thread-pool-8080-(7)" - Thread t_at_109
   java.lang.Thread.State: TIMED_WAITING on java.lang.Object_at_70729a2b
        at java.lang.Object.wait(Native Method)
        at com.sun.grizzly.util.SyncThreadPool$SyncThreadWorker.getTask(SyncThreadPool.java:445)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:323)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.run(AbstractThreadPool.java:309)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"http-thread-pool-8080-(6)" - Thread t_at_108
   java.lang.Thread.State: TIMED_WAITING on java.lang.Object_at_70729a2b
        at java.lang.Object.wait(Native Method)
        at com.sun.grizzly.util.SyncThreadPool$SyncThreadWorker.getTask(SyncThreadPool.java:445)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:323)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.run(AbstractThreadPool.java:309)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"http-thread-pool-8080-(5)" - Thread t_at_107
   java.lang.Thread.State: TIMED_WAITING on java.lang.Object_at_70729a2b
        at java.lang.Object.wait(Native Method)
        at com.sun.grizzly.util.SyncThreadPool$SyncThreadWorker.getTask(SyncThreadPool.java:445)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:323)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.run(AbstractThreadPool.java:309)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"http-thread-pool-8080-(4)" - Thread t_at_106
   java.lang.Thread.State: TIMED_WAITING on java.lang.Object_at_70729a2b
        at java.lang.Object.wait(Native Method)
        at com.sun.grizzly.util.SyncThreadPool$SyncThreadWorker.getTask(SyncThreadPool.java:445)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:323)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.run(AbstractThreadPool.java:309)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"http-thread-pool-8080-(3)" - Thread t_at_105
   java.lang.Thread.State: TIMED_WAITING on java.lang.Object_at_70729a2b
        at java.lang.Object.wait(Native Method)
        at com.sun.grizzly.util.SyncThreadPool$SyncThreadWorker.getTask(SyncThreadPool.java:445)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:323)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.run(AbstractThreadPool.java:309)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"{felix.fileinstall.poll=5000,
felix.fileinstall.bundles.new.start=true,
service.pid=org.apache.felix.fileinstall.a34379e5-de1c-45d1-9f8c-03413af105de,
felix.fileinstall.dir=/home/glassfishv3/glassfish/domains/domain1/autodeploy/bundles/,
felix.fileinstall.filename=org.apache.felix.fileinstall-autodeploy-bundles.cfg,
service.factorypid=org.apache.felix.fileinstall,
felix.fileinstall.debug=1}" - Thread t_at_104
   java.lang.Thread.State: TIMED_WAITING on
org.apache.felix.fileinstall.internal.DirectoryWatcher_at_25d71aaf
        at java.lang.Object.wait(Native Method)
        at org.apache.felix.fileinstall.internal.DirectoryWatcher.run(DirectoryWatcher.java:210)

   Locked ownable synchronizers:
        - None

"http-thread-pool-8080-(2)" - Thread t_at_103
   java.lang.Thread.State: TIMED_WAITING on java.lang.Object_at_70729a2b
        at java.lang.Object.wait(Native Method)
        at com.sun.grizzly.util.SyncThreadPool$SyncThreadWorker.getTask(SyncThreadPool.java:445)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:323)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.run(AbstractThreadPool.java:309)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"RMI RenewClean-[88.198.39.111:8686]" - Thread t_at_100
   java.lang.Thread.State: TIMED_WAITING on
java.lang.ref.ReferenceQueue$Lock_at_4d18ea91
        at java.lang.Object.wait(Native Method)
        at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:118)
        at sun.rmi.transport.DGCClient$EndpointEntry$RenewCleanThread.run(DGCClient.java:516)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"RMI Scheduler(0)" - Thread t_at_98
   java.lang.Thread.State: TIMED_WAITING on
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject_at_1ecffd
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963)
        at java.util.concurrent.DelayQueue.take(DelayQueue.java:164)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:583)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:576)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"SCR Component Actor" - Thread t_at_97
   java.lang.Thread.State: WAITING on java.util.LinkedList_at_68f3d2fb
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:485)
        at org.apache.felix.scr.impl.ComponentActorThread.run(ComponentActorThread.java:63)

   Locked ownable synchronizers:
        - None

"{felix.fileinstall.poll=5000,
felix.fileinstall.bundles.new.start=true,
felix.fileinstall.dir=/home/glassfishv3/glassfish/modules/autostart/,
felix.fileinstall.debug=1}" - Thread t_at_96
   java.lang.Thread.State: TIMED_WAITING on
org.apache.felix.fileinstall.internal.DirectoryWatcher_at_5fa09291
        at java.lang.Object.wait(Native Method)
        at org.apache.felix.fileinstall.internal.DirectoryWatcher.run(DirectoryWatcher.java:210)

   Locked ownable synchronizers:
        - None

"Configuration Updater" - Thread t_at_94
   java.lang.Thread.State: WAITING on java.util.LinkedList_at_928f351
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:485)
        at org.apache.felix.cm.impl.UpdateThread.run(UpdateThread.java:67)

   Locked ownable synchronizers:
        - None

"telnetconsole.Listener" - Thread t_at_93
   java.lang.Thread.State: RUNNABLE
        at java.net.PlainSocketImpl.socketAccept(Native Method)
        at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:390)
        - locked java.net.SocksSocketImpl_at_f664a0e
        at java.net.ServerSocket.implAccept(ServerSocket.java:453)
        at java.net.ServerSocket.accept(ServerSocket.java:421)
        at org.apache.felix.shell.remote.Listener$Acceptor.run(Listener.java:137)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"GC Daemon" - Thread t_at_92
   java.lang.Thread.State: TIMED_WAITING on sun.misc.GC$LatencyLock_at_5f7e4f89
        at java.lang.Object.wait(Native Method)
        at sun.misc.GC$Daemon.run(GC.java:100)

   Locked ownable synchronizers:
        - None

"RMI Reaper" - Thread t_at_91
   java.lang.Thread.State: WAITING on java.lang.ref.ReferenceQueue$Lock_at_5d97dc59
        at java.lang.Object.wait(Native Method)
        at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:118)
        at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:134)
        at sun.rmi.transport.ObjectTable$Reaper.run(ObjectTable.java:333)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"RMI TCP Accept-8686" - Thread t_at_90
   java.lang.Thread.State: RUNNABLE
        at java.net.PlainSocketImpl.socketAccept(Native Method)
        at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:390)
        - locked java.net.SocksSocketImpl_at_487e508
        at java.net.ServerSocket.implAccept(ServerSocket.java:453)
        at java.net.ServerSocket.accept(ServerSocket.java:421)
        at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:369)
        at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:341)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"GlassFish Kernel Main Thread" - Thread t_at_89
   java.lang.Thread.State: WAITING on
com.sun.enterprise.v3.server.AppServerStartup$1_at_1d038710
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:485)
        at com.sun.enterprise.v3.server.AppServerStartup$1.run(AppServerStartup.java:146)

   Locked ownable synchronizers:
        - None

"AutoDeployer" - Thread t_at_88
   java.lang.Thread.State: TIMED_WAITING on java.util.TaskQueue_at_52983a51
        at java.lang.Object.wait(Native Method)
        at java.util.TimerThread.mainLoop(Timer.java:509)
        at java.util.TimerThread.run(Timer.java:462)

   Locked ownable synchronizers:
        - None

"DynamicReloader" - Thread t_at_86
   java.lang.Thread.State: TIMED_WAITING on java.util.TaskQueue_at_7fd1c60
        at java.lang.Object.wait(Native Method)
        at java.util.TimerThread.mainLoop(Timer.java:509)
        at java.util.TimerThread.run(Timer.java:462)

   Locked ownable synchronizers:
        - None

"ContainerBackgroundProcessor[StandardEngine[com.sun.appserv].StandardHost[server].StandardContext[]]"
- Thread t_at_65
   java.lang.Thread.State: TIMED_WAITING
        at java.lang.Thread.sleep(Native Method)
        at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.run(ContainerBase.java:1774)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"MultiThreadedHttpConnectionManager cleanup" - Thread t_at_64
   java.lang.Thread.State: WAITING on java.lang.ref.ReferenceQueue$Lock_at_170278c5
        at java.lang.Object.wait(Native Method)
        at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:118)
        at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:134)
        at org.apache.commons.httpclient.MultiThreadedHttpConnectionManager$ReferenceQueueThread.run(MultiThreadedHttpConnectionManager.java:1122)

   Locked ownable synchronizers:
        - None

"connector-timer-proxy" - Thread t_at_63
   java.lang.Thread.State: TIMED_WAITING on java.util.TaskQueue_at_2346d950
        at java.lang.Object.wait(Native Method)
        at java.util.TimerThread.mainLoop(Timer.java:509)
        at java.util.TimerThread.run(Timer.java:462)

   Locked ownable synchronizers:
        - None

"Timer-1" - Thread t_at_62
   java.lang.Thread.State: WAITING on java.util.TaskQueue_at_652548
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:485)
        at java.util.TimerThread.mainLoop(Timer.java:483)
        at java.util.TimerThread.run(Timer.java:462)

   Locked ownable synchronizers:
        - None

"http-thread-pool-8080-(1)" - Thread t_at_61
   java.lang.Thread.State: TIMED_WAITING on java.lang.Object_at_70729a2b
        at java.lang.Object.wait(Native Method)
        at com.sun.grizzly.util.SyncThreadPool$SyncThreadWorker.getTask(SyncThreadPool.java:445)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:323)
        at com.sun.grizzly.util.AbstractThreadPool$Worker.run(AbstractThreadPool.java:309)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"p: thread-pool-1; w: 16" - Thread t_at_60
   java.lang.Thread.State: TIMED_WAITING on
com.sun.corba.ee.impl.orbutil.threadpool.WorkQueueImpl_at_73e7b6b9
        at java.lang.Object.wait(Native Method)
        at com.sun.corba.ee.impl.orbutil.threadpool.WorkQueueImpl.requestWork(WorkQueueImpl.java:120)
        at com.sun.corba.ee.impl.orbutil.threadpool.ThreadPoolImpl$WorkerThread.run(ThreadPoolImpl.java:506)

   Locked ownable synchronizers:
        - None

"p: thread-pool-1; w: 15" - Thread t_at_59
   java.lang.Thread.State: TIMED_WAITING on
com.sun.corba.ee.impl.orbutil.threadpool.WorkQueueImpl_at_73e7b6b9
        at java.lang.Object.wait(Native Method)
        at com.sun.corba.ee.impl.orbutil.threadpool.WorkQueueImpl.requestWork(WorkQueueImpl.java:120)
        at com.sun.corba.ee.impl.orbutil.threadpool.ThreadPoolImpl$WorkerThread.run(ThreadPoolImpl.java:506)

   Locked ownable synchronizers:
        - None

"p: thread-pool-1; w: 14" - Thread t_at_58
   java.lang.Thread.State: TIMED_WAITING on
com.sun.corba.ee.impl.orbutil.threadpool.WorkQueueImpl_at_73e7b6b9
        at java.lang.Object.wait(Native Method)
        at com.sun.corba.ee.impl.orbutil.threadpool.WorkQueueImpl.requestWork(WorkQueueImpl.java:120)
        at com.sun.corba.ee.impl.orbutil.threadpool.ThreadPoolImpl$WorkerThread.run(ThreadPoolImpl.java:506)

   Locked ownable synchronizers:
        - None

"p: thread-pool-1; w: 13" - Thread t_at_57
   java.lang.Thread.State: TIMED_WAITING on
com.sun.corba.ee.impl.orbutil.threadpool.WorkQueueImpl_at_73e7b6b9
        at java.lang.Object.wait(Native Method)
        at com.sun.corba.ee.impl.orbutil.threadpool.WorkQueueImpl.requestWork(WorkQueueImpl.java:120)
        at com.sun.corba.ee.impl.orbutil.threadpool.ThreadPoolImpl$WorkerThread.run(ThreadPoolImpl.java:506)

   Locked ownable synchronizers:
        - None

"p: thread-pool-1; w: 12" - Thread t_at_56
   java.lang.Thread.State: TIMED_WAITING on
com.sun.corba.ee.impl.orbutil.threadpool.WorkQueueImpl_at_73e7b6b9
        at java.lang.Object.wait(Native Method)
        at com.sun.corba.ee.impl.orbutil.threadpool.WorkQueueImpl.requestWork(WorkQueueImpl.java:120)
        at com.sun.corba.ee.impl.orbutil.threadpool.ThreadPoolImpl$WorkerThread.run(ThreadPoolImpl.java:506)

   Locked ownable synchronizers:
        - None

"p: thread-pool-1; w: 11" - Thread t_at_55
   java.lang.Thread.State: TIMED_WAITING on
com.sun.corba.ee.impl.orbutil.threadpool.WorkQueueImpl_at_73e7b6b9
        at java.lang.Object.wait(Native Method)
        at com.sun.corba.ee.impl.orbutil.threadpool.WorkQueueImpl.requestWork(WorkQueueImpl.java:120)
        at com.sun.corba.ee.impl.orbutil.threadpool.ThreadPoolImpl$WorkerThread.run(ThreadPoolImpl.java:506)

   Locked ownable synchronizers:
        - None

"p: thread-pool-1; w: 10" - Thread t_at_54
   java.lang.Thread.State: TIMED_WAITING on
com.sun.corba.ee.impl.orbutil.threadpool.WorkQueueImpl_at_73e7b6b9
        at java.lang.Object.wait(Native Method)
        at com.sun.corba.ee.impl.orbutil.threadpool.WorkQueueImpl.requestWork(WorkQueueImpl.java:120)
        at com.sun.corba.ee.impl.orbutil.threadpool.ThreadPoolImpl$WorkerThread.run(ThreadPoolImpl.java:506)

   Locked ownable synchronizers:
        - None

"p: thread-pool-1; w: 9" - Thread t_at_53
   java.lang.Thread.State: TIMED_WAITING on
com.sun.corba.ee.impl.orbutil.threadpool.WorkQueueImpl_at_73e7b6b9
        at java.lang.Object.wait(Native Method)
        at com.sun.corba.ee.impl.orbutil.threadpool.WorkQueueImpl.requestWork(WorkQueueImpl.java:120)
        at com.sun.corba.ee.impl.orbutil.threadpool.ThreadPoolImpl$WorkerThread.run(ThreadPoolImpl.java:506)

   Locked ownable synchronizers:
        - None

"p: thread-pool-1; w: 8" - Thread t_at_52
   java.lang.Thread.State: TIMED_WAITING on
com.sun.corba.ee.impl.orbutil.threadpool.WorkQueueImpl_at_73e7b6b9
        at java.lang.Object.wait(Native Method)
        at com.sun.corba.ee.impl.orbutil.threadpool.WorkQueueImpl.requestWork(WorkQueueImpl.java:120)
        at com.sun.corba.ee.impl.orbutil.threadpool.ThreadPoolImpl$WorkerThread.run(ThreadPoolImpl.java:506)

   Locked ownable synchronizers:
        - None

"p: thread-pool-1; w: 7" - Thread t_at_51
   java.lang.Thread.State: TIMED_WAITING on
com.sun.corba.ee.impl.orbutil.threadpool.WorkQueueImpl_at_73e7b6b9
        at java.lang.Object.wait(Native Method)
        at com.sun.corba.ee.impl.orbutil.threadpool.WorkQueueImpl.requestWork(WorkQueueImpl.java:120)
        at com.sun.corba.ee.impl.orbutil.threadpool.ThreadPoolImpl$WorkerThread.run(ThreadPoolImpl.java:506)

   Locked ownable synchronizers:
        - None

"p: thread-pool-1; w: 6" - Thread t_at_50
   java.lang.Thread.State: TIMED_WAITING on
com.sun.corba.ee.impl.orbutil.threadpool.WorkQueueImpl_at_73e7b6b9
        at java.lang.Object.wait(Native Method)
        at com.sun.corba.ee.impl.orbutil.threadpool.WorkQueueImpl.requestWork(WorkQueueImpl.java:120)
        at com.sun.corba.ee.impl.orbutil.threadpool.ThreadPoolImpl$WorkerThread.run(ThreadPoolImpl.java:506)

   Locked ownable synchronizers:
        - None

"p: thread-pool-1; w: 5" - Thread t_at_49
   java.lang.Thread.State: TIMED_WAITING on
com.sun.corba.ee.impl.orbutil.threadpool.WorkQueueImpl_at_73e7b6b9
        at java.lang.Object.wait(Native Method)
        at com.sun.corba.ee.impl.orbutil.threadpool.WorkQueueImpl.requestWork(WorkQueueImpl.java:120)
        at com.sun.corba.ee.impl.orbutil.threadpool.ThreadPoolImpl$WorkerThread.run(ThreadPoolImpl.java:506)

   Locked ownable synchronizers:
        - None

"p: thread-pool-1; w: 4" - Thread t_at_48
   java.lang.Thread.State: TIMED_WAITING on
com.sun.corba.ee.impl.orbutil.threadpool.WorkQueueImpl_at_73e7b6b9
        at java.lang.Object.wait(Native Method)
        at com.sun.corba.ee.impl.orbutil.threadpool.WorkQueueImpl.requestWork(WorkQueueImpl.java:120)
        at com.sun.corba.ee.impl.orbutil.threadpool.ThreadPoolImpl$WorkerThread.run(ThreadPoolImpl.java:506)

   Locked ownable synchronizers:
        - None

"p: thread-pool-1; w: 3" - Thread t_at_47
   java.lang.Thread.State: TIMED_WAITING on
com.sun.corba.ee.impl.orbutil.threadpool.WorkQueueImpl_at_73e7b6b9
        at java.lang.Object.wait(Native Method)
        at com.sun.corba.ee.impl.orbutil.threadpool.WorkQueueImpl.requestWork(WorkQueueImpl.java:120)
        at com.sun.corba.ee.impl.orbutil.threadpool.ThreadPoolImpl$WorkerThread.run(ThreadPoolImpl.java:506)

   Locked ownable synchronizers:
        - None

"p: thread-pool-1; w: 2" - Thread t_at_46
   java.lang.Thread.State: TIMED_WAITING on
com.sun.corba.ee.impl.orbutil.threadpool.WorkQueueImpl_at_73e7b6b9
        at java.lang.Object.wait(Native Method)
        at com.sun.corba.ee.impl.orbutil.threadpool.WorkQueueImpl.requestWork(WorkQueueImpl.java:120)
        at com.sun.corba.ee.impl.orbutil.threadpool.ThreadPoolImpl$WorkerThread.run(ThreadPoolImpl.java:506)

   Locked ownable synchronizers:
        - None

"p: thread-pool-1; w: 1" - Thread t_at_45
   java.lang.Thread.State: TIMED_WAITING on
com.sun.corba.ee.impl.orbutil.threadpool.WorkQueueImpl_at_73e7b6b9
        at java.lang.Object.wait(Native Method)
        at com.sun.corba.ee.impl.orbutil.threadpool.WorkQueueImpl.requestWork(WorkQueueImpl.java:120)
        at com.sun.corba.ee.impl.orbutil.threadpool.ThreadPoolImpl$WorkerThread.run(ThreadPoolImpl.java:506)

   Locked ownable synchronizers:
        - None

"ContainerBackgroundProcessor[StandardEngine[com.sun.appserv]]" - Thread t_at_44
   java.lang.Thread.State: TIMED_WAITING
        at java.lang.Thread.sleep(Native Method)
        at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.run(ContainerBase.java:1774)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"Timer-0" - Thread t_at_42
   java.lang.Thread.State: TIMED_WAITING on java.util.TaskQueue_at_718d6721
        at java.lang.Object.wait(Native Method)
        at java.util.TimerThread.mainLoop(Timer.java:509)
        at java.util.TimerThread.run(Timer.java:462)

   Locked ownable synchronizers:
        - None

"transaction-manager" - Thread t_at_36
   java.lang.Thread.State: WAITING on java.util.TaskQueue_at_7a6de1c4
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:485)
        at java.util.TimerThread.mainLoop(Timer.java:483)
        at java.util.TimerThread.run(Timer.java:462)

   Locked ownable synchronizers:
        - None

"Thread-16" - Thread t_at_35
   java.lang.Thread.State: RUNNABLE
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
        - locked sun.nio.ch.Util$1_at_75cd728a
        - locked java.util.Collections$UnmodifiableSet_at_31afe72
        - locked sun.nio.ch.EPollSelectorImpl_at_6136eff5
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
        at com.sun.grizzly.TCPSelectorHandler.select(TCPSelectorHandler.java:546)
        at com.sun.grizzly.SelectorHandlerRunner.doSelect(SelectorHandlerRunner.java:185)
        at com.sun.grizzly.SelectorHandlerRunner.run(SelectorHandlerRunner.java:130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - locked java.util.concurrent.locks.ReentrantLock$NonfairSync_at_5dc3efaf

"Thread-15" - Thread t_at_34
   java.lang.Thread.State: RUNNABLE
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
        - locked sun.nio.ch.Util$1_at_6aacfa40
        - locked java.util.Collections$UnmodifiableSet_at_261b677
        - locked sun.nio.ch.EPollSelectorImpl_at_5d123fd9
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
        at com.sun.grizzly.TCPSelectorHandler.select(TCPSelectorHandler.java:546)
        at com.sun.grizzly.SelectorHandlerRunner.doSelect(SelectorHandlerRunner.java:185)
        at com.sun.grizzly.SelectorHandlerRunner.run(SelectorHandlerRunner.java:130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - locked java.util.concurrent.locks.ReentrantLock$NonfairSync_at_591def4b

"Thread-14" - Thread t_at_33
   java.lang.Thread.State: RUNNABLE
        at sun.nio.ch.FileDispatcher.preClose0(Native Method)
        at sun.nio.ch.SocketDispatcher.preClose(SocketDispatcher.java:41)
        at sun.nio.ch.SocketChannelImpl.implCloseSelectableChannel(SocketChannelImpl.java:684)
        - locked java.lang.Object_at_2f208139
        at java.nio.channels.spi.AbstractSelectableChannel.implCloseChannel(AbstractSelectableChannel.java:201)
        at java.nio.channels.spi.AbstractInterruptibleChannel.close(AbstractInterruptibleChannel.java:97)
        - locked java.lang.Object_at_7f03dd5a
        at sun.nio.ch.SocketAdaptor.close(SocketAdaptor.java:352)
        at com.sun.grizzly.TCPSelectorHandler.closeChannel(TCPSelectorHandler.java:1354)
        at com.sun.grizzly.BaseSelectionKeyHandler.doAfterKeyCancel(BaseSelectionKeyHandler.java:229)
        at com.sun.grizzly.BaseSelectionKeyHandler.cancel(BaseSelectionKeyHandler.java:216)
        at com.sun.grizzly.http.SelectorThreadKeyHandler.cancel(SelectorThreadKeyHandler.java:80)
        at com.sun.enterprise.v3.services.impl.monitor.MonitorableSelectionKeyHandler.cancel(MonitorableSelectionKeyHandler.java:80)
        at com.sun.grizzly.TCPSelectorHandler.addPendingKeyCancel(TCPSelectorHandler.java:613)
        at com.sun.grizzly.http.SelectorThreadKeyHandler.expire(SelectorThreadKeyHandler.java:136)
        at com.sun.grizzly.TCPSelectorHandler.postSelect(TCPSelectorHandler.java:556)
        at com.sun.grizzly.SelectorHandlerRunner.doSelect(SelectorHandlerRunner.java:206)
        at com.sun.grizzly.SelectorHandlerRunner.run(SelectorHandlerRunner.java:130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - locked java.util.concurrent.locks.ReentrantLock$NonfairSync_at_19198f2f

"Thread-11" - Thread t_at_32
   java.lang.Thread.State: RUNNABLE
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
        - locked sun.nio.ch.Util$1_at_5d076388
        - locked java.util.Collections$UnmodifiableSet_at_1017a56f
        - locked sun.nio.ch.EPollSelectorImpl_at_6e22aa
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
        at com.sun.grizzly.TCPSelectorHandler.select(TCPSelectorHandler.java:546)
        at com.sun.grizzly.SelectorHandlerRunner.doSelect(SelectorHandlerRunner.java:185)
        at com.sun.grizzly.SelectorHandlerRunner.run(SelectorHandlerRunner.java:130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - locked java.util.concurrent.locks.ReentrantLock$NonfairSync_at_3ea5c71e

"Thread-13" - Thread t_at_31
   java.lang.Thread.State: RUNNABLE
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
        - locked sun.nio.ch.Util$1_at_75b3a2b3
        - locked java.util.Collections$UnmodifiableSet_at_647ed223
        - locked sun.nio.ch.EPollSelectorImpl_at_42162b60
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
        at com.sun.grizzly.TCPSelectorHandler.select(TCPSelectorHandler.java:546)
        at com.sun.grizzly.SelectorHandlerRunner.doSelect(SelectorHandlerRunner.java:185)
        at com.sun.grizzly.SelectorHandlerRunner.run(SelectorHandlerRunner.java:130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - locked java.util.concurrent.locks.ReentrantLock$NonfairSync_at_3d79cd85

"Thread-10" - Thread t_at_28
   java.lang.Thread.State: TIMED_WAITING on
java.util.concurrent.atomic.AtomicInteger_at_440149a7
        at java.lang.Object.wait(Native Method)
        at com.sun.grizzly.Controller.waitUntilSeletorHandlersStop(Controller.java:1092)
        at com.sun.grizzly.Controller.start(Controller.java:843)
        at com.sun.grizzly.http.SelectorThread.startListener(SelectorThread.java:1241)
        - locked java.lang.Object_at_7a3f64b
        at com.sun.grizzly.http.SelectorThread.run(SelectorThread.java:1087)
        at com.sun.grizzly.http.SelectorThread.startEndpoint(SelectorThread.java:1168)
        at com.sun.enterprise.v3.services.impl.GrizzlyListener.startEndpoint(GrizzlyListener.java:129)
        at com.sun.enterprise.v3.services.impl.GrizzlyProxy$2.run(GrizzlyProxy.java:317)

   Locked ownable synchronizers:
        - None

"Thread-7" - Thread t_at_26
   java.lang.Thread.State: TIMED_WAITING on
java.util.concurrent.atomic.AtomicInteger_at_308a95d2
        at java.lang.Object.wait(Native Method)
        at com.sun.grizzly.Controller.waitUntilSeletorHandlersStop(Controller.java:1092)
        at com.sun.grizzly.Controller.start(Controller.java:843)
        at com.sun.grizzly.http.SelectorThread.startListener(SelectorThread.java:1241)
        - locked java.lang.Object_at_1d7eb9d8
        at com.sun.grizzly.http.SelectorThread.run(SelectorThread.java:1087)
        at com.sun.grizzly.http.SelectorThread.startEndpoint(SelectorThread.java:1168)
        at com.sun.enterprise.v3.services.impl.GrizzlyListener.startEndpoint(GrizzlyListener.java:129)
        at com.sun.enterprise.v3.services.impl.GrizzlyProxy$2.run(GrizzlyProxy.java:317)

   Locked ownable synchronizers:
        - None

"Thread-6" - Thread t_at_25
   java.lang.Thread.State: TIMED_WAITING on
java.util.concurrent.atomic.AtomicInteger_at_5a7226e3
        at java.lang.Object.wait(Native Method)
        at com.sun.grizzly.Controller.waitUntilSeletorHandlersStop(Controller.java:1092)
        at com.sun.grizzly.Controller.start(Controller.java:843)
        at com.sun.grizzly.http.SelectorThread.startListener(SelectorThread.java:1241)
        - locked java.lang.Object_at_6adb2308
        at com.sun.grizzly.http.SelectorThread.run(SelectorThread.java:1087)
        at com.sun.grizzly.http.SelectorThread.startEndpoint(SelectorThread.java:1168)
        at com.sun.enterprise.v3.services.impl.GrizzlyListener.startEndpoint(GrizzlyListener.java:131)
        at com.sun.enterprise.v3.services.impl.GrizzlyProxy$2.run(GrizzlyProxy.java:317)

   Locked ownable synchronizers:
        - None

"Thread-4" - Thread t_at_22
   java.lang.Thread.State: TIMED_WAITING on
java.util.concurrent.atomic.AtomicInteger_at_7c572f78
        at java.lang.Object.wait(Native Method)
        at com.sun.grizzly.Controller.waitUntilSeletorHandlersStop(Controller.java:1092)
        at com.sun.grizzly.Controller.start(Controller.java:843)
        at com.sun.grizzly.http.SelectorThread.startListener(SelectorThread.java:1241)
        - locked java.lang.Object_at_58d91906
        at com.sun.grizzly.http.SelectorThread.run(SelectorThread.java:1087)
        at com.sun.grizzly.http.SelectorThread.startEndpoint(SelectorThread.java:1168)
        at com.sun.enterprise.v3.services.impl.GrizzlyListener.startEndpoint(GrizzlyListener.java:131)
        at com.sun.enterprise.v3.services.impl.GrizzlyProxy$2.run(GrizzlyProxy.java:317)

   Locked ownable synchronizers:
        - None

"Thread-2" - Thread t_at_19
   java.lang.Thread.State: TIMED_WAITING on
java.util.concurrent.atomic.AtomicInteger_at_3fedaf0e
        at java.lang.Object.wait(Native Method)
        at com.sun.grizzly.Controller.waitUntilSeletorHandlersStop(Controller.java:1092)
        at com.sun.grizzly.Controller.start(Controller.java:843)
        at com.sun.grizzly.http.SelectorThread.startListener(SelectorThread.java:1241)
        - locked java.lang.Object_at_14fa067c
        at com.sun.grizzly.http.SelectorThread.run(SelectorThread.java:1087)
        at com.sun.grizzly.http.SelectorThread.startEndpoint(SelectorThread.java:1168)
        at com.sun.enterprise.v3.services.impl.GrizzlyListener.startEndpoint(GrizzlyListener.java:131)
        at com.sun.enterprise.v3.services.impl.GrizzlyProxy$2.run(GrizzlyProxy.java:317)

   Locked ownable synchronizers:
        - None

"Thread-1" - Thread t_at_17
   java.lang.Thread.State: WAITING on
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject_at_d7fe0e
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
        at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:317)
        at com.sun.enterprise.server.logging.GFFileHandler.log(GFFileHandler.java:510)
        at com.sun.enterprise.server.logging.GFFileHandler$3.run(GFFileHandler.java:255)

   Locked ownable synchronizers:
        - None

"pool-2-thread-1" - Thread t_at_16
   java.lang.Thread.State: TIMED_WAITING on
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject_at_4687ac
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963)
        at java.util.concurrent.DelayQueue.take(DelayQueue.java:164)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:583)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:576)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"FelixPackageAdmin" - Thread t_at_15
   java.lang.Thread.State: WAITING on
org.apache.felix.framework.PackageAdminImpl_at_7566ca4c
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:485)
        at org.apache.felix.framework.PackageAdminImpl.run(PackageAdminImpl.java:314)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"FelixStartLevel" - Thread t_at_14
   java.lang.Thread.State: WAITING on java.util.ArrayList_at_81cc634
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:485)
        at org.apache.felix.framework.StartLevelImpl.run(StartLevelImpl.java:242)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"FelixDispatchQueue" - Thread t_at_13
   java.lang.Thread.State: WAITING on java.util.ArrayList_at_2f6b7fcb
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:485)
        at org.apache.felix.framework.util.EventDispatcher.run(EventDispatcher.java:931)
        at org.apache.felix.framework.util.EventDispatcher.access$000(EventDispatcher.java:54)
        at org.apache.felix.framework.util.EventDispatcher$1.run(EventDispatcher.java:106)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"DestroyJavaVM" - Thread t_at_11
   java.lang.Thread.State: RUNNABLE

   Locked ownable synchronizers:
        - None

"OSGi Framework Launcher" - Thread t_at_10
   java.lang.Thread.State: WAITING on
org.apache.felix.framework.util.ThreadGate_at_19188415
        at java.lang.Object.wait(Native Method)
        at org.apache.felix.framework.util.ThreadGate.await(ThreadGate.java:79)
        at org.apache.felix.framework.Felix.waitForStop(Felix.java:794)
        at org.apache.felix.main.Main.main(Main.java:299)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
        at java.lang.reflect.Method.invoke(Method.java:597)
        at com.sun.enterprise.glassfish.bootstrap.ASMainFelix$1.run(ASMainFelix.java:116)
        at java.lang.Thread.run(Thread.java:619)

   Locked ownable synchronizers:
        - None

"Signal Dispatcher" - Thread t_at_5
   java.lang.Thread.State: RUNNABLE

   Locked ownable synchronizers:
        - None

"Finalizer" - Thread t_at_3
   java.lang.Thread.State: WAITING on java.lang.ref.ReferenceQueue$Lock_at_91619a4
        at java.lang.Object.wait(Native Method)
        at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:118)
        at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:134)
        at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159)

   Locked ownable synchronizers:
        - None

"Reference Handler" - Thread t_at_2
   java.lang.Thread.State: WAITING on java.lang.ref.Reference$Lock_at_9ed62a5
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:485)
        at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116)

   Locked ownable synchronizers:
        - None