After a couple of hours running SJSAS 9.1 (9.0 had the same problem) the application server stops responding on port 8080, yet still responds on the admin port 4848. Running asadmin> list-domains reports back that domain1 is actually running.
I set --verbose and used jstack to get the following trace:
#######################
Full thread dump Java HotSpot(TM) Client VM (1.6.0_03-b05 mixed mode):
"Attach Listener" daemon prio=10 tid=0x08b86400 nid=0x5982 waiting on condition [0x00000000..0x00000000]
java.lang.Thread.State: RUNNABLE
"Thread-34" daemon prio=10 tid=0x09497800 nid=0x57b1 waiting on condition [0x607fe000..0x607ff020]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at com.sun.enterprise.admin.jmx.remote.server.notification.ServerNotificationManager.run(ServerNotificationManager.java:228)
at java.lang.Thread.run(Thread.java:619)
"Java2D Disposer" daemon prio=10 tid=0x08e34800 nid=0x4e4a in Object.wait() [0x60eb2000..0x60eb30a0]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x77d94b80> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:116)
- locked <0x77d94b80> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:132)
at sun.java2d.Disposer.run(Disposer.java:125)
at java.lang.Thread.run(Thread.java:619)
"httpSSLWorkerThread-8080-4" daemon prio=10 tid=0x08a76800 nid=0x4e46 in Object.wait() [0x610fb000..0x610fc120]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x76e0f910> (a java.lang.Object)
at java.lang.Object.wait(Object.java:485)
at com.sun.appserv.util.cache.BaseCache.waitRefresh(BaseCache.java:766)
- locked <0x76e0f910> (a java.lang.Object)
at com.sun.appserv.web.cache.filter.CachingFilter.doFilter(CachingFilter.java:191)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:230)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:198)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:288)
at org.apache.catalina.core.StandardContextValve.invokeInternal(StandardContextValve.java:271)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:202)
at org.apache.catalina.core.StandardPipeline.doInvoke(StandardPipeline.java:632)
at org.apache.catalina.core.StandardPipeline.doInvoke(StandardPipeline.java:577)
at com.sun.enterprise.web.WebPipeline.invoke(WebPipeline.java:94)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:206)
at org.apache.catalina.core.StandardPipeline.doInvoke(StandardPipeline.java:632)
at org.apache.catalina.core.StandardPipeline.doInvoke(StandardPipeline.java:577)
at org.apache.catalina.core.StandardPipeline.invoke(StandardPipeline.java:571)
at org.apache.catalina.core.ContainerBase.invoke(ContainerBase.java:1080)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:150)
at org.apache.catalina.core.StandardPipeline.doInvoke(StandardPipeline.java:632)
at org.apache.catalina.core.StandardPipeline.doInvoke(StandardPipeline.java:577)
at org.apache.catalina.core.StandardPipeline.invoke(StandardPipeline.java:571)
at org.apache.catalina.core.ContainerBase.invoke(ContainerBase.java:1080)
at org.apache.coyote.tomcat5.CoyoteAdapter.service(CoyoteAdapter.java:270)
at com.sun.enterprise.web.connector.grizzly.DefaultProcessorTask.invokeAdapter(DefaultProcessorTask.java:637)
at com.sun.enterprise.web.connector.grizzly.DefaultProcessorTask.doProcess(DefaultProcessorTask.java:568)
at com.sun.enterprise.web.connector.grizzly.DefaultProcessorTask.process(DefaultProcessorTask.java:813)
at com.sun.enterprise.web.connector.grizzly.DefaultReadTask.executeProcessorTask(DefaultReadTask.java:339)
at com.sun.enterprise.web.connector.grizzly.DefaultReadTask.doTask(DefaultReadTask.java:261)
at com.sun.enterprise.web.connector.grizzly.DefaultReadTask.doTask(DefaultReadTask.java:212)
at com.sun.enterprise.web.portunif.PortUnificationPipeline$PUTask.doTask(PortUnificationPipeline.java:361)
at com.sun.enterprise.web.connector.grizzly.TaskBase.run(TaskBase.java:265)
at com.sun.enterprise.web.connector.grizzly.ssl.SSLWorkerThread.run(SSLWorkerThread.java:106)
"MySQL Statement Cancellation Timer" daemon prio=10 tid=0x08d5c000 nid=0x4e3d in Object.wait() [0x6114c000..0x6114cfa0]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x77dbb1c8> (a java.util.TaskQueue)
at java.lang.Object.wait(Object.java:485)
at java.util.TimerThread.mainLoop(Timer.java:483)
- locked <0x77dbb1c8> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:462)
"DestroyJavaVM" prio=10 tid=0x0805c400 nid=0x4d67 waiting on condition [0x00000000..0xb7feb070]
java.lang.Thread.State: RUNNABLE
"httpSSLWorkerThread-8080-3" daemon prio=10 tid=0x09452400 nid=0x4e35 in Object.wait() [0x6119d000..0x6119e020]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x76e0f910> (a java.lang.Object)
at java.lang.Object.wait(Object.java:485)
at com.sun.appserv.util.cache.BaseCache.waitRefresh(BaseCache.java:766)
- locked <0x76e0f910> (a java.lang.Object)
at com.sun.appserv.web.cache.filter.CachingFilter.doFilter(CachingFilter.java:191)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:230)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:198)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:288)
at org.apache.catalina.core.StandardContextValve.invokeInternal(StandardContextValve.java:271)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:202)
at org.apache.catalina.core.StandardPipeline.doInvoke(StandardPipeline.java:632)
at org.apache.catalina.core.StandardPipeline.doInvoke(StandardPipeline.java:577)
at com.sun.enterprise.web.WebPipeline.invoke(WebPipeline.java:94)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:206)
at org.apache.catalina.core.StandardPipeline.doInvoke(StandardPipeline.java:632)
at org.apache.catalina.core.StandardPipeline.doInvoke(StandardPipeline.java:577)
at org.apache.catalina.core.StandardPipeline.invoke(StandardPipeline.java:571)
at org.apache.catalina.core.ContainerBase.invoke(ContainerBase.java:1080)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:150)
at org.apache.catalina.core.StandardPipeline.doInvoke(StandardPipeline.java:632)
at org.apache.catalina.core.StandardPipeline.doInvoke(StandardPipeline.java:577)
at org.apache.catalina.core.StandardPipeline.invoke(StandardPipeline.java:571)
at org.apache.catalina.core.ContainerBase.invoke(ContainerBase.java:1080)
at org.apache.coyote.tomcat5.CoyoteAdapter.service(CoyoteAdapter.java:270)
at com.sun.enterprise.web.connector.grizzly.DefaultProcessorTask.invokeAdapter(DefaultProcessorTask.java:637)
at com.sun.enterprise.web.connector.grizzly.DefaultProcessorTask.doProcess(DefaultProcessorTask.java:568)
at com.sun.enterprise.web.connector.grizzly.DefaultProcessorTask.process(DefaultProcessorTask.java:813)
at com.sun.enterprise.web.connector.grizzly.DefaultReadTask.executeProcessorTask(DefaultReadTask.java:339)
at com.sun.enterprise.web.connector.grizzly.DefaultReadTask.doTask(DefaultReadTask.java:261)
at com.sun.enterprise.web.connector.grizzly.DefaultReadTask.doTask(DefaultReadTask.java:212)
at com.sun.enterprise.web.portunif.PortUnificationPipeline$PUTask.doTask(PortUnificationPipeline.java:361)
at com.sun.enterprise.web.connector.grizzly.TaskBase.run(TaskBase.java:265)
at com.sun.enterprise.web.connector.grizzly.ssl.SSLWorkerThread.run(SSLWorkerThread.java:106)
"httpSSLWorkerThread-8080-2" daemon prio=10 tid=0x08de9000 nid=0x4e34 in Object.wait() [0x611ee000..0x611eeea0]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x76e0f910> (a java.lang.Object)
at java.lang.Object.wait(Object.java:485)
at com.sun.appserv.util.cache.BaseCache.waitRefresh(BaseCache.java:766)
- locked <0x76e0f910> (a java.lang.Object)
at com.sun.appserv.web.cache.filter.CachingFilter.doFilter(CachingFilter.java:191)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:230)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:198)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:288)
at org.apache.catalina.core.StandardContextValve.invokeInternal(StandardContextValve.java:271)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:202)
at org.apache.catalina.core.StandardPipeline.doInvoke(StandardPipeline.java:632)
at org.apache.catalina.core.StandardPipeline.doInvoke(StandardPipeline.java:577)
at com.sun.enterprise.web.WebPipeline.invoke(WebPipeline.java:94)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:206)
at org.apache.catalina.core.StandardPipeline.doInvoke(StandardPipeline.java:632)
at org.apache.catalina.core.StandardPipeline.doInvoke(StandardPipeline.java:577)
at org.apache.catalina.core.StandardPipeline.invoke(StandardPipeline.java:571)
at org.apache.catalina.core.ContainerBase.invoke(ContainerBase.java:1080)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:150)
at org.apache.catalina.core.StandardPipeline.doInvoke(StandardPipeline.java:632)
at org.apache.catalina.core.StandardPipeline.doInvoke(StandardPipeline.java:577)
at org.apache.catalina.core.StandardPipeline.invoke(StandardPipeline.java:571)
at org.apache.catalina.core.ContainerBase.invoke(ContainerBase.java:1080)
at org.apache.coyote.tomcat5.CoyoteAdapter.service(CoyoteAdapter.java:270)
at com.sun.enterprise.web.connector.grizzly.DefaultProcessorTask.invokeAdapter(DefaultProcessorTask.java:637)
at com.sun.enterprise.web.connector.grizzly.DefaultProcessorTask.doProcess(DefaultProcessorTask.java:568)
at com.sun.enterprise.web.connector.grizzly.DefaultProcessorTask.process(DefaultProcessorTask.java:813)
at com.sun.enterprise.web.connector.grizzly.DefaultReadTask.executeProcessorTask(DefaultReadTask.java:339)
at com.sun.enterprise.web.connector.grizzly.DefaultReadTask.doTask(DefaultReadTask.java:261)
at com.sun.enterprise.web.connector.grizzly.DefaultReadTask.doTask(DefaultReadTask.java:212)
at com.sun.enterprise.web.portunif.PortUnificationPipeline$PUTask.doTask(PortUnificationPipeline.java:361)
at com.sun.enterprise.web.connector.grizzly.TaskBase.run(TaskBase.java:265)
at com.sun.enterprise.web.connector.grizzly.ssl.SSLWorkerThread.run(SSLWorkerThread.java:106)
"Timer-7" prio=10 tid=0x08db9800 nid=0x4e33 in Object.wait() [0x6123f000..0x6123ff20]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x77b383e0> (a java.util.TaskQueue)
at java.util.TimerThread.mainLoop(Timer.java:509)
- locked <0x77b383e0> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:462)
"Timer-6" daemon prio=10 tid=0x080d4000 nid=0x4e2f in Object.wait() [0x61290000..0x61290da0]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x77aed000> (a java.util.TaskQueue)
at java.util.TimerThread.mainLoop(Timer.java:509)
- locked <0x77aed000> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:462)
"derby.rawStoreDaemon" daemon prio=10 tid=0x08dee800 nid=0x4e2e in Object.wait() [0x612e1000..0x612e1e20]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x77abe488> (a org.apache.derby.impl.services.daemon.BasicDaemon)
at org.apache.derby.impl.services.daemon.BasicDaemon.rest(Unknown Source)
- locked <0x77abe488> (a org.apache.derby.impl.services.daemon.BasicDaemon)
at org.apache.derby.impl.services.daemon.BasicDaemon.run(Unknown Source)
at java.lang.Thread.run(Thread.java:619)
...skipping...
- waiting on <0x75515b88> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:116)
- locked <0x75515b88> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:132)
at sun.rmi.transport.ObjectTable$Reaper.run(ObjectTable.java:333)
at java.lang.Thread.run(Thread.java:619)
"RMI TCP Accept-0" daemon prio=10 tid=0x08174c00 nid=0x4d76 runnable [0x63be4000..0x63be4da0]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:145)
- locked <0x75515c88> (a java.lang.Object)
at sun.nio.ch.ServerSocketAdaptor.accept(ServerSocketAdaptor.java:84)
- locked <0x75515c90> (a java.lang.Object)
at com.sun.enterprise.server.ss.provider.ASServerSocket.acceptSocket(ASServerSocket.java:188)
at com.sun.enterprise.server.ss.provider.ASServerSocket.accept(ASServerSocket.java:168)
- locked <0x75515cb8> (a com.sun.enterprise.server.ss.provider.ASServerSocket)
at com.sun.enterprise.server.ss.provider.ASServerSocketImpl.accept(ASServerSocketImpl.java:174)
at java.net.ServerSocket.implAccept(ServerSocket.java:453)
at java.net.ServerSocket.accept(ServerSocket.java:421)
at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:369)
at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:341)
at java.lang.Thread.run(Thread.java:619)
"Low Memory Detector" daemon prio=10 tid=0x080c3c00 nid=0x4d70 runnable [0x00000000..0x00000000]
java.lang.Thread.State: RUNNABLE
"CompilerThread0" daemon prio=10 tid=0x080c2400 nid=0x4d6f waiting on condition [0x00000000..0x64223ab8]
java.lang.Thread.State: RUNNABLE
"Signal Dispatcher" daemon prio=10 tid=0x080c1000 nid=0x4d6e runnable [0x00000000..0x64274b80]
java.lang.Thread.State: RUNNABLE
"Finalizer" daemon prio=10 tid=0x080adc00 nid=0x4d6d in Object.wait() [0x6459e000..0x6459ef20]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x752c55d8> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:116)
- locked <0x752c55d8> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:132)
at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159)
"Reference Handler" daemon prio=10 tid=0x080acc00 nid=0x4d6c in Object.wait() [0x645ef000..0x645efda0]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x752c5668> (a java.lang.ref.Reference$Lock)
at java.lang.Object.wait(Object.java:485)
at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116)
- locked <0x752c5668> (a java.lang.ref.Reference$Lock)
"VM Thread" prio=10 tid=0x080a9800 nid=0x4d68 runnable
"VM Periodic Task Thread" prio=10 tid=0x080c5400 nid=0x4d71 waiting on condition
JNI global references: 1505
#################
Any advice?
Thanks
Dan
[Message sent by forum member 'danielrhoades' (danielrhoades)]
http://forums.java.net/jive/thread.jspa?messageID=246377