I did a kill -3 on the pid before I killed my Glassfish server. Now when I
restart I get this error. The server worked fine for weeks. I think
someone tried to deploy something and hosed it. Running Fedora 10,
jdk1.6.0_12, Glassfish V2UR2
I've included a partial server log and the kill -3 output.
-Mike
============================================= partial server.log
=====================================================
[#|2009-03-04T17:29:07.725-0500|WARNING|sun-appserver9.1|javax.enterprise.system.core|_ThreadID=10;_ThreadName=main;_RequestID=d03e1fc5-decf-4707-b12d-8edadb1bd787;|CORE5061:
Exception :
java.lang.NullPointerException
at
com.sun.enterprise.util.InvocationManagerImpl.preInvoke(InvocationManagerImpl.java:132)
at
com.sun.appserv.server.ServerLifecycleModule.preInvoke(ServerLifecycleModule.java:178)
at
com.sun.appserv.server.ServerLifecycleModule.postEvent(ServerLifecycleModule.java:219)
at
com.sun.appserv.server.ServerLifecycleModule.onShutdown(ServerLifecycleModule.java:273)
at
com.sun.appserv.server.LifecycleModuleService.onShutdown(LifecycleModuleService.java:487)
at
com.sun.enterprise.server.ApplicationServer.onShutdown(ApplicationServer.java:549)
at
com.sun.enterprise.server.ondemand.OnDemandServer.onShutdown(OnDemandServer.java:131)
at com.sun.enterprise.server.PEMain.run(PEMain.java:429)
at com.sun.enterprise.server.PEMain.main(PEMain.java:338)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at com.sun.enterprise.server.PELaunch.main(PELaunch.java:412)
|#]
[#|2009-03-04T17:29:07.729-0500|WARNING|sun-appserver9.1|javax.enterprise.system.core|_ThreadID=10;_ThreadName=main;_RequestID=d03e1fc5-decf-4707-b12d-8edadb1bd787;|CORE5061:
Exception :
com.sun.appserv.server.ServerLifecycleException
at
com.sun.enterprise.admin.server.core.JmxConnectorLifecycle.onShutdown(JmxConnectorLifecycle.java:171)
at
com.sun.enterprise.server.ApplicationServer.onShutdown(ApplicationServer.java:549)
at
com.sun.enterprise.server.ondemand.OnDemandServer.onShutdown(OnDemandServer.java:131)
at com.sun.enterprise.server.PEMain.run(PEMain.java:429)
at com.sun.enterprise.server.PEMain.main(PEMain.java:338)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at com.sun.enterprise.server.PELaunch.main(PELaunch.java:412)
|#]
[#|2009-03-04T17:29:07.731-0500|FINE|sun-appserver9.1|javax.enterprise.system.tools.admin|_ThreadID=10;_ThreadName=main;ClassName=com.sun.enterprise.admin.server.core.ManualChangeTracker;MethodName=stop;_RequestID=d03e1fc5-decf-4707-b12d-8edadb1bd787;|ADM0103:
Stopping the thread for tracking manual changes|#]
[#|2009-03-04T17:29:07.731-0500|WARNING|sun-appserver9.1|javax.enterprise.system.tools.admin|_ThreadID=10;_ThreadName=main;_RequestID=d03e1fc5-decf-4707-b12d-8edadb1bd787;|core.tmp_folder_deletion_failed|#]
[#|2009-03-04T17:29:07.731-0500|FINE|sun-appserver9.1|javax.enterprise.resource.resourceadapter|_ThreadID=10;_ThreadName=main;ClassName=com.sun.enterprise.resource.PoolManagerImpl;MethodName=logFine;_RequestID=d03e1fc5-decf-4707-b12d-8edadb1bd787;|---Killing
all pools|#]
[#|2009-03-04T17:29:07.869-0500|FINE|sun-appserver9.1|javax.enterprise.resource.resourceadapter|_ThreadID=10;_ThreadName=main;ClassName=com.sun.enterprise.resource.PoolManagerImpl;MethodName=stopEmbeddedDerby;_RequestID=d03e1fc5-decf-4707-b12d-8edadb1bd787;|RAR7104:
Exception while stopping Derby :
java.sql.SQLException: Derby system shutdown.
at
com.sun.enterprise.resource.PoolManagerImpl.stopEmbeddedDerby(PoolManagerImpl.java:887)
at
com.sun.enterprise.resource.PoolManagerImpl.killAllPools(PoolManagerImpl.java:874)
at
com.sun.enterprise.server.ApplicationServer.onTermination(ApplicationServer.java:574)
at com.sun.enterprise.server.PEMain.run(PEMain.java:434)
at com.sun.enterprise.server.PEMain.main(PEMain.java:338)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at com.sun.enterprise.server.PELaunch.main(PELaunch.java:412)
Caused by: java.sql.SQLException: Derby system shutdown.
at
org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown
Source)
at
org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown
Source)
at
org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown
Source)
at org.apache.derby.impl.jdbc.Util.newEmbedSQLException(Unknown Source)
at org.apache.derby.impl.jdbc.Util.newEmbedSQLException(Unknown Source)
at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown
Source)
at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source)
at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source)
at java.sql.DriverManager.getConnection(DriverManager.java:582)
at java.sql.DriverManager.getConnection(DriverManager.java:207)
at
com.sun.enterprise.resource.PoolManagerImpl.stopEmbeddedDerby(PoolManagerImpl.java:880)
... 9 more
|#]
[#|2009-03-04T17:29:07.870-0500|WARNING|sun-appserver9.1|javax.enterprise.system.core|_ThreadID=10;_ThreadName=main;_RequestID=d03e1fc5-decf-4707-b12d-8edadb1bd787;|CORE5061:
Exception :
java.lang.NullPointerException
at
com.sun.enterprise.util.InvocationManagerImpl.preInvoke(InvocationManagerImpl.java:132)
at
com.sun.appserv.server.ServerLifecycleModule.preInvoke(ServerLifecycleModule.java:178)
at
com.sun.appserv.server.ServerLifecycleModule.postEvent(ServerLifecycleModule.java:219)
at
com.sun.appserv.server.ServerLifecycleModule.onTermination(ServerLifecycleModule.java:277)
at
com.sun.appserv.server.LifecycleModuleService.onTermination(LifecycleModuleService.java:496)
at
com.sun.enterprise.server.ApplicationServer.onTermination(ApplicationServer.java:587)
at com.sun.enterprise.server.PEMain.run(PEMain.java:434)
at com.sun.enterprise.server.PEMain.main(PEMain.java:338)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at com.sun.enterprise.server.PELaunch.main(PELaunch.java:412)
|#]
[#|2009-03-04T17:29:07.871-0500|SEVERE|sun-appserver9.1|javax.enterprise.system.core|_ThreadID=10;_ThreadName=main;_RequestID=d03e1fc5-decf-4707-b12d-8edadb1bd787;|Server
stopped due to Server startup failure.|#]
============================================= kill -3
=========================================================
[prod_at_sol1 ~]$ /opt/glassfish/bin/asadmin start-domain
Starting Domain domain1, please wait.
Log redirected to /opt/glassfish/domains/domain1/logs/server.log.
2009-03-04 17:04:00
Full thread dump Java HotSpot(TM) 64-Bit Server VM (11.2-b01 mixed mode):
"RMI RenewClean-[127.0.0.1:39292
,com.sun.enterprise.admin.server.core.channel.LocalRMIClientSocketFactory_at_52]"
daemon prio=10 tid=0x00007f6ecc1b0400 nid=0x1877 runnable
[0x000000001562a000..0x000000001562acf0]
java.lang.Thread.State: RUNNABLE
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.read(SocketInputStream.java:129)
at java.io.BufferedInputStream.fill(BufferedInputStream.java:218)
at java.io.BufferedInputStream.read(BufferedInputStream.java:237)
- locked <0x00007f6f28d76f00> (a java.io.BufferedInputStream)
at java.io.DataInputStream.readByte(DataInputStream.java:248)
at
sun.rmi.transport.tcp.TCPChannel.createConnection(TCPChannel.java:228)
at sun.rmi.transport.tcp.TCPChannel.newConnection(TCPChannel.java:184)
at sun.rmi.server.UnicastRef.newCall(UnicastRef.java:322)
at sun.rmi.transport.DGCImpl_Stub.dirty(Unknown Source)
at
sun.rmi.transport.DGCClient$EndpointEntry.makeDirtyCall(DGCClient.java:342)
at
sun.rmi.transport.DGCClient$EndpointEntry.access$1600(DGCClient.java:153)
at
sun.rmi.transport.DGCClient$EndpointEntry$RenewCleanThread.run(DGCClient.java:555)
at java.lang.Thread.run(Thread.java:619)
"GC Daemon" daemon prio=10 tid=0x00007f6ecc289000 nid=0x1866 in
Object.wait() [0x0000000001e6a000..0x0000000001e6ad70]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00007f6ed562abe8> (a sun.misc.GC$LatencyLock)
at sun.misc.GC$Daemon.run(GC.java:100)
- locked <0x00007f6ed562abe8> (a sun.misc.GC$LatencyLock)
"RMI RenewClean-[127.0.0.1:39292
,com.sun.enterprise.admin.server.core.channel.LocalRMIClientSocketFactory_at_42]"
daemon prio=10 tid=0x00007f6ecc07e400 nid=0x1865 runnable
[0x0000000001d69000..0x0000000001d69bf0]
java.lang.Thread.State: RUNNABLE
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.read(SocketInputStream.java:129)
at java.io.BufferedInputStream.fill(BufferedInputStream.java:218)
at java.io.BufferedInputStream.read(BufferedInputStream.java:237)
- locked <0x00007f6f28cdb9b8> (a java.io.BufferedInputStream)
at java.io.DataInputStream.readByte(DataInputStream.java:248)
at
sun.rmi.transport.tcp.TCPChannel.createConnection(TCPChannel.java:228)
at sun.rmi.transport.tcp.TCPChannel.newConnection(TCPChannel.java:184)
at sun.rmi.server.UnicastRef.newCall(UnicastRef.java:322)
at sun.rmi.transport.DGCImpl_Stub.dirty(Unknown Source)
at
sun.rmi.transport.DGCClient$EndpointEntry.makeDirtyCall(DGCClient.java:342)
at
sun.rmi.transport.DGCClient$EndpointEntry.access$1600(DGCClient.java:153)
at
sun.rmi.transport.DGCClient$EndpointEntry$RenewCleanThread.run(DGCClient.java:555)
at java.lang.Thread.run(Thread.java:619)
"Low Memory Detector" daemon prio=10 tid=0x000000004126fc00 nid=0x1862
runnable [0x0000000000000000..0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"CompilerThread1" daemon prio=10 tid=0x000000004126cc00 nid=0x1861 waiting
on condition [0x0000000000000000..0x0000000001b511f0]
java.lang.Thread.State: RUNNABLE
"CompilerThread0" daemon prio=10 tid=0x000000004126a400 nid=0x1860 waiting
on condition [0x0000000000000000..0x0000000001a502b0]
java.lang.Thread.State: RUNNABLE
"Signal Dispatcher" daemon prio=10 tid=0x0000000041268800 nid=0x185f waiting
on condition [0x0000000000000000..0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Finalizer" daemon prio=10 tid=0x000000004124a400 nid=0x185e in
Object.wait() [0x0000000001950000..0x0000000001950d70]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00007f6ed562adf0> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:116)
- locked <0x00007f6ed562adf0> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:132)
at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159)
"Reference Handler" daemon prio=10 tid=0x0000000041243000 nid=0x185d in
Object.wait() [0x000000000a8b9000..0x000000000a8b9bf0]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00007f6ed562abd8> (a java.lang.ref.Reference$Lock)
at java.lang.Object.wait(Object.java:485)
at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116)
- locked <0x00007f6ed562abd8> (a java.lang.ref.Reference$Lock)
"main" prio=10 tid=0x00000000411b7c00 nid=0x1857 runnable
[0x0000000000826000..0x0000000000826eb0]
java.lang.Thread.State: RUNNABLE
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.read(SocketInputStream.java:129)
at java.io.BufferedInputStream.fill(BufferedInputStream.java:218)
at java.io.BufferedInputStream.read(BufferedInputStream.java:237)
- locked <0x00007f6f28be22a8> (a java.io.BufferedInputStream)
at java.io.DataInputStream.readByte(DataInputStream.java:248)
at
sun.rmi.transport.tcp.TCPChannel.createConnection(TCPChannel.java:228)
at sun.rmi.transport.tcp.TCPChannel.newConnection(TCPChannel.java:184)
at sun.rmi.server.UnicastRef.invoke(UnicastRef.java:110)
at
com.sun.enterprise.admin.server.core.channel.AdminChannelServer_Stub.getServerStatusCode(Unknown
Source)
at
com.sun.enterprise.admin.server.core.channel.RMIClient.getInstanceStatusCode(RMIClient.java:278)
at
com.sun.enterprise.admin.servermgmt.pe.PEInstancesManager.getInstanceStatus(PEInstancesManager.java:785)
at
com.sun.enterprise.cli.commands.StartDomainCommand.isStopping(StartDomainCommand.java:412)
at
com.sun.enterprise.cli.commands.StartDomainCommand.checkIfStopping(StartDomainCommand.java:517)
at
com.sun.enterprise.cli.commands.StartDomainCommand.startDomain(StartDomainCommand.java:126)
at
com.sun.enterprise.cli.commands.StartDomainCommand.runCommand(StartDomainCommand.java:215)
at
com.sun.enterprise.cli.framework.CLIMain.invokeCommand(CLIMain.java:171)
at com.sun.enterprise.cli.framework.CLIMain.main(CLIMain.java:79)
"VM Thread" prio=10 tid=0x000000004123d800 nid=0x185c runnable
"GC task thread#0 (ParallelGC)" prio=10 tid=0x00000000411c2800 nid=0x1858
runnable
"GC task thread#1 (ParallelGC)" prio=10 tid=0x00000000411c4000 nid=0x1859
runnable
"GC task thread#2 (ParallelGC)" prio=10 tid=0x00000000411c5c00 nid=0x185a
runnable
"GC task thread#3 (ParallelGC)" prio=10 tid=0x00000000411c7400 nid=0x185b
runnable
"VM Periodic Task Thread" prio=10 tid=0x0000000041272000 nid=0x1863 waiting
on condition
JNI global references: 1209
Heap
PSYoungGen total 37376K, used 2386K [0x00007f6f28bc0000,
0x00007f6f2b570000, 0x00007f6f52710000)
eden space 32064K, 7% used
[0x00007f6f28bc0000,0x00007f6f28e14958,0x00007f6f2ab10000)
from space 5312K, 0% used
[0x00007f6f2ab10000,0x00007f6f2ab10000,0x00007f6f2b040000)
to space 5312K, 0% used
[0x00007f6f2b040000,0x00007f6f2b040000,0x00007f6f2b570000)
PSOldGen total 85440K, used 2101K [0x00007f6ed5510000,
0x00007f6eda880000, 0x00007f6f28bc0000)
object space 85440K, 2% used
[0x00007f6ed5510000,0x00007f6ed571d6c0,0x00007f6eda880000)
PSPermGen total 23872K, used 11971K [0x00007f6ed0110000,
0x00007f6ed1860000, 0x00007f6ed5510000)
object space 23872K, 50% used
[0x00007f6ed0110000,0x00007f6ed0cc0d18,0x00007f6ed1860000)