I rebooted my machine yesterday and this seemed to fix my problem. Some
process must have been holding onto a file, or something weird like that.
-Mike
On Wed, Mar 4, 2009 at 5:34 PM, Michael Mellinger <mmellinger66_at_gmail.com>wrote:
> I did a kill -3 on the pid before I killed my Glassfish server. Now when I
> restart I get this error. The server worked fine for weeks. I think
> someone tried to deploy something and hosed it. Running Fedora 10,
> jdk1.6.0_12, Glassfish V2UR2
>
> I've included a partial server log and the kill -3 output.
>
> -Mike
>
> ============================================= partial server.log
> =====================================================
> [#|2009-03-04T17:29:07.725-0500|WARNING|sun-appserver9.1|javax.enterprise.system.core|_ThreadID=10;_ThreadName=main;_RequestID=d03e1fc5-decf-4707-b12d-8edadb1bd787;|CORE5061:
> Exception :
> java.lang.NullPointerException
> at
> com.sun.enterprise.util.InvocationManagerImpl.preInvoke(InvocationManagerImpl.java:132)
> at
> com.sun.appserv.server.ServerLifecycleModule.preInvoke(ServerLifecycleModule.java:178)
> at
> com.sun.appserv.server.ServerLifecycleModule.postEvent(ServerLifecycleModule.java:219)
> at
> com.sun.appserv.server.ServerLifecycleModule.onShutdown(ServerLifecycleModule.java:273)
> at
> com.sun.appserv.server.LifecycleModuleService.onShutdown(LifecycleModuleService.java:487)
> at
> com.sun.enterprise.server.ApplicationServer.onShutdown(ApplicationServer.java:549)
> at
> com.sun.enterprise.server.ondemand.OnDemandServer.onShutdown(OnDemandServer.java:131)
> at com.sun.enterprise.server.PEMain.run(PEMain.java:429)
> at com.sun.enterprise.server.PEMain.main(PEMain.java:338)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> at java.lang.reflect.Method.invoke(Method.java:597)
> at com.sun.enterprise.server.PELaunch.main(PELaunch.java:412)
> |#]
>
> [#|2009-03-04T17:29:07.729-0500|WARNING|sun-appserver9.1|javax.enterprise.system.core|_ThreadID=10;_ThreadName=main;_RequestID=d03e1fc5-decf-4707-b12d-8edadb1bd787;|CORE5061:
> Exception :
> com.sun.appserv.server.ServerLifecycleException
> at
> com.sun.enterprise.admin.server.core.JmxConnectorLifecycle.onShutdown(JmxConnectorLifecycle.java:171)
> at
> com.sun.enterprise.server.ApplicationServer.onShutdown(ApplicationServer.java:549)
> at
> com.sun.enterprise.server.ondemand.OnDemandServer.onShutdown(OnDemandServer.java:131)
> at com.sun.enterprise.server.PEMain.run(PEMain.java:429)
> at com.sun.enterprise.server.PEMain.main(PEMain.java:338)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> at java.lang.reflect.Method.invoke(Method.java:597)
> at com.sun.enterprise.server.PELaunch.main(PELaunch.java:412)
> |#]
>
> [#|2009-03-04T17:29:07.731-0500|FINE|sun-appserver9.1|javax.enterprise.system.tools.admin|_ThreadID=10;_ThreadName=main;ClassName=com.sun.enterprise.admin.server.core.ManualChangeTracker;MethodName=stop;_RequestID=d03e1fc5-decf-4707-b12d-8edadb1bd787;|ADM0103:
> Stopping the thread for tracking manual changes|#]
>
>
> [#|2009-03-04T17:29:07.731-0500|WARNING|sun-appserver9.1|javax.enterprise.system.tools.admin|_ThreadID=10;_ThreadName=main;_RequestID=d03e1fc5-decf-4707-b12d-8edadb1bd787;|core.tmp_folder_deletion_failed|#]
>
> [#|2009-03-04T17:29:07.731-0500|FINE|sun-appserver9.1|javax.enterprise.resource.resourceadapter|_ThreadID=10;_ThreadName=main;ClassName=com.sun.enterprise.resource.PoolManagerImpl;MethodName=logFine;_RequestID=d03e1fc5-decf-4707-b12d-8edadb1bd787;|---Killing
> all pools|#]
>
> [#|2009-03-04T17:29:07.869-0500|FINE|sun-appserver9.1|javax.enterprise.resource.resourceadapter|_ThreadID=10;_ThreadName=main;ClassName=com.sun.enterprise.resource.PoolManagerImpl;MethodName=stopEmbeddedDerby;_RequestID=d03e1fc5-decf-4707-b12d-8edadb1bd787;|RAR7104:
> Exception while stopping Derby :
> java.sql.SQLException: Derby system shutdown.
> at
> com.sun.enterprise.resource.PoolManagerImpl.stopEmbeddedDerby(PoolManagerImpl.java:887)
> at
> com.sun.enterprise.resource.PoolManagerImpl.killAllPools(PoolManagerImpl.java:874)
> at
> com.sun.enterprise.server.ApplicationServer.onTermination(ApplicationServer.java:574)
> at com.sun.enterprise.server.PEMain.run(PEMain.java:434)
> at com.sun.enterprise.server.PEMain.main(PEMain.java:338)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> at java.lang.reflect.Method.invoke(Method.java:597)
> at com.sun.enterprise.server.PELaunch.main(PELaunch.java:412)
> Caused by: java.sql.SQLException: Derby system shutdown.
> at
> org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown
> Source)
> at
> org.apache.derby.impl.jdbc.SQLExceptionFactory40.wrapArgsForTransportAcrossDRDA(Unknown
> Source)
> at
> org.apache.derby.impl.jdbc.SQLExceptionFactory40.getSQLException(Unknown
> Source)
> at org.apache.derby.impl.jdbc.Util.newEmbedSQLException(Unknown Source)
> at org.apache.derby.impl.jdbc.Util.newEmbedSQLException(Unknown Source)
> at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown
> Source)
> at org.apache.derby.jdbc.InternalDriver.connect(Unknown Source)
> at org.apache.derby.jdbc.AutoloadedDriver.connect(Unknown Source)
> at java.sql.DriverManager.getConnection(DriverManager.java:582)
> at java.sql.DriverManager.getConnection(DriverManager.java:207)
> at
> com.sun.enterprise.resource.PoolManagerImpl.stopEmbeddedDerby(PoolManagerImpl.java:880)
> ... 9 more
> |#]
>
> [#|2009-03-04T17:29:07.870-0500|WARNING|sun-appserver9.1|javax.enterprise.system.core|_ThreadID=10;_ThreadName=main;_RequestID=d03e1fc5-decf-4707-b12d-8edadb1bd787;|CORE5061:
> Exception :
> java.lang.NullPointerException
> at
> com.sun.enterprise.util.InvocationManagerImpl.preInvoke(InvocationManagerImpl.java:132)
> at
> com.sun.appserv.server.ServerLifecycleModule.preInvoke(ServerLifecycleModule.java:178)
> at
> com.sun.appserv.server.ServerLifecycleModule.postEvent(ServerLifecycleModule.java:219)
> at
> com.sun.appserv.server.ServerLifecycleModule.onTermination(ServerLifecycleModule.java:277)
> at
> com.sun.appserv.server.LifecycleModuleService.onTermination(LifecycleModuleService.java:496)
> at
> com.sun.enterprise.server.ApplicationServer.onTermination(ApplicationServer.java:587)
> at com.sun.enterprise.server.PEMain.run(PEMain.java:434)
> at com.sun.enterprise.server.PEMain.main(PEMain.java:338)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> at java.lang.reflect.Method.invoke(Method.java:597)
> at com.sun.enterprise.server.PELaunch.main(PELaunch.java:412)
> |#]
>
> [#|2009-03-04T17:29:07.871-0500|SEVERE|sun-appserver9.1|javax.enterprise.system.core|_ThreadID=10;_ThreadName=main;_RequestID=d03e1fc5-decf-4707-b12d-8edadb1bd787;|Server
> stopped due to Server startup failure.|#]
>
> ============================================= kill -3
> =========================================================
> [prod_at_sol1 ~]$ /opt/glassfish/bin/asadmin start-domain
> Starting Domain domain1, please wait.
> Log redirected to /opt/glassfish/domains/domain1/logs/server.log.
>
> 2009-03-04 17:04:00
> Full thread dump Java HotSpot(TM) 64-Bit Server VM (11.2-b01 mixed mode):
>
> "RMI RenewClean-[127.0.0.1:39292
> ,com.sun.enterprise.admin.server.core.channel.LocalRMIClientSocketFactory_at_52]"
> daemon prio=10 tid=0x00007f6ecc1b0400 nid=0x1877 runnable
> [0x000000001562a000..0x000000001562acf0]
> java.lang.Thread.State: RUNNABLE
> at java.net.SocketInputStream.socketRead0(Native Method)
> at java.net.SocketInputStream.read(SocketInputStream.java:129)
> at java.io.BufferedInputStream.fill(BufferedInputStream.java:218)
> at java.io.BufferedInputStream.read(BufferedInputStream.java:237)
> - locked <0x00007f6f28d76f00> (a java.io.BufferedInputStream)
> at java.io.DataInputStream.readByte(DataInputStream.java:248)
> at
> sun.rmi.transport.tcp.TCPChannel.createConnection(TCPChannel.java:228)
> at sun.rmi.transport.tcp.TCPChannel.newConnection(TCPChannel.java:184)
> at sun.rmi.server.UnicastRef.newCall(UnicastRef.java:322)
> at sun.rmi.transport.DGCImpl_Stub.dirty(Unknown Source)
> at
> sun.rmi.transport.DGCClient$EndpointEntry.makeDirtyCall(DGCClient.java:342)
> at
> sun.rmi.transport.DGCClient$EndpointEntry.access$1600(DGCClient.java:153)
> at
> sun.rmi.transport.DGCClient$EndpointEntry$RenewCleanThread.run(DGCClient.java:555)
> at java.lang.Thread.run(Thread.java:619)
>
> "GC Daemon" daemon prio=10 tid=0x00007f6ecc289000 nid=0x1866 in
> Object.wait() [0x0000000001e6a000..0x0000000001e6ad70]
> java.lang.Thread.State: TIMED_WAITING (on object monitor)
> at java.lang.Object.wait(Native Method)
> - waiting on <0x00007f6ed562abe8> (a sun.misc.GC$LatencyLock)
> at sun.misc.GC$Daemon.run(GC.java:100)
> - locked <0x00007f6ed562abe8> (a sun.misc.GC$LatencyLock)
>
> "RMI RenewClean-[127.0.0.1:39292
> ,com.sun.enterprise.admin.server.core.channel.LocalRMIClientSocketFactory_at_42]"
> daemon prio=10 tid=0x00007f6ecc07e400 nid=0x1865 runnable
> [0x0000000001d69000..0x0000000001d69bf0]
> java.lang.Thread.State: RUNNABLE
> at java.net.SocketInputStream.socketRead0(Native Method)
> at java.net.SocketInputStream.read(SocketInputStream.java:129)
> at java.io.BufferedInputStream.fill(BufferedInputStream.java:218)
> at java.io.BufferedInputStream.read(BufferedInputStream.java:237)
> - locked <0x00007f6f28cdb9b8> (a java.io.BufferedInputStream)
> at java.io.DataInputStream.readByte(DataInputStream.java:248)
> at
> sun.rmi.transport.tcp.TCPChannel.createConnection(TCPChannel.java:228)
> at sun.rmi.transport.tcp.TCPChannel.newConnection(TCPChannel.java:184)
> at sun.rmi.server.UnicastRef.newCall(UnicastRef.java:322)
> at sun.rmi.transport.DGCImpl_Stub.dirty(Unknown Source)
> at
> sun.rmi.transport.DGCClient$EndpointEntry.makeDirtyCall(DGCClient.java:342)
> at
> sun.rmi.transport.DGCClient$EndpointEntry.access$1600(DGCClient.java:153)
> at
> sun.rmi.transport.DGCClient$EndpointEntry$RenewCleanThread.run(DGCClient.java:555)
> at java.lang.Thread.run(Thread.java:619)
>
> "Low Memory Detector" daemon prio=10 tid=0x000000004126fc00 nid=0x1862
> runnable [0x0000000000000000..0x0000000000000000]
> java.lang.Thread.State: RUNNABLE
>
> "CompilerThread1" daemon prio=10 tid=0x000000004126cc00 nid=0x1861 waiting
> on condition [0x0000000000000000..0x0000000001b511f0]
> java.lang.Thread.State: RUNNABLE
>
> "CompilerThread0" daemon prio=10 tid=0x000000004126a400 nid=0x1860 waiting
> on condition [0x0000000000000000..0x0000000001a502b0]
> java.lang.Thread.State: RUNNABLE
>
> "Signal Dispatcher" daemon prio=10 tid=0x0000000041268800 nid=0x185f
> waiting on condition [0x0000000000000000..0x0000000000000000]
> java.lang.Thread.State: RUNNABLE
>
> "Finalizer" daemon prio=10 tid=0x000000004124a400 nid=0x185e in
> Object.wait() [0x0000000001950000..0x0000000001950d70]
> java.lang.Thread.State: WAITING (on object monitor)
> at java.lang.Object.wait(Native Method)
> - waiting on <0x00007f6ed562adf0> (a java.lang.ref.ReferenceQueue$Lock)
> at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:116)
> - locked <0x00007f6ed562adf0> (a java.lang.ref.ReferenceQueue$Lock)
> at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:132)
> at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159)
>
> "Reference Handler" daemon prio=10 tid=0x0000000041243000 nid=0x185d in
> Object.wait() [0x000000000a8b9000..0x000000000a8b9bf0]
> java.lang.Thread.State: WAITING (on object monitor)
> at java.lang.Object.wait(Native Method)
> - waiting on <0x00007f6ed562abd8> (a java.lang.ref.Reference$Lock)
> at java.lang.Object.wait(Object.java:485)
> at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116)
> - locked <0x00007f6ed562abd8> (a java.lang.ref.Reference$Lock)
>
> "main" prio=10 tid=0x00000000411b7c00 nid=0x1857 runnable
> [0x0000000000826000..0x0000000000826eb0]
> java.lang.Thread.State: RUNNABLE
> at java.net.SocketInputStream.socketRead0(Native Method)
> at java.net.SocketInputStream.read(SocketInputStream.java:129)
> at java.io.BufferedInputStream.fill(BufferedInputStream.java:218)
> at java.io.BufferedInputStream.read(BufferedInputStream.java:237)
> - locked <0x00007f6f28be22a8> (a java.io.BufferedInputStream)
> at java.io.DataInputStream.readByte(DataInputStream.java:248)
> at
> sun.rmi.transport.tcp.TCPChannel.createConnection(TCPChannel.java:228)
> at sun.rmi.transport.tcp.TCPChannel.newConnection(TCPChannel.java:184)
> at sun.rmi.server.UnicastRef.invoke(UnicastRef.java:110)
> at
> com.sun.enterprise.admin.server.core.channel.AdminChannelServer_Stub.getServerStatusCode(Unknown
> Source)
> at
> com.sun.enterprise.admin.server.core.channel.RMIClient.getInstanceStatusCode(RMIClient.java:278)
> at
> com.sun.enterprise.admin.servermgmt.pe.PEInstancesManager.getInstanceStatus(PEInstancesManager.java:785)
> at
> com.sun.enterprise.cli.commands.StartDomainCommand.isStopping(StartDomainCommand.java:412)
> at
> com.sun.enterprise.cli.commands.StartDomainCommand.checkIfStopping(StartDomainCommand.java:517)
> at
> com.sun.enterprise.cli.commands.StartDomainCommand.startDomain(StartDomainCommand.java:126)
> at
> com.sun.enterprise.cli.commands.StartDomainCommand.runCommand(StartDomainCommand.java:215)
> at
> com.sun.enterprise.cli.framework.CLIMain.invokeCommand(CLIMain.java:171)
> at com.sun.enterprise.cli.framework.CLIMain.main(CLIMain.java:79)
>
> "VM Thread" prio=10 tid=0x000000004123d800 nid=0x185c runnable
>
> "GC task thread#0 (ParallelGC)" prio=10 tid=0x00000000411c2800 nid=0x1858
> runnable
>
> "GC task thread#1 (ParallelGC)" prio=10 tid=0x00000000411c4000 nid=0x1859
> runnable
>
> "GC task thread#2 (ParallelGC)" prio=10 tid=0x00000000411c5c00 nid=0x185a
> runnable
>
> "GC task thread#3 (ParallelGC)" prio=10 tid=0x00000000411c7400 nid=0x185b
> runnable
>
> "VM Periodic Task Thread" prio=10 tid=0x0000000041272000 nid=0x1863 waiting
> on condition
>
> JNI global references: 1209
>
> Heap
> PSYoungGen total 37376K, used 2386K [0x00007f6f28bc0000,
> 0x00007f6f2b570000, 0x00007f6f52710000)
> eden space 32064K, 7% used
> [0x00007f6f28bc0000,0x00007f6f28e14958,0x00007f6f2ab10000)
> from space 5312K, 0% used
> [0x00007f6f2ab10000,0x00007f6f2ab10000,0x00007f6f2b040000)
> to space 5312K, 0% used
> [0x00007f6f2b040000,0x00007f6f2b040000,0x00007f6f2b570000)
> PSOldGen total 85440K, used 2101K [0x00007f6ed5510000,
> 0x00007f6eda880000, 0x00007f6f28bc0000)
> object space 85440K, 2% used
> [0x00007f6ed5510000,0x00007f6ed571d6c0,0x00007f6eda880000)
> PSPermGen total 23872K, used 11971K [0x00007f6ed0110000,
> 0x00007f6ed1860000, 0x00007f6ed5510000)
> object space 23872K, 50% used
> [0x00007f6ed0110000,0x00007f6ed0cc0d18,0x00007f6ed1860000)
>
>