[#|2012-01-04T17:08:37.630-0800|WARNING|glassfish3.1.2|com.sun.grizzly.config.GrizzlyServiceListener|_ThreadID=13;_ThreadName=Thread-2;|GRIZZLY0023: Interrupting idle Thread: admin-thread-pool-4848(563).|#] [#|2012-01-04T17:08:37.631-0800|WARNING|glassfish3.1.2|com.sun.grizzly.config.GrizzlyServiceListener|_ThreadID=13;_ThreadName=Thread-2;|GRIZZLY0023: Interrupting idle Thread: admin-thread-pool-4848(565).|#] [#|2012-01-04T17:08:38.640-0800|WARNING|glassfish3.1.2|com.sun.grizzly.config.GrizzlyServiceListener|_ThreadID=13;_ThreadName=Thread-2;|GRIZZLY0023: Interrupting idle Thread: admin-thread-pool-4848(563).|#] [#|2012-01-04T17:08:38.641-0800|WARNING|glassfish3.1.2|com.sun.grizzly.config.GrizzlyServiceListener|_ThreadID=13;_ThreadName=Thread-2;|GRIZZLY0023: Interrupting idle Thread: admin-thread-pool-4848(565).|#] [#|2012-01-04T17:08:38.649-0800|WARNING|glassfish3.1.2|javax.enterprise.system.container.web.com.sun.enterprise.web|_ThreadID=165;_ThreadName=Thread-2;|StandardWrapperValve[DownloadServlet]: PWC1406: Servlet.service() for servlet DownloadServlet threw exception java.lang.RuntimeException: com.sun.jersey.api.client.ClientHandlerException: java.io.InterruptedIOException: Operation interrupted at org.glassfish.admingui.common.servlet.LogFilesContentSource.getInputStream(LogFilesContentSource.java:110) at org.glassfish.admingui.common.servlet.DownloadServlet.writeContent(DownloadServlet.java:277) at org.glassfish.admingui.common.servlet.DownloadServlet.doPost(DownloadServlet.java:174) at org.glassfish.admingui.common.servlet.DownloadServlet.doGet(DownloadServlet.java:155) at javax.servlet.http.HttpServlet.service(HttpServlet.java:668) at javax.servlet.http.HttpServlet.service(HttpServlet.java:770) at org.apache.catalina.core.StandardWrapper.service(StandardWrapper.java:1542) at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:281) at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:175) at org.apache.catalina.core.StandardPipeline.doInvoke(StandardPipeline.java:655) at org.apache.catalina.core.StandardPipeline.invoke(StandardPipeline.java:595) at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:161) at org.apache.catalina.connector.CoyoteAdapter.doService(CoyoteAdapter.java:331) at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:231) at com.sun.enterprise.v3.services.impl.ContainerMapper.service(ContainerMapper.java:232) at com.sun.grizzly.http.ProcessorTask.invokeAdapter(ProcessorTask.java:849) at com.sun.grizzly.http.ProcessorTask.doProcess(ProcessorTask.java:746) at com.sun.grizzly.http.ProcessorTask.process(ProcessorTask.java:1045) at com.sun.grizzly.http.DefaultProtocolFilter.execute(DefaultProtocolFilter.java:228) at com.sun.grizzly.DefaultProtocolChain.executeProtocolFilter(DefaultProtocolChain.java:137) at com.sun.grizzly.DefaultProtocolChain.execute(DefaultProtocolChain.java:104) at com.sun.grizzly.DefaultProtocolChain.execute(DefaultProtocolChain.java:90) at com.sun.grizzly.http.HttpProtocolChain.execute(HttpProtocolChain.java:79) at com.sun.grizzly.ProtocolChainContextTask.doCall(ProtocolChainContextTask.java:54) at com.sun.grizzly.SelectionKeyContextTask.call(SelectionKeyContextTask.java:59) at com.sun.grizzly.ContextTask.run(ContextTask.java:71) at com.sun.grizzly.util.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:532) at com.sun.grizzly.util.AbstractThreadPool$Worker.run(AbstractThreadPool.java:513) at java.lang.Thread.run(Thread.java:662) Caused by: com.sun.jersey.api.client.ClientHandlerException: java.io.InterruptedIOException: Operation interrupted at com.sun.jersey.client.urlconnection.URLConnectionClientHandler.handle(URLConnectionClientHandler.java:149) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.Client.handle(Client.java:648) at com.sun.jersey.api.client.WebResource.handle(WebResource.java:670) at com.sun.jersey.api.client.WebResource.access$200(WebResource.java:74) at com.sun.jersey.api.client.WebResource$Builder.post(WebResource.java:563) at org.glassfish.admingui.common.util.RestUtil.postRestRequestFromServlet(RestUtil.java:712) at org.glassfish.admingui.common.servlet.LogFilesContentSource.getInputStream(LogFilesContentSource.java:106) ... 28 more Caused by: java.io.InterruptedIOException: Operation interrupted at java.net.SocketInputStream.socketRead0(Native Method) at java.net.SocketInputStream.read(SocketInputStream.java:129) at com.sun.net.ssl.internal.ssl.InputRecord.readFully(InputRecord.java:293) at com.sun.net.ssl.internal.ssl.InputRecord.read(InputRecord.java:331) at com.sun.net.ssl.internal.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:798) at com.sun.net.ssl.internal.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:755) at com.sun.net.ssl.internal.ssl.AppInputStream.read(AppInputStream.java:75) at java.io.BufferedInputStream.fill(BufferedInputStream.java:218) at java.io.BufferedInputStream.read1(BufferedInputStream.java:258) at java.io.BufferedInputStream.read(BufferedInputStream.java:317) at sun.net.www.http.HttpClient.parseHTTPHeader(HttpClient.java:695) at sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:640) at sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:660) at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1195) at java.net.HttpURLConnection.getResponseCode(HttpURLConnection.java:379) at sun.net.www.protocol.https.HttpsURLConnectionImpl.getResponseCode(HttpsURLConnectionImpl.java:318) at com.sun.jersey.client.urlconnection.URLConnectionClientHandler._invoke(URLConnectionClientHandler.java:240) at com.sun.jersey.client.urlconnection.URLConnectionClientHandler.handle(URLConnectionClientHandler.java:147) ... 84 more |#] [#|2012-01-04T17:23:38.450-0800|WARNING|glassfish3.1.2|com.sun.grizzly.config.GrizzlyServiceListener|_ThreadID=13;_ThreadName=Thread-2;|GRIZZLY0023: Interrupting idle Thread: admin-thread-pool-4848(4).|#] [#|2012-01-04T17:23:39.460-0800|WARNING|glassfish3.1.2|com.sun.grizzly.config.GrizzlyServiceListener|_ThreadID=13;_ThreadName=Thread-2;|GRIZZLY0023: Interrupting idle Thread: admin-thread-pool-4848(4).|#] [#|2012-01-04T17:23:56.974-0800|INFO|glassfish3.1.2|javax.enterprise.system.std.com.sun.enterprise.server.logging|_ThreadID=22;_ThreadName=Thread-2;|Stopping Cluster ....|#] [#|2012-01-04T17:23:56.977-0800|INFO|glassfish3.1.2|org.glassfish.admingui|_ThreadID=22;_ThreadName=Thread-2;|https://localhost:4848/management/domain/clusters/cluster/cld/stop-cluster|#] [#|2012-01-04T17:23:57.004-0800|INFO|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=166;_ThreadName=Thread-2;|Stopping cluster cld|#] [#|2012-01-04T17:23:57.015-0800|INFO|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=166;_ThreadName=Thread-2;|Executing stop-instance on 2 instances using a thread pool of size 2: jedy:cld2 jedy:cld1|#] [#|2012-01-04T17:23:57.016-0800|INFO|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=166;_ThreadName=Thread-2;|stop-instance cld2|#] [#|2012-01-04T17:23:57.018-0800|INFO|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=166;_ThreadName=Thread-2;|stop-instance cld1|#] [#|2012-01-04T17:23:57.035-0800|INFO|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=167;_ThreadName=Thread-2;|Instance cld1 shutdown initiated|#] [#|2012-01-04T17:23:57.036-0800|INFO|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=168;_ThreadName=Thread-2;|Instance cld2 shutdown initiated|#] [#|2012-01-04T17:24:33.066-0800|INFO|glassfish3.1.2|javax.enterprise.system.core.org.glassfish.gms.bootstrap|_ThreadID=169;_ThreadName=Thread-2;|GMSBS2001: Adding instance clt2 to health history table.|#] [#|2012-01-04T17:24:36.233-0800|INFO|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=170;_ThreadName=Thread-2;|Command _validate-das-options executed successfully.|#] [#|2012-01-04T17:24:38.553-0800|INFO|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=170;_ThreadName=Thread-2;|Using DAS host lancerl and port 4848 from existing das.properties for node tuppy. To use a different DAS, create a new node using create-node-ssh or create-node-config. Create the instance with the new node and correct host and port: asadmin --host das_host --port das_port create-local-instance --node node_name instance_name. Command _create-instance-filesystem executed successfully.|#] [#|2012-01-04T17:25:12.591-0800|INFO|glassfish3.1.2|javax.enterprise.system.std.com.sun.enterprise.server.logging|_ThreadID=171;_ThreadName=Thread-2;|Starting Cluster ....|#] [#|2012-01-04T17:25:12.594-0800|INFO|glassfish3.1.2|org.glassfish.admingui|_ThreadID=171;_ThreadName=Thread-2;|https://localhost:4848/management/domain/clusters/cluster/cl/start-cluster|#] [#|2012-01-04T17:25:12.622-0800|INFO|glassfish3.1.2|ShoalLogger|_ThreadID=172;_ThreadName=Thread-2;|GMS1098: GMS:Announcing GroupStartup[INITIATED] for group: cl members: clt1,cll2,clt2,|#] [#|2012-01-04T17:25:12.624-0800|INFO|glassfish3.1.2|ShoalLogger|_ThreadID=172;_ThreadName=Thread-2;|GMS1062: GroupStart for group: cl State: INITIATED Starting Members: clt1,cll2,clt2,|#] [#|2012-01-04T17:25:12.624-0800|INFO|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=172;_ThreadName=Thread-2;|Starting cluster cl|#] [#|2012-01-04T17:25:12.634-0800|INFO|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=172;_ThreadName=Thread-2;|Executing start-instance on 3 instances using a thread pool of size 3: tuppy:clt1 localhost-domain1:cll2 tuppy:clt2|#] [#|2012-01-04T17:25:12.636-0800|INFO|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=172;_ThreadName=Thread-2;|start-instance clt1|#] [#|2012-01-04T17:25:12.637-0800|INFO|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=172;_ThreadName=Thread-2;|start-instance cll2|#] [#|2012-01-04T17:25:12.638-0800|INFO|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=172;_ThreadName=Thread-2;|start-instance clt2|#] [#|2012-01-04T17:25:47.912-0800|WARNING|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin|_ThreadID=22;_ThreadName=Thread-2;|Command: _synchronize-files failed to acquire a command lock. REASON: time out (current lock acquired on Wed, 04 Jan 2012 16:11:42 PST)|#] [#|2012-01-04T17:25:47.978-0800|INFO|glassfish3.1.2|javax.enterprise.system.std.com.sun.enterprise.server.logging|_ThreadID=173;_ThreadName=Thread-2;|Warning: Synchronization with DAS failed, continuing startup...|#] [#|2012-01-04T17:25:51.027-0800|WARNING|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin|_ThreadID=174;_ThreadName=Thread-2;|Command: _synchronize-files failed to acquire a command lock. REASON: time out (current lock acquired on Wed, 04 Jan 2012 16:11:42 PST)|#] [#|2012-01-04T17:25:51.060-0800|WARNING|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin|_ThreadID=175;_ThreadName=Thread-2;|Command: _synchronize-files failed to acquire a command lock. REASON: time out (current lock acquired on Wed, 04 Jan 2012 16:11:42 PST)|#] [#|2012-01-04T17:26:14.668-0800|INFO|glassfish3.1.2|ShoalLogger|_ThreadID=60;_ThreadName=Thread-2;|GMS1092: GMS View Change Received for group: cl : Members in view for ADD_EVENT(before change analysis) are : 1: MemberId: cll2, MemberType: CORE, Address: 10.133.162.59:9197:228.9.199.152:17621:cl:cll2 2: MemberId: server, MemberType: SPECTATOR, Address: 10.133.162.59:9199:228.9.199.152:17621:cl:server |#] [#|2012-01-04T17:26:14.670-0800|INFO|glassfish3.1.2|ShoalLogger|_ThreadID=60;_ThreadName=Thread-2;|GMS1016: Analyzing new membership snapshot received as part of event: ADD_EVENT for member: cll2 of group: cl|#] [#|2012-01-04T17:26:17.717-0800|INFO|glassfish3.1.2|ShoalLogger|_ThreadID=60;_ThreadName=Thread-2;|GMS1024: Adding Join member: cll2 group: cl StartupState: GROUP_STARTUP |#] [#|2012-01-04T17:26:17.718-0800|INFO|glassfish3.1.2|ShoalLogger|_ThreadID=60;_ThreadName=Thread-2;|GMS1092: GMS View Change Received for group: cl : Members in view for ADD_EVENT(before change analysis) are : 1: MemberId: cll2, MemberType: CORE, Address: 10.133.162.59:9197:228.9.199.152:17621:cl:cll2 2: MemberId: server, MemberType: SPECTATOR, Address: 10.133.162.59:9199:228.9.199.152:17621:cl:server |#] [#|2012-01-04T17:26:17.719-0800|INFO|glassfish3.1.2|ShoalLogger|_ThreadID=60;_ThreadName=Thread-2;|GMS1016: Analyzing new membership snapshot received as part of event: ADD_EVENT for member: cll2 of group: cl|#] [#|2012-01-04T17:26:20.522-0800|INFO|glassfish3.1.2|javax.enterprise.system.std.com.sun.enterprise.server.logging|_ThreadID=173;_ThreadName=Thread-2;|Waiting for cll2 to start ............................|#] [#|2012-01-04T17:26:20.523-0800|INFO|glassfish3.1.2|javax.enterprise.system.std.com.sun.enterprise.server.logging|_ThreadID=173;_ThreadName=Thread-2;|Successfully started the instance: cll2|#] [#|2012-01-04T17:26:20.524-0800|INFO|glassfish3.1.2|javax.enterprise.system.std.com.sun.enterprise.server.logging|_ThreadID=173;_ThreadName=Thread-2;|instance Location: /export/home/j2eetest/3.1.2/glassfish3/glassfish/nodes/localhost-domain1/cll2|#] [#|2012-01-04T17:26:20.524-0800|INFO|glassfish3.1.2|javax.enterprise.system.std.com.sun.enterprise.server.logging|_ThreadID=173;_ThreadName=Thread-2;|Log File: /export/home/j2eetest/3.1.2/glassfish3/glassfish/nodes/localhost-domain1/cll2/logs/server.log|#] [#|2012-01-04T17:26:20.525-0800|INFO|glassfish3.1.2|javax.enterprise.system.std.com.sun.enterprise.server.logging|_ThreadID=173;_ThreadName=Thread-2;|Admin Port: 24849|#] [#|2012-01-04T17:26:20.526-0800|INFO|glassfish3.1.2|javax.enterprise.system.std.com.sun.enterprise.server.logging|_ThreadID=173;_ThreadName=Thread-2;|Command start-local-instance executed successfully.|#] [#|2012-01-04T17:26:20.724-0800|INFO|glassfish3.1.2|ShoalLogger|_ThreadID=60;_ThreadName=Thread-2;|GMS1024: Adding Join member: cll2 group: cl StartupState: GROUP_STARTUP |#] [#|2012-01-04T17:26:20.726-0800|INFO|glassfish3.1.2|ShoalLogger|_ThreadID=60;_ThreadName=Thread-2;|GMS1092: GMS View Change Received for group: cl : Members in view for JOINED_AND_READY_EVENT(before change analysis) are : 1: MemberId: cll2, MemberType: CORE, Address: 10.133.162.59:9197:228.9.199.152:17621:cl:cll2 2: MemberId: server, MemberType: SPECTATOR, Address: 10.133.162.59:9199:228.9.199.152:17621:cl:server |#] [#|2012-01-04T17:26:20.726-0800|INFO|glassfish3.1.2|ShoalLogger|_ThreadID=60;_ThreadName=Thread-2;|GMS1016: Analyzing new membership snapshot received as part of event: JOINED_AND_READY_EVENT for member: cll2 of group: cl|#] [#|2012-01-04T17:26:20.727-0800|INFO|glassfish3.1.2|ShoalLogger|_ThreadID=60;_ThreadName=Thread-2;|GMS1025: Adding Joined And Ready member: cll2 group: cl StartupState: GROUP_STARTUP |#] [#|2012-01-04T17:26:21.235-0800|INFO|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=176;_ThreadName=Thread-2;|Warning: Synchronization with DAS failed, continuing startup... Waiting for clt1 to start ......................... Successfully started the instance: clt1 instance Location: /export/home/j2eetest/3.1.2/glassfish3/glassfish/nodes/tuppy/clt1 Log File: /export/home/j2eetest/3.1.2/glassfish3/glassfish/nodes/tuppy/clt1/logs/server.log Admin Port: 24848 Command start-local-instance executed successfully.|#] [#|2012-01-04T17:26:21.417-0800|INFO|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=177;_ThreadName=Thread-2;|Warning: Synchronization with DAS failed, continuing startup... Waiting for cll2 to start ............................ Successfully started the instance: cll2 instance Location: /export/home/j2eetest/3.1.2/glassfish3/glassfish/nodes/localhost-domain1/cll2 Log File: /export/home/j2eetest/3.1.2/glassfish3/glassfish/nodes/localhost-domain1/cll2/logs/server.log Admin Port: 24849 Command start-local-instance executed successfully.|#] [#|2012-01-04T17:26:35.698-0800|WARNING|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=178;_ThreadName=Thread-2;|Could not start instance clt2 on node tuppy (tuppy).: Command ' /export/home/j2eetest/3.1.2/glassfish3/glassfish/bin/asadmin --_auxinput - --interactive=false start-local-instance --node tuppy --sync normal clt2' failed on node tuppy (tuppy): Warning: Synchronization with DAS failed, continuing startup... Waiting for clt2 to start .......................................Command start-local-instance failed. Error starting instance clt2. The server exited prematurely with exit code 0. Before it died, it produced the following output: Launching GlassFish on Felix platform Jan 4, 2012 6:02:55 PM com.sun.common.util.logging.LoggingConfigImpl copyLoggingPropertiesFile WARNING: Logging.properties file not found, creating new file using DAS logging.properties [#|2012-01-04T18:02:55.844-0800|INFO|glassfish3.1.2|com.sun.enterprise.server.logging.GFFileHandler|_ThreadID=1;_ThreadName=main;|Running GlassFish Version: GlassFish Server Open Source Edition 3.1.2-b16 (build 16)|#] [#|2012-01-04T18:02:59.077-0800|INFO|glassfish3.1.2|javax.enterprise.system.core.transaction.com.sun.jts.CosTransactions|_ThreadID=10;_ThreadName=main;|JTS5014: Recoverable JTS instance, serverId = [100]|#] [#|2012-01-04T18:03:00.228-0800|INFO|glassfish3.1.2|javax.enterprise.system.core.com.sun.enterprise.v3.services.impl|_ThreadID=11;_ThreadName=Grizzly-kernel-thread(1);|Grizzly Framework 1.9.43 started in: 281ms - bound to [0.0.0.0:28081]|#] [#|2012-01-04T18:03:00.301-0800|INFO|glassfish3.1.2|javax.enterprise.system.core.com.sun.enterprise.v3.services.impl|_ThreadID=13;_ThreadName=Grizzly-kernel-thread(1);|Grizzly Framework 1.9.43 started in: 117ms - bound to [0.0.0.0:27677]|#] [#|2012-01-04T18:03:00.290-0800|INFO|glassfish3.1.2|javax.enterprise.system.core.com.sun.enterprise.v3.services.impl|_ThreadID=12;_ThreadName=Grizzly-kernel-thread(1);|Grizzly Framework 1.9.43 started in: 163ms - bound to [0.0.0.0:24849]|#] [#|2012-01-04T18:03:00.309-0800|INFO|glassfish3.1.2|javax.enterprise.system.core.com.sun.enterprise.v3.services.impl|_ThreadID=14;_ThreadName=Grizzly-kernel-thread(1);|Grizzly Framework 1.9.43 started in: 410ms - bound to [0.0.0.0:28182]|#] [#|2012-01-04T18:03:00.648-0800|SEVERE|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.container.common|_ThreadID=10;_ThreadName=main;|The server requires a valid admin password to be set before it can start. Please set a password using the change-admin-password command.|#] [#|2012-01-04T18:03:00.651-0800|SEVERE|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.container.common|_ThreadID=10;_ThreadName=main;|The server requires a valid admin password to be set before it can start. Please set a password using the change-admin-password command.|#] [#|2012-01-04T18:03:00.654-0800|SEVERE|glassfish3.1.2|javax.enterprise.system.core.com.sun.enterprise.v3.services.impl|_ThreadID=10;_ThreadName=main;|Unable to start v3. Closing all ports org.jvnet.hk2.component.ComponentException: injection failed on com.sun.enterprise.v3.admin.AdminAdapter.authenticator with interface org.glassfish.internal.api.AdminAccessController at org.jvnet.hk2.component.InjectionManager.error_injectionException(InjectionManager.java:284) at org.jvnet.hk2.component.InjectionManager.inject(InjectionManager.java:165) at org.jvnet.hk2.component.InjectionManager.inject(InjectionManager.java:93) at com.sun.hk2.component.AbstractCreatorImpl.inject(AbstractCreatorImpl.java:126) at com.sun.hk2.component.ConstructorCreator.initialize(ConstructorCreator.java:91) at com.sun.hk2.component.AbstractCreatorImpl.get(AbstractCreatorImpl.java:82) at com.sun.hk2.component.SingletonInhabitant.get(SingletonInhabitant.java:67) at com.sun.hk2.component.EventPublishingInhabitant.get(EventPublishingInhabitant.java:139) at com.sun.hk2.component.AbstractInhabitantImpl.get(AbstractInhabitantImpl.java:78) at org.jvnet.hk2.component.Habitat$5.get(Habitat.java:703) at java.util.AbstractList$Itr.next(AbstractList.java:345) at com.sun.enterprise.v3.services.impl.GrizzlyService.registerNetworkProxy(GrizzlyService.java:492) at com.sun.enterprise.v3.services.impl.GrizzlyService.postConstruct(GrizzlyService.java:407) at com.sun.hk2.component.AbstractCreatorImpl.inject(AbstractCreatorImpl.java:131) at com.sun.hk2.component.ConstructorCreator.initialize(ConstructorCreator.java:91) at com.sun.hk2.component.AbstractCreatorImpl.get(AbstractCreatorImpl.java:82) at com.sun.hk2.component.SingletonInhabitant.get(SingletonInhabitant.java:67) at com.sun.hk2.component.EventPublishingInhabitant.get(EventPublishingInhabitant.java:139) at com.sun.hk2.component.AbstractInhabitantImpl.get(AbstractInhabitantImpl.java:78) at org.jvnet.hk2.component.Habitat.getBy(Habitat.java:1056) at org.jvnet.hk2.component.Habitat.getByType(Habitat.java:1037) at com.sun.hk2.component.InjectInjectionResolver.getComponentInjectValue(InjectInjectionResolver.java:159) at com.sun.hk2.component.InjectInjectionResolver.getValue(InjectInjectionResolver.java:90) at org.jvnet.hk2.component.InjectionManager.inject(InjectionManager.java:143) at org.jvnet.hk2.component.InjectionManager.inject(InjectionManager.java:93) at com.sun.hk2.component.AbstractCreatorImpl.inject(AbstractCreatorImpl.java:126) at com.sun.hk2.component.ConstructorCreator.initialize(ConstructorCreator.java:91) at com.sun.hk2.component.AbstractCreatorImpl.get(AbstractCreatorImpl.java:82) at com.sun.hk2.component.SingletonInhabitant.get(SingletonInhabitant.java:67) at com.sun.hk2.component.EventPublishingInhabitant.get(EventPublishingInhabitant.java:139) at com.sun.hk2.component.AbstractInhabitantImpl.get(AbstractInhabitantImpl.java:78) at org.glassfish.internal.data.EngineInfo.getContainer(EngineInfo.java:93) at com.sun.enterprise.v3.services.impl.WebContainerStarter.startWebContainer(WebContainerStarter.java:202) at com.sun.enterprise.v3.services.impl.WebContainerStarter.postConstruct(WebContainerStarter.java:134) at com.sun.hk2.component.AbstractCreatorImpl.inject(AbstractCreatorImpl.java:131) at com.sun.hk2.component.ConstructorCreator.initialize(ConstructorCreator.java:91) at com.sun.hk2.component.AbstractCreatorImpl.get(AbstractCreatorImpl.java:82) at com.sun.hk2.component.SingletonInhabitant.get(SingletonInhabitant.java:67) at com.sun.hk2.component.EventPublishingInhabitant.get(EventPublishingInhabitant.java:139) at com.sun.hk2.component.AbstractInhabitantImpl.get(AbstractInhabitantImpl.java:78) at com.sun.enterprise.v3.server.AppServerStartup.run(AppServerStartup.java:253) at com.sun.enterprise.v3.server.AppServerStartup.doStart(AppServerStartup.java:145) at com.sun.enterprise.v3.server.AppServerStartup.start(AppServerStartup.java:136) at com.sun.enterprise.glassfish.bootstrap.GlassFishImpl.start(GlassFishImpl.java:79) at com.sun.enterprise.glassfish.bootstrap.GlassFishDecorator.start(GlassFishDecorator.java:63) at com.sun.enterprise.glassfish.bootstrap.osgi.OSGiGlassFishImpl.start(OSGiGlassFishImpl.java:69) at com.sun.enterprise.glassfish.bootstrap.GlassFishMain$Launcher.launch(GlassFishMain.java:117) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at com.sun.enterprise.glassfish.bootstrap.GlassFishMain.main(GlassFishMain.java:97) at com.sun.enterprise.glassfish.bootstrap.ASMain.main(ASMain.java:55) Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: The server requires a valid admin password to be set before it can start. Please set a password using the change-admin-password command. at com.sun.enterprise.container.common.GenericAdminAuthenticator.postConstruct(GenericAdminAuthenticator.java:167) at com.sun.hk2.component.AbstractCreatorImpl.inject(AbstractCreatorImpl.java:131) at com.sun.hk2.component.ConstructorCreator.initialize(ConstructorCreator.java:91) at com.sun.hk2.component.AbstractCreatorImpl.get(AbstractCreatorImpl.java:82) at com.sun.hk2.component.SingletonInhabitant.get(SingletonInhabitant.java:67) at com.sun.hk2.component.EventPublishingInhabitant.get(EventPublishingInhabitant.java:139) at com.sun.hk2.component.AbstractInhabitantImpl.get(AbstractInhabitantImpl.java:78) at org.jvnet.hk2.component.Habitat.getComponent(Habitat.java:798) at com.sun.hk2.component.InjectInjectionResolver.getServiceInjectValue(InjectInjectionResolver.java:147) at com.sun.hk2.component.InjectInjectionResolver.getValue(InjectInjectionResolver.java:88) at org.jvnet.hk2.component.InjectionManager.inject(InjectionManager.java:143) ... 51 more Caused by: java.lang.IllegalStateException: The server requires a valid admin password to be set before it can start. Please set a password using the change-admin-password command. at com.sun.enterprise.container.common.GenericAdminAuthenticator.postConstruct(GenericAdminAuthenticator.java:162) ... 61 more |#] [#|2012-01-04T18:03:03.328-0800|SEVERE|glassfish3.1.2|javax.enterprise.system.core.com.sun.enterprise.v3.server|_ThreadID=10;_ThreadName=main;|Startup service failed to start : com.sun.enterprise.v3.services.impl.WebContainerStarter|#] [#|2012-01-04T18:03:03.447-0800|INFO|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=15;_ThreadName=Thread-19;|Server shutdown initiated|#] [#|2012-01-04T18:03:03.451-0800|INFO|glassfish3.1.2|javax.enterprise.system.core.com.sun.enterprise.v3.server|_ThreadID=15;_ThreadName=Thread-19;|Already stopped, so just returning|#]: To complete this operation run the following command locally on host tuppy from the GlassFish install location /export/home/j2eetest/3.1.2/glassfish3: bin/asadmin start-local-instance --node tuppy --sync normal clt2|#] [#|2012-01-04T17:26:35.705-0800|SEVERE|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=172;_ThreadName=Thread-2;|clt2: Could not start instance clt2 on node tuppy (tuppy). Command failed on node tuppy (tuppy): Warning: Synchronization with DAS failed, continuing startup... Waiting for clt2 to start .......................................Command start-local-instance failed. Error starting instance clt2. The server exited prematurely with exit code 0. Before it died, it produced the following output: Launching GlassFish on Felix platform Jan 4, 2012 6:02:55 PM com.sun.common.util.logging.LoggingConfigImpl copyLoggingPropertiesFile WARNING: Logging.properties file not found, creating new file using DAS logging.properties [#|2012-01-04T18:02:55.844-0800|INFO|glassfish3.1.2|com.sun.enterprise.server.logging.GFFileHandler|_ThreadID=1;_ThreadName=main;|Running GlassFish Version: GlassFish Server Open Source Edition 3.1.2-b16 (build 16)|#] [#|2012-01-04T18:02:59.077-0800|INFO|glassfish3.1.2|javax.enterprise.system.core.transaction.com.sun.jts.CosTransactions|_ThreadID=10;_ThreadName=main;|JTS5014: Recoverable JTS instance, serverId = [100]|#] [#|2012-01-04T18:03:00.228-0800|INFO|glassfish3.1.2|javax.enterprise.system.core.com.sun.enterprise.v3.services.impl|_ThreadID=11;_ThreadName=Grizzly-kernel-thread(1);|Grizzly Framework 1.9.43 started in: 281ms - bound to [0.0.0.0:28081]|#] [#|2012-01-04T18:03:00.301-0800|INFO|glassfish3.1.2|javax.enterprise.system.core.com.sun.enterprise.v3.services.impl|_ThreadID=13;_ThreadName=Grizzly-kernel-thread(1);|Grizzly Framework 1.9.43 started in: 117ms - bound to [0.0.0.0:27677]|#] [#|2012-01-04T18:03:00.290-0800|INFO|glassfish3.1.2|javax.enterprise.system.core.com.sun.enterprise.v3.services.impl|_ThreadID=12;_ThreadName=Grizzly-kernel-thread(1);|Grizzly Framework 1.9.43 started in: 163ms - bound to [0.0.0.0:24849]|#] [#|2012-01-04T18:03:00.309-0800|INFO|glassfish3.1.2|javax.enterprise.system.core.com.sun.enterprise.v3.services.impl|_ThreadID=14;_ThreadName=Grizzly-kernel-thread(1);|Grizzly Framework 1.9.43 started in: 410ms - bound to [0.0.0.0:28182]|#] [#|2012-01-04T18:03:00.648-0800|SEVERE|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.container.common|_ThreadID=10;_ThreadName=main;|The server requires a valid admin password to be set before it can start. Please set a password using the change-admin-password command.|#] [#|2012-01-04T18:03:00.651-0800|SEVERE|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.container.common|_ThreadID=10;_ThreadName=main;|The server requires a valid admin password to be set before it can start. Please set a password using the change-admin-password command.|#] [#|2012-01-04T18:03:00.654-0800|SEVERE|glassfish3.1.2|javax.enterprise.system.core.com.sun.enterprise.v3.services.impl|_ThreadID=10;_ThreadName=main;|Unable to start v3. Closing all ports org.jvnet.hk2.component.ComponentException: injection failed on com.sun.enterprise.v3.admin.AdminAdapter.authenticator with interface org.glassfish.internal.api.AdminAccessController at org.jvnet.hk2.component.InjectionManager.error_injectionException(InjectionManager.java:284) at org.jvnet.hk2.component.InjectionManager.inject(InjectionManager.java:165) at org.jvnet.hk2.component.InjectionManager.inject(InjectionManager.java:93) at com.sun.hk2.component.AbstractCreatorImpl.inject(AbstractCreatorImpl.java:126) at com.sun.hk2.component.ConstructorCreator.initialize(ConstructorCreator.java:91) at com.sun.hk2.component.AbstractCreatorImpl.get(AbstractCreatorImpl.java:82) at com.sun.hk2.component.SingletonInhabitant.get(SingletonInhabitant.java:67) at com.sun.hk2.component.EventPublishingInhabitant.get(EventPublishingInhabitant.java:139) at com.sun.hk2.component.AbstractInhabitantImpl.get(AbstractInhabitantImpl.java:78) at org.jvnet.hk2.component.Habitat$5.get(Habitat.java:703) at java.util.AbstractList$Itr.next(AbstractList.java:345) at com.sun.enterprise.v3.services.impl.GrizzlyService.registerNetworkProxy(GrizzlyService.java:492) at com.sun.enterprise.v3.services.impl.GrizzlyService.postConstruct(GrizzlyService.java:407) at com.sun.hk2.component.AbstractCreatorImpl.inject(AbstractCreatorImpl.java:131) at com.sun.hk2.component.ConstructorCreator.initialize(ConstructorCreator.java:91) at com.sun.hk2.component.AbstractCreatorImpl.get(AbstractCreatorImpl.java:82) at com.sun.hk2.component.SingletonInhabitant.get(SingletonInhabitant.java:67) at com.sun.hk2.component.EventPublishingInhabitant.get(EventPublishingInhabitant.java:139) at com.sun.hk2.component.AbstractInhabitantImpl.get(AbstractInhabitantImpl.java:78) at org.jvnet.hk2.component.Habitat.getBy(Habitat.java:1056) at org.jvnet.hk2.component.Habitat.getByType(Habitat.java:1037) at com.sun.hk2.component.InjectInjectionResolver.getComponentInjectValue(InjectInjectionResolver.java:159) at com.sun.hk2.component.InjectInjectionResolver.getValue(InjectInjectionResolver.java:90) at org.jvnet.hk2.component.InjectionManager.inject(InjectionManager.java:143) at org.jvnet.hk2.component.InjectionManager.inject(InjectionManager.java:93) at com.sun.hk2.component.AbstractCreatorImpl.inject(AbstractCreatorImpl.java:126) at com.sun.hk2.component.ConstructorCreator.initialize(ConstructorCreator.java:91) at com.sun.hk2.component.AbstractCreatorImpl.get(AbstractCreatorImpl.java:82) at com.sun.hk2.component.SingletonInhabitant.get(SingletonInhabitant.java:67) at com.sun.hk2.component.EventPublishingInhabitant.get(EventPublishingInhabitant.java:139) at com.sun.hk2.component.AbstractInhabitantImpl.get(AbstractInhabitantImpl.java:78) at org.glassfish.internal.data.EngineInfo.getContainer(EngineInfo.java:93) at com.sun.enterprise.v3.services.impl.WebContainerStarter.startWebContainer(WebContainerStarter.java:202) at com.sun.enterprise.v3.services.impl.WebContainerStarter.postConstruct(WebContainerStarter.java:134) at com.sun.hk2.component.AbstractCreatorImpl.inject(AbstractCreatorImpl.java:131) at com.sun.hk2.component.ConstructorCreator.initialize(ConstructorCreator.java:91) at com.sun.hk2.component.AbstractCreatorImpl.get(AbstractCreatorImpl.java:82) at com.sun.hk2.component.SingletonInhabitant.get(SingletonInhabitant.java:67) at com.sun.hk2.component.EventPublishingInhabitant.get(EventPublishingInhabitant.java:139) at com.sun.hk2.component.AbstractInhabitantImpl.get(AbstractInhabitantImpl.java:78) at com.sun.enterprise.v3.server.AppServerStartup.run(AppServerStartup.java:253) at com.sun.enterprise.v3.server.AppServerStartup.doStart(AppServerStartup.java:145) at com.sun.enterprise.v3.server.AppServerStartup.start(AppServerStartup.java:136) at com.sun.enterprise.glassfish.bootstrap.GlassFishImpl.start(GlassFishImpl.java:79) at com.sun.enterprise.glassfish.bootstrap.GlassFishDecorator.start(GlassFishDecorator.java:63) at com.sun.enterprise.glassfish.bootstrap.osgi.OSGiGlassFishImpl.start(OSGiGlassFishImpl.java:69) at com.sun.enterprise.glassfish.bootstrap.GlassFishMain$Launcher.launch(GlassFishMain.java:117) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at com.sun.enterprise.glassfish.bootstrap.GlassFishMain.main(GlassFishMain.java:97) at com.sun.enterprise.glassfish.bootstrap.ASMain.main(ASMain.java:55) Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: The server requires a valid admin password to be set before it can start. Please set a password using the change-admin-password command. at com.sun.enterprise.container.common.GenericAdminAuthenticator.postConstruct(GenericAdminAuthenticator.java:167) at com.sun.hk2.component.AbstractCreatorImpl.inject(AbstractCreatorImpl.java:131) at com.sun.hk2.component.ConstructorCreator.initialize(ConstructorCreator.java:91) at com.sun.hk2.component.AbstractCreatorImpl.get(AbstractCreatorImpl.java:82) at com.sun.hk2.component.SingletonInhabitant.get(SingletonInhabitant.java:67) at com.sun.hk2.component.EventPublishingInhabitant.get(EventPublishingInhabitant.java:139) at com.sun.hk2.component.AbstractInhabitantImpl.get(AbstractInhabitantImpl.java:78) at org.jvnet.hk2.component.Habitat.getComponent(Habitat.java:798) at com.sun.hk2.component.InjectInjectionResolver.getServiceInjectValue(InjectInjectionResolver.java:147) at com.sun.hk2.component.InjectInjectionResolver.getValue(InjectInjectionResolver.java:88) at org.jvnet.hk2.component.InjectionManager.inject(InjectionManager.java:143) ... 51 more Caused by: java.lang.IllegalStateException: The server requires a valid admin password to be set before it can start. Please set a password using the change-admin-password command. at com.sun.enterprise.container.common.GenericAdminAuthenticator.postConstruct(GenericAdminAuthenticator.java:162) ... 61 more |#] [#|2012-01-04T18:03:03.328-0800|SEVERE|glassfish3.1.2|javax.enterprise.system.core.com.sun.enterprise.v3.server|_ThreadID=10;_ThreadName=main;|Startup service failed to start : com.sun.enterprise.v3.services.impl.WebContainerStarter|#] [#|2012-01-04T18:03:03.447-0800|INFO|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=15;_ThreadName=Thread-19;|Server shutdown initiated|#] [#|2012-01-04T18:03:03.451-0800|INFO|glassfish3.1.2|javax.enterprise.system.core.com.sun.enterprise.v3.server|_ThreadID=15;_ThreadName=Thread-19;|Already stopped, so just returning|#] To complete this operation run the following command locally on host tuppy from the GlassFish install location /export/home/j2eetest/3.1.2/glassfish3: bin/asadmin start-local-instance --node tuppy --sync normal clt2|#] [#|2012-01-04T17:26:35.712-0800|INFO|glassfish3.1.2|javax.org.glassfish.gms.org.glassfish.gms.admin|_ThreadID=172;_ThreadName=Thread-2;|GMSAD3001: GMSAnnounceAfterStartClusterCommand: exitCode:WARNING members null clusterMembers:[clt1, cll2, clt2]|#] [#|2012-01-04T17:26:35.713-0800|INFO|glassfish3.1.2|ShoalLogger|_ThreadID=172;_ThreadName=Thread-2;|GMS1098: GMS:Announcing GroupStartup[COMPLETED_FAILED] for group: cl members: clt2,|#] [#|2012-01-04T17:26:35.715-0800|INFO|glassfish3.1.2|ShoalLogger|_ThreadID=172;_ThreadName=Thread-2;|GMS1062: GroupStart for group: cl State: COMPLETED_FAILED Failed Members: clt2,|#] [#|2012-01-04T17:26:35.731-0800|WARNING|glassfish3.1.2|org.glassfish.admingui|_ThreadID=171;_ThreadName=Thread-2;|RestResponse.getResponse() gives WARNING. endpoint = 'https://localhost:4848/management/domain/clusters/cluster/cl/start-cluster'; attrs = '{}'|#] [#|2012-01-04T17:27:27.950-0800|INFO|glassfish3.1.2|javax.enterprise.system.std.com.sun.enterprise.server.logging|_ThreadID=179;_ThreadName=Thread-2;|Stopping Cluster ....|#] [#|2012-01-04T17:27:27.954-0800|INFO|glassfish3.1.2|org.glassfish.admingui|_ThreadID=179;_ThreadName=Thread-2;|https://localhost:4848/management/domain/clusters/cluster/cl/stop-cluster|#] [#|2012-01-04T17:27:27.981-0800|INFO|glassfish3.1.2|ShoalLogger|_ThreadID=180;_ThreadName=Thread-2;|GMS1097: GMS:Announcing GroupShutdown to group: cl with State = INITIATED|#] [#|2012-01-04T17:27:27.988-0800|INFO|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=180;_ThreadName=Thread-2;|Stopping cluster cl|#] [#|2012-01-04T17:27:28.001-0800|INFO|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=180;_ThreadName=Thread-2;|Executing stop-instance on 3 instances using a thread pool of size 3: tuppy:clt1 localhost-domain1:cll2 tuppy:clt2|#] [#|2012-01-04T17:27:28.003-0800|INFO|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=180;_ThreadName=Thread-2;|stop-instance clt1|#] [#|2012-01-04T17:27:28.004-0800|INFO|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=180;_ThreadName=Thread-2;|stop-instance cll2|#] [#|2012-01-04T17:27:28.005-0800|INFO|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=180;_ThreadName=Thread-2;|stop-instance clt2|#] [#|2012-01-04T17:27:28.020-0800|INFO|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=181;_ThreadName=Thread-2;|Instance cll2 shutdown initiated|#] [#|2012-01-04T17:27:28.031-0800|INFO|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=182;_ThreadName=Thread-2;|Instance clt1 shutdown initiated|#] [#|2012-01-04T17:27:28.217-0800|INFO|glassfish3.1.2|ShoalLogger|_ThreadID=60;_ThreadName=Thread-2;|GMS1092: GMS View Change Received for group: cl : Members in view for CLUSTER_STOP_EVENT(before change analysis) are : 1: MemberId: server, MemberType: SPECTATOR, Address: 10.133.162.59:9199:228.9.199.152:17621:cl:server |#] [#|2012-01-04T17:27:28.219-0800|INFO|glassfish3.1.2|ShoalLogger|_ThreadID=60;_ThreadName=Thread-2;|GMS1016: Analyzing new membership snapshot received as part of event: CLUSTER_STOP_EVENT for member: cll2 of group: cl|#] [#|2012-01-04T17:27:28.219-0800|INFO|glassfish3.1.2|ShoalLogger|_ThreadID=60;_ThreadName=Thread-2;|GMS1017: Received PlannedShutdownEvent Announcement from member: cll2 with shutdown type: GROUP_SHUTDOWN of group: cl|#] [#|2012-01-04T17:27:28.220-0800|INFO|glassfish3.1.2|ShoalLogger|_ThreadID=134;_ThreadName=Thread-2;|GMS1008: Sending PlannedShutdownSignals to registered Actions for shutdownType GROUP_SHUTDOWN member: cll2 ...|#] [#|2012-01-04T17:27:36.257-0800|INFO|glassfish3.1.2|ShoalLogger|_ThreadID=180;_ThreadName=Thread-2;|GMS1097: GMS:Announcing GroupShutdown to group: cl with State = COMPLETED|#] [#|2012-01-04T17:27:58.367-0800|INFO|glassfish3.1.2|org.glassfish.admingui|_ThreadID=183;_ThreadName=Thread-2;|https://localhost:4848/management/domain/servers/server/clt2/delete-instance|#] [#|2012-01-04T17:28:03.273-0800|INFO|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=184;_ThreadName=Thread-2;|Command _delete-instance-filesystem executed successfully.|#] [#|2012-01-04T17:28:03.302-0800|INFO|glassfish3.1.2|javax.enterprise.system.core.org.glassfish.gms.bootstrap|_ThreadID=185;_ThreadName=Thread-2;|GMSBS2003: Deleting instance clt2 from health history table.|#] [#|2012-01-04T17:28:23.996-0800|INFO|glassfish3.1.2|javax.enterprise.system.std.com.sun.enterprise.server.logging|_ThreadID=186;_ThreadName=Thread-2;|Starting Cluster ....|#] [#|2012-01-04T17:28:23.999-0800|INFO|glassfish3.1.2|org.glassfish.admingui|_ThreadID=186;_ThreadName=Thread-2;|https://localhost:4848/management/domain/clusters/cluster/cl/start-cluster|#] [#|2012-01-04T17:28:24.027-0800|INFO|glassfish3.1.2|ShoalLogger|_ThreadID=165;_ThreadName=Thread-2;|GMS1098: GMS:Announcing GroupStartup[INITIATED] for group: cl members: clt1,cll2,|#] [#|2012-01-04T17:28:24.028-0800|INFO|glassfish3.1.2|ShoalLogger|_ThreadID=165;_ThreadName=Thread-2;|GMS1062: GroupStart for group: cl State: INITIATED Starting Members: clt1,cll2,|#] [#|2012-01-04T17:28:24.028-0800|INFO|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=165;_ThreadName=Thread-2;|Starting cluster cl|#] [#|2012-01-04T17:28:24.039-0800|INFO|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=165;_ThreadName=Thread-2;|Executing start-instance on 2 instances using a thread pool of size 2: tuppy:clt1 localhost-domain1:cll2|#] [#|2012-01-04T17:28:24.040-0800|INFO|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=165;_ThreadName=Thread-2;|start-instance clt1|#] [#|2012-01-04T17:28:24.042-0800|INFO|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=165;_ThreadName=Thread-2;|start-instance cll2|#] [#|2012-01-04T17:28:56.900-0800|WARNING|glassfish3.1.2|com.sun.grizzly.config.GrizzlyServiceListener|_ThreadID=13;_ThreadName=Thread-2;|GRIZZLY0023: Interrupting idle Thread: admin-thread-pool-4848(5).|#] [#|2012-01-04T17:28:56.901-0800|WARNING|glassfish3.1.2|com.sun.grizzly.config.GrizzlyServiceListener|_ThreadID=13;_ThreadName=Thread-2;|GRIZZLY0023: Interrupting idle Thread: admin-thread-pool-4848(611).|#] [#|2012-01-04T17:28:58.210-0800|WARNING|glassfish3.1.2|com.sun.grizzly.config.GrizzlyServiceListener|_ThreadID=13;_ThreadName=Thread-2;|GRIZZLY0023: Interrupting idle Thread: admin-thread-pool-4848(611).|#] [#|2012-01-04T17:28:58.217-0800|WARNING|glassfish3.1.2|javax.enterprise.system.container.web.com.sun.enterprise.web|_ThreadID=187;_ThreadName=Thread-2;|StandardWrapperValve[DownloadServlet]: PWC1406: Servlet.service() for servlet DownloadServlet threw exception java.lang.RuntimeException: com.sun.jersey.api.client.ClientHandlerException: java.io.InterruptedIOException: Operation interrupted at org.glassfish.admingui.common.servlet.LogFilesContentSource.getInputStream(LogFilesContentSource.java:110) at org.glassfish.admingui.common.servlet.DownloadServlet.writeContent(DownloadServlet.java:277) at org.glassfish.admingui.common.servlet.DownloadServlet.doPost(DownloadServlet.java:174) at org.glassfish.admingui.common.servlet.DownloadServlet.doGet(DownloadServlet.java:155) at javax.servlet.http.HttpServlet.service(HttpServlet.java:668) at javax.servlet.http.HttpServlet.service(HttpServlet.java:770) at org.apache.catalina.core.StandardWrapper.service(StandardWrapper.java:1542) at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:281) at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:175) at org.apache.catalina.core.StandardPipeline.doInvoke(StandardPipeline.java:655) at org.apache.catalina.core.StandardPipeline.invoke(StandardPipeline.java:595) at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:161) at org.apache.catalina.connector.CoyoteAdapter.doService(CoyoteAdapter.java:331) at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:231) at com.sun.enterprise.v3.services.impl.ContainerMapper.service(ContainerMapper.java:232) at com.sun.grizzly.http.ProcessorTask.invokeAdapter(ProcessorTask.java:849) at com.sun.grizzly.http.ProcessorTask.doProcess(ProcessorTask.java:746) at com.sun.grizzly.http.ProcessorTask.process(ProcessorTask.java:1045) at com.sun.grizzly.http.DefaultProtocolFilter.execute(DefaultProtocolFilter.java:228) at com.sun.grizzly.DefaultProtocolChain.executeProtocolFilter(DefaultProtocolChain.java:137) at com.sun.grizzly.DefaultProtocolChain.execute(DefaultProtocolChain.java:104) at com.sun.grizzly.DefaultProtocolChain.execute(DefaultProtocolChain.java:90) at com.sun.grizzly.http.HttpProtocolChain.execute(HttpProtocolChain.java:79) at com.sun.grizzly.ProtocolChainContextTask.doCall(ProtocolChainContextTask.java:54) at com.sun.grizzly.SelectionKeyContextTask.call(SelectionKeyContextTask.java:59) at com.sun.grizzly.ContextTask.run(ContextTask.java:71) at com.sun.grizzly.util.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:532) at com.sun.grizzly.util.AbstractThreadPool$Worker.run(AbstractThreadPool.java:513) at java.lang.Thread.run(Thread.java:662) Caused by: com.sun.jersey.api.client.ClientHandlerException: java.io.InterruptedIOException: Operation interrupted at com.sun.jersey.client.urlconnection.URLConnectionClientHandler.handle(URLConnectionClientHandler.java:149) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.filter.CsrfProtectionFilter.handle(CsrfProtectionFilter.java:97) at com.sun.jersey.api.client.Client.handle(Client.java:648) at com.sun.jersey.api.client.WebResource.handle(WebResource.java:670) at com.sun.jersey.api.client.WebResource.access$200(WebResource.java:74) at com.sun.jersey.api.client.WebResource$Builder.post(WebResource.java:563) at org.glassfish.admingui.common.util.RestUtil.postRestRequestFromServlet(RestUtil.java:712) at org.glassfish.admingui.common.servlet.LogFilesContentSource.getInputStream(LogFilesContentSource.java:106) ... 28 more Caused by: java.io.InterruptedIOException: Operation interrupted at java.net.SocketInputStream.socketRead0(Native Method) at java.net.SocketInputStream.read(SocketInputStream.java:129) at com.sun.net.ssl.internal.ssl.InputRecord.readFully(InputRecord.java:293) at com.sun.net.ssl.internal.ssl.InputRecord.read(InputRecord.java:331) at com.sun.net.ssl.internal.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:798) at com.sun.net.ssl.internal.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:755) at com.sun.net.ssl.internal.ssl.AppInputStream.read(AppInputStream.java:75) at java.io.BufferedInputStream.fill(BufferedInputStream.java:218) at java.io.BufferedInputStream.read1(BufferedInputStream.java:258) at java.io.BufferedInputStream.read(BufferedInputStream.java:317) at sun.net.www.http.HttpClient.parseHTTPHeader(HttpClient.java:695) at sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:640) at sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:660) at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1195) at java.net.HttpURLConnection.getResponseCode(HttpURLConnection.java:379) at sun.net.www.protocol.https.HttpsURLConnectionImpl.getResponseCode(HttpsURLConnectionImpl.java:318) at com.sun.jersey.client.urlconnection.URLConnectionClientHandler._invoke(URLConnectionClientHandler.java:240) at com.sun.jersey.client.urlconnection.URLConnectionClientHandler.handle(URLConnectionClientHandler.java:147) ... 84 more |#] [#|2012-01-04T17:29:27.150-0800|WARNING|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin|_ThreadID=188;_ThreadName=Thread-2;|Command: _synchronize-files failed to acquire a command lock. REASON: time out (current lock acquired on Wed, 04 Jan 2012 17:28:56 PST)|#] [#|2012-01-04T17:29:27.167-0800|INFO|glassfish3.1.2|javax.enterprise.system.std.com.sun.enterprise.server.logging|_ThreadID=189;_ThreadName=Thread-2;|CLI802 Synchronization failed for directory applications, caused by:|#] [#|2012-01-04T17:29:27.168-0800|INFO|glassfish3.1.2|javax.enterprise.system.std.com.sun.enterprise.server.logging|_ThreadID=189;_ThreadName=Thread-2;| remote failure: Command timed out. Unable to acquire a lock to access the domain. Another command acquired exclusive access to the domain on Wed, 04 Jan 2012 17:28:56 PST. Retry the command at a later time.|#] [#|2012-01-04T17:29:27.169-0800|INFO|glassfish3.1.2|javax.enterprise.system.std.com.sun.enterprise.server.logging|_ThreadID=190;_ThreadName=Thread-2;|Command start-local-instance failed.|#] [#|2012-01-04T17:29:27.224-0800|WARNING|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=191;_ThreadName=Thread-2;|Could not start instance cll2 on node localhost-domain1 (localhost).: Command ' /export/home/j2eetest/3.1.2/glassfish3/glassfish/bin/asadmin --_auxinput - --interactive=false start-local-instance --node localhost-domain1 --sync normal cll2' failed on node localhost-domain1 (localhost): Command start-local-instance failed. CLI802 Synchronization failed for directory applications, caused by: remote failure: Command timed out. Unable to acquire a lock to access the domain. Another command acquired exclusive access to the domain on Wed, 04 Jan 2012 17:28:56 PST. Retry the command at a later time.: To complete this operation run the following command locally on host localhost from the GlassFish install location /export/home/j2eetest/3.1.2/glassfish3: bin/asadmin start-local-instance --node localhost-domain1 --sync normal cll2|#] [#|2012-01-04T17:29:27.227-0800|SEVERE|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=165;_ThreadName=Thread-2;|cll2: Could not start instance cll2 on node localhost-domain1 (localhost). Command failed on node localhost-domain1 (localhost): Command start-local-instance failed. CLI802 Synchronization failed for directory applications, caused by: remote failure: Command timed out. Unable to acquire a lock to access the domain. Another command acquired exclusive access to the domain on Wed, 04 Jan 2012 17:28:56 PST. Retry the command at a later time. To complete this operation run the following command locally on host localhost from the GlassFish install location /export/home/j2eetest/3.1.2/glassfish3: bin/asadmin start-local-instance --node localhost-domain1 --sync normal cll2|#] [#|2012-01-04T17:29:27.270-0800|WARNING|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin|_ThreadID=22;_ThreadName=Thread-2;|Command: _synchronize-files failed to acquire a command lock. REASON: time out (current lock acquired on Wed, 04 Jan 2012 17:28:56 PST)|#] [#|2012-01-04T17:29:27.416-0800|WARNING|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=192;_ThreadName=Thread-2;|Could not start instance clt1 on node tuppy (tuppy).: Command ' /export/home/j2eetest/3.1.2/glassfish3/glassfish/bin/asadmin --_auxinput - --interactive=false start-local-instance --node tuppy --sync normal clt1' failed on node tuppy (tuppy): Command start-local-instance failed. CLI802 Synchronization failed for directory applications, caused by: remote failure: Command timed out. Unable to acquire a lock to access the domain. Another command acquired exclusive access to the domain on Wed, 04 Jan 2012 17:28:56 PST. Retry the command at a later time.: To complete this operation run the following command locally on host tuppy from the GlassFish install location /export/home/j2eetest/3.1.2/glassfish3: bin/asadmin start-local-instance --node tuppy --sync normal clt1|#] [#|2012-01-04T17:29:27.419-0800|SEVERE|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=165;_ThreadName=Thread-2;|clt1: Could not start instance clt1 on node tuppy (tuppy). Command failed on node tuppy (tuppy): Command start-local-instance failed. CLI802 Synchronization failed for directory applications, caused by: remote failure: Command timed out. Unable to acquire a lock to access the domain. Another command acquired exclusive access to the domain on Wed, 04 Jan 2012 17:28:56 PST. Retry the command at a later time. To complete this operation run the following command locally on host tuppy from the GlassFish install location /export/home/j2eetest/3.1.2/glassfish3: bin/asadmin start-local-instance --node tuppy --sync normal clt1|#] [#|2012-01-04T17:29:27.435-0800|SEVERE|glassfish3.1.2|org.glassfish.admingui|_ThreadID=186;_ThreadName=Thread-2;|RestResponse.getResponse() gives FAILURE. endpoint = 'https://localhost:4848/management/domain/clusters/cluster/cl/start-cluster'; attrs = '{}'|#] [#|2012-01-04T17:31:21.381-0800|INFO|glassfish3.1.2|javax.enterprise.system.std.com.sun.enterprise.server.logging|_ThreadID=172;_ThreadName=Thread-2;|Stopping instance ....|#] [#|2012-01-04T17:31:21.389-0800|INFO|glassfish3.1.2|javax.enterprise.system.std.com.sun.enterprise.server.logging|_ThreadID=172;_ThreadName=Thread-2;|https://localhost:4848/management/domain/servers/server/inl1/stop-instance|#] [#|2012-01-04T17:31:21.426-0800|INFO|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=166;_ThreadName=Thread-2;|Instance inl1 shutdown initiated|#] [#|2012-01-04T17:31:29.112-0800|WARNING|glassfish3.1.2|org.glassfish.admingui|_ThreadID=186;_ThreadName=Thread-2;|RestResponse.getResponse() gives WARNING. endpoint = 'https://localhost:4848/management/domain/get-runtime-info'; attrs = '{target=inl1}'|#] [#|2012-01-04T17:31:46.921-0800|INFO|glassfish3.1.2|javax.enterprise.system.std.com.sun.enterprise.server.logging|_ThreadID=184;_ThreadName=Thread-2;|Starting Cluster ....|#] [#|2012-01-04T17:31:46.925-0800|INFO|glassfish3.1.2|org.glassfish.admingui|_ThreadID=184;_ThreadName=Thread-2;|https://localhost:4848/management/domain/clusters/cluster/cl/start-cluster|#] [#|2012-01-04T17:31:46.951-0800|INFO|glassfish3.1.2|ShoalLogger|_ThreadID=179;_ThreadName=Thread-2;|GMS1098: GMS:Announcing GroupStartup[INITIATED] for group: cl members: clt1,cll2,|#] [#|2012-01-04T17:31:46.952-0800|INFO|glassfish3.1.2|ShoalLogger|_ThreadID=179;_ThreadName=Thread-2;|GMS1062: GroupStart for group: cl State: INITIATED Starting Members: clt1,cll2,|#] [#|2012-01-04T17:31:46.953-0800|INFO|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=179;_ThreadName=Thread-2;|Starting cluster cl|#] [#|2012-01-04T17:31:46.961-0800|INFO|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=179;_ThreadName=Thread-2;|Executing start-instance on 2 instances using a thread pool of size 2: tuppy:clt1 localhost-domain1:cll2|#] [#|2012-01-04T17:31:46.963-0800|INFO|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=179;_ThreadName=Thread-2;|start-instance clt1|#] [#|2012-01-04T17:31:46.966-0800|INFO|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=179;_ThreadName=Thread-2;|start-instance cll2|#] [#|2012-01-04T17:31:48.984-0800|INFO|glassfish3.1.2|javax.enterprise.system.std.com.sun.enterprise.server.logging|_ThreadID=193;_ThreadName=Thread-2;|Previous synchronization failed at Jan 4, 2012 5:28:26 PM|#] [#|2012-01-04T17:31:48.986-0800|INFO|glassfish3.1.2|javax.enterprise.system.std.com.sun.enterprise.server.logging|_ThreadID=193;_ThreadName=Thread-2;|Will perform full synchronization.|#] [#|2012-01-04T17:31:48.987-0800|INFO|glassfish3.1.2|javax.enterprise.system.std.com.sun.enterprise.server.logging|_ThreadID=193;_ThreadName=Thread-2;|Removing all cached state for instance cll2.|#] [#|2012-01-04T17:32:21.600-0800|WARNING|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin|_ThreadID=194;_ThreadName=Thread-2;|Command: _synchronize-files failed to acquire a command lock. REASON: time out (current lock acquired on Wed, 04 Jan 2012 17:28:56 PST)|#] [#|2012-01-04T17:32:21.636-0800|INFO|glassfish3.1.2|javax.enterprise.system.std.com.sun.enterprise.server.logging|_ThreadID=195;_ThreadName=Thread-2;|CLI802 Synchronization failed for directory config, caused by:|#] [#|2012-01-04T17:32:21.638-0800|INFO|glassfish3.1.2|javax.enterprise.system.std.com.sun.enterprise.server.logging|_ThreadID=195;_ThreadName=Thread-2;| remote failure: Command timed out. Unable to acquire a lock to access the domain. Another command acquired exclusive access to the domain on Wed, 04 Jan 2012 17:28:56 PST. Retry the command at a later time.|#] [#|2012-01-04T17:32:21.638-0800|INFO|glassfish3.1.2|javax.enterprise.system.std.com.sun.enterprise.server.logging|_ThreadID=193;_ThreadName=Thread-2;|Command start-local-instance failed.|#] [#|2012-01-04T17:32:21.679-0800|WARNING|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=196;_ThreadName=Thread-2;|Could not start instance cll2 on node localhost-domain1 (localhost).: Command ' /export/home/j2eetest/3.1.2/glassfish3/glassfish/bin/asadmin --_auxinput - --interactive=false start-local-instance --node localhost-domain1 --sync normal cll2' failed on node localhost-domain1 (localhost): Previous synchronization failed at Jan 4, 2012 5:28:26 PM Will perform full synchronization. Removing all cached state for instance cll2. Command start-local-instance failed. CLI802 Synchronization failed for directory config, caused by: remote failure: Command timed out. Unable to acquire a lock to access the domain. Another command acquired exclusive access to the domain on Wed, 04 Jan 2012 17:28:56 PST. Retry the command at a later time.: To complete this operation run the following command locally on host localhost from the GlassFish install location /export/home/j2eetest/3.1.2/glassfish3: bin/asadmin start-local-instance --node localhost-domain1 --sync normal cll2|#] [#|2012-01-04T17:32:21.682-0800|SEVERE|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=179;_ThreadName=Thread-2;|cll2: Could not start instance cll2 on node localhost-domain1 (localhost). Command failed on node localhost-domain1 (localhost): Previous synchronization failed at Jan 4, 2012 5:28:26 PM Will perform full synchronization. Removing all cached state for instance cll2. Command start-local-instance failed. CLI802 Synchronization failed for directory config, caused by: remote failure: Command timed out. Unable to acquire a lock to access the domain. Another command acquired exclusive access to the domain on Wed, 04 Jan 2012 17:28:56 PST. Retry the command at a later time. To complete this operation run the following command locally on host localhost from the GlassFish install location /export/home/j2eetest/3.1.2/glassfish3: bin/asadmin start-local-instance --node localhost-domain1 --sync normal cll2|#] [#|2012-01-04T17:32:22.860-0800|WARNING|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin|_ThreadID=197;_ThreadName=Thread-2;|Command: _synchronize-files failed to acquire a command lock. REASON: time out (current lock acquired on Wed, 04 Jan 2012 17:28:56 PST)|#] [#|2012-01-04T17:32:23.015-0800|WARNING|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=198;_ThreadName=Thread-2;|Could not start instance clt1 on node tuppy (tuppy).: Command ' /export/home/j2eetest/3.1.2/glassfish3/glassfish/bin/asadmin --_auxinput - --interactive=false start-local-instance --node tuppy --sync normal clt1' failed on node tuppy (tuppy): Previous synchronization failed at Jan 4, 2012 6:04:59 PM Will perform full synchronization. Removing all cached state for instance clt1. CLI802 Synchronization failed for directory config, caused by: remote failure: Command timed out. Unable to acquire a lock to access the domain. Another command acquired exclusive access to the domain on Wed, 04 Jan 2012 17:28:56 PST. Retry the command at a later time. Command start-local-instance failed.: To complete this operation run the following command locally on host tuppy from the GlassFish install location /export/home/j2eetest/3.1.2/glassfish3: bin/asadmin start-local-instance --node tuppy --sync normal clt1|#] [#|2012-01-04T17:32:23.017-0800|SEVERE|glassfish3.1.2|javax.enterprise.system.tools.admin.com.sun.enterprise.v3.admin.cluster|_ThreadID=179;_ThreadName=Thread-2;|clt1: Could not start instance clt1 on node tuppy (tuppy). Command failed on node tuppy (tuppy): Previous synchronization failed at Jan 4, 2012 6:04:59 PM Will perform full synchronization. Removing all cached state for instance clt1. CLI802 Synchronization failed for directory config, caused by: remote failure: Command timed out. Unable to acquire a lock to access the domain. Another command acquired exclusive access to the domain on Wed, 04 Jan 2012 17:28:56 PST. Retry the command at a later time. Command start-local-instance failed. To complete this operation run the following command locally on host tuppy from the GlassFish install location /export/home/j2eetest/3.1.2/glassfish3: bin/asadmin start-local-instance --node tuppy --sync normal clt1|#] [#|2012-01-04T17:32:23.037-0800|SEVERE|glassfish3.1.2|org.glassfish.admingui|_ThreadID=184;_ThreadName=Thread-2;|RestResponse.getResponse() gives FAILURE. endpoint = 'https://localhost:4848/management/domain/clusters/cluster/cl/start-cluster'; attrs = '{}'|#]