/*
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
*
* Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
*
* The contents of this file are subject to the terms of either the GNU
* General Public License Version 2 only ("GPL") or the Common Development
* and Distribution License("CDDL") (collectively, the "License"). You
* may not use this file except in compliance with the License. You can obtain
* a copy of the License at https://glassfish.dev.java.net/public/CDDL+GPL.html
* or glassfish/bootstrap/legal/LICENSE.txt. See the License for the specific
* language governing permissions and limitations under the License.
*
* When distributing the software, include this License Header Notice in each
* file and include the License file at glassfish/bootstrap/legal/LICENSE.txt.
* Sun designates this particular file as subject to the "Classpath" exception
* as provided by Sun in the GPL Version 2 section of the License file that
* accompanied this code. If applicable, add the following below the License
* Header, with the fields enclosed by brackets [] replaced by your own
* identifying information: "Portions Copyrighted [year]
* [name of copyright owner]"
*
* Contributor(s):
*
* If you wish your version of this file to be governed by only the CDDL or
* only the GPL Version 2, indicate your decision by adding "[Contributor]
* elects to include this software in this distribution under the [CDDL or GPL
* Version 2] license." If you don't indicate a single choice of license, a
* recipient has the option to distribute your version of this file under
* either the CDDL, the GPL Version 2 or to extend the choice of license to
* its licensees as provided above. However, if you add GPL Version 2 code
* and therefore, elected the GPL Version 2 license, then the option applies
* only if the new code is made subject to such option by the copyright
* holder.
*/
package com.sun.enterprise.jxtamgmt;
import net.jxta.document.*;
import net.jxta.endpoint.Message;
import net.jxta.endpoint.MessageElement;
import net.jxta.endpoint.TextDocumentMessageElement;
import net.jxta.id.ID;
import net.jxta.peer.PeerID;
import net.jxta.pipe.*;
import net.jxta.protocol.PipeAdvertisement;
import net.jxta.protocol.RouteAdvertisement;
import net.jxta.impl.pipe.BlockingWireOutputPipe;
import net.jxta.endpoint.StringMessageElement;
import java.io.IOException;
import java.text.MessageFormat;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.net.InetAddress;
import java.net.Socket;
import java.net.URI;
/**
* HealthMonitor utilizes MasterNode to determine self designation. All nodes
* cache other node's states, and can act as a master node at any given
* point in time. The intention behind the designation is that no node other than
* the master node should determine collective state and communicate it to
* group members.
* <p/>
* TODO: Convert the InDoubt Peer Determination and Failure Verification into
* Callable FutureTask using java.util.concurrent
*/
public class HealthMonitor implements PipeMsgListener, Runnable {
private static final Logger LOG = JxtaUtil.getLogger(HealthMonitor.class.getName());
// Default health reporting period
private long timeout = 10 * 1000L;
private long verifyTimeout = 10 * 1000L;
private int maxMissedBeats = 3;
private final String threadLock = new String("threadLock");
private final ConcurrentHashMap<PeerID, HealthMessage.Entry> cache = new ConcurrentHashMap<PeerID, HealthMessage.Entry>();
private MasterNode masterNode = null;
private ClusterManager manager = null;
private final PeerID localPeerID;
InputPipe inputPipe = null;
private OutputPipe outputPipe = null;
private PipeAdvertisement pipeAdv = null;
private final PipeService pipeService;
private volatile boolean started = false;
private volatile boolean stop = false;
private Thread healthMonitorThread = null;
private Thread failureDetectorThread = null;
private static final String NODEADV = "NAD";
private InDoubtPeerDetector inDoubtPeerDetector;
private final String[] states = {"starting",
"started",
"alive",
"clusterstopping",
"peerstopping",
"stopped",
"dead",
"indoubt",
"unknown",
"ready",
"aliveandready"};
// "alive_in_isolation"};
private static final short STARTING = 0;
private static final short ALIVE = 2;
private static final short CLUSTERSTOPPING = 3;
private static final short PEERSTOPPING = 4;
private static final short STOPPED = 5;
private static final short DEAD = 6;
private static final short INDOUBT = 7;
private static final short UNKNOWN = 8;
private static final short READY = 9;
private static final short ALIVEANDREADY = 10;
//private static final short ALIVE_IN_ISOLATION = 10;
private static final String HEALTHM = "HM";
private static final String NAMESPACE = "HEALTH";
private static final String cacheLock = "cacheLock";
private static final String verifierLock = "verifierLock";
private static final String MEMBER_STATE_QUERY = "MEMBERSTATEQUERY";
private static final String MEMBER_STATE_RESPONSE = "MEMBERSTATERESPONSE";
private boolean readyStateComplete = false;
private Message aliveMsg = null;
private Message aliveAndReadyMsg = null;
private transient Map<ID, OutputPipe> pipeCache = new Hashtable<ID, OutputPipe>();
//counter for keeping track of the seq ids of the health messages
AtomicLong hmSeqID = new AtomicLong();
//use LWRMulticast to send messages for getting the member state
LWRMulticast mcast = null;
int lwrTimeout = 6000;
private static final String memberStateLock = new String("memberStateLock");
private String memberState;
private final String hwFailureDetectionthreadLock = new String ("hwFailureDetectionthreadLock");
private static final String CONNECTION_REFUSED = "Connection refused";
private long failureDetectionTCPTimeout;
private int failureDetectionTCPPort;
private final ThreadPoolExecutor isConnectedPool;
//private ShutdownHook shutdownHook;
/**
* Constructor for the HealthMonitor object
*
* @param manager the ClusterManager
* @param maxMissedBeats Maximum retries before failure
* @param verifyTimeout timeout in milliseconds that the health monitor
* waits before finalizing that the in doubt peer is dead.
* @param timeout in milliseconds that the health monitor waits before
* retrying an indoubt peer's availability.
*/
public HealthMonitor(final ClusterManager manager, final long timeout,
final int maxMissedBeats, final long verifyTimeout,
final long failureDetectionTCPTimeout,
final int failureDetectionTCPPort) {
this.timeout = timeout;
this.maxMissedBeats = maxMissedBeats;
this.verifyTimeout = verifyTimeout;
this.manager = manager;
this.masterNode = manager.getMasterNode();
this.localPeerID = manager.getNetPeerGroup().getPeerID();
this.pipeService = manager.getNetPeerGroup().getPipeService();
this.failureDetectionTCPTimeout = failureDetectionTCPTimeout;
this.failureDetectionTCPPort = failureDetectionTCPPort;
isConnectedPool = (ThreadPoolExecutor)Executors.newCachedThreadPool();
try {
mcast = new LWRMulticast(manager,
createPipeAdv(),
this);
mcast.setSoTimeout(lwrTimeout);
} catch (IOException e) {
LOG.warning("Cound not instantiate LWRMulticast : " + e.getMessage());
}
//this.shutdownHook = new ShutdownHook();
//Runtime.getRuntime().addShutdownHook(shutdownHook);
}
void fine(String msg, Object[] obj) {
if (LOG.isLoggable(Level.FINE)) {
LOG.log(Level.FINE, msg, obj);
}
}
void fine(String msg) {
if (LOG.isLoggable(Level.FINE)) {
LOG.log(Level.FINE, msg);
}
}
/**
* in the event of a failure or planned shutdown, remove the
* pipe from the pipeCache
*/
public void removePipeFromCache(ID token) {
pipeCache.remove(token);
}
/**
* Creates a Message containing this node's state
*
* @param state member state
* @return a Message containing this node's state
*/
private Message createHealthMessage(final short state) {
Message msg = createMessage(state, HEALTHM, manager.getSystemAdvertisement());
masterNode.addRoute(msg);
return msg;
}
private Message createMessage(final short state, final String tag,
final SystemAdvertisement adv) {
final Message msg = new Message();
final HealthMessage hm = new HealthMessage();
hm.setSrcID(localPeerID);
final HealthMessage.Entry entry = new HealthMessage.Entry(adv, states[state], hmSeqID.incrementAndGet());
hm.add(entry);
msg.addMessageElement(NAMESPACE, new TextDocumentMessageElement(tag,
(XMLDocument) hm.getDocument(MimeMediaType.XMLUTF8), null));
//add this state to the local health cache.
cache.put(entry.id, entry);
return msg;
}
private Message getAliveMessage() {
if (aliveMsg == null) {
aliveMsg = createHealthMessage(ALIVE);
}
return aliveMsg;
}
private Message getAliveAndReadyMessage() {
if (aliveAndReadyMsg == null) {
aliveAndReadyMsg = createHealthMessage(ALIVEANDREADY);
}
return aliveAndReadyMsg;
}
/**
* Given a pipeid it returns a HealthMonitor pipe advertisement of propagate type
*
* @return a HealthMonitor pipe advertisement of propagate type
*/
private PipeAdvertisement createPipeAdv() {
final PipeAdvertisement pipeAdv;
// create the pipe advertisement, to be used in creating the pipe
pipeAdv = (PipeAdvertisement)
AdvertisementFactory.newAdvertisement(
PipeAdvertisement.getAdvertisementType());
pipeAdv.setPipeID(manager.getNetworkManager().getHealthPipeID());
pipeAdv.setType(PipeService.PropagateType);
return pipeAdv;
}
/**
* {@inheritDoc}
*/
public void pipeMsgEvent(final PipeMsgEvent event) {
//if this peer is stopping, then stop processing incoming health messages.
if (manager.isStopping()) {
return;
}
if (started) {
final Message msg;
MessageElement msgElement;
try {
// grab the message from the event
msg = event.getMessage();
if (msg != null) {
final Message.ElementIterator iter = msg.getMessageElements();
while (iter.hasNext()) {
msgElement = iter.next();
if (msgElement != null && msgElement.getElementName().equals(HEALTHM)) {
HealthMessage hm = getHealthMessage(msgElement);
if (!hm.getSrcID().equals(localPeerID)) {
masterNode.processRoute(msg);
}
process(hm);
} else if (msgElement != null && msgElement.getElementName().equals(MEMBER_STATE_QUERY)) {
processMemberStateQuery(msg);
} else if (msgElement != null && msgElement.getElementName().equals(MEMBER_STATE_RESPONSE)) {
processMemberStateResponse(msg);
}
}
}
} catch (IOException ex) {
if (LOG.isLoggable(Level.FINE)) {
ex.printStackTrace();
}
LOG.log(Level.WARNING, "HealthMonitor:Caught IOException : " + ex.getLocalizedMessage());
} catch (Throwable e) {
if (LOG.isLoggable(Level.FINE)) {
e.printStackTrace();
}
LOG.log(Level.WARNING, e.getLocalizedMessage());
}
}
}
private void processMemberStateQuery(Message msg) {
boolean foundNodeAdv = false;
LOG.fine(" received a MemberStateQuery...");
try {
//SystemAdvertisement adv = masterNode.processNodeAdvertisement(msg);
Message.ElementIterator iter = msg.getMessageElements();
MessageElement msgElement = null;
while (iter.hasNext()) {
msgElement = iter.next();
if (msgElement.getElementName().equals(NODEADV)) {
foundNodeAdv = true;
break;
}
}
if (foundNodeAdv == true) {
final StructuredDocument asDoc;
asDoc = StructuredDocumentFactory.newStructuredDocument(msgElement.getMimeType(), msgElement.getStream());
final SystemAdvertisement adv = new SystemAdvertisement(asDoc);
if (!adv.getID().equals(localPeerID)) {
LOG.log(Level.FINER, "Received a System advertisment Name :" + adv.getName());
}
if (adv != null) {
ID sender = adv.getID(); //sender of this query
String state = getStateFromCache(localPeerID);
Message response = createMemberStateResponse(state);
LOG.fine(" sending via LWR response to " + sender.toString() + " with state " + state + " for " + localPeerID);
mcast.send((PeerID) sender, response); //send the response back to the query sender
}
} else {
LOG.warning("Don't know where this query came from. SysAdv is null");
}
} catch (IOException e) {
if (LOG.isLoggable(Level.FINE)) {
e.printStackTrace();
}
LOG.warning("Could not send the message via LWRMulticast : " + e.getMessage());
}
}
private void processMemberStateResponse(Message msg) {
LOG.fine("received a MemberStateResponse...");
memberState = msg.getMessageElement(NAMESPACE, MEMBER_STATE_RESPONSE).toString();
LOG.fine(" member state in processMemberStateResponse() is " + memberState);
synchronized (memberStateLock) {
memberStateLock.notify();
}
}
/**
* creates a node query message specially for querying the member state
*
* @return msg Message
*/
private Message createMemberStateQuery() {
Message msg = new Message();
msg.addMessageElement(NAMESPACE, new TextDocumentMessageElement(NODEADV,
(XMLDocument) manager.getSystemAdvertisement()
.getDocument(MimeMediaType.XMLUTF8), null));
msg.addMessageElement(NAMESPACE, new StringMessageElement(MEMBER_STATE_QUERY, "member state query", null));
LOG.log(Level.FINE, "Created a Member State Query Message ");
return msg;
}
/**
* creates a node response message specially for sending the member state
*
* @return msg Message
*/
private Message createMemberStateResponse(String myState) {
Message msg = new Message();
msg.addMessageElement(NAMESPACE, new StringMessageElement(MEMBER_STATE_RESPONSE, myState, null));
msg.addMessageElement(NAMESPACE, new TextDocumentMessageElement(NODEADV,
(XMLDocument) manager.getSystemAdvertisement()
.getDocument(MimeMediaType.XMLUTF8), null));
LOG.log(Level.FINE, "Created a Member State Response Message with " + myState);
return msg;
}
/**
* process a health message received on the wire into cache
*
* @param hm Health message to process
*/
private void process(final HealthMessage hm) {
//discard loopback messages
if (!hm.getSrcID().equals(localPeerID)) {
//current assumption is that there is only 1 entry
//per health message
for (HealthMessage.Entry entry : hm.getEntries()) {
LOG.log(Level.FINEST, "Processing Health Message " + entry.getSeqID() + " for entry " + entry.adv.getName());
LOG.log(Level.FINEST, "Getting the cachedEntry " + entry.id);
HealthMessage.Entry cachedEntry = cache.get(entry.id);
if (cachedEntry != null) {
LOG.log(Level.FINEST, "cachedEntry is not null");
if (entry.getSeqID() <= cachedEntry.getSeqID()) {
LOG.log(Level.FINER, MessageFormat.format("Received an older health message sequence {0}." +
" Current sequence id is {1}. ",
entry.getSeqID(), cachedEntry.getSeqID()));
if (entry.state.equals(states[CLUSTERSTOPPING]) || entry.state.equals(states[PEERSTOPPING])) {
//dont discard the message
//and don't alter the state if the cachedEntry's state is stopped
LOG.log(Level.FINER, "Received out of order health message " +
"with clusterstopping state." +
" Calling handleStopEvent() to handle shutdown state.");
handleStopEvent(entry);
} else if (entry.state.equals(states[READY])) {
LOG.finer("Received out of order health message with Joined and Ready state. " +
"Calling handleReadyEvent() for handling the peer's ready state");
handleReadyEvent(entry);
} else {
LOG.log(Level.FINER, "Discarding out of sequence health message");
}
return;
}
}
LOG.log(Level.FINEST, "Putting into cache " + entry.adv.getName() + " state = " + entry.state + " peerid = " + entry.id);
cache.put(entry.id, entry);
if (!manager.getClusterViewManager().containsKey(entry.id) &&
(!entry.state.equals(states[CLUSTERSTOPPING]) &&
!entry.state.equals(states[PEERSTOPPING]) &&
!entry.state.equals(states[STOPPED]) &&
!entry.state.equals(states[DEAD]))) {
try {
masterNode.probeNode(entry);
} catch (IOException e) {
if (LOG.isLoggable(Level.FINE)) {
e.printStackTrace();
}
LOG.warning("IOException occured while sending probeNode() Message in HealthMonitor:"+e.getLocalizedMessage());
}
}
if (entry.state.equals(states[READY])) {
handleReadyEvent( entry );
}
if (entry.state.equals(states[PEERSTOPPING]) || entry.state.equals(states[CLUSTERSTOPPING])) {
handleStopEvent(entry);
}
if (entry.state.equals(states[INDOUBT]) || entry.state.equals(states[DEAD])) {
if (entry.id.equals(localPeerID)) {
if (readyStateComplete) {
reportMyState(ALIVEANDREADY,hm.getSrcID());
} else {
reportMyState(ALIVE, hm.getSrcID());
}
} else {
if (entry.state.equals(states[INDOUBT])) {
LOG.log(Level.FINE, "Peer " + entry.id.toString() + " is suspected failed. Its state is " + entry.state);
notifyLocalListeners(entry.state, entry.adv);
}
if (entry.state.equals(states[DEAD])) {
LOG.log(Level.FINE, "Peer " + entry.id.toString() + " has failed. Its state is " + entry.state);
cleanAllCaches(entry);
}
}
}
}
}
}
private void handleReadyEvent(final HealthMessage.Entry entry) {
cache.put(entry.id, entry);
if(entry.id.equals(masterNode.getMasterNodeID())){
//if this is a ready state sent out by master, take no action here as master would send out a view to the group.
return;
}
//if this is a ready state sent by a non-master, and if I am the assigned master, send out a new view and notify local listeners.
if(masterNode.isMaster() && masterNode.isMasterAssigned()){
LOG.log(Level.FINEST, MessageFormat.format("Handling Ready Event for peer :{0}", entry.adv.getName()));
final ClusterViewEvent cvEvent = masterNode.sendReadyEventView(entry.adv);
manager.getClusterViewManager().notifyListeners(cvEvent);
}
}
private void handleStopEvent(final HealthMessage.Entry entry) {
LOG.log(Level.FINEST, MessageFormat.format("Handling Stop Event for peer :{0}", entry.adv.getName()));
short stateByte = PEERSTOPPING;
if (entry.state.equals(states[CLUSTERSTOPPING])) {
stateByte = CLUSTERSTOPPING;
}
if (entry.adv.getID().equals(masterNode.getMasterNodeID())) {
//if masternode is resigning, remove master node from view and start discovery
LOG.log(Level.FINER, MessageFormat.format("Removing master node {0} from view as it has stopped.", entry.adv.getName()));
removeMasterAdv(entry, stateByte);
masterNode.resetMaster();
masterNode.appointMasterNode();
} else if (masterNode.isMaster() && masterNode.isMasterAssigned()) {
removeMasterAdv(entry, stateByte);
LOG.log(Level.FINE, "Announcing Peer Stop Event of " + entry.adv.getName() + " to group ...");
final ClusterViewEvent cvEvent;
if (entry.state.equals(states[CLUSTERSTOPPING])) {
cvEvent = new ClusterViewEvent(ClusterViewEvents.CLUSTER_STOP_EVENT, entry.adv);
} else {
cvEvent = new ClusterViewEvent(ClusterViewEvents.PEER_STOP_EVENT, entry.adv);
}
masterNode.viewChanged(cvEvent);
}
cleanAllCaches(entry);
}
private void cleanAllCaches(HealthMessage.Entry entry) {
LOG.fine("HealthMonitor.cleanAllCaches : removing pipes and route from cache..." + entry.id);
removePipeFromCache(entry.id);
manager.removePipeFromCache(entry.id);
manager.removeRouteFromCache(entry.id);
masterNode.removePipeFromCache(entry.id);
}
private Map<PeerID, HealthMessage.Entry> getCacheCopy() {
ConcurrentHashMap<PeerID, HealthMessage.Entry> clone = new ConcurrentHashMap<PeerID, HealthMessage.Entry>();
clone.putAll(cache);
return clone;
}
/**
* Reports on the wire the specified state
*
* @param state specified state can be
* ALIVE|SLEEP|HIBERNATING|SHUTDOWN|DEAD
* @param id destination node ID, if null broadcast to group
*/
private void reportMyState(final short state, final PeerID id) {
LOG.log(Level.FINER, MessageFormat.format("Sending state {0} to {1}", states[state], id));
if (state == ALIVE) {
send(id, getAliveMessage());
} else {
if (state == ALIVEANDREADY) {
send(id, getAliveAndReadyMessage());
} else
send(id, createHealthMessage(state));
}
}
private void reportOtherPeerState(final short state, final SystemAdvertisement adv) {
final Message msg = createMessage(state, HEALTHM, adv);
LOG.log(Level.FINEST, MessageFormat.format("Reporting {0} health state as {1}", adv.getName(), states[state]));
send(null, msg);
}
/**
* Main processing method for the HealthMonitor object
*/
public void run() {
//At start, the starting state is reported. But reporting ready state is dependent on
//the parent application making the call that it is in the ready state, which is not
//necessarily deterministic. Hence we move on from starting state to reporting
//this peer's alive state. For join notification event, the liveness state of a peer
// would be one of starting or ready or alive (other than peerstopping, etc).
reportMyState(STARTING, null);
//System.out.println("Running HealthMonitor Thread at interval :"+actualto);
while (!stop) {
try {
synchronized (threadLock) {
threadLock.wait(timeout);
}
if (!stop) {
if (readyStateComplete) {
reportMyState(ALIVEANDREADY, null);
} else {
reportMyState(ALIVE, null);
}
}
} catch (InterruptedException e) {
stop = true;
LOG.log(Level.FINEST, "Shoal Health Monitor Thread Stopping as the thread is now interrupted...:"+e.getLocalizedMessage());
break;
} catch (Throwable all) {
LOG.log(Level.WARNING, "Uncaught Throwable in healthMonitorThread " + Thread.currentThread().getName() + ":" + all);
}
}
}
/**
* Send a message to a specific node. In case the peerId is null the
* message is multicast to the group
*
* @param peerid Peer ID to send massage to
* @param msg the message to send
*/
private void send(final PeerID peerid, final Message msg) {
try {
if (peerid != null) {
// Unicast datagram
// create a op pipe to the destination peer
LOG.log(Level.FINE, "Unicasting Message to :" + peerid.toString());
OutputPipe output = null;
if (!pipeCache.containsKey(peerid)) {
RouteAdvertisement route = manager.getCachedRoute((PeerID) peerid);
if (route != null) {
output = new BlockingWireOutputPipe(manager.getNetPeerGroup(), pipeAdv, (PeerID) peerid, route);
}
if (output == null) {
// Unicast datagram
// create a op pipe to the destination peer
output = pipeService.createOutputPipe(pipeAdv, Collections.singleton(peerid), 1);
}
pipeCache.put(peerid, output);
} else {
output = pipeCache.get(peerid);
if (output.isClosed()) {
output = pipeService.createOutputPipe(pipeAdv, Collections.singleton(peerid), 1);
pipeCache.put(peerid, output);
}
}
output.send(msg);
} else {
outputPipe.send(msg);
}
} catch (IOException io) {
LOG.log(Level.WARNING, "Failed to send message", io);
}
}
/**
* Creates both input/output pipes, and starts monitoring service
*/
void start() {
if (!started) {
// 2008.06.10, added by carryel
// ensure whether discovery is done.
// If masterNode.probeNode(entry) is called before master's discovery is not done, unexpected results can be occurred in the master selection algorithm.
if( masterNode.isDiscoveryInProgress() ) {
synchronized( masterNode.discoveryLock ) {
try {
masterNode.discoveryLock.wait();
LOG.log( Level.FINEST, "start() waiting for masternode discovery to finish..." );
} catch( InterruptedException e ) {
LOG.log( Level.FINEST, "MasterNode's DiscoveryLock Thread is interrupted " + e );
}
}
}
LOG.log(Level.FINE, "Starting HealthMonitor");
try {
// create the pipe advertisement, to be used in creating the pipe
pipeAdv = createPipeAdv();
// create input
inputPipe = pipeService.createInputPipe(pipeAdv, this);
// create output
outputPipe = pipeService.createOutputPipe(pipeAdv, 1);
this.healthMonitorThread = new Thread(this, "HealthMonitor");
healthMonitorThread.start();
inDoubtPeerDetector = new InDoubtPeerDetector();
inDoubtPeerDetector.start();
started = true;
} catch (IOException ioe) {
LOG.log(Level.WARNING, "Failed to create health monitoring pipe advertisement :" + ioe);
}
}
}
/**
* Announces Stop event to all members indicating that this peer
* will gracefully exit the group.
* TODO: Make this a synchronous call or a simulated synchronous call so that responses from majority of members can be collected before returning from this method
*
* @param isClusterShutdown boolean value indicating whether this
* announcement is in the context of a clusterwide shutdown or a shutdown of
* this peer only.
*/
void announceStop(final boolean isClusterShutdown) {
//System.out.println("Announcing Shutdown");
LOG.log(Level.FINE, MessageFormat.format("Announcing stop event to group with clusterShutdown set to {0}", isClusterShutdown));
if (isClusterShutdown) {
reportMyState(CLUSTERSTOPPING, null);
} else {
reportMyState(PEERSTOPPING, null);
}
}
/**
* Stops this service
*
* @param isClusterShutdown true if the cluster is shutting down
*/
void stop(boolean isClusterShutdown) {
stop = true;
started = false;
announceStop(isClusterShutdown);
reportMyState(STOPPED, null);
LOG.log(Level.FINE, "Stopping HealthMonitor");
final Thread tmpThread = healthMonitorThread;
healthMonitorThread = null;
if (tmpThread != null) {
tmpThread.interrupt();
}
inDoubtPeerDetector.stop();
inputPipe.close();
outputPipe.close();
pipeCache.clear();
manager.clearAllCaches();
masterNode.clearPipeCache();
}
private HealthMessage getHealthMessage(final MessageElement msgElement) throws IOException {
final HealthMessage hm;
hm = new HealthMessage(getStructuredDocument(msgElement), hmSeqID.incrementAndGet());
return hm;
}
private static StructuredTextDocument getStructuredDocument(
final MessageElement msgElement) throws IOException {
return (StructuredTextDocument) StructuredDocumentFactory.newStructuredDocument(MimeMediaType.XMLUTF8,
msgElement.getStream());
}
private void notifyLocalListeners(final String state, final SystemAdvertisement adv) {
if (state.equals(states[INDOUBT])) {
manager.getClusterViewManager().setInDoubtPeerState(adv);
} else if (state.equals(states[ALIVE])) {
manager.getClusterViewManager().setPeerNoLongerInDoubtState(adv);
} else if (state.equals(states[ALIVEANDREADY])) {
manager.getClusterViewManager().setPeerNoLongerInDoubtState(adv);
} else if (state.equals(states[CLUSTERSTOPPING])) {
manager.getClusterViewManager().setClusterStoppingState(adv);
} else if (state.equals(states[PEERSTOPPING])) {
manager.getClusterViewManager().setPeerStoppingState(adv);
} else if (state.equals(states[READY])) {
manager.getClusterViewManager().setPeerReadyState(adv);
} /*else if (state.equals(states[ALIVE_IN_ISOLATION])) {
manager.getClusterViewManager().setInIsolationState(adv);
} */
}
public String getMemberState(final ID peerID) {
//don't get the cached member state for that instance
//instead send a query to that instance to get the most up-to-date state
LOG.fine("inside getMemberState for " + peerID.toString());
Message msg = createMemberStateQuery();
//send it via LWRMulticast
try {
mcast.send((PeerID) peerID, msg);
LOG.fine("send message in getMemberState via LWR...");
} catch (IOException e) {
LOG.warning("Could not send the LWR Multicast message to get the member state of " + peerID.toString() + " IOException : " + e.getMessage());
}
synchronized (memberStateLock) {
try {
memberStateLock.wait(timeout);
} catch (InterruptedException e) {
LOG.warning("wait() was interrupted : " + e.getMessage());
}
}
if (memberState != null) {
String state = memberState;
memberState = null; //nullify this string before returning so that the next time this method is accessed
// memberState is null before it is assigned a value in processMemberStateResponse
LOG.fine("inside getMemberState got state via lwr " + state);
return state;
} else { //if timeout happened even before notify() was called
String state = getStateFromCache(peerID);
LOG.fine("inside getMemberState got state after timeout " + state);
return state;
}
}
String getStateFromCache(final ID peerID) {
HealthMessage.Entry entry;
final String state;
entry = cache.get((PeerID) peerID);
if (entry != null) {
state = entry.state;
} else {
if (((PeerID) peerID).equals(localPeerID)) {
if (!started) {
state = states[STARTING];
} else if (readyStateComplete) {
state = states[ALIVEANDREADY];
} else {
state = states[ALIVE];
}
} else {
entry = cache.get((PeerID) peerID);
if (entry != null) {
state = entry.state;
} else {
if (manager.getClusterViewManager().containsKey(peerID)) {
state = states[STARTING];//we assume that the peer is in starting state hence its state is not yet known in this peer
} else {
state = states[UNKNOWN];
}
}
}
}
return state;
}
public void reportJoinedAndReadyState() {
//if I am the master send out a new view with ready event but wait for master discovery to be over.Also send out a health message
//with state set to READY. Recipients of health message would only update their health state cache.
//Master Node discovery completion and assignment is necessary for the ready state of a peer to be sent out as an event with a view.
//if I am not the master, report my state to the group as READY. MasterNode will send out a
//JOINED_AND_READY_EVENT view to the group on receipt of this health state message.
if(masterNode.isDiscoveryInProgress()){
synchronized (masterNode.discoveryLock){
try {
masterNode.discoveryLock.wait();
LOG.log(Level.FINEST, "reportJoinedAndReadyState() waiting for masternode discovery to finish...");
} catch (InterruptedException e) {
LOG.log(Level.FINEST, "MasterNode's DiscoveryLock Thread is interrupted "+e);
}
}
}
if(masterNode.isMaster() && masterNode.isMasterAssigned()){
LOG.log(Level.FINEST, "Sending Ready Event View for "+ manager.getSystemAdvertisement().getName());
ClusterViewEvent cvEvent = masterNode.sendReadyEventView(manager.getSystemAdvertisement());
LOG.log(Level.FINEST, MessageFormat.format("Notifying Local listeners about " +
"Joined and Ready Event View for peer :{0}", manager.getSystemAdvertisement().getName()));
manager.getClusterViewManager().notifyListeners(cvEvent);
}
LOG.log(Level.FINEST, "Calling reportMyState() with READY...");
reportMyState(READY, null);
readyStateComplete = true;
}
/**
* Detects suspected failures and then delegates final decision to
* FailureVerifier
*
* @author : Shreedhar Ganapathy
*/
private class InDoubtPeerDetector implements Runnable {
void start() {
failureDetectorThread = new Thread(this, "InDoubtPeerDetector Thread");
LOG.log(Level.FINE, "Starting InDoubtPeerDetector Thread");
failureDetectorThread.start();
FailureVerifier fverifier = new FailureVerifier();
final Thread fvThread = new Thread(fverifier, "FailureVerifier Thread");
LOG.log(Level.FINE, "Starting FailureVerifier Thread");
fvThread.start();
}
void stop() {
final Thread tmpThread = failureDetectorThread;
failureDetectorThread = null;
if (tmpThread != null) {
tmpThread.interrupt();
}
synchronized (verifierLock) {
verifierLock.notify();
}
}
public void run() {
while (!stop) {
synchronized (cacheLock) {
try {
//System.out.println("InDoubtPeerDetector failureDetectorThread waiting for :"+timeout);
//wait for specified timeout or until woken up
cacheLock.wait(timeout);
//LOG.log(Level.FINEST, "Analyzing cache for health...");
//get the copy of the states cache
if (!manager.isStopping()) {
processCacheUpdate();
}
} catch (InterruptedException ex) {
LOG.log(Level.FINEST, "InDoubtPeerDetector Thread stopping as it is now interrupted :"+ex.getLocalizedMessage());
break;
}
catch(Throwable all){
if (LOG.isLoggable(Level.FINE)) {
all.printStackTrace();
}
LOG.warning("Uncaught Throwable in failureDetectorThread " + Thread.currentThread().getName() + ":" + all);
}
}
}
}
/**
* computes the number of heart beats missed based on an entry's timestamp
*
* @param entry the Health entry
* @return the number heart beats missed
*/
int computeMissedBeat(HealthMessage.Entry entry) {
return (int) ((System.currentTimeMillis() - entry.timestamp) / timeout);
}
private void processCacheUpdate() {
final Map<PeerID, HealthMessage.Entry> cacheCopy = getCacheCopy();
//for each peer id
for (HealthMessage.Entry entry : cacheCopy.values()) {
//don't check for isConnected with your own self
if (!entry.id.equals(manager.getSystemAdvertisement().getID())) {
LOG.fine("processCacheUpdate : " + entry.adv.getName() + " 's state is " + entry.state);
if (entry.state.equals(states[ALIVE]) ||
(entry.state.equals(states[ALIVEANDREADY])) ) {
//if there is a record, then get the number of
//retries performed in an earlier iteration
try {
determineInDoubtPeers(entry);
} catch (NumberFormatException nfe) {
if (LOG.isLoggable(Level.FINE)) {
nfe.printStackTrace();
}
LOG.log(Level.WARNING, "Exception occurred during time stamp conversion : " + nfe.getLocalizedMessage());
}
}
}
}
}
private void determineInDoubtPeers(final HealthMessage.Entry entry) {
//if current time exceeds the last state update timestamp from this peer id, by more than the
//the specified max timeout
if (!stop) {
if (computeMissedBeat(entry) >= maxMissedBeats && !isConnected(entry)) {
LOG.log(Level.FINEST, "timeDiff > maxTime");
if (canProcessInDoubt(entry)) {
LOG.log(Level.FINER, "Designating InDoubtState");
designateInDoubtState(entry);
//delegate verification to Failure Verifier
LOG.log(Level.FINER, "Notifying FailureVerifier for "+entry.adv.getName());
synchronized (verifierLock) {
verifierLock.notify();
LOG.log(Level.FINER, "Done Notifying FailureVerifier for "+entry.adv.getName());
}
}
} else {
//dont suspect self
if (!entry.id.equals(localPeerID)) {
if (canProcessInDoubt(entry)) {
LOG.log(Level.FINE, MessageFormat.format("For instance = {0}; last recorded heart-beat = {1}ms ago, heart-beat # {2} out of a max of {3}",
entry.adv.getName(), (System.currentTimeMillis() - entry.timestamp), computeMissedBeat(entry), maxMissedBeats));
}
}
}
}
}
private boolean canProcessInDoubt(final HealthMessage.Entry entry) {
boolean canProcessIndoubt = false;
if (masterNode.getMasterNodeID().equals(entry.id)) {
canProcessIndoubt = true;
} else if (masterNode.isMaster()) {
canProcessIndoubt = true;
}
return canProcessIndoubt;
}
private void designateInDoubtState(final HealthMessage.Entry entry) {
entry.state = states[INDOUBT];
cache.put(entry.id, entry);
if (masterNode.isMaster()) {
//do this only when masternode is not suspect.
// When masternode is suspect, all members update their states
// anyways so no need to report
//Send group message announcing InDoubt State
LOG.log(Level.FINE, "Sending INDOUBT state message about node ID: " + entry.id + " to the cluster...");
reportOtherPeerState(INDOUBT, entry.adv);
}
LOG.log(Level.FINEST, "Notifying Local Listeners of designated indoubt state for "+entry.adv.getName());
notifyLocalListeners(entry.state, entry.adv);
}
}
private class FailureVerifier implements Runnable {
private final long buffer = 500;
public void run() {
try {
synchronized (verifierLock) {
while (!stop) {
LOG.log(Level.FINER, "FV: Entering verifierLock Wait....");
verifierLock.wait();
LOG.log(Level.FINER, "FV: Woken up from verifierLock Wait by a notify ....");
if (!stop) {
LOG.log(Level.FINER, "FV: Calling verify() ....");
verify();
LOG.log(Level.FINER, "FV: Done verifying ....");
}
}
}
} catch (InterruptedException ex) {
LOG.log(Level.FINEST, MessageFormat.format("failure Verifier Thread stopping as it is now interrupted: {0}", ex.getLocalizedMessage()));
}
}
void verify() throws InterruptedException {
//wait for the specified timeout for verification
Thread.sleep(verifyTimeout + buffer);
HealthMessage.Entry entry;
for (HealthMessage.Entry entry1 : getCacheCopy().values()) {
entry = entry1;
LOG.log(Level.FINER, "FV: Verifying state of "+entry.adv.getName()+" state = "+entry.state);
if(entry.state.equals(states[INDOUBT]) && !isConnected(entry)){
LOG.log(Level.FINER, "FV: Assigning and reporting failure ....");
assignAndReportFailure(entry);
}
}
}
}
private void assignAndReportFailure(final HealthMessage.Entry entry) {
if (entry != null) {
entry.state = states[DEAD];
cache.put(entry.id, entry);
if (masterNode.isMaster()) {
LOG.log(Level.FINE, MessageFormat.format("Reporting Failed Node {0}", entry.id.toString()));
reportOtherPeerState(DEAD, entry.adv);
}
final boolean masterFailed = (masterNode.getMasterNodeID()).equals(entry.id);
if (masterNode.isMaster() && masterNode.isMasterAssigned()) {
LOG.log(Level.FINE, MessageFormat.format("Removing System Advertisement :{0} for name {1}", entry.id.toString(), entry.adv.getName()));
removeMasterAdv(entry, DEAD);
LOG.log(Level.FINE, MessageFormat.format("Announcing Failure Event of {0} for name {1}...", entry.id, entry.adv.getName()));
final ClusterViewEvent cvEvent = new ClusterViewEvent(ClusterViewEvents.FAILURE_EVENT, entry.adv);
masterNode.viewChanged(cvEvent);
} else if (masterFailed) {
//remove the failed node
LOG.log(Level.FINE, MessageFormat.format("Master Failed. Removing System Advertisement :{0} for master named {1}", entry.id.toString(), entry.adv.getName()));
removeMasterAdv(entry, DEAD);
//manager.getClusterViewManager().remove(entry.id);
masterNode.resetMaster();
masterNode.appointMasterNode();
}
cleanAllCaches(entry);
}
}
private void removeMasterAdv(HealthMessage.Entry entry, short state) {
manager.getClusterViewManager().remove(entry.adv);
if (entry.adv != null) {
switch (state) {
case DEAD:
LOG.log(Level.FINER, "FV: Notifying local listeners of Failure of "+entry.adv.getName());
manager.getClusterViewManager().notifyListeners(
new ClusterViewEvent(ClusterViewEvents.FAILURE_EVENT, entry.adv));
break;
case PEERSTOPPING:
LOG.log(Level.FINER, "FV: Notifying local listeners of Shutdown of "+entry.adv.getName());
manager.getClusterViewManager().notifyListeners(
new ClusterViewEvent(ClusterViewEvents.PEER_STOP_EVENT, entry.adv));
break;
case CLUSTERSTOPPING:
LOG.log(Level.FINER, "FV: Notifying local listeners of Cluster_Stopping of "+entry.adv.getName());
manager.getClusterViewManager().notifyListeners(
new ClusterViewEvent(ClusterViewEvents.CLUSTER_STOP_EVENT, entry.adv));
break;
default:
LOG.log(Level.FINEST, MessageFormat.format("Invalid State for removing adv from view {0}", state));
}
} else {
LOG.log(Level.WARNING, states[state] + " peer: " + entry.id + " does not exist in local ClusterView");
}
}
/**
* This method is for designating myself in network isolation
* if my network interface is not up.
* It does not matter if I am the master or not.
* @param entry
*/
/* private void designateInIsolationState(final HealthMessage.Entry entry) {
entry.state = states[ALIVE_IN_ISOLATION];
cache.put(entry.id, entry);
LOG.log(Level.FINE, "Sending ALIVE_IN_ISOLATION state message about node ID: " + entry.id + " to the cluster...");
reportMyState(ALIVE_IN_ISOLATION, entry.id);
LOG.log(Level.FINE, "Notifying Local Listeners of designated ALIVE_IN_ISOLATION state for " + entry.adv.getName());
notifyLocalListeners(entry.state, entry.adv);
} */
/**
* Determines whether a connection to a specific node exists, or one can be created
*
* @param pid Node ID
* @return true, if a connection already exists, or a new was sucessfully created
*/
/* public boolean isConnected(PeerID pid) {
//if System property for InetAddress.isReachable() is set, then check for the following:
//if InetAddress.isReachable() is true, then check for isConnected()
//if InetAddress.isReachable() is false, then simply return false
return masterNode.getRouteControl().isConnected(pid, manager.getCachedRoute(pid));
} */
public boolean isConnected(HealthMessage.Entry entry) {
//TODO : put only during the first heartbeat that comes in for that entry
//TODO : cleaup needed if entry goes down
//TODO : map host to uris. and entries to host
List<URI> list = entry.adv.getURIs();
List<Future> array = new ArrayList<Future>();
for (URI uri : list) {
LOG.info("Checking for machine status for network interface : " + uri.toString());
CheckConnectionToPeerMachine connectionToPeer = new CheckConnectionToPeerMachine(entry, uri.getHost());
Future future = isConnectedPool.submit(connectionToPeer);
array.add(future);
}
try {
synchronized (hwFailureDetectionthreadLock) {
hwFailureDetectionthreadLock.wait(failureDetectionTCPTimeout);
}
} catch (InterruptedException e) {
fine("InterruptedException occurred..." + e.getMessage(), new Object[] {e});
}
//check if the interrupt() call really does interrupt the thread
//for (int i = 0; i < array.size(); i++) {
for (Future future : array) {
try {
//if the task has completed after waiting for the
//specified timeout
if (future.isDone()) {
if (future.get().equals(Boolean.TRUE)) {
fine("Peer Machine for " + entry.adv.getName() + " is up");
boolean connection = masterNode.getRouteControl().isConnected(entry.id, manager.getCachedRoute(entry.id));
fine("routeControl.isConnected() for " + entry.adv.getName() + " is => " + connection);
return connection;
}
} else {
//interrupt the thread which is running the task submitted to it
fine("Going to cancel the future task...");
future.cancel(true);
isConnectedPool.purge();
fine("Finished cancelling and purging the future task...");
//cancelling the task is as good as getting false
//underlying socket that got created will still be alive.
}
} catch (ExecutionException e) {
fine("Exception occurred while getting the return value from the Future object " + e.getMessage());
return false;
} catch (InterruptedException e) {
fine("InterruptedException occurred..." + e.getMessage(), new Object[]{e});
}
return false;
}
fine("Peer Machine for " + entry.adv.getName() + " is down!");
return false;
}
//This is the Callable Object that gets executed by the ExecutorService
private class CheckConnectionToPeerMachine implements Callable<Object> {
HealthMessage.Entry entry;
String address;
CheckConnectionToPeerMachine(HealthMessage.Entry entry, String address) {
this.entry = entry;
this.address = address;
}
public Object call() {
Socket socket = null;
try {
fine("Going to create a socket at " + address + "port = " + failureDetectionTCPPort);
socket = new Socket(InetAddress.getByName(address), failureDetectionTCPPort);
fine("Socket created at " + address + "port = " + failureDetectionTCPPort);
synchronized (hwFailureDetectionthreadLock) {
hwFailureDetectionthreadLock.notify();
}
//for whatever reason a socket got created, close it and return true
//i.e. this instance was able to create a socket on that machine so it is up
return Boolean.valueOf(true);
} catch (IOException e) {
fine("IOException occurred while trying to connect to peer " + entry.adv.getName() +
"'s machine : " + e.getMessage(), new Object[] {e});
if (e.getMessage().trim().equals(CONNECTION_REFUSED)) {
fine("Going to call notify since the peer machine is up");
synchronized (hwFailureDetectionthreadLock) {
hwFailureDetectionthreadLock.notify();
}
return Boolean.valueOf(true);
}
return Boolean.valueOf(false);
} finally {
if (socket != null) {
try {
socket.close();
} catch (IOException e) {
fine("Could not close the socket due to " + e.getMessage());
}
}
}
}
}
/**
* Determines whether a connection to a specific node exists, or one can be created
*
* @param entry HealthMessage.Entry
* le@return true, if a connection already exists, or a new was sucessfully created
*/
/* public boolean isConnected(HealthMessage.Entry entry) {
//if System property for InetAddress.isReachable() is set, then check for the following:
//if InetAddress.isReachable() is true, then check for isConnected()
//if InetAddress.isReachable() is false, then simply return false
//check if using JDK 5 or 6. isUp() API available only in 6
Method method ;
try {
Class c = Class.forName("java.net.NetworkInterface");
method = c.getMethod("isUp", new Class[]{});
} catch (NoSuchMethodException nsme) {
//we are using JDK version < 6
return masterNode.getRouteControl().isConnected(entry.id, manager.getCachedRoute(entry.id));
} catch (SecurityException s) {
return masterNode.getRouteControl().isConnected(entry.id, manager.getCachedRoute(entry.id));
} catch (ClassNotFoundException c) {
return masterNode.getRouteControl().isConnected(entry.id, manager.getCachedRoute(entry.id));
}
try {
String ipAddr = manager.getSystemAdvertisement().getIP(); //get my IP address
LOG.fine("ipAddr in isConnected => " + ipAddr);
NetworkInterface ni = NetworkInterface.getByInetAddress(InetAddress.getByName(ipAddr));
//if (ni.isUp()) {
if (((Boolean) method.invoke(ni, new Object[]{})).booleanValue()) {
LOG.fine("The network interface " + ni.getDisplayName() + " is up");
return masterNode.getRouteControl().isConnected(entry.id, manager.getCachedRoute(entry.id));
} else {
LOG.fine("The network interface " + ni.getDisplayName() + " is NOT up");
MasterNode.INSTANCE_IN_NETWORK_ISOLATION = true;
designateInIsolationState((HealthMessage.Entry) cache.get(manager.getSystemAdvertisement().getID())); //put myself in network isolation category
return false;
}
} catch (Exception e) {
return false;
}
} */
/*
private void shutdown() {
}
private class ShutdownHook extends Thread {
public void run() {
shutdown();
}
}
*/
}