1
// **********************************************************************
3
// Copyright (c) 2003-2011 ZeroC, Inc. All rights reserved.
5
// This copy of Ice is licensed to you under the terms described in the
6
// ICE_LICENSE file included in this distribution.
8
// **********************************************************************
10
#include <IceStorm/NodeI.h>
11
#include <IceStorm/Observers.h>
12
#include <IceStorm/TraceLevels.h>
14
using namespace IceStorm;
15
using namespace IceStormElection;
21
bool operator==(const GroupNodeInfo& info, int id)
26
class CheckTask : public IceUtil::TimerTask
32
CheckTask(const NodeIPtr& node) : _node(node) { }
33
virtual void runTimerTask()
39
class MergeTask : public IceUtil::TimerTask
46
MergeTask(const NodeIPtr& node, const set<int>& s) : _node(node), _s(s) { }
47
virtual void runTimerTask()
53
class MergeContinueTask : public IceUtil::TimerTask
59
MergeContinueTask(const NodeIPtr& node) : _node(node) { }
60
virtual void runTimerTask()
62
_node->mergeContinue();
66
class TimeoutTask: public IceUtil::TimerTask
72
TimeoutTask(const NodeIPtr& node) : _node(node) { }
73
virtual void runTimerTask()
84
LogUpdate emptyLU = {0, 0};
88
GroupNodeInfo::GroupNodeInfo(int i) :
93
GroupNodeInfo::GroupNodeInfo(int i, LogUpdate l, const Ice::ObjectPrx& o) :
94
id(i), llu(l), observer(o)
99
GroupNodeInfo::operator<(const GroupNodeInfo& rhs) const
105
GroupNodeInfo::operator==(const GroupNodeInfo& rhs) const
112
//cout << "~Replica" << endl;
118
getTimeout(const string& key, int def, const Ice::PropertiesPtr& properties, const TraceLevelsPtr& traceLevels)
120
int t = properties->getPropertyAsIntWithDefault(key, def);
123
Ice::Warning out(traceLevels->logger);
124
out << traceLevels->electionCat << ": " << key << " < 0; Adjusted to 1";
127
return IceUtil::Time::seconds(t);
131
toString(const set<int>& s)
135
for(set<int>::const_iterator p = s.begin(); p != s.end(); ++p)
149
NodeI::NodeI(const InstancePtr& instance,
150
const ReplicaPtr& replica,
151
const Ice::ObjectPrx& replicaProxy,
152
int id, const map<int, NodePrx>& nodes) :
153
_timer(instance->timer()),
154
_traceLevels(instance->traceLevels()),
155
_observers(instance->observers()),
157
_replicaProxy(replicaProxy),
160
_state(NodeStateInactive),
166
map<int, NodePrx> oneway;
167
for(map<int, NodePrx>::const_iterator p = _nodes.begin(); p != _nodes.end(); ++p)
169
oneway[p->first] = NodePrx::uncheckedCast(p->second->ice_oneway());
171
const_cast<map<int, NodePrx>& >(_nodesOneway) = oneway;
173
Ice::PropertiesPtr properties = instance->communicator()->getProperties();
174
const_cast<IceUtil::Time&>(_masterTimeout) = getTimeout(
175
instance->serviceName() + ".Election.MasterTimeout", 10, properties, _traceLevels);
176
const_cast<IceUtil::Time&>(_electionTimeout) = getTimeout(
177
instance->serviceName() + ".Election.ElectionTimeout", 10, properties, _traceLevels);
178
const_cast<IceUtil::Time&>(_mergeTimeout) = getTimeout(
179
instance->serviceName() + ".Election.ResponseTimeout", 10, properties, _traceLevels);
184
//cout << "~NodeI" << endl;
190
// As an optimization we want the initial election to occur as
193
// However, if we have the node trigger the election immediately
194
// upon startup then we'll have a clash with lower priority nodes
195
// starting an election denying a higher priority node the
196
// opportunity to start the election that results in it becoming
197
// the leader. Of course, things will eventually reach a stable
198
// state but it will take longer.
200
// As such as we schedule the initial election check inversely
201
// proportional to our priority.
203
// By setting _checkTask first we stop recovery() from setting it
204
// to the regular election interval.
208
// We use this lock to ensure that recovery is called before CheckTask
209
// is scheduled, even if timeout is 0
213
_checkTask = new CheckTask(this);
214
_timer->schedule(_checkTask, IceUtil::Time::seconds((_nodes.size() - _id) * 2));
229
if(_state == NodeStateElection || _state == NodeStateReorganization || _coord != _id)
232
_timer->schedule(_checkTask, _electionTimeout);
236
// Next get the set of nodes that were detected as unreachable
237
// from the replica and remove them from our slave list.
239
_observers->getReapedSlaves(dead);
242
for(vector<int>::const_iterator p = dead.begin(); p != dead.end(); ++p)
244
set<GroupNodeInfo>::iterator q = _up.find(GroupNodeInfo(*p));
247
if(_traceLevels->election > 0)
249
Ice::Trace out(_traceLevels->logger, _traceLevels->electionCat);
250
out << "node " << _id << ": reaping slave " << *p;
256
// If we no longer have the majority of the nodes under our
257
// care then we need to stop our replica.
258
if(_up.size() < _nodes.size()/2)
260
if(_traceLevels->election > 0)
262
Ice::Trace out(_traceLevels->logger, _traceLevels->electionCat);
263
out << "node " << _id << ": stopping replica";
265
// Clear _checkTask -- recovery() will reset the
275
// See if other groups exist for possible merge.
278
for(map<int, NodePrx>::const_iterator p = _nodes.begin(); p != _nodes.end(); ++p)
286
if(p->second->areYouCoordinator())
292
tmpset.insert(p->first);
295
catch(const Ice::Exception& ex)
297
if(_traceLevels->election > 0)
299
Ice::Trace out(_traceLevels->logger, _traceLevels->electionCat);
300
out << "node " << _id << ": call on node " << p->first << " failed: " << ex;
307
// If the node state has changed while the mutex has been released
308
// then bail. We don't schedule a re-check since we're either
309
// destroyed in which case we're going to terminate or the end of
310
// the election/reorg will re-schedule the check.
311
if(_destroy || _state == NodeStateElection || _state == NodeStateReorganization || _coord != _id)
317
// If we didn't find any coordinators then we're done. Reschedule
318
// the next check and terminate.
322
_timer->schedule(_checkTask, _electionTimeout);
326
// _checkTask == 0 means that the check isn't scheduled.
329
if(_traceLevels->election > 0)
331
Ice::Trace out(_traceLevels->logger, _traceLevels->electionCat);
332
out << "node " << _id << ": highest priority node count: " << max;
335
IceUtil::Time delay = IceUtil::Time::seconds(0);
338
// Reschedule timer proportial to p-i.
339
delay = _mergeTimeout + _mergeTimeout * (max - _id);
340
if(_traceLevels->election > 0)
342
Ice::Trace out(_traceLevels->logger, _traceLevels->electionCat);
343
out << "node " << _id << ": scheduling merge in " << delay.toDuration()
349
_mergeTask = new MergeTask(this, tmpset);
350
_timer->schedule(_mergeTask, delay);
353
// Called if the node has not heard from the coordinator in some time.
361
// If we're destroyed or we are our own coordinator then we're
363
if(_destroy || _coord == _id)
374
map<int, NodePrx>::const_iterator p = _nodes.find(myCoord);
375
assert(p != _nodes.end());
376
if(!p->second->areYouThere(myGroup, _id))
378
if(_traceLevels->election > 0)
380
Ice::Trace out(_traceLevels->logger, _traceLevels->electionCat);
381
out << "node " << _id << ": lost connection to coordinator " << myCoord
382
<< ": areYouThere returned false";
387
catch(const Ice::Exception& ex)
389
if(_traceLevels->election > 0)
391
Ice::Trace out(_traceLevels->logger, _traceLevels->electionCat);
392
out << "node " << _id << ": lost connection to coordinator " << myCoord << ": " << ex;
403
NodeI::merge(const set<int>& coordinatorSet)
411
// If the node is currently in an election, or reorganizing
413
if(_state == NodeStateElection || _state == NodeStateReorganization)
418
// This state change prevents this node from accepting
419
// invitations while the merge is executing.
420
setState(NodeStateElection);
422
// No more replica changes are permitted.
423
while(!_destroy && _updateCounter > 0)
433
os << _id << ":" << IceUtil::generateUUID();
437
_invitesAccepted.clear();
438
_invitesIssued.clear();
440
// Construct a set of node ids to invite. This is the union of
441
// _up and set of coordinators gathered in the check stage.
442
invited = coordinatorSet;
443
for(set<GroupNodeInfo>::const_iterator p = _up.begin(); p != _up.end(); ++p)
445
invited.insert(p->id);
451
if(_traceLevels->election > 0)
453
Ice::Trace out(_traceLevels->logger, _traceLevels->electionCat);
454
out << "node " << _id << ": inviting " << toString(invited) << " to group " << _group;
458
set<int>::iterator p = invited.begin();
459
while(p != invited.end())
463
if(_traceLevels->election > 0)
465
Ice::Trace out(_traceLevels->logger, _traceLevels->electionCat);
466
out << "node " << _id << ": inviting node " << *p << " to group " << gp;
468
map<int, NodePrx>::const_iterator node = _nodesOneway.find(*p);
469
assert(node != _nodesOneway.end());
470
node->second->invitation(_id, gp);
473
catch(const Ice::Exception&)
479
// Now we wait for responses to our invitation.
487
// Add each of the invited nodes in the invites issed set.
488
_invitesIssued.insert(invited.begin(), invited.end());
490
if(_traceLevels->election > 0)
492
Ice::Trace out(_traceLevels->logger, _traceLevels->electionCat);
493
out << "node " << _id << ": invites pending: " << toString(_invitesIssued);
496
// Schedule the mergeContinueTask.
497
assert(_mergeContinueTask == 0);
498
_mergeContinueTask = new MergeContinueTask(this);
500
// At this point we may have already accepted all of the
501
// invitations, if so then we want to schedule the
502
// mergeContinue immediately.
503
IceUtil::Time timeout = _mergeTimeout;
504
if(_up.size() == _nodes.size()-1 || _invitesIssued == _invitesAccepted)
506
timeout = IceUtil::Time::seconds(0);
508
_timer->schedule(_mergeContinueTask, timeout);
513
NodeI::mergeContinue()
516
set<GroupNodeInfo> tmpSet;
524
// Copy variables for thread safety.
528
assert(_mergeContinueTask);
529
_mergeContinueTask = 0;
531
// The node is now reorganizing.
532
assert(_state == NodeStateElection);
533
setState(NodeStateReorganization);
535
if(_traceLevels->election > 0)
537
Ice::Trace out(_traceLevels->logger, _traceLevels->electionCat);
538
out << "node " << _id << ": coordinator for " << (tmpSet.size() +1) << " nodes (including myself)";
541
// Now we need to decide whether we can start serving content. If
542
// we're on initial startup then we need all nodes to participate
543
// in the election. If we're running a subsequent election then we
544
// need a majority of the nodes to be active in order to start
546
unsigned int ingroup = static_cast<unsigned int>(tmpSet.size());
547
if((_max != _nodes.size() && ingroup != _nodes.size() -1) || ingroup < _nodes.size()/2)
549
if(_traceLevels->election > 0)
551
Ice::Trace out(_traceLevels->logger, _traceLevels->electionCat);
552
out << "node " << _id << ": not enough nodes " << (ingroup+1) << "/" << _nodes.size()
553
<< " for replication to commence";
554
if(_max != _nodes.size())
556
out << " (require full participation for startup)";
564
// Find out who has the highest available set of database
567
LogUpdate maxllu = { -1, 0 };
568
set<GroupNodeInfo>::const_iterator p;
569
for(p = tmpSet.begin(); p != tmpSet.end(); ++p)
571
if(_traceLevels->election > 0)
573
Ice::Trace out(_traceLevels->logger, _traceLevels->electionCat);
574
out << "node id=" << p->id << " llu=" << p->llu.generation << "/" << p->llu.iteration;
576
if(p->llu.generation > maxllu.generation ||
577
(p->llu.generation == maxllu.generation && p->llu.iteration > maxllu.iteration))
584
LogUpdate myLlu = _replica->getLastLogUpdate();
585
if(_traceLevels->election > 0)
587
Ice::Trace out(_traceLevels->logger, _traceLevels->electionCat);
588
out << "node id=" << _id << " llu=" << myLlu.generation << "/" << myLlu.iteration;
591
// If its not us then we have to get the latest database data from
592
// the replica with the latest set.
593
//if(maxllu > _replica->getLastLogUpdate())
596
if(_traceLevels->election > 0)
598
Ice::Trace out(_traceLevels->logger, _traceLevels->electionCat);
599
out << "node " << _id << ": syncing database state with node " << maxid;
603
map<int, NodePrx>::const_iterator node = _nodes.find(maxid);
604
assert(node != _nodes.end());
605
_replica->sync(node->second->sync());
607
catch(const Ice::Exception& ex)
609
if(_traceLevels->election > 0)
611
Ice::Trace out(_traceLevels->logger, _traceLevels->electionCat);
612
out << "node " << _id << ": syncing database state with node "
613
<< maxid << " failed: " << ex;
621
if(_traceLevels->election > 0)
623
Ice::Trace out(_traceLevels->logger, _traceLevels->electionCat);
624
out << "node " << _id << ": I have the latest database state.";
628
// At this point we've ensured that we have the latest database
629
// state, as such we can set our _max flag.
630
unsigned int max = static_cast<unsigned int>(tmpSet.size()) + 1;
640
// Prepare the LogUpdate for this generation.
642
maxllu.iteration = 0;
646
// Tell the replica that it is now the master with the given
647
// set of slaves and llu generation.
648
_replica->initMaster(tmpSet, maxllu);
650
catch(const Ice::Exception& ex)
652
if(_traceLevels->election > 0)
654
Ice::Trace out(_traceLevels->logger, _traceLevels->electionCat);
655
out << "node " << _id << ": initMaster failed: " << ex;
661
// Tell each node to go.
662
for(p = tmpSet.begin(); p != tmpSet.end(); ++p)
666
map<int, NodePrx>::const_iterator node = _nodes.find(p->id);
667
assert(node != _nodes.end());
668
node->second->ready(_id, gp, _replicaProxy, max, maxllu.generation);
670
catch(const Ice::Exception& ex)
672
if(_traceLevels->election > 0)
674
Ice::Trace out(_traceLevels->logger, _traceLevels->electionCat);
675
out << "node " << _id << ": error calling ready on " << p->id << " ex: " << ex;
688
if(_traceLevels->election > 0)
690
Ice::Trace out(_traceLevels->logger, _traceLevels->electionCat);
691
out << "node " << _id << ": reporting for duty in group " << _group << " as coordinator. ";
692
out << "replication commencing with " << _up.size()+1 << "/" << _nodes.size()
693
<< " nodes with llu generation: " << maxllu.generation;
695
setState(NodeStateNormal);
696
_coordinatorProxy = 0;
698
_generation = maxllu.generation;
701
_checkTask = new CheckTask(this);
702
_timer->schedule(_checkTask, _electionTimeout);
707
NodeI::invitation(int j, const string& gn, const Ice::Current&)
709
if(_traceLevels->election > 0)
711
Ice::Trace out(_traceLevels->logger, _traceLevels->electionCat);
712
out << "node " << _id << ": invitation from " << j << " to group " << gn;
715
// Verify that j exists in our node set.
716
if(_nodes.find(j) == _nodes.end())
718
Ice::Warning warn(_traceLevels->logger);
719
warn << _traceLevels->electionCat << ": ignoring invitation from unknown node " << j;
725
set<GroupNodeInfo> tmpSet;
732
// If we're in the election or reorg state a merge has already
733
// started, so ignore the invitation.
734
if(_state == NodeStateElection || _state == NodeStateReorganization)
736
if(_traceLevels->election > 0)
738
Ice::Trace out(_traceLevels->logger, _traceLevels->electionCat);
739
out << "node " << _id << ": invitation ignored";
745
// Upon receipt of an invitation we cancel any pending merge
750
// If the timer doesn't cancel it means that the timer has
751
// fired and the merge is currently in-progress in which
752
// case we should reject the invitation.
753
if(!_timer->cancel(_mergeTask))
755
// The merge task is cleared in the merge. This
756
// ensures two invitations cannot cause a race with
764
// We're now joining with another group. If we are active we
765
// must stop serving as a master or slave.
766
setState(NodeStateElection);
767
while(!_destroy && _updateCounter > 0)
784
Ice::IntSeq forwardedInvites;
785
if(tmpCoord == _id) // Forward invitation to my old members.
787
for(set<GroupNodeInfo>::const_iterator p = tmpSet.begin(); p != tmpSet.end(); ++p)
791
map<int, NodePrx>::const_iterator node = _nodesOneway.find(p->id);
792
assert(node != _nodesOneway.end());
793
node->second->invitation(j, gn);
794
forwardedInvites.push_back(p->id);
796
catch(const Ice::Exception&)
802
// Set the state and timer before calling accept. This ensures
803
// that if ready is called directly after accept is called then
804
// everything is fine. Setting the state *after* calling accept
812
assert(_state == NodeStateElection);
813
setState(NodeStateReorganization);
816
_timeoutTask = new TimeoutTask(this);
817
_timer->scheduleRepeated(_timeoutTask, _masterTimeout);
823
map<int, NodePrx>::const_iterator node = _nodesOneway.find(j);
824
assert(node != _nodesOneway.end());
825
node->second->accept(_id, gn, forwardedInvites, _replica->getObserver(), _replica->getLastLogUpdate(), max);
827
catch(const Ice::Exception&)
835
NodeI::ready(int j, const string& gn, const Ice::ObjectPrx& coordinator, int max, Ice::Long generation,
839
if(!_destroy && _state == NodeStateReorganization && _group == gn)
841
// The coordinator must be j (this was set in the invitation).
844
Ice::Warning warn(_traceLevels->logger);
845
warn << _traceLevels->electionCat << ": ignoring ready call from replica node " << j
846
<< " (real coordinator is " << _coord << ")";
850
// Here we've already validated j in the invite call
851
// (otherwise _group != gn).
852
if(_traceLevels->election > 0)
854
Ice::Trace out(_traceLevels->logger, _traceLevels->electionCat);
855
out << "node " << _id << ": reporting for duty in group " << gn << " with coordinator " << j;
858
if(static_cast<unsigned int>(max) > _max)
862
_generation = generation;
864
// Activate the replica here since the replica is now ready
866
setState(NodeStateNormal);
867
_coordinatorProxy = coordinator;
871
_checkTask = new CheckTask(this);
872
_timer->schedule(_checkTask, _electionTimeout);
878
NodeI::accept(int j, const string& gn, const Ice::IntSeq& forwardedInvites, const Ice::ObjectPrx& observer,
879
const LogUpdate& llu, int max, const Ice::Current&)
881
// Verify that j exists in our node set.
882
if(_nodes.find(j) == _nodes.end())
884
Ice::Warning warn(_traceLevels->logger);
885
warn << _traceLevels->electionCat << ": ignoring accept from unknown node " << j;
890
if(!_destroy && _state == NodeStateElection && _group == gn && _coord == _id)
892
_up.insert(GroupNodeInfo(j, llu, observer));
894
if(static_cast<unsigned int>(max) > _max)
899
if(_traceLevels->election > 0)
901
Ice::Trace out(_traceLevels->logger, _traceLevels->electionCat);
902
out << "node " << _id << ": accept " << j << " forward invites (";
903
for(Ice::IntSeq::const_iterator p = forwardedInvites.begin(); p != forwardedInvites.end(); ++p)
905
if(p != forwardedInvites.begin())
912
<< llu.generation << "/" << llu.iteration << " into group " << gn
913
<< " group size " << (_up.size() + 1);
916
// Add each of the forwarded invites to the list of issued
917
// invitations. This doesn't use set_union since
918
// forwardedInvites may not be sorted.
919
_invitesIssued.insert(forwardedInvites.begin(), forwardedInvites.end());
920
// We've accepted the invitation from node j.
921
_invitesAccepted.insert(j);
923
if(_traceLevels->election > 0)
925
Ice::Trace out(_traceLevels->logger, _traceLevels->electionCat);
926
out << "node " << _id << ": invites pending: " << toString(_invitesIssued)
927
<< " invites accepted: " << toString(_invitesAccepted);
930
// If invitations have been accepted from all nodes and the
931
// merge task has already been scheduled then reschedule the
932
// merge continue immediately. Otherwise, we let the existing
933
// merge() schedule continue.
934
if((_up.size() == _nodes.size()-1 || _invitesIssued == _invitesAccepted) &&
935
_mergeContinueTask && _timer->cancel(_mergeContinueTask))
937
_timer->schedule(_mergeContinueTask, IceUtil::Time::seconds(0));
943
NodeI::areYouCoordinator(const Ice::Current&) const
946
return _state != NodeStateElection && _state != NodeStateReorganization && _coord == _id;
950
NodeI::areYouThere(const string& gn, int j, const Ice::Current&) const
953
return _group == gn && _coord == _id && _up.find(GroupNodeInfo(j)) != _up.end();
957
NodeI::sync(const Ice::Current&) const
959
return _replica->getSync();
963
NodeI::nodes(const Ice::Current&) const
966
for(map<int, NodePrx>::const_iterator q = _nodes.begin(); q != _nodes.end(); ++q)
978
NodeI::query(const Ice::Current&) const
985
info.replica = _replicaProxy;
989
for(set<GroupNodeInfo>::const_iterator p = _up.begin(); p != _up.end(); ++p)
994
info.up.push_back(gi);
1001
NodeI::recovery(Ice::Long generation)
1005
// Ignore the recovery if the node has already advanced a
1007
if(generation != -1 && generation != _generation)
1012
setState(NodeStateInactive);
1013
while(!_destroy && _updateCounter > 0)
1023
os << _id << ":" << IceUtil::generateUUID();
1030
if(_traceLevels->election > 0)
1032
Ice::Trace out(_traceLevels->logger, _traceLevels->electionCat);
1033
out << "node " << _id << ": creating new self-coordinated group " << _group;
1036
// Reset the timer states.
1039
_timer->cancel(_mergeTask);
1044
_timer->cancel(_timeoutTask);
1049
_checkTask = new CheckTask(this);
1050
_timer->schedule(_checkTask, _electionTimeout);
1060
while(_updateCounter > 0)
1067
// Cancel the timers.
1070
_timer->cancel(_checkTask);
1076
_timer->cancel(_timeoutTask);
1082
_timer->cancel(_mergeTask);
1087
// A node should only receive an observer init call if the node is
1088
// reorganizing and its not the coordinator.
1090
NodeI::checkObserverInit(Ice::Long generation)
1093
if(_state != NodeStateReorganization)
1095
throw ObserverInconsistencyException("init cannot block when state != NodeStateReorganization");
1099
throw ObserverInconsistencyException("init called on coordinator");
1103
// Notify the node that we're about to start an update.
1105
NodeI::startUpdate(Ice::Long& generation, const char* file, int line)
1107
bool majority = _observers->check();
1111
// If we've actively replicating & lost the majority of our replicas then recover.
1112
if(!_coordinatorProxy && !_destroy && _state == NodeStateNormal && !majority)
1117
while(!_destroy && _state != NodeStateNormal)
1123
throw Ice::UnknownException(file, line);
1125
if(!_coordinatorProxy)
1129
generation = _generation;
1130
return _coordinatorProxy;
1134
NodeI::updateMaster(const char* file, int line)
1136
bool majority = _observers->check();
1140
// If the node is destroyed, or is not a coordinator then we're
1142
if(_destroy || _coordinatorProxy)
1147
// If we've lost the majority of our replicas then recover.
1148
if(_state == NodeStateNormal && !majority)
1153
// If we're not replicating then we're done.
1154
if(_state != NodeStateNormal)
1159
// Otherwise adjust the update counter, and return success.
1165
NodeI::startCachedRead(Ice::Long& generation, const char* file, int line)
1168
while(!_destroy && _state != NodeStateNormal)
1174
throw Ice::UnknownException(file, line);
1176
generation = _generation;
1178
return _coordinatorProxy;
1182
NodeI::startObserverUpdate(Ice::Long generation, const char* file, int line)
1187
throw Ice::UnknownException(file, line);
1189
if(_state != NodeStateNormal)
1191
throw ObserverInconsistencyException("update called on inactive node");
1193
if(!_coordinatorProxy)
1195
throw ObserverInconsistencyException("update called on the master");
1197
if(generation != _generation)
1199
throw ObserverInconsistencyException("invalid generation");
1205
NodeI::finishUpdate()
1210
assert(_updateCounter >= 0);
1211
if(_updateCounter == 0)
1220
stateToString(NodeState s)
1224
case NodeStateInactive:
1226
case NodeStateElection:
1228
case NodeStateReorganization:
1229
return "reorganization";
1230
case NodeStateNormal:
1238
NodeI::setState(NodeState s)
1242
if(_traceLevels->election > 0)
1244
Ice::Trace out(_traceLevels->logger, _traceLevels->electionCat);
1245
out << "node " << _id << ": transition from " << stateToString(_state) << " to "
1246
<< stateToString(s);
1249
if(_state == NodeStateNormal)
added
removed
.pc/ice_for_gcc4.7.patch/cpp/src/IceStorm/NodeI.cpp
