Skip to content

Commit d3cdd74

Browse files
Emanuel Muckenhuberbstansberry
Emanuel Muckenhuber
authored and
bstansberry
committed
[AS7-5887] reconnect servers automatically
1 parent ba90c9f commit d3cdd74

File tree

12 files changed

+123
-42
lines changed

12 files changed

+123
-42
lines changed

‎host-controller/src/main/java/org/jboss/as/host/controller/DomainModelControllerService.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -656,8 +656,8 @@ public ServerStatus startServer(String serverName, ModelNode domainModel, boolea
656656
return serverInventory.startServer(serverName, domainModel, blocking);
657657
}
658658

659-
public void reconnectServer(String serverName, ModelNode domainModel, boolean running) {
660-
serverInventory.reconnectServer(serverName, domainModel, running);
659+
public void reconnectServer(String serverName, ModelNode domainModel, byte[] authKey, boolean running) {
660+
serverInventory.reconnectServer(serverName, domainModel, authKey, running);
661661
}
662662

663663
public ServerStatus restartServer(String serverName, int gracefulTimeout, ModelNode domainModel) {

‎host-controller/src/main/java/org/jboss/as/host/controller/ManagedServer.java‎

Lines changed: 49 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -119,9 +119,9 @@ public static String getServerName(String serverProcessName) {
119119
private volatile InternalState requiredState = InternalState.STOPPED;
120120
private volatile InternalState internalState = InternalState.STOPPED;
121121

122-
ManagedServer(final String hostControllerName, final String serverName, final ProcessControllerClient processControllerClient,
123-
final InetSocketAddress managementSocket, final ManagedServer.ManagedServerBootConfiguration bootConfiguration,
124-
final TransformationTarget transformationTarget) {
122+
ManagedServer(final String hostControllerName, final String serverName, final byte[] authKey,
123+
final ProcessControllerClient processControllerClient, final InetSocketAddress managementSocket,
124+
final ManagedServer.ManagedServerBootConfiguration bootConfiguration, final TransformationTarget transformationTarget) {
125125

126126
assert hostControllerName != null : "hostControllerName is null";
127127
assert serverName != null : "serverName is null";
@@ -136,8 +136,6 @@ public static String getServerName(String serverProcessName) {
136136
this.bootConfiguration = bootConfiguration;
137137
this.transformationTarget = transformationTarget;
138138

139-
final byte[] authKey = new byte[16];
140-
new Random(new SecureRandom().nextLong()).nextBytes(authKey);
141139
this.authKey = authKey;
142140
serverPath = PathElement.pathElement(RUNNING_SERVER, serverName);
143141
}
@@ -226,6 +224,14 @@ protected synchronized void reconnectServerProcess() {
226224
}
227225
}
228226

227+
/**
228+
* On host controller reload, remove a not running server registered in the process controller declared as down.
229+
*/
230+
protected synchronized void removeServerProcess() {
231+
this.requiredState = InternalState.STOPPED;
232+
internalSetState(new ProcessRemoveTask(), InternalState.STOPPED, InternalState.PROCESS_REMOVING);
233+
}
234+
229235
/**
230236
* Await a state.
231237
*
@@ -272,18 +278,25 @@ protected void processStarted() {
272278
finishTransition(InternalState.PROCESS_STARTING, InternalState.PROCESS_STARTED);
273279
}
274280

275-
protected synchronized void channelRegistered(final ManagementChannelHandler channelAssociation) {
276-
internalSetState(new TransitionTask() {
277-
@Override
278-
public void execute(final ManagedServer server) throws Exception {
279-
280-
server.proxyController = TransformingProxyController.Factory.create(channelAssociation,
281-
Transformers.Factory.create(transformationTarget),
282-
PathAddress.pathAddress(PathElement.pathElement(HOST, hostControllerName), serverPath),
283-
ProxyOperationAddressTranslator.SERVER);
284-
}
285-
// TODO we just check that we are in the correct state, perhaps introduce a new state
286-
}, InternalState.SERVER_STARTING, InternalState.SERVER_STARTING);
281+
protected synchronized ProxyController channelRegistered(final ManagementChannelHandler channelAssociation) {
282+
final InternalState current = this.internalState;
283+
final TransformingProxyController proxy = TransformingProxyController.Factory.create(channelAssociation,
284+
Transformers.Factory.create(transformationTarget),
285+
PathAddress.pathAddress(PathElement.pathElement(HOST, hostControllerName), serverPath),
286+
ProxyOperationAddressTranslator.SERVER);
287+
// TODO better handling for server :reload operation
288+
if(current == InternalState.SERVER_STARTED && proxyController == null) {
289+
this.proxyController = proxy;
290+
} else {
291+
internalSetState(new TransitionTask() {
292+
@Override
293+
public void execute(final ManagedServer server) throws Exception {
294+
server.proxyController = proxy;
295+
}
296+
// TODO we just check that we are in the correct state, perhaps introduce a new state
297+
}, InternalState.SERVER_STARTING, InternalState.SERVER_STARTING);
298+
}
299+
return proxyController;
287300
}
288301

289302
protected synchronized void serverStarted(final TransitionTask task) {
@@ -296,9 +309,27 @@ protected synchronized void serverStartFailed() {
296309

297310
/**
298311
* Unregister the mgmt channel.
312+
*
313+
* @param shuttingDown whether the server inventory is shutting down
299314
*/
300-
protected synchronized void callbackUnregistered() {
315+
protected synchronized void callbackUnregistered(final boolean shuttingDown) {
301316
this.proxyController = null;
317+
// If the connection dropped without us stopping the process ask for reconnection
318+
if(! shuttingDown && requiredState == InternalState.SERVER_STARTED) {
319+
final InternalState state = internalState;
320+
if(state == InternalState.PROCESS_STOPPED
321+
|| state == InternalState.PROCESS_STOPPING
322+
|| state == InternalState.STOPPED) {
323+
// In case it stopped we don't reconnect
324+
return;
325+
}
326+
try {
327+
HostControllerLogger.ROOT_LOGGER.tracef("trying to reconnect to %s current-state (%s) required-state (%s)", serverName, state, requiredState);
328+
internalSetState(new ReconnectTask(), state, InternalState.SERVER_STARTING);
329+
} catch (Exception e) {
330+
HostControllerLogger.ROOT_LOGGER.debugf(e, "failed to send reconnect task");
331+
}
332+
}
302333
}
303334

304335
/**

‎host-controller/src/main/java/org/jboss/as/host/controller/ServerInventory.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -157,10 +157,11 @@ public interface ServerInventory {
157157
*
158158
* @param serverName the name of the server
159159
* @param domainModel the configuration model for the domain
160+
* @param authKey the authentication key
160161
* @param running whether the process was running. If {@code false}, the existence of the server will be
161162
* recorded but no attempt to contact it will be made
162163
*/
163-
void reconnectServer(final String serverName, final ModelNode domainModel, final boolean running);
164+
void reconnectServer(String serverName, ModelNode domainModel, byte[] authKey, boolean running);
164165

165166
/**
166167
* Gets a callback handler security services can use for handling authentication data provided by

‎host-controller/src/main/java/org/jboss/as/host/controller/ServerInventoryImpl.java‎

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -35,11 +35,13 @@
3535
import java.net.InetSocketAddress;
3636
import java.nio.charset.Charset;
3737
import java.security.NoSuchAlgorithmException;
38+
import java.security.SecureRandom;
3839
import java.util.HashMap;
3940
import java.util.LinkedList;
4041
import java.util.List;
4142
import java.util.Locale;
4243
import java.util.Map;
44+
import java.util.Random;
4345
import java.util.concurrent.ConcurrentHashMap;
4446
import java.util.concurrent.ConcurrentMap;
4547
import java.util.concurrent.CountDownLatch;
@@ -168,7 +170,11 @@ public ServerStatus startServer(final String serverName, final ModelNode domainM
168170
}
169171
ManagedServer server = servers.get(serverName);
170172
if(server == null) {
171-
final ManagedServer newServer = createManagedServer(serverName, domainModel);
173+
// Create a new authKey
174+
final byte[] authKey = new byte[16];
175+
new Random(new SecureRandom().nextLong()).nextBytes(authKey);
176+
// Create the managed server
177+
final ManagedServer newServer = createManagedServer(serverName, domainModel, authKey);
172178
server = servers.putIfAbsent(serverName, newServer);
173179
if(server == null) {
174180
server = newServer;
@@ -231,7 +237,7 @@ public ServerStatus stopServer(final String serverName, final int gracefulTimeou
231237
}
232238

233239
@Override
234-
public void reconnectServer(final String serverName, final ModelNode domainModel, final boolean running) {
240+
public void reconnectServer(final String serverName, final ModelNode domainModel, final byte[] authKey, final boolean running) {
235241
if(shutdown || connectionFinished) {
236242
throw HostControllerMessages.MESSAGES.hostAlreadyShutdown();
237243
}
@@ -240,13 +246,15 @@ public void reconnectServer(final String serverName, final ModelNode domainModel
240246
ROOT_LOGGER.existingServerWithState(serverName, existing.getState());
241247
return;
242248
}
243-
final ManagedServer server = createManagedServer(serverName, domainModel);
249+
final ManagedServer server = createManagedServer(serverName, domainModel, authKey);
244250
if(servers.putIfAbsent(serverName, server) != null) {
245251
ROOT_LOGGER.existingServerWithState(serverName, existing.getState());
246252
return;
247253
}
248254
if(running) {
249255
server.reconnectServerProcess();
256+
} else {
257+
server.removeServerProcess();
250258
}
251259
synchronized (shutdownCondition) {
252260
shutdownCondition.notifyAll();
@@ -294,7 +302,7 @@ public void stopServers(final int gracefulTimeout, final boolean blockUntilStopp
294302
}
295303
}
296304

297-
public void shutdown(final int gracefulTimeout, final boolean blockUntilStopped) {
305+
void shutdown(final boolean shutdownServers, final int gracefulTimeout, final boolean blockUntilStopped) {
298306
final boolean shutdown = this.shutdown;
299307
this.shutdown = true;
300308
if(! shutdown) {
@@ -303,7 +311,10 @@ public void shutdown(final int gracefulTimeout, final boolean blockUntilStopped)
303311
// nor can expect to receive any further notifications notifications.
304312
return;
305313
}
306-
stopServers(gracefulTimeout, blockUntilStopped);
314+
if(shutdownServers) {
315+
// Shutdown the servers as well
316+
stopServers(gracefulTimeout, blockUntilStopped);
317+
}
307318
}
308319
}
309320

@@ -323,7 +334,8 @@ public void serverCommunicationRegistered(final String serverProcessName, final
323334
channel.addCloseHandler(new CloseHandler<Channel>() {
324335

325336
public void handleClose(final Channel closed, final IOException exception) {
326-
server.callbackUnregistered();
337+
final boolean shuttingDown = shutdown || connectionFinished;
338+
server.callbackUnregistered(shuttingDown);
327339
domainController.unregisterRunningServer(server.getServerName());
328340
}
329341
});
@@ -479,14 +491,14 @@ public void processInventory(final Map<String, ProcessInfo> processInfos) {
479491
}
480492
}
481493

482-
private ManagedServer createManagedServer(final String serverName, final ModelNode domainModel) {
494+
private ManagedServer createManagedServer(final String serverName, final ModelNode domainModel, final byte[] authKey) {
483495
final String hostControllerName = domainController.getLocalHostInfo().getLocalHostName();
484496
final ModelNode hostModel = domainModel.require(HOST).require(hostControllerName);
485497
final ModelCombiner combiner = new ModelCombiner(serverName, domainModel, hostModel, domainController, environment);
486498
final ManagedServer.ManagedServerBootConfiguration configuration = combiner.createConfiguration();
487499
final ModelNode subsystems = resolveSubsystems(extensionRegistry);
488500
final TransformationTarget target = TransformationTargetImpl.create(extensionRegistry.getTransformerRegistry(), ModelVersion.create(Version.MANAGEMENT_MAJOR_VERSION, Version.MANAGEMENT_MINOR_VERSION, Version.MANAGEMENT_MICRO_VERSION), subsystems, TransformationTarget.TransformationTargetType.SERVER);
489-
return new ManagedServer(hostControllerName, serverName, processControllerClient, managementAddress, configuration, target);
501+
return new ManagedServer(hostControllerName, serverName, authKey, processControllerClient, managementAddress, configuration, target);
490502
}
491503

492504
public static ModelNode resolveSubsystems(final ExtensionRegistry extensionRegistry) {

‎host-controller/src/main/java/org/jboss/as/host/controller/ServerInventoryService.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -117,14 +117,15 @@ public synchronized void start(StartContext context) throws StartException {
117117
*/
118118
@Override
119119
public synchronized void stop(final StopContext context) {
120-
if (runningModeControl.getRestartMode() == RestartMode.SERVERS) {
120+
final boolean shutdownServers = runningModeControl.getRestartMode() == RestartMode.SERVERS;
121+
if (shutdownServers) {
121122
context.asynchronous();
122123
Runnable r = new Runnable() {
123124

124125
@Override
125126
public void run() {
126127
try {
127-
serverInventory.shutdown(-1, true); // TODO graceful shutdown // TODO async
128+
serverInventory.shutdown(true, -1, true); // TODO graceful shutdown
128129
serverInventory = null;
129130
// client.getValue().setServerInventory(null);
130131
} finally {
@@ -134,6 +135,10 @@ public void run() {
134135
}
135136
};
136137
executorService.getValue().execute(r);
138+
} else {
139+
// We have to set the shutdown flag in any case
140+
serverInventory.shutdown(false, -1, true);
141+
serverInventory = null;
137142
}
138143
}
139144

‎host-controller/src/main/java/org/jboss/as/host/controller/mgmt/ServerToHostOperationHandlerFactoryService.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import org.jboss.as.protocol.mgmt.FlushableDataOutput;
4141
import org.jboss.as.protocol.mgmt.ManagementChannelHandler;
4242
import org.jboss.as.protocol.mgmt.ManagementChannelReceiver;
43+
import org.jboss.as.protocol.mgmt.ManagementPongRequestHandler;
4344
import org.jboss.as.protocol.mgmt.ManagementProtocol;
4445
import org.jboss.as.protocol.mgmt.ManagementProtocolHeader;
4546
import org.jboss.as.protocol.mgmt.ManagementRequestContext;
@@ -175,6 +176,8 @@ public void run() {
175176
CONTROLLER_MANAGEMENT_LOGGER.serverRegistered(serverName, channel);
176177
// Create the server mgmt handler
177178
final ManagementChannelHandler handler = new ManagementChannelHandler(channel, executorService, new ServerHandlerFactory(serverName));
179+
// Ping/Pong
180+
handler.addHandlerFactory(new ManagementPongRequestHandler());
178181
// Register the communication channel
179182
inventory.serverCommunicationRegistered(serverName, handler);
180183
// Send the response once the server is fully registered
@@ -201,6 +204,8 @@ public void run() {
201204
} else {
202205
param = DomainServerProtocol.PARAM_RESTART_REQUIRED;
203206
}
207+
// Ping/Pong
208+
handler.addHandlerFactory(new ManagementPongRequestHandler());
204209
// Notify the server whether configuration is still in sync or it requires a reload
205210
safeWriteResponse(channel, header, param);
206211
// Onto the next message

‎host-controller/src/main/java/org/jboss/as/host/controller/operations/StartServersHandler.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,8 @@ private void restartedHcStartOrReconnectServers(final ModelNode servers, final M
132132
}
133133
} else if (info != null){
134134
//Reconnect the server
135-
serverInventory.reconnectServer(serverName, domainModel, info.isRunning());
135+
final byte[] authkey = info.getAuthKey();
136+
serverInventory.reconnectServer(serverName, domainModel, authkey, info.isRunning());
136137
}
137138
}
138139
}

‎process-controller/src/main/java/org/jboss/as/process/ManagedProcess.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,9 @@ public void reconnect(String hostName, int port, boolean managementSubsystemEndp
169169
stdin.write(asAuthKey);
170170
stdin.flush();
171171
} catch (IOException e) {
172-
log.failedToSendReconnect(e, processName);
172+
if(state == State.STARTED) {
173+
log.failedToSendReconnect(e, processName);
174+
}
173175
}
174176
}
175177

‎process-controller/src/main/java/org/jboss/as/process/ProcessController.java‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,6 @@ public final class ProcessController {
5454
*/
5555
private final Object lock = new Object();
5656

57-
private final Random rng;
5857
private final ProtocolServer server;
5958
private final Map<String, ManagedProcess> processes = new HashMap<String, ManagedProcess>();
6059
private final Map<Key, ManagedProcess> processesByKey = new HashMap<Key, ManagedProcess>();
@@ -68,7 +67,6 @@ public final class ProcessController {
6867
public ProcessController(final ProtocolServer.Configuration configuration, final PrintStream stdout, final PrintStream stderr) throws IOException {
6968
this.stdout = stdout;
7069
this.stderr = stderr;
71-
rng = new Random(new SecureRandom().nextLong());
7270
//noinspection ThisEscapedInObjectConstruction
7371
configuration.setConnectionHandler(new ProcessControllerServerHandler(this));
7472
final ProtocolServer server = new ProtocolServer(configuration);
@@ -90,6 +88,14 @@ void removeManagedConnection(final Connection connection) {
9088
}
9189

9290
public void addProcess(final String processName, final List<String> command, final Map<String, String> env, final String workingDirectory, final boolean isPrivileged, final boolean respawn) {
91+
// Create a new authKey
92+
final byte[] authKey = new byte[16];
93+
new Random(new SecureRandom().nextLong()).nextBytes(authKey);
94+
//
95+
addProcess(processName, authKey, command, env, workingDirectory, isPrivileged, respawn);
96+
}
97+
98+
public void addProcess(final String processName, final byte[] authKey, final List<String> command, final Map<String, String> env, final String workingDirectory, final boolean isPrivileged, final boolean respawn) {
9399
for (String s : command) {
94100
if (s == null) {
95101
throw MESSAGES.nullCommandComponent();
@@ -105,8 +111,6 @@ public void addProcess(final String processName, final List<String> command, fin
105111
// ignore
106112
return;
107113
}
108-
final byte[] authKey = new byte[16];
109-
rng.nextBytes(authKey);
110114
final ManagedProcess process = new ManagedProcess(processName, command, env, workingDirectory, lock, this, authKey, isPrivileged, respawn);
111115
processes.put(processName, process);
112116
processesByKey.put(new Key(authKey), process);
@@ -292,7 +296,6 @@ void processRemoved(final String processName) {
292296
}
293297
} catch (IOException e) {
294298
ROOT_LOGGER.failedToWriteMessage("PROCESS_REMOVED " + processName, e);
295-
e.printStackTrace();
296299
removeManagedConnection(connection);
297300
}
298301
}

‎process-controller/src/main/java/org/jboss/as/process/ProcessControllerClient.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ public void handleMessage(final Connection connection, final InputStream dataStr
111111
final byte[] processAuthCode = new byte[16];
112112
readFully(dataStream, processAuthCode);
113113
final boolean processRunning = StreamUtils.readBoolean(dataStream);
114-
inventory.put(processName, new ProcessInfo(processName, authCode, processRunning));
114+
inventory.put(processName, new ProcessInfo(processName, processAuthCode, processRunning));
115115
}
116116
dataStream.close();
117117
CLIENT_LOGGER.tracef("Received process_inventory");

0 commit comments

Comments
 (0)