public synchronized void setInstanceDescriptor(UbaCoordinate ubaCoordinate, InstanceDescriptor id)
throws Exception {
InstanceDescriptor got = instanceDescriptor.get();
if (got != null && !got.equals(id)) {
instancePath.writeInstanceDescriptor(ubaCoordinate, id);
ensureCerts(id);
startupId.incrementAndGet();
unexpectedRestartTimestamp.set(-1);
redeploy.set(true);
LOG.info("Instance changed from " + got + " to " + id);
} else if (!instancePath.script("status").exists()) {
startupId.incrementAndGet();
unexpectedRestartTimestamp.set(-1);
redeploy.set(true);
LOG.info("Missing status script from " + got + " to " + id);
}
if (!redeploy.get()) {
LOG.debug("Service:" + instancePath.toHumanReadableName() + " has NOT changed.");
} else {
instanceDescriptor.set(id);
}
if (id.restartTimestampGMTMillis > lastRestart.get()) {
restartAtTimestamp.set(id.restartTimestampGMTMillis);
}
}
public Nanny(
PasswordStore passwordStore,
UpenaClient upenaClient,
RepositoryProvider repositoryProvider,
InstanceDescriptor instanceDescriptor,
InstancePath instancePath,
DeployableValidator deployableValidator,
DeployLog deployLog,
HealthLog healthLog,
DeployableScriptInvoker invokeScript,
UbaLog ubaLog,
Cache<String, Boolean> haveRunConfigExtractionCache) {
this.passwordStore = passwordStore;
this.upenaClient = upenaClient;
this.repositoryProvider = repositoryProvider;
this.instanceDescriptor = new AtomicReference<>(instanceDescriptor);
this.instancePath = instancePath;
this.deployableValidator = deployableValidator;
this.deployLog = deployLog;
this.healthLog = healthLog;
this.invokeScript = invokeScript;
this.ubaLog = ubaLog;
linkedBlockingQueue = new LinkedBlockingQueue<>(10);
threadPoolExecutor =
new ThreadPoolExecutor(1, 1, 1000, TimeUnit.MILLISECONDS, linkedBlockingQueue);
boolean exists = instancePath.deployLog().exists();
LOG.info("Stats script for {} exists == {}", instanceDescriptor, exists);
redeploy = new AtomicBoolean(!exists);
destroyed = new AtomicBoolean(false);
this.haveRunConfigExtractionCache = haveRunConfigExtractionCache;
}
@POST
@Path("/topology/shift")
@Produces(MediaType.TEXT_HTML)
@Consumes(MediaType.APPLICATION_FORM_URLENCODED)
public Response shiftTopologies(
@FormParam("logicalName") String logicalName,
@FormParam("direction") int direction,
@FormParam("unhealthyPct") @DefaultValue("0.24") float unhealthyPct,
@FormParam("probability") @DefaultValue("0.10") float probability) {
try {
ShiftPredicate shiftPredicate;
boolean caterpillar;
if (direction == 0) {
// "zero" means evac
caterpillar = false;
shiftPredicate =
new UnhealthyTopologyShiftPredicate(unhealthyPct); // TODO should be passed in
} else {
// "non-zero" means shift by N places
caterpillar = true;
shiftPredicate = new RandomShiftPredicate(probability);
}
rebalanceDirector.shiftTopologies(
Optional.of(new MiruHost(logicalName)),
shiftPredicate,
new CaterpillarSelectHostsStrategy(caterpillar, direction, false));
return Response.ok("success").build();
} catch (Throwable t) {
LOG.error(
"POST /topology/shift {} {} {} {} {}",
new Object[] {logicalName, direction, unhealthyPct, probability},
t);
return Response.serverError().entity(t.getMessage()).build();
}
}
public void commitRows(PartitionName partitionName, List<Row> rows) throws Exception {
LOG.info("Received from partition:{} rows:{}", partitionName, rows.size());
PartitionClient client = partitionClientProvider.getPartition(partitionName);
Consistency consistency =
consistencyCache.computeIfAbsent(
partitionName,
partitionName1 -> {
try {
RingPartitionProperties properties =
partitionClientProvider.getProperties(partitionName1);
return properties != null ? properties.partitionProperties.consistency : null;
} catch (Exception e) {
LOG.error("Failed to get properties for partition:{}", partitionName1);
return null;
}
});
if (consistency == null) {
throw new RuntimeException("Missing consistency for partition: " + partitionName);
}
PeekingIterator<Row> iter = Iterators.peekingIterator(rows.iterator());
while (iter.hasNext()) {
byte[] prefix = iter.peek().prefix;
client.commit(
consistency,
prefix,
commitKeyValueStream -> {
while (iter.hasNext()) {
byte[] peek = iter.peek().prefix;
if ((prefix == null && peek == null)
|| (prefix != null && peek != null && Arrays.equals(prefix, peek))) {
Row row = iter.next();
commitKeyValueStream.commit(
row.key, row.value, row.valueTimestamp, row.valueTombstoned);
} else {
break;
}
}
return true;
},
additionalSolverAfterNMillis,
abandonSolutionAfterNMillis,
Optional.empty());
}
}
String invalidateRouting(String tenantId) {
try {
LOG.info("invalidating tenant routing for tenatId:" + tenantId + " on " + this);
StringBuilder curl = new StringBuilder();
curl.append("localhost:");
curl.append(instanceDescriptor.get().ports.get("manage"));
curl.append("/tenant/routing/invalidate?");
curl.append("tenantId=").append(tenantId).append('&');
curl.append("connectToServiceId=*").append('&');
curl.append("portName=*");
String response = Curl.create().curl(curl.toString());
LOG.info(response);
return response;
} catch (IOException x) {
LOG.warn("failed to invalidate tenant routing for tenantId:" + tenantId + " on " + this);
return "failed to invalidate tenant routing for tenantId:" + tenantId + " on " + this;
}
}
@DELETE
@Path("/hosts/{logicalName}")
@Produces(MediaType.TEXT_HTML)
@Consumes(MediaType.APPLICATION_FORM_URLENCODED)
public Response removeHost(@PathParam("logicalName") String logicalName) {
try {
registryClusterClient.removeHost(new MiruHost(logicalName));
return Response.ok("success").build();
} catch (Throwable t) {
LOG.error("DELETE /hosts/{}", new Object[] {logicalName}, t);
return Response.serverError().entity(t.getMessage()).build();
}
}
@GET
@Path("/topology/debugTenant/{tenantId}")
@Produces(MediaType.TEXT_HTML)
public Response debugTenant(@PathParam("tenantId") String tenantId) {
try {
StringBuilder stringBuilder = new StringBuilder();
rebalanceDirector.debugTenant(
new MiruTenantId(tenantId.getBytes(Charsets.UTF_8)), stringBuilder);
return Response.ok(stringBuilder.toString()).build();
} catch (Throwable t) {
LOG.error("GET /topology/debugTenant/{}", new Object[] {tenantId}, t);
return Response.serverError().entity(t.getMessage()).build();
}
}
public List<Future<?>> index(
final MiruContext<BM, IBM, ?> context,
MiruTenantId tenantId,
final Future<List<BloomWork>> bloomWorksFuture,
boolean repair,
ExecutorService indexExecutor)
throws ExecutionException, InterruptedException {
StackBuffer stackBuffer = new StackBuffer();
List<BloomWork> bloomWorks = bloomWorksFuture.get();
final MiruFieldIndex<BM, IBM> bloomFieldIndex =
context.getFieldIndexProvider().getFieldIndex(MiruFieldType.bloom);
int callableCount = 0;
List<Future<?>> futures = Lists.newArrayList();
for (final BloomWork bloomWork : bloomWorks) {
futures.add(
indexExecutor.submit(
() -> {
log.inc("count", bloomWork.bloomFieldValues.size());
log.inc("count", bloomWork.bloomFieldValues.size(), tenantId.toString());
MiruFieldDefinition bloomFieldDefinition =
context.schema.getFieldDefinition(bloomWork.bloomFieldId);
MiruTermId compositeBloomId =
indexUtil.makeBloomTerm(bloomWork.fieldValue, bloomFieldDefinition.name);
MiruInvertedIndex<BM, IBM> invertedIndex =
bloomFieldIndex.getOrCreateInvertedIndex(
"indexBloom", bloomWork.fieldId, compositeBloomId);
bloomIndex.put(invertedIndex, bloomWork.bloomFieldValues, stackBuffer);
return null;
}));
callableCount++;
}
log.trace("Submitted {} bloom callables", callableCount);
return futures;
}
@GET
@Path("/tenants/diff/{tenantId}/{partitionId}")
@Produces(MediaType.TEXT_PLAIN)
public Response getTenantsForTenant(
@PathParam("tenantId") String tenantId, @PathParam("partitionId") int partitionId) {
try {
String rendered =
rebalanceDirector.diffTenantPartition(
new MiruTenantId(tenantId.getBytes(Charsets.UTF_8)), MiruPartitionId.of(partitionId));
return Response.ok(rendered).build();
} catch (Throwable t) {
LOG.error("GET /tenants/diff/{}/{}", new Object[] {tenantId, partitionId}, t);
return Response.serverError().entity(t.getMessage()).build();
}
}
@GET
@Path("/{tenantId}/{guid}")
@Produces(MediaType.TEXT_HTML)
public Response wiki(
@PathParam("tenantId") @DefaultValue("") String tenantId,
@PathParam("guid") @DefaultValue("") String guid) {
try {
String rendered =
wikiMiruService.renderPlugin(pluginRegion, new WikiWikiPluginRegionInput(tenantId, guid));
return Response.ok(rendered).build();
} catch (Exception x) {
LOG.error("Failed to generating query ui.", x);
return Response.serverError().build();
}
}
@POST
@Path("/topology/repair")
@Produces(MediaType.TEXT_HTML)
@Consumes(MediaType.APPLICATION_FORM_URLENCODED)
public Response repairTopologies() {
try {
rebalanceDirector.shiftTopologies(
Optional.<MiruHost>absent(),
(tenantId, partitionId, hostHeartbeats, partitions) -> true,
new CaterpillarSelectHostsStrategy(true, 0, false));
return Response.ok("success").build();
} catch (Throwable t) {
LOG.error("POST /topology/repair", t);
return Response.serverError().entity(t.getMessage()).build();
}
}
@POST
@Path("/tenants/rebuild")
@Produces(MediaType.TEXT_HTML)
@Consumes(MediaType.APPLICATION_FORM_URLENCODED)
public Response rebuildTenantPartition(
@FormParam("logicalName") String logicalName,
@FormParam("tenantId") String tenantId,
@FormParam("partitionId") int partitionId) {
try {
rebalanceDirector.rebuildTenantPartition(
new MiruHost(logicalName),
new MiruTenantId(tenantId.getBytes(Charsets.UTF_8)),
MiruPartitionId.of(partitionId));
return Response.ok("success").build();
} catch (Throwable t) {
LOG.error(
"POST /tenants/rebuild {} {} {} {}",
new Object[] {logicalName, tenantId, partitionId},
t);
return Response.serverError().entity(t.getMessage()).build();
}
}
public synchronized boolean ensureCerts(InstanceDescriptor id) throws Exception {
boolean sslEnabled = false;
for (Map.Entry<String, InstanceDescriptor.InstanceDescriptorPort> entry : id.ports.entrySet()) {
if (entry.getValue().sslEnabled) {
sslEnabled = true;
break;
}
}
boolean generated = false;
if (sslEnabled) {
SelfSigningCertGenerator generator = new SelfSigningCertGenerator();
String password = passwordStore.password(id.instanceKey);
File certFile = instancePath.certs("sslKeystore");
boolean generateSSL = false;
try {
if (!certFile.exists() || !generator.validate(id.instanceKey, password, certFile)) {
generateSSL = true;
}
} catch (IOException x) {
if (Throwables.getRootCause(x) instanceof UnrecoverableKeyException) {
LOG.warn(
"Looks like password changed so existing certs will be replaced with regenerate certs.");
generateSSL = true;
} else {
throw x;
}
}
if (generateSSL) {
FileUtils.deleteQuietly(certFile);
generator.create(id.instanceKey, password, certFile);
generated = true;
}
}
File oauthKeystoreFile = instancePath.certs("oauthKeystore");
File oauthPublicKeyFile = instancePath.certs("oauthPublicKey");
String password = passwordStore.password(id.instanceKey);
RSAKeyPairGenerator generator = new RSAKeyPairGenerator();
boolean generateOauth = false;
try {
if (id.publicKey == null
|| !id.publicKey.equals(
generator.getPublicKey(
id.instanceKey, password, oauthKeystoreFile, oauthPublicKeyFile))) {
generateOauth = true;
}
} catch (IOException x) {
if (Throwables.getRootCause(x) instanceof UnrecoverableKeyException) {
LOG.warn(
"Looks like password changed so existing oauth certs will be replaced with regenerate certs.");
generateOauth = true;
} else {
throw x;
}
}
if (generateOauth) {
FileUtils.deleteQuietly(oauthKeystoreFile);
FileUtils.deleteQuietly(oauthPublicKeyFile);
generator.create(id.instanceKey, password, oauthKeystoreFile, oauthPublicKeyFile);
upenaClient.updateKeyPair(
id.instanceKey,
generator.getPublicKey(id.instanceKey, password, oauthKeystoreFile, oauthPublicKeyFile));
}
return generated;
}