/** Warn about the given disk usage if the low or high watermark has been passed */
private void warnAboutDiskIfNeeded(DiskUsage usage) {
// Check absolute disk values
if (usage.getFreeBytes() < DiskThresholdDecider.this.freeBytesThresholdHigh.bytes()) {
logger.warn(
"high disk watermark [{}] exceeded on {}, shards will be relocated away from this node",
DiskThresholdDecider.this.freeBytesThresholdHigh,
usage);
} else if (usage.getFreeBytes() < DiskThresholdDecider.this.freeBytesThresholdLow.bytes()) {
logger.info(
"low disk watermark [{}] exceeded on {}, replicas will not be assigned to this node",
DiskThresholdDecider.this.freeBytesThresholdLow,
usage);
}
// Check percentage disk values
if (usage.getFreeDiskAsPercentage() < DiskThresholdDecider.this.freeDiskThresholdHigh) {
logger.warn(
"high disk watermark [{}] exceeded on {}, shards will be relocated away from this node",
Strings.format1Decimals(100.0 - DiskThresholdDecider.this.freeDiskThresholdHigh, "%"),
usage);
} else if (usage.getFreeDiskAsPercentage() < DiskThresholdDecider.this.freeDiskThresholdLow) {
logger.info(
"low disk watermark [{}] exceeded on {}, replicas will not be assigned to this node",
Strings.format1Decimals(100.0 - DiskThresholdDecider.this.freeDiskThresholdLow, "%"),
usage);
}
}
@Override
public String toString() {
long bytes = getBytes();
double value = bytes;
String suffix = "b";
if (bytes >= ByteSizeUnit.C5) {
value = getPbFrac();
suffix = "pb";
} else if (bytes >= ByteSizeUnit.C4) {
value = getTbFrac();
suffix = "tb";
} else if (bytes >= ByteSizeUnit.C3) {
value = getGbFrac();
suffix = "gb";
} else if (bytes >= ByteSizeUnit.C2) {
value = getMbFrac();
suffix = "mb";
} else if (bytes >= ByteSizeUnit.C1) {
value = getKbFrac();
suffix = "kb";
}
return Strings.format1Decimals(value, suffix);
}
@Override
public Decision canAllocate(
ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
final Decision decision = earlyTerminate(allocation);
if (decision != null) {
return decision;
}
final double usedDiskThresholdLow = 100.0 - DiskThresholdDecider.this.freeDiskThresholdLow;
final double usedDiskThresholdHigh = 100.0 - DiskThresholdDecider.this.freeDiskThresholdHigh;
ClusterInfo clusterInfo = allocation.clusterInfo();
Map<String, DiskUsage> usages = clusterInfo.getNodeMostAvailableDiskUsages();
DiskUsage usage = getDiskUsage(node, allocation, usages);
// First, check that the node currently over the low watermark
double freeDiskPercentage = usage.getFreeDiskAsPercentage();
// Cache the used disk percentage for displaying disk percentages consistent with documentation
double usedDiskPercentage = usage.getUsedDiskAsPercentage();
long freeBytes = usage.getFreeBytes();
if (logger.isTraceEnabled()) {
logger.trace("node [{}] has {}% used disk", node.nodeId(), usedDiskPercentage);
}
// a flag for whether the primary shard has been previously allocated
boolean primaryHasBeenAllocated =
shardRouting.primary() && shardRouting.allocatedPostIndexCreate();
// checks for exact byte comparisons
if (freeBytes < freeBytesThresholdLow.bytes()) {
// If the shard is a replica or has a primary that has already been allocated before, check
// the low threshold
if (!shardRouting.primary() || (shardRouting.primary() && primaryHasBeenAllocated)) {
if (logger.isDebugEnabled()) {
logger.debug(
"less than the required {} free bytes threshold ({} bytes free) on node {}, preventing allocation",
freeBytesThresholdLow,
freeBytes,
node.nodeId());
}
return allocation.decision(
Decision.NO,
NAME,
"less than required [%s] free on node, free: [%s]",
freeBytesThresholdLow,
new ByteSizeValue(freeBytes));
} else if (freeBytes > freeBytesThresholdHigh.bytes()) {
// Allow the shard to be allocated because it is primary that
// has never been allocated if it's under the high watermark
if (logger.isDebugEnabled()) {
logger.debug(
"less than the required {} free bytes threshold ({} bytes free) on node {}, "
+ "but allowing allocation because primary has never been allocated",
freeBytesThresholdLow,
freeBytes,
node.nodeId());
}
return allocation.decision(Decision.YES, NAME, "primary has never been allocated before");
} else {
// Even though the primary has never been allocated, the node is
// above the high watermark, so don't allow allocating the shard
if (logger.isDebugEnabled()) {
logger.debug(
"less than the required {} free bytes threshold ({} bytes free) on node {}, "
+ "preventing allocation even though primary has never been allocated",
freeBytesThresholdHigh,
freeBytes,
node.nodeId());
}
return allocation.decision(
Decision.NO,
NAME,
"less than required [%s] free on node, free: [%s]",
freeBytesThresholdHigh,
new ByteSizeValue(freeBytes));
}
}
// checks for percentage comparisons
if (freeDiskPercentage < freeDiskThresholdLow) {
// If the shard is a replica or has a primary that has already been allocated before, check
// the low threshold
if (!shardRouting.primary() || (shardRouting.primary() && primaryHasBeenAllocated)) {
if (logger.isDebugEnabled()) {
logger.debug(
"more than the allowed {} used disk threshold ({} used) on node [{}], preventing allocation",
Strings.format1Decimals(usedDiskThresholdLow, "%"),
Strings.format1Decimals(usedDiskPercentage, "%"),
node.nodeId());
}
return allocation.decision(
Decision.NO,
NAME,
"more than allowed [%s%%] used disk on node, free: [%s%%]",
usedDiskThresholdLow,
freeDiskPercentage);
} else if (freeDiskPercentage > freeDiskThresholdHigh) {
// Allow the shard to be allocated because it is primary that
// has never been allocated if it's under the high watermark
if (logger.isDebugEnabled()) {
logger.debug(
"more than the allowed {} used disk threshold ({} used) on node [{}], "
+ "but allowing allocation because primary has never been allocated",
Strings.format1Decimals(usedDiskThresholdLow, "%"),
Strings.format1Decimals(usedDiskPercentage, "%"),
node.nodeId());
}
return allocation.decision(Decision.YES, NAME, "primary has never been allocated before");
} else {
// Even though the primary has never been allocated, the node is
// above the high watermark, so don't allow allocating the shard
if (logger.isDebugEnabled()) {
logger.debug(
"less than the required {} free bytes threshold ({} bytes free) on node {}, "
+ "preventing allocation even though primary has never been allocated",
Strings.format1Decimals(freeDiskThresholdHigh, "%"),
Strings.format1Decimals(freeDiskPercentage, "%"),
node.nodeId());
}
return allocation.decision(
Decision.NO,
NAME,
"more than allowed [%s%%] used disk on node, free: [%s%%]",
usedDiskThresholdHigh,
freeDiskPercentage);
}
}
// Secondly, check that allocating the shard to this node doesn't put it above the high
// watermark
final long shardSize = getShardSize(shardRouting, allocation.clusterInfo());
double freeSpaceAfterShard = freeDiskPercentageAfterShardAssigned(usage, shardSize);
long freeBytesAfterShard = freeBytes - shardSize;
if (freeBytesAfterShard < freeBytesThresholdHigh.bytes()) {
logger.warn(
"after allocating, node [{}] would have less than the required {} free bytes threshold ({} bytes free), preventing allocation",
node.nodeId(),
freeBytesThresholdHigh,
freeBytesAfterShard);
return allocation.decision(
Decision.NO,
NAME,
"after allocation less than required [%s] free on node, free: [%s]",
freeBytesThresholdLow,
new ByteSizeValue(freeBytesAfterShard));
}
if (freeSpaceAfterShard < freeDiskThresholdHigh) {
logger.warn(
"after allocating, node [{}] would have more than the allowed {} free disk threshold ({} free), preventing allocation",
node.nodeId(),
Strings.format1Decimals(freeDiskThresholdHigh, "%"),
Strings.format1Decimals(freeSpaceAfterShard, "%"));
return allocation.decision(
Decision.NO,
NAME,
"after allocation more than allowed [%s%%] used disk on node, free: [%s%%]",
usedDiskThresholdLow,
freeSpaceAfterShard);
}
return allocation.decision(
Decision.YES,
NAME,
"enough disk for shard on node, free: [%s]",
new ByteSizeValue(freeBytes));
}
