private int findLatestIndex() throws IOException {
ImmutableMap<String, BlobMetaData> blobs = metaDataBlobContainer.listBlobsByPrefix("metadata-");
int index = -1;
for (BlobMetaData md : blobs.values()) {
if (logger.isTraceEnabled()) {
logger.trace("[findLatestMetadata]: Processing [" + md.name() + "]");
}
String name = md.name();
int fileIndex = Integer.parseInt(name.substring(name.indexOf('-') + 1));
if (fileIndex >= index) {
// try and read the meta data
byte[] data = null;
try {
data = metaDataBlobContainer.readBlobFully(name);
readMetaData(data);
index = fileIndex;
} catch (IOException e) {
logger.warn(
"[findLatestMetadata]: failed to read metadata from [{}], data_length [{}] ignoring...",
e,
name,
data == null ? "na" : data.length);
}
}
}
return index;
}
private boolean commitPointFileExistsInBlobs(
CommitPoint.FileInfo fileInfo, ImmutableMap<String, BlobMetaData> blobs) {
BlobMetaData blobMetaData = blobs.get(fileInfo.name());
if (blobMetaData != null) {
if (blobMetaData.length() != fileInfo.length()) {
return false;
}
} else if (blobs.containsKey(fileInfo.name() + ".part0")) {
// multi part file sum up the size and check
int part = 0;
long totalSize = 0;
while (true) {
blobMetaData = blobs.get(fileInfo.name() + ".part" + part++);
if (blobMetaData == null) {
break;
}
totalSize += blobMetaData.length();
}
if (totalSize != fileInfo.length()) {
return false;
}
} else {
// no file, not exact and not multipart
return false;
}
return true;
}
@Override
public void deleteBlobsByFilter(BlobNameFilter filter) throws IOException {
ImmutableMap<String, BlobMetaData> blobs = listBlobs();
for (BlobMetaData blob : blobs.values()) {
if (filter.accept(blob.name())) {
deleteBlob(blob.name());
}
}
}
@Override
public ImmutableMap<String, BlobMetaData> listBlobsByPrefix(String blobNamePrefix)
throws IOException {
ImmutableMap<String, BlobMetaData> allBlobs = listBlobs();
ImmutableMap.Builder<String, BlobMetaData> blobs = ImmutableMap.builder();
for (BlobMetaData blob : allBlobs.values()) {
if (blob.name().startsWith(blobNamePrefix)) {
blobs.put(blob.name(), blob);
}
}
return blobs.build();
}
public CommitPoint findCommitPoint(String index, int shardId) throws IOException {
BlobPath path = BlobStoreIndexGateway.shardPath(basePath, index, shardId);
ImmutableBlobContainer container = blobStore.immutableBlobContainer(path);
ImmutableMap<String, BlobMetaData> blobs = container.listBlobs();
List<CommitPoint> commitPointsList = Lists.newArrayList();
for (BlobMetaData md : blobs.values()) {
if (md.length() == 0) { // a commit point that was not flushed yet...
continue;
}
if (md.name().startsWith("commit-")) {
try {
commitPointsList.add(CommitPoints.fromXContent(container.readBlobFully(md.name())));
} catch (Exception e) {
logger.warn("failed to read commit point at path {} with name [{}]", e, path, md.name());
}
}
}
CommitPoints commitPoints = new CommitPoints(commitPointsList);
if (commitPoints.commits().isEmpty()) {
return null;
}
return commitPoints.commits().get(0);
}