/**
* Return an error message describing the types and number of failures in the BatchWrite.
*
* @param failedRows the rows which failed, and their respective error messages
* @return error message describing the failed rows
*/
private String getFailedRowsMessage(IntObjectOpenHashMap<WriteResult> failedRows) {
if (failedRows != null && failedRows.size() > 0) {
// Aggregate the error counts by code.
HashMap<Code, Integer> errorCodeToCountMap = new HashMap<>();
for (IntObjectCursor<WriteResult> failedRowCursor : failedRows) {
WriteResult wr = failedRowCursor.value;
Code errorCode = (wr == null ? null : wr.getCode());
Integer errorCount = errorCodeToCountMap.get(errorCode);
errorCodeToCountMap.put(
errorCode, (errorCode == null || errorCount == null ? 1 : errorCount + 1));
}
// Make a string out of the error map.
StringBuilder buf = new StringBuilder();
buf.append("{ ");
boolean first = true;
for (Map.Entry<Code, Integer> entry : errorCodeToCountMap.entrySet()) {
if (!first) {
buf.append(", ");
} else {
first = false;
}
buf.append(String.format("%s=%s", entry.getKey(), entry.getValue()));
}
buf.append(" }");
return buf.toString();
} else {
return "NONE";
}
}
public static void main(String[] args) throws UnknownHostException {
MongoClient client = new MongoClient();
DB database = client.getDB("school");
DBCollection collection = database.getCollection("students");
/*
Hint/spoiler: With the new schema, this problem is a lot harder
and that is sort of the point. One way is to find the lowest
homework in code and then update the scores array with the low
homework pruned. If you are struggling with the Node.js side of
this, look at the .slice() operator, which can remove an element
from an array in-place.
*/
DBCursor cursor = collection.find();
try {
while (cursor.hasNext()) {
BasicDBObject student = (BasicDBObject) cursor.next();
int studentId = student.getInt("_id");
String name = student.getString("name");
BasicDBList scores = (BasicDBList) student.get("scores");
System.out.printf("_id[%d], name[%s], scores%s %n", studentId, name, scores);
DBObject scoreToRemove = null;
double minScoreValue = 100.0;
for (Object obj : scores) {
BasicDBObject score = (BasicDBObject) obj;
String type = score.getString("type");
if (!"homework".equals(type)) {
continue;
}
double curScoreValue = score.getDouble("score");
System.out.printf("type[%s], current score value[%f] %n", type, curScoreValue);
if (curScoreValue < minScoreValue) {
scoreToRemove = score;
minScoreValue = curScoreValue;
}
}
System.out.printf("score to remove[%s] %n", scoreToRemove);
if (scoreToRemove != null) {
scores.remove(scoreToRemove);
BasicDBObject query = new BasicDBObject("_id", studentId);
BasicDBObject scoresUpdate =
new BasicDBObject("$set", new BasicDBObject("scores", scores));
WriteResult result = collection.update(query, scoresUpdate);
System.out.printf("update count[%d] %n", result.getN());
}
}
} finally {
cursor.close();
}
}
public void run(
Level logLevel,
ExecutorService executor,
FileReadUtil fileReader,
PerformanceTrackers perfTracking)
throws IOException, FileFormatException {
HashSet<List<String>> missingNamespaces = new HashSet<>();
Logging log =
this.logFile != null
? new LoggingImpl(
logLevel,
new PrintStream(this.logFile.toFile()),
LoggingPrefixFormat.DATETIME_LEVEL_AND_CLASS)
: null;
val loadRes = SourceFiles.load(this.sources);
if (log != null) {
loadRes.log(log, logLevel, true);
}
// TODO debugging
long start = System.nanoTime();
val parseRes = ParsedResult.parse(loadRes.getSources(), executor, fileReader, perfTracking);
// TODO debugging
System.out.println(
"load() time: "
+ TimeUnitUtil.convert(
TimeUnit.NANOSECONDS, (System.nanoTime() - start), TimeUnit.MILLISECONDS)
+ " "
+ TimeUnitUtil.abbreviation(TimeUnit.MILLISECONDS, true, false));
if (log != null) {
parseRes.log(log, logLevel, true);
}
val resolvedRes = ResolvedResult.resolve(parseRes.compilationUnits, missingNamespaces);
if (log != null) {
resolvedRes.log(log, logLevel, true);
}
val filterRes = FilterResult.filter(resolvedRes.compilationUnits, this.destinations);
if (log != null) {
filterRes.log(log, logLevel, true);
}
WriteResult.write(filterRes.filterSets, missingNamespaces);
}
/**
* Method writes the {@link WritableMessage} to the specific address.
*
* @param connection the {@link org.glassfish.grizzly.Connection} to write to
* @param dstAddress the destination address the <tt>message</tt> will be sent to
* @param message the {@link WritableMessage}, from which the data will be written
* @param completionHandler {@link org.glassfish.grizzly.CompletionHandler}, which will get
* notified, when write will be completed
*/
public void write(
Connection<SocketAddress> connection,
SocketAddress dstAddress,
WritableMessage message,
CompletionHandler<WriteResult<WritableMessage, SocketAddress>> completionHandler,
final org.glassfish.grizzly.asyncqueue.PushBackHandler pushBackHandler,
long timeout,
TimeUnit timeunit) {
if (message == null) {
failure(new IllegalStateException("Message cannot be null"), completionHandler);
return;
}
if (connection == null || !(connection instanceof NIOConnection)) {
failure(
new IllegalStateException("Connection should be NIOConnection and cannot be null"),
completionHandler);
return;
}
final NIOConnection nioConnection = (NIOConnection) connection;
final WriteResult<WritableMessage, SocketAddress> writeResult =
WriteResult.create(connection, message, dstAddress, 0);
try {
write0(nioConnection, dstAddress, message, writeResult, timeout, timeunit);
// Call PushBackHandler after data is written.
// IMO It has more sense for blocking case
if (pushBackHandler != null) {
pushBackHandler.onAccept(connection, message);
}
if (completionHandler != null) {
completionHandler.completed(writeResult);
}
message.release();
} catch (IOException e) {
failure(e, completionHandler);
}
}
@Override
protected Tuple<BulkShardResponse, BulkShardRequest> shardOperationOnPrimary(
ClusterState clusterState, PrimaryOperationRequest shardRequest) {
final BulkShardRequest request = shardRequest.request;
final IndexService indexService = indicesService.indexServiceSafe(request.index());
final IndexShard indexShard = indexService.shardSafe(shardRequest.shardId.id());
long[] preVersions = new long[request.items().length];
VersionType[] preVersionTypes = new VersionType[request.items().length];
Translog.Location location = null;
for (int requestIndex = 0; requestIndex < request.items().length; requestIndex++) {
BulkItemRequest item = request.items()[requestIndex];
if (item.request() instanceof IndexRequest) {
IndexRequest indexRequest = (IndexRequest) item.request();
preVersions[requestIndex] = indexRequest.version();
preVersionTypes[requestIndex] = indexRequest.versionType();
try {
WriteResult<IndexResponse> result =
shardIndexOperation(request, indexRequest, clusterState, indexShard, true);
location = locationToSync(location, result.location);
// add the response
IndexResponse indexResponse = result.response();
setResponse(
item,
new BulkItemResponse(item.id(), indexRequest.opType().lowercase(), indexResponse));
} catch (Throwable e) {
// rethrow the failure if we are going to retry on primary and let parent failure to
// handle it
if (retryPrimaryException(e)) {
// restore updated versions...
for (int j = 0; j < requestIndex; j++) {
applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]);
}
throw (ElasticsearchException) e;
}
if (ExceptionsHelper.status(e) == RestStatus.CONFLICT) {
logger.trace(
"{} failed to execute bulk item (index) {}", e, shardRequest.shardId, indexRequest);
} else {
logger.debug(
"{} failed to execute bulk item (index) {}", e, shardRequest.shardId, indexRequest);
}
// if its a conflict failure, and we already executed the request on a primary (and we
// execute it
// again, due to primary relocation and only processing up to N bulk items when the shard
// gets closed)
// then just use the response we got from the successful execution
if (item.getPrimaryResponse() != null && isConflictException(e)) {
setResponse(item, item.getPrimaryResponse());
} else {
setResponse(
item,
new BulkItemResponse(
item.id(),
indexRequest.opType().lowercase(),
new BulkItemResponse.Failure(
request.index(), indexRequest.type(), indexRequest.id(), e)));
}
}
} else if (item.request() instanceof DeleteRequest) {
DeleteRequest deleteRequest = (DeleteRequest) item.request();
preVersions[requestIndex] = deleteRequest.version();
preVersionTypes[requestIndex] = deleteRequest.versionType();
try {
// add the response
final WriteResult<DeleteResponse> writeResult =
shardDeleteOperation(request, deleteRequest, indexShard);
DeleteResponse deleteResponse = writeResult.response();
location = locationToSync(location, writeResult.location);
setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_DELETE, deleteResponse));
} catch (Throwable e) {
// rethrow the failure if we are going to retry on primary and let parent failure to
// handle it
if (retryPrimaryException(e)) {
// restore updated versions...
for (int j = 0; j < requestIndex; j++) {
applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]);
}
throw (ElasticsearchException) e;
}
if (ExceptionsHelper.status(e) == RestStatus.CONFLICT) {
logger.trace(
"{} failed to execute bulk item (delete) {}",
e,
shardRequest.shardId,
deleteRequest);
} else {
logger.debug(
"{} failed to execute bulk item (delete) {}",
e,
shardRequest.shardId,
deleteRequest);
}
// if its a conflict failure, and we already executed the request on a primary (and we
// execute it
// again, due to primary relocation and only processing up to N bulk items when the shard
// gets closed)
// then just use the response we got from the successful execution
if (item.getPrimaryResponse() != null && isConflictException(e)) {
setResponse(item, item.getPrimaryResponse());
} else {
setResponse(
item,
new BulkItemResponse(
item.id(),
OP_TYPE_DELETE,
new BulkItemResponse.Failure(
request.index(), deleteRequest.type(), deleteRequest.id(), e)));
}
}
} else if (item.request() instanceof UpdateRequest) {
UpdateRequest updateRequest = (UpdateRequest) item.request();
preVersions[requestIndex] = updateRequest.version();
preVersionTypes[requestIndex] = updateRequest.versionType();
// We need to do the requested retries plus the initial attempt. We don't do <
// 1+retry_on_conflict because retry_on_conflict may be Integer.MAX_VALUE
for (int updateAttemptsCount = 0;
updateAttemptsCount <= updateRequest.retryOnConflict();
updateAttemptsCount++) {
UpdateResult updateResult;
try {
updateResult = shardUpdateOperation(clusterState, request, updateRequest, indexShard);
} catch (Throwable t) {
updateResult = new UpdateResult(null, null, false, t, null);
}
if (updateResult.success()) {
if (updateResult.writeResult != null) {
location = locationToSync(location, updateResult.writeResult.location);
}
switch (updateResult.result.operation()) {
case UPSERT:
case INDEX:
WriteResult<IndexResponse> result = updateResult.writeResult;
IndexRequest indexRequest = updateResult.request();
BytesReference indexSourceAsBytes = indexRequest.source();
// add the response
IndexResponse indexResponse = result.response();
UpdateResponse updateResponse =
new UpdateResponse(
indexResponse.getShardInfo(),
indexResponse.getIndex(),
indexResponse.getType(),
indexResponse.getId(),
indexResponse.getVersion(),
indexResponse.isCreated());
if (updateRequest.fields() != null && updateRequest.fields().length > 0) {
Tuple<XContentType, Map<String, Object>> sourceAndContent =
XContentHelper.convertToMap(indexSourceAsBytes, true);
updateResponse.setGetResult(
updateHelper.extractGetResult(
updateRequest,
shardRequest.request.index(),
indexResponse.getVersion(),
sourceAndContent.v2(),
sourceAndContent.v1(),
indexSourceAsBytes));
}
item =
request.items()[requestIndex] =
new BulkItemRequest(request.items()[requestIndex].id(), indexRequest);
setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResponse));
break;
case DELETE:
WriteResult<DeleteResponse> writeResult = updateResult.writeResult;
DeleteResponse response = writeResult.response();
DeleteRequest deleteRequest = updateResult.request();
updateResponse =
new UpdateResponse(
response.getShardInfo(),
response.getIndex(),
response.getType(),
response.getId(),
response.getVersion(),
false);
updateResponse.setGetResult(
updateHelper.extractGetResult(
updateRequest,
shardRequest.request.index(),
response.getVersion(),
updateResult.result.updatedSourceAsMap(),
updateResult.result.updateSourceContentType(),
null));
// Replace the update request to the translated delete request to execute on the
// replica.
item =
request.items()[requestIndex] =
new BulkItemRequest(request.items()[requestIndex].id(), deleteRequest);
setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResponse));
break;
case NONE:
setResponse(
item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResult.noopResult));
item.setIgnoreOnReplica(); // no need to go to the replica
break;
}
// NOTE: Breaking out of the retry_on_conflict loop!
break;
} else if (updateResult.failure()) {
Throwable t = updateResult.error;
if (updateResult.retry) {
// updateAttemptCount is 0 based and marks current attempt, if it's equal to
// retryOnConflict we are going out of the iteration
if (updateAttemptsCount >= updateRequest.retryOnConflict()) {
setResponse(
item,
new BulkItemResponse(
item.id(),
OP_TYPE_UPDATE,
new BulkItemResponse.Failure(
request.index(), updateRequest.type(), updateRequest.id(), t)));
}
} else {
// rethrow the failure if we are going to retry on primary and let parent failure to
// handle it
if (retryPrimaryException(t)) {
// restore updated versions...
for (int j = 0; j < requestIndex; j++) {
applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]);
}
throw (ElasticsearchException) t;
}
// if its a conflict failure, and we already executed the request on a primary (and we
// execute it
// again, due to primary relocation and only processing up to N bulk items when the
// shard gets closed)
// then just use the response we got from the successful execution
if (item.getPrimaryResponse() != null && isConflictException(t)) {
setResponse(item, item.getPrimaryResponse());
} else if (updateResult.result == null) {
setResponse(
item,
new BulkItemResponse(
item.id(),
OP_TYPE_UPDATE,
new BulkItemResponse.Failure(
shardRequest.request.index(),
updateRequest.type(),
updateRequest.id(),
t)));
} else {
switch (updateResult.result.operation()) {
case UPSERT:
case INDEX:
IndexRequest indexRequest = updateResult.request();
if (ExceptionsHelper.status(t) == RestStatus.CONFLICT) {
logger.trace(
"{} failed to execute bulk item (index) {}",
t,
shardRequest.shardId,
indexRequest);
} else {
logger.debug(
"{} failed to execute bulk item (index) {}",
t,
shardRequest.shardId,
indexRequest);
}
setResponse(
item,
new BulkItemResponse(
item.id(),
OP_TYPE_UPDATE,
new BulkItemResponse.Failure(
request.index(), indexRequest.type(), indexRequest.id(), t)));
break;
case DELETE:
DeleteRequest deleteRequest = updateResult.request();
if (ExceptionsHelper.status(t) == RestStatus.CONFLICT) {
logger.trace(
"{} failed to execute bulk item (delete) {}",
t,
shardRequest.shardId,
deleteRequest);
} else {
logger.debug(
"{} failed to execute bulk item (delete) {}",
t,
shardRequest.shardId,
deleteRequest);
}
setResponse(
item,
new BulkItemResponse(
item.id(),
OP_TYPE_DELETE,
new BulkItemResponse.Failure(
request.index(), deleteRequest.type(), deleteRequest.id(), t)));
break;
}
}
// NOTE: Breaking out of the retry_on_conflict loop!
break;
}
}
}
} else {
throw new IllegalStateException("Unexpected index operation: " + item.request());
}
assert item.getPrimaryResponse() != null;
assert preVersionTypes[requestIndex] != null;
}
processAfter(request.refresh(), indexShard, location);
BulkItemResponse[] responses = new BulkItemResponse[request.items().length];
BulkItemRequest[] items = request.items();
for (int i = 0; i < items.length; i++) {
responses[i] = items[i].getPrimaryResponse();
}
return new Tuple<>(
new BulkShardResponse(shardRequest.shardId, responses), shardRequest.request);
}