@Override
void writeBody(final BufferedSource in, final BufferedSink out) throws IOException {
try {
final ResponseBody data = body();
if (data != null) {
final long length = data.contentLength();
if (length > 0) {
out.write(data.source(), data.contentLength());
out.flush();
} else if (length < 0) {
final Buffer buffer = new Buffer();
final long step = 65536;
long read;
while ((read = data.source().read(buffer, step)) > -1) {
buffer.flush();
out.write(buffer, read);
out.flush();
}
}
}
} finally {
// try { in.close(); } catch (final IOException ignore) {}
// try { out.close(); } catch (final IOException ignore) {}
// try { socket.close(); } catch (final IOException ignore) {}
}
}
/**
* Sets the response body.
*
* @param body the response body.
* @return this
*/
public Builder body(final ResponseBody body) {
this.mBody = body;
if (body == null) {
contentLength(0);
} else {
contentType(body.contentType());
contentLength(body.contentLength());
}
return this;
}
@Override
public BufferedSource source() {
if (mBufferedSource == null) {
mBufferedSource = Okio.buffer(source(mResponseBody.source()));
}
return mBufferedSource;
}
@Override
public T convert(ResponseBody value) throws IOException {
String returned = value.string();
try {
return gson.fromJson(returned, type);
} catch (JsonParseException e) {
return (T) returned;
}
}
@Override
public byte[] download(String url, String accepts) {
HttpUrl httpUrl = HttpUrl.parse(url);
if (httpUrl == null) {
App.log.log(Level.SEVERE, "Invalid external resource URL", url);
return null;
}
String host = httpUrl.host();
if (host == null) {
App.log.log(Level.SEVERE, "External resource URL doesn't specify a host name", url);
return null;
}
OkHttpClient resourceClient = HttpClient.create();
// authenticate only against a certain host, and only upon request
resourceClient =
HttpClient.addAuthentication(
resourceClient, baseUrl.host(), settings.username(), settings.password());
// allow redirects
resourceClient = resourceClient.newBuilder().followRedirects(true).build();
try {
Response response =
resourceClient.newCall(new Request.Builder().get().url(httpUrl).build()).execute();
ResponseBody body = response.body();
if (body != null) {
@Cleanup InputStream stream = body.byteStream();
if (response.isSuccessful() && stream != null) {
return IOUtils.toByteArray(stream);
} else App.log.severe("Couldn't download external resource");
}
} catch (IOException e) {
App.log.log(Level.SEVERE, "Couldn't download external resource", e);
}
return null;
}
/*
* (non-Javadoc)
*
* @see okhttp3.ws.WebSocketListener#onMessage(okhttp3.ResponseBody)
*/
@Override
public void onMessage(ResponseBody response) throws IOException {
String message = response.string();
if (message == null) return;
try {
JsonObject json = new JsonParser().parse(message).getAsJsonObject();
if (json.has(ERROR)) {
callback.onError(new RuntimeException(json.get(ERROR).getAsString()));
} else if (json.has(RESULTS)) {
callback.onTranscription(GSON.fromJson(message, SpeechResults.class));
} else if (json.has(STATE)) {
if (!audioSent) {
sendInputSteam(stream);
socket.sendMessage(RequestBody.create(WebSocket.TEXT, buildStopMessage().toString()));
audioSent = true;
} else {
socket.close(CLOSE_NORMAL, "Transcription completed");
}
}
} catch (JsonParseException e) {
throw new RuntimeException("Error parsing the incoming message: " + response.string());
}
}
@Override
public long contentLength() {
return mResponseBody.contentLength();
}
@Override
public MediaType contentType() {
return mResponseBody.contentType();
}
@Override
protected void downloadRemote()
throws IOException, HttpException, DavException, ContactsStorageException {
App.log.info("Downloading " + toDownload.size() + " contacts (" + MAX_MULTIGET + " at once)");
// prepare downloader which may be used to download external resource like contact photos
Contact.Downloader downloader = new ResourceDownloader(collectionURL);
// download new/updated VCards from server
for (DavResource[] bunch :
ArrayUtils.partition(
toDownload.toArray(new DavResource[toDownload.size()]), MAX_MULTIGET)) {
if (Thread.interrupted()) return;
App.log.info("Downloading " + StringUtils.join(bunch, ", "));
if (bunch.length == 1) {
// only one contact, use GET
DavResource remote = bunch[0];
ResponseBody body =
remote.get("text/vcard;version=4.0, text/vcard;charset=utf-8;q=0.8, text/vcard;q=0.5");
String eTag = ((GetETag) remote.properties.get(GetETag.NAME)).eTag;
Charset charset = Charsets.UTF_8;
MediaType contentType = body.contentType();
if (contentType != null) charset = contentType.charset(Charsets.UTF_8);
@Cleanup InputStream stream = body.byteStream();
processVCard(remote.fileName(), eTag, stream, charset, downloader);
} else {
// multiple contacts, use multi-get
List<HttpUrl> urls = new LinkedList<>();
for (DavResource remote : bunch) urls.add(remote.location);
davAddressBook().multiget(urls.toArray(new HttpUrl[urls.size()]), hasVCard4);
// process multiget results
for (DavResource remote : davCollection.members) {
String eTag;
GetETag getETag = (GetETag) remote.properties.get(GetETag.NAME);
if (getETag != null) eTag = getETag.eTag;
else throw new DavException("Received multi-get response without ETag");
Charset charset = Charsets.UTF_8;
GetContentType getContentType =
(GetContentType) remote.properties.get(GetContentType.NAME);
if (getContentType != null && getContentType.type != null) {
MediaType type = MediaType.parse(getContentType.type);
if (type != null) charset = type.charset(Charsets.UTF_8);
}
AddressData addressData = (AddressData) remote.properties.get(AddressData.NAME);
if (addressData == null || addressData.vCard == null)
throw new DavException("Received multi-get response without address data");
@Cleanup InputStream stream = new ByteArrayInputStream(addressData.vCard.getBytes());
processVCard(remote.fileName(), eTag, stream, charset, downloader);
}
}
}
}
@Override
public void onTrigger(final ProcessContext context, final ProcessSession session)
throws ProcessException {
FlowFile flowFile = null;
if (context.hasIncomingConnection()) {
flowFile = session.get();
// If we have no FlowFile, and all incoming connections are self-loops then we can continue
// on.
// However, if we have no FlowFile and we have connections coming from other Processors, then
// we know that we should run only if we have a FlowFile.
if (flowFile == null && context.hasNonLoopConnection()) {
return;
}
}
OkHttpClient okHttpClient = getClient();
if (flowFile == null) {
flowFile = session.create();
}
final String index =
context.getProperty(INDEX).evaluateAttributeExpressions(flowFile).getValue();
final String docId =
context.getProperty(DOC_ID).evaluateAttributeExpressions(flowFile).getValue();
final String docType =
context.getProperty(TYPE).evaluateAttributeExpressions(flowFile).getValue();
final String fields =
context.getProperty(FIELDS).isSet()
? context.getProperty(FIELDS).evaluateAttributeExpressions(flowFile).getValue()
: null;
// Authentication
final String username =
context.getProperty(USERNAME).evaluateAttributeExpressions(flowFile).getValue();
final String password = context.getProperty(PASSWORD).evaluateAttributeExpressions().getValue();
final ComponentLog logger = getLogger();
Response getResponse = null;
try {
logger.debug("Fetching {}/{}/{} from Elasticsearch", new Object[] {index, docType, docId});
// read the url property from the context
final String urlstr =
StringUtils.trimToEmpty(
context.getProperty(ES_URL).evaluateAttributeExpressions().getValue());
final URL url = buildRequestURL(urlstr, docId, index, docType, fields);
final long startNanos = System.nanoTime();
getResponse = sendRequestToElasticsearch(okHttpClient, url, username, password, "GET", null);
final int statusCode = getResponse.code();
if (isSuccess(statusCode)) {
ResponseBody body = getResponse.body();
final byte[] bodyBytes = body.bytes();
JsonNode responseJson = parseJsonResponse(new ByteArrayInputStream(bodyBytes));
boolean found = responseJson.get("found").asBoolean(false);
String retrievedIndex = responseJson.get("_index").asText();
String retrievedType = responseJson.get("_type").asText();
String retrievedId = responseJson.get("_id").asText();
if (found) {
JsonNode source = responseJson.get("_source");
flowFile = session.putAttribute(flowFile, "filename", retrievedId);
flowFile = session.putAttribute(flowFile, "es.index", retrievedIndex);
flowFile = session.putAttribute(flowFile, "es.type", retrievedType);
if (source != null) {
flowFile =
session.write(
flowFile,
out -> {
out.write(source.toString().getBytes());
});
}
logger.debug("Elasticsearch document " + retrievedId + " fetched, routing to success");
// emit provenance event
final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
if (context.hasNonLoopConnection()) {
session.getProvenanceReporter().fetch(flowFile, url.toExternalForm(), millis);
} else {
session.getProvenanceReporter().receive(flowFile, url.toExternalForm(), millis);
}
session.transfer(flowFile, REL_SUCCESS);
} else {
logger.warn(
"Failed to read {}/{}/{} from Elasticsearch: Document not found",
new Object[] {index, docType, docId});
// We couldn't find the document, so send it to "not found"
session.transfer(flowFile, REL_NOT_FOUND);
}
} else {
if (statusCode == 404) {
logger.warn(
"Failed to read {}/{}/{} from Elasticsearch: Document not found",
new Object[] {index, docType, docId});
// We couldn't find the document, so penalize it and send it to "not found"
session.transfer(flowFile, REL_NOT_FOUND);
} else {
// 5xx -> RETRY, but a server error might last a while, so yield
if (statusCode / 100 == 5) {
logger.warn(
"Elasticsearch returned code {} with message {}, transferring flow file to retry. This is likely a server problem, yielding...",
new Object[] {statusCode, getResponse.message()});
session.transfer(flowFile, REL_RETRY);
context.yield();
} else if (context.hasIncomingConnection()) { // 1xx, 3xx, 4xx -> NO RETRY
logger.warn(
"Elasticsearch returned code {} with message {}, transferring flow file to failure",
new Object[] {statusCode, getResponse.message()});
session.transfer(flowFile, REL_FAILURE);
} else {
logger.warn(
"Elasticsearch returned code {} with message {}",
new Object[] {statusCode, getResponse.message()});
session.remove(flowFile);
}
}
}
} catch (IOException ioe) {
logger.error(
"Failed to read from Elasticsearch due to {}, this may indicate an error in configuration "
+ "(hosts, username/password, etc.). Routing to retry",
new Object[] {ioe.getLocalizedMessage()},
ioe);
if (context.hasIncomingConnection()) {
session.transfer(flowFile, REL_RETRY);
} else {
session.remove(flowFile);
}
context.yield();
} catch (Exception e) {
logger.error(
"Failed to read {} from Elasticsearch due to {}",
new Object[] {flowFile, e.getLocalizedMessage()},
e);
if (context.hasIncomingConnection()) {
session.transfer(flowFile, REL_FAILURE);
} else {
session.remove(flowFile);
}
context.yield();
} finally {
if (getResponse != null) {
getResponse.close();
}
}
}
