Example #1
0
 @Override
 void writeBody(final BufferedSource in, final BufferedSink out) throws IOException {
   try {
     final ResponseBody data = body();
     if (data != null) {
       final long length = data.contentLength();
       if (length > 0) {
         out.write(data.source(), data.contentLength());
         out.flush();
       } else if (length < 0) {
         final Buffer buffer = new Buffer();
         final long step = 65536;
         long read;
         while ((read = data.source().read(buffer, step)) > -1) {
           buffer.flush();
           out.write(buffer, read);
           out.flush();
         }
       }
     }
   } finally {
     //        try { in.close(); } catch (final IOException ignore) {}
     //        try { out.close(); } catch (final IOException ignore) {}
     //        try { socket.close(); } catch (final IOException ignore) {}
   }
 }
Example #2
0
 /**
  * Sets the response body.
  *
  * @param body the response body.
  * @return this
  */
 public Builder body(final ResponseBody body) {
   this.mBody = body;
   if (body == null) {
     contentLength(0);
   } else {
     contentType(body.contentType());
     contentLength(body.contentLength());
   }
   return this;
 }
 @Override
 public BufferedSource source() {
   if (mBufferedSource == null) {
     mBufferedSource = Okio.buffer(source(mResponseBody.source()));
   }
   return mBufferedSource;
 }
 @Override
 public T convert(ResponseBody value) throws IOException {
   String returned = value.string();
   try {
     return gson.fromJson(returned, type);
   } catch (JsonParseException e) {
     return (T) returned;
   }
 }
    @Override
    public byte[] download(String url, String accepts) {
      HttpUrl httpUrl = HttpUrl.parse(url);

      if (httpUrl == null) {
        App.log.log(Level.SEVERE, "Invalid external resource URL", url);
        return null;
      }

      String host = httpUrl.host();
      if (host == null) {
        App.log.log(Level.SEVERE, "External resource URL doesn't specify a host name", url);
        return null;
      }

      OkHttpClient resourceClient = HttpClient.create();

      // authenticate only against a certain host, and only upon request
      resourceClient =
          HttpClient.addAuthentication(
              resourceClient, baseUrl.host(), settings.username(), settings.password());

      // allow redirects
      resourceClient = resourceClient.newBuilder().followRedirects(true).build();

      try {
        Response response =
            resourceClient.newCall(new Request.Builder().get().url(httpUrl).build()).execute();

        ResponseBody body = response.body();
        if (body != null) {
          @Cleanup InputStream stream = body.byteStream();
          if (response.isSuccessful() && stream != null) {
            return IOUtils.toByteArray(stream);
          } else App.log.severe("Couldn't download external resource");
        }
      } catch (IOException e) {
        App.log.log(Level.SEVERE, "Couldn't download external resource", e);
      }
      return null;
    }
Example #6
0
    /*
     * (non-Javadoc)
     *
     * @see okhttp3.ws.WebSocketListener#onMessage(okhttp3.ResponseBody)
     */
    @Override
    public void onMessage(ResponseBody response) throws IOException {
      String message = response.string();
      if (message == null) return;

      try {
        JsonObject json = new JsonParser().parse(message).getAsJsonObject();
        if (json.has(ERROR)) {
          callback.onError(new RuntimeException(json.get(ERROR).getAsString()));
        } else if (json.has(RESULTS)) {
          callback.onTranscription(GSON.fromJson(message, SpeechResults.class));
        } else if (json.has(STATE)) {
          if (!audioSent) {
            sendInputSteam(stream);
            socket.sendMessage(RequestBody.create(WebSocket.TEXT, buildStopMessage().toString()));
            audioSent = true;
          } else {
            socket.close(CLOSE_NORMAL, "Transcription completed");
          }
        }
      } catch (JsonParseException e) {
        throw new RuntimeException("Error parsing the incoming message: " + response.string());
      }
    }
 @Override
 public long contentLength() {
   return mResponseBody.contentLength();
 }
 @Override
 public MediaType contentType() {
   return mResponseBody.contentType();
 }
  @Override
  protected void downloadRemote()
      throws IOException, HttpException, DavException, ContactsStorageException {
    App.log.info("Downloading " + toDownload.size() + " contacts (" + MAX_MULTIGET + " at once)");

    // prepare downloader which may be used to download external resource like contact photos
    Contact.Downloader downloader = new ResourceDownloader(collectionURL);

    // download new/updated VCards from server
    for (DavResource[] bunch :
        ArrayUtils.partition(
            toDownload.toArray(new DavResource[toDownload.size()]), MAX_MULTIGET)) {
      if (Thread.interrupted()) return;

      App.log.info("Downloading " + StringUtils.join(bunch, ", "));

      if (bunch.length == 1) {
        // only one contact, use GET
        DavResource remote = bunch[0];

        ResponseBody body =
            remote.get("text/vcard;version=4.0, text/vcard;charset=utf-8;q=0.8, text/vcard;q=0.5");
        String eTag = ((GetETag) remote.properties.get(GetETag.NAME)).eTag;

        Charset charset = Charsets.UTF_8;
        MediaType contentType = body.contentType();
        if (contentType != null) charset = contentType.charset(Charsets.UTF_8);

        @Cleanup InputStream stream = body.byteStream();
        processVCard(remote.fileName(), eTag, stream, charset, downloader);

      } else {
        // multiple contacts, use multi-get
        List<HttpUrl> urls = new LinkedList<>();
        for (DavResource remote : bunch) urls.add(remote.location);
        davAddressBook().multiget(urls.toArray(new HttpUrl[urls.size()]), hasVCard4);

        // process multiget results
        for (DavResource remote : davCollection.members) {
          String eTag;
          GetETag getETag = (GetETag) remote.properties.get(GetETag.NAME);
          if (getETag != null) eTag = getETag.eTag;
          else throw new DavException("Received multi-get response without ETag");

          Charset charset = Charsets.UTF_8;
          GetContentType getContentType =
              (GetContentType) remote.properties.get(GetContentType.NAME);
          if (getContentType != null && getContentType.type != null) {
            MediaType type = MediaType.parse(getContentType.type);
            if (type != null) charset = type.charset(Charsets.UTF_8);
          }

          AddressData addressData = (AddressData) remote.properties.get(AddressData.NAME);
          if (addressData == null || addressData.vCard == null)
            throw new DavException("Received multi-get response without address data");

          @Cleanup InputStream stream = new ByteArrayInputStream(addressData.vCard.getBytes());
          processVCard(remote.fileName(), eTag, stream, charset, downloader);
        }
      }
    }
  }
Example #10
0
  @Override
  public void onTrigger(final ProcessContext context, final ProcessSession session)
      throws ProcessException {

    FlowFile flowFile = null;
    if (context.hasIncomingConnection()) {
      flowFile = session.get();

      // If we have no FlowFile, and all incoming connections are self-loops then we can continue
      // on.
      // However, if we have no FlowFile and we have connections coming from other Processors, then
      // we know that we should run only if we have a FlowFile.
      if (flowFile == null && context.hasNonLoopConnection()) {
        return;
      }
    }

    OkHttpClient okHttpClient = getClient();

    if (flowFile == null) {
      flowFile = session.create();
    }

    final String index =
        context.getProperty(INDEX).evaluateAttributeExpressions(flowFile).getValue();
    final String docId =
        context.getProperty(DOC_ID).evaluateAttributeExpressions(flowFile).getValue();
    final String docType =
        context.getProperty(TYPE).evaluateAttributeExpressions(flowFile).getValue();
    final String fields =
        context.getProperty(FIELDS).isSet()
            ? context.getProperty(FIELDS).evaluateAttributeExpressions(flowFile).getValue()
            : null;

    // Authentication
    final String username =
        context.getProperty(USERNAME).evaluateAttributeExpressions(flowFile).getValue();
    final String password = context.getProperty(PASSWORD).evaluateAttributeExpressions().getValue();

    final ComponentLog logger = getLogger();

    Response getResponse = null;

    try {
      logger.debug("Fetching {}/{}/{} from Elasticsearch", new Object[] {index, docType, docId});

      // read the url property from the context
      final String urlstr =
          StringUtils.trimToEmpty(
              context.getProperty(ES_URL).evaluateAttributeExpressions().getValue());
      final URL url = buildRequestURL(urlstr, docId, index, docType, fields);
      final long startNanos = System.nanoTime();

      getResponse = sendRequestToElasticsearch(okHttpClient, url, username, password, "GET", null);
      final int statusCode = getResponse.code();

      if (isSuccess(statusCode)) {
        ResponseBody body = getResponse.body();
        final byte[] bodyBytes = body.bytes();
        JsonNode responseJson = parseJsonResponse(new ByteArrayInputStream(bodyBytes));
        boolean found = responseJson.get("found").asBoolean(false);
        String retrievedIndex = responseJson.get("_index").asText();
        String retrievedType = responseJson.get("_type").asText();
        String retrievedId = responseJson.get("_id").asText();

        if (found) {
          JsonNode source = responseJson.get("_source");
          flowFile = session.putAttribute(flowFile, "filename", retrievedId);
          flowFile = session.putAttribute(flowFile, "es.index", retrievedIndex);
          flowFile = session.putAttribute(flowFile, "es.type", retrievedType);
          if (source != null) {
            flowFile =
                session.write(
                    flowFile,
                    out -> {
                      out.write(source.toString().getBytes());
                    });
          }
          logger.debug("Elasticsearch document " + retrievedId + " fetched, routing to success");

          // emit provenance event
          final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
          if (context.hasNonLoopConnection()) {
            session.getProvenanceReporter().fetch(flowFile, url.toExternalForm(), millis);
          } else {
            session.getProvenanceReporter().receive(flowFile, url.toExternalForm(), millis);
          }
          session.transfer(flowFile, REL_SUCCESS);
        } else {
          logger.warn(
              "Failed to read {}/{}/{} from Elasticsearch: Document not found",
              new Object[] {index, docType, docId});

          // We couldn't find the document, so send it to "not found"
          session.transfer(flowFile, REL_NOT_FOUND);
        }
      } else {
        if (statusCode == 404) {
          logger.warn(
              "Failed to read {}/{}/{} from Elasticsearch: Document not found",
              new Object[] {index, docType, docId});

          // We couldn't find the document, so penalize it and send it to "not found"
          session.transfer(flowFile, REL_NOT_FOUND);
        } else {
          // 5xx -> RETRY, but a server error might last a while, so yield
          if (statusCode / 100 == 5) {

            logger.warn(
                "Elasticsearch returned code {} with message {}, transferring flow file to retry. This is likely a server problem, yielding...",
                new Object[] {statusCode, getResponse.message()});
            session.transfer(flowFile, REL_RETRY);
            context.yield();
          } else if (context.hasIncomingConnection()) { // 1xx, 3xx, 4xx -> NO RETRY
            logger.warn(
                "Elasticsearch returned code {} with message {}, transferring flow file to failure",
                new Object[] {statusCode, getResponse.message()});
            session.transfer(flowFile, REL_FAILURE);
          } else {
            logger.warn(
                "Elasticsearch returned code {} with message {}",
                new Object[] {statusCode, getResponse.message()});
            session.remove(flowFile);
          }
        }
      }
    } catch (IOException ioe) {
      logger.error(
          "Failed to read from Elasticsearch due to {}, this may indicate an error in configuration "
              + "(hosts, username/password, etc.). Routing to retry",
          new Object[] {ioe.getLocalizedMessage()},
          ioe);
      if (context.hasIncomingConnection()) {
        session.transfer(flowFile, REL_RETRY);
      } else {
        session.remove(flowFile);
      }
      context.yield();

    } catch (Exception e) {
      logger.error(
          "Failed to read {} from Elasticsearch due to {}",
          new Object[] {flowFile, e.getLocalizedMessage()},
          e);
      if (context.hasIncomingConnection()) {
        session.transfer(flowFile, REL_FAILURE);
      } else {
        session.remove(flowFile);
      }
      context.yield();
    } finally {
      if (getResponse != null) {
        getResponse.close();
      }
    }
  }