예제 #1
0
 @Test
 public void parseNameValueBlock() throws IOException {
   List<Header> headerBlock =
       headerEntries(
           "cache-control", "no-cache, no-store",
           "set-cookie", "Cookie1\u0000Cookie2",
           ":status", "200 OK",
           ":version", "HTTP/1.1");
   Request request = new Request.Builder().url("http://square.com/").build();
   Response response = Http2xStream.readSpdy3HeadersList(headerBlock).request(request).build();
   Headers headers = response.headers();
   assertEquals(3, headers.size());
   Assert.assertEquals(Protocol.SPDY_3, response.protocol());
   assertEquals(200, response.code());
   assertEquals("OK", response.message());
   assertEquals("no-cache, no-store", headers.get("cache-control"));
   assertEquals("Cookie2", headers.get("set-cookie"));
   assertEquals("cache-control", headers.name(0));
   assertEquals("no-cache, no-store", headers.value(0));
   assertEquals("set-cookie", headers.name(1));
   assertEquals("Cookie1", headers.value(1));
   assertEquals("set-cookie", headers.name(2));
   assertEquals("Cookie2", headers.value(2));
   assertNull(headers.get(":status"));
   assertNull(headers.get(":version"));
 }
  private String authorize() {
    try {
      OkHttpClient.Builder builder = client.newBuilder();
      builder.interceptors().remove(this);
      OkHttpClient clone = builder.build();

      String credential = Credentials.basic(config.getUsername(), new String(config.getPassword()));
      URL url = new URL(URLUtils.join(config.getMasterUrl(), AUTHORIZE_PATH));
      Response response =
          clone
              .newCall(
                  new Request.Builder().get().url(url).header(AUTHORIZATION, credential).build())
              .execute();

      response.body().close();
      response = response.priorResponse() != null ? response.priorResponse() : response;
      response = response.networkResponse() != null ? response.networkResponse() : response;
      String token = response.header(LOCATION);
      if (token == null || token.isEmpty()) {
        throw new KubernetesClientException(
            "Unexpected response ("
                + response.code()
                + " "
                + response.message()
                + "), to the authorization request. Missing header:["
                + LOCATION
                + "]!");
      }
      token = token.substring(token.indexOf(BEFORE_TOKEN) + BEFORE_TOKEN.length());
      token = token.substring(0, token.indexOf(AFTER_TOKEN));
      return token;
    } catch (Exception e) {
      throw KubernetesClientException.launderThrowable(e);
    }
  }
예제 #3
0
  private InputStream prepareConnection() throws IOException {
    Request request = new Request.Builder().url(url).build();

    Response response = CLIENT.newCall(request).execute();

    if (response.code() != 200) {
      Log.e("ERROR", "Server returned HTTP " + response.message());
      return null;
    }

    return response.body().byteStream();
  }
예제 #4
0
  @Override
  public void onTrigger(final ProcessContext context, final ProcessSession session)
      throws ProcessException {

    FlowFile flowFile = null;
    if (context.hasIncomingConnection()) {
      flowFile = session.get();

      // If we have no FlowFile, and all incoming connections are self-loops then we can continue
      // on.
      // However, if we have no FlowFile and we have connections coming from other Processors, then
      // we know that we should run only if we have a FlowFile.
      if (flowFile == null && context.hasNonLoopConnection()) {
        return;
      }
    }

    OkHttpClient okHttpClient = getClient();

    if (flowFile == null) {
      flowFile = session.create();
    }

    final String index =
        context.getProperty(INDEX).evaluateAttributeExpressions(flowFile).getValue();
    final String docId =
        context.getProperty(DOC_ID).evaluateAttributeExpressions(flowFile).getValue();
    final String docType =
        context.getProperty(TYPE).evaluateAttributeExpressions(flowFile).getValue();
    final String fields =
        context.getProperty(FIELDS).isSet()
            ? context.getProperty(FIELDS).evaluateAttributeExpressions(flowFile).getValue()
            : null;

    // Authentication
    final String username =
        context.getProperty(USERNAME).evaluateAttributeExpressions(flowFile).getValue();
    final String password = context.getProperty(PASSWORD).evaluateAttributeExpressions().getValue();

    final ComponentLog logger = getLogger();

    Response getResponse = null;

    try {
      logger.debug("Fetching {}/{}/{} from Elasticsearch", new Object[] {index, docType, docId});

      // read the url property from the context
      final String urlstr =
          StringUtils.trimToEmpty(
              context.getProperty(ES_URL).evaluateAttributeExpressions().getValue());
      final URL url = buildRequestURL(urlstr, docId, index, docType, fields);
      final long startNanos = System.nanoTime();

      getResponse = sendRequestToElasticsearch(okHttpClient, url, username, password, "GET", null);
      final int statusCode = getResponse.code();

      if (isSuccess(statusCode)) {
        ResponseBody body = getResponse.body();
        final byte[] bodyBytes = body.bytes();
        JsonNode responseJson = parseJsonResponse(new ByteArrayInputStream(bodyBytes));
        boolean found = responseJson.get("found").asBoolean(false);
        String retrievedIndex = responseJson.get("_index").asText();
        String retrievedType = responseJson.get("_type").asText();
        String retrievedId = responseJson.get("_id").asText();

        if (found) {
          JsonNode source = responseJson.get("_source");
          flowFile = session.putAttribute(flowFile, "filename", retrievedId);
          flowFile = session.putAttribute(flowFile, "es.index", retrievedIndex);
          flowFile = session.putAttribute(flowFile, "es.type", retrievedType);
          if (source != null) {
            flowFile =
                session.write(
                    flowFile,
                    out -> {
                      out.write(source.toString().getBytes());
                    });
          }
          logger.debug("Elasticsearch document " + retrievedId + " fetched, routing to success");

          // emit provenance event
          final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
          if (context.hasNonLoopConnection()) {
            session.getProvenanceReporter().fetch(flowFile, url.toExternalForm(), millis);
          } else {
            session.getProvenanceReporter().receive(flowFile, url.toExternalForm(), millis);
          }
          session.transfer(flowFile, REL_SUCCESS);
        } else {
          logger.warn(
              "Failed to read {}/{}/{} from Elasticsearch: Document not found",
              new Object[] {index, docType, docId});

          // We couldn't find the document, so send it to "not found"
          session.transfer(flowFile, REL_NOT_FOUND);
        }
      } else {
        if (statusCode == 404) {
          logger.warn(
              "Failed to read {}/{}/{} from Elasticsearch: Document not found",
              new Object[] {index, docType, docId});

          // We couldn't find the document, so penalize it and send it to "not found"
          session.transfer(flowFile, REL_NOT_FOUND);
        } else {
          // 5xx -> RETRY, but a server error might last a while, so yield
          if (statusCode / 100 == 5) {

            logger.warn(
                "Elasticsearch returned code {} with message {}, transferring flow file to retry. This is likely a server problem, yielding...",
                new Object[] {statusCode, getResponse.message()});
            session.transfer(flowFile, REL_RETRY);
            context.yield();
          } else if (context.hasIncomingConnection()) { // 1xx, 3xx, 4xx -> NO RETRY
            logger.warn(
                "Elasticsearch returned code {} with message {}, transferring flow file to failure",
                new Object[] {statusCode, getResponse.message()});
            session.transfer(flowFile, REL_FAILURE);
          } else {
            logger.warn(
                "Elasticsearch returned code {} with message {}",
                new Object[] {statusCode, getResponse.message()});
            session.remove(flowFile);
          }
        }
      }
    } catch (IOException ioe) {
      logger.error(
          "Failed to read from Elasticsearch due to {}, this may indicate an error in configuration "
              + "(hosts, username/password, etc.). Routing to retry",
          new Object[] {ioe.getLocalizedMessage()},
          ioe);
      if (context.hasIncomingConnection()) {
        session.transfer(flowFile, REL_RETRY);
      } else {
        session.remove(flowFile);
      }
      context.yield();

    } catch (Exception e) {
      logger.error(
          "Failed to read {} from Elasticsearch due to {}",
          new Object[] {flowFile, e.getLocalizedMessage()},
          e);
      if (context.hasIncomingConnection()) {
        session.transfer(flowFile, REL_FAILURE);
      } else {
        session.remove(flowFile);
      }
      context.yield();
    } finally {
      if (getResponse != null) {
        getResponse.close();
      }
    }
  }
예제 #5
0
  public static InputStream openUri(Context context, Uri uri, String reqContentTypeSubstring)
      throws OpenUriException {

    if (uri == null) {
      throw new IllegalArgumentException("Uri cannot be empty");
    }

    String scheme = uri.getScheme();
    if (scheme == null) {
      throw new OpenUriException(false, new IOException("Uri had no scheme"));
    }

    InputStream in = null;
    if ("content".equals(scheme)) {
      try {
        in = context.getContentResolver().openInputStream(uri);
      } catch (FileNotFoundException | SecurityException e) {
        throw new OpenUriException(false, e);
      }

    } else if ("file".equals(scheme)) {
      List<String> segments = uri.getPathSegments();
      if (segments != null && segments.size() > 1 && "android_asset".equals(segments.get(0))) {
        AssetManager assetManager = context.getAssets();
        StringBuilder assetPath = new StringBuilder();
        for (int i = 1; i < segments.size(); i++) {
          if (i > 1) {
            assetPath.append("/");
          }
          assetPath.append(segments.get(i));
        }
        try {
          in = assetManager.open(assetPath.toString());
        } catch (IOException e) {
          throw new OpenUriException(false, e);
        }
      } else {
        try {
          in = new FileInputStream(new File(uri.getPath()));
        } catch (FileNotFoundException e) {
          throw new OpenUriException(false, e);
        }
      }

    } else if ("http".equals(scheme) || "https".equals(scheme)) {
      OkHttpClient client =
          new OkHttpClient.Builder()
              .connectTimeout(DEFAULT_CONNECT_TIMEOUT, TimeUnit.SECONDS)
              .readTimeout(DEFAULT_READ_TIMEOUT, TimeUnit.SECONDS)
              .build();
      Request request;
      int responseCode = 0;
      String responseMessage = null;
      try {
        request = new Request.Builder().url(new URL(uri.toString())).build();
      } catch (MalformedURLException e) {
        throw new OpenUriException(false, e);
      }

      try {
        Response response = client.newCall(request).execute();
        responseCode = response.code();
        responseMessage = response.message();
        if (!(responseCode >= 200 && responseCode < 300)) {
          throw new IOException("HTTP error response.");
        }
        if (reqContentTypeSubstring != null) {
          String contentType = response.header("Content-Type");
          if (contentType == null || !contentType.contains(reqContentTypeSubstring)) {
            throw new IOException(
                "HTTP content type '"
                    + contentType
                    + "' didn't match '"
                    + reqContentTypeSubstring
                    + "'.");
          }
        }
        in = response.body().byteStream();

      } catch (IOException e) {
        if (responseCode > 0) {
          throw new OpenUriException(500 <= responseCode && responseCode < 600, responseMessage, e);
        } else {
          throw new OpenUriException(false, e);
        }
      }
    }

    if (in == null) {
      throw new OpenUriException(false, "Null input stream for URI: " + uri, null);
    }

    return in;
  }