@Test
public void parseNameValueBlock() throws IOException {
List<Header> headerBlock =
headerEntries(
"cache-control", "no-cache, no-store",
"set-cookie", "Cookie1\u0000Cookie2",
":status", "200 OK",
":version", "HTTP/1.1");
Request request = new Request.Builder().url("http://square.com/").build();
Response response = Http2xStream.readSpdy3HeadersList(headerBlock).request(request).build();
Headers headers = response.headers();
assertEquals(3, headers.size());
Assert.assertEquals(Protocol.SPDY_3, response.protocol());
assertEquals(200, response.code());
assertEquals("OK", response.message());
assertEquals("no-cache, no-store", headers.get("cache-control"));
assertEquals("Cookie2", headers.get("set-cookie"));
assertEquals("cache-control", headers.name(0));
assertEquals("no-cache, no-store", headers.value(0));
assertEquals("set-cookie", headers.name(1));
assertEquals("Cookie1", headers.value(1));
assertEquals("set-cookie", headers.name(2));
assertEquals("Cookie2", headers.value(2));
assertNull(headers.get(":status"));
assertNull(headers.get(":version"));
}
@Override
public Response intercept(Interceptor.Chain chain) throws IOException {
Request request = chain.request();
Response response = null;
boolean responseOK = false;
int tryCount = 0;
while (!responseOK && tryCount < MAX_RETRY) {
try {
response = chain.proceed(request);
int statusCode = response.code();
if (statusCode == 200) {
responseOK = response.isSuccessful();
} else {
// rate is limited by the number of calls per min
Thread.sleep(API_LIMIT_INTERVAL);
}
} catch (Exception e) {
Log.d(LOG_TAG, "Request is not successful - " + tryCount);
} finally {
tryCount++;
}
}
// otherwise just pass the original response on
return response;
}
private String authorize() {
try {
OkHttpClient.Builder builder = client.newBuilder();
builder.interceptors().remove(this);
OkHttpClient clone = builder.build();
String credential = Credentials.basic(config.getUsername(), new String(config.getPassword()));
URL url = new URL(URLUtils.join(config.getMasterUrl(), AUTHORIZE_PATH));
Response response =
clone
.newCall(
new Request.Builder().get().url(url).header(AUTHORIZATION, credential).build())
.execute();
response.body().close();
response = response.priorResponse() != null ? response.priorResponse() : response;
response = response.networkResponse() != null ? response.networkResponse() : response;
String token = response.header(LOCATION);
if (token == null || token.isEmpty()) {
throw new KubernetesClientException(
"Unexpected response ("
+ response.code()
+ " "
+ response.message()
+ "), to the authorization request. Missing header:["
+ LOCATION
+ "]!");
}
token = token.substring(token.indexOf(BEFORE_TOKEN) + BEFORE_TOKEN.length());
token = token.substring(0, token.indexOf(AFTER_TOKEN));
return token;
} catch (Exception e) {
throw KubernetesClientException.launderThrowable(e);
}
}
@Override
public Response intercept(Chain chain) throws IOException {
Request request = chain.request();
// Build new request
Request.Builder builder = request.newBuilder();
builder.header("Accept", "application/json");
String token = oauthToken.get();
if (Utils.isNotNullOrEmpty(token)) {
setAuthHeader(builder, token);
}
request = builder.build();
Response response = chain.proceed(request);
// If response is Forbidden or Unauthorized, try to obtain a token via authorize() or via
// config.
if (response.code() != 401 && response.code() != 403) {
return response;
} else if (Utils.isNotNullOrEmpty(config.getUsername())
&& Utils.isNotNullOrEmpty(config.getPassword())) {
synchronized (client) {
token = authorize();
if (token != null) {
oauthToken.set(token);
}
}
} else if (Utils.isNotNullOrEmpty(config.getOauthToken())) {
token = config.getOauthToken();
oauthToken.set(token);
}
// If token was obtain, then retry request using the obtained token.
if (Utils.isNotNullOrEmpty(token)) {
// Close the previous response to prevent leaked connections.
response.body().close();
setAuthHeader(builder, token);
request = builder.build();
return chain.proceed(request); // repeat request with new token
} else {
return response;
}
}
private InputStream prepareConnection() throws IOException {
Request request = new Request.Builder().url(url).build();
Response response = CLIENT.newCall(request).execute();
if (response.code() != 200) {
Log.e("ERROR", "Server returned HTTP " + response.message());
return null;
}
return response.body().byteStream();
}
@Override
public long open(DataSpec dataSpec) throws HttpDataSourceException {
this.dataSpec = dataSpec;
this.bytesRead = 0;
this.bytesSkipped = 0;
Request request = makeRequest(dataSpec);
try {
response = okHttpClient.newCall(request).execute();
responseByteStream = response.body().byteStream();
} catch (IOException e) {
throw new HttpDataSourceException(
"Unable to connect to " + dataSpec.uri.toString(), e, dataSpec);
}
int responseCode = response.code();
// Check for a valid response code.
if (!response.isSuccessful()) {
Map<String, List<String>> headers = request.headers().toMultimap();
closeConnectionQuietly();
throw new InvalidResponseCodeException(responseCode, headers, dataSpec);
}
// Check for a valid content type.
MediaType mediaType = response.body().contentType();
String contentType = mediaType != null ? mediaType.toString() : null;
if (contentTypePredicate != null && !contentTypePredicate.evaluate(contentType)) {
closeConnectionQuietly();
throw new InvalidContentTypeException(contentType, dataSpec);
}
// If we requested a range starting from a non-zero position and received a 200 rather than a
// 206, then the server does not support partial requests. We'll need to manually skip to the
// requested position.
bytesToSkip = responseCode == 200 && dataSpec.position != 0 ? dataSpec.position : 0;
// Determine the length of the data to be read, after skipping.
long contentLength = response.body().contentLength();
bytesToRead =
dataSpec.length != C.LENGTH_UNBOUNDED
? dataSpec.length
: contentLength != -1 ? contentLength - bytesToSkip : C.LENGTH_UNBOUNDED;
opened = true;
if (listener != null) {
listener.onTransferStart();
}
return bytesToRead;
}
public static <R> R parseJson(okhttp3.Response response, Type bodyType)
throws HasuraJsonException {
int code = response.code();
try {
String rawBody = response.body().string();
System.out.println(rawBody);
return gson.fromJson(rawBody, bodyType);
} catch (JsonSyntaxException e) {
String msg =
"FATAL : JSON strucutre not as expected. Schema changed maybe? : " + e.getMessage();
throw new HasuraJsonException(code, msg, e);
} catch (JsonParseException e) {
String msg = "FATAL : Server didn't return vaild JSON : " + e.getMessage();
throw new HasuraJsonException(code, msg, e);
} catch (IOException e) {
String msg = "FATAL : Decoding response body failed : " + e.getMessage();
throw new HasuraJsonException(code, msg, e);
}
}
/**
* 加载图片 返回 InputStream
*
* @param url
* @return
*/
public static InputStream loadImageStream(String url) {
throwException();
Request.Builder builder = new Request.Builder().url(url);
try {
Response response = mOkHttpClient.newCall(builder.build()).execute();
int responseCode = response.code();
if (responseCode >= 300) {
try {
response.body().close();
} catch (Exception e) {
e.printStackTrace();
}
}
try {
return response.body().byteStream();
} catch (Exception e) {
e.printStackTrace();
}
} catch (Exception e) {
}
return null;
}
@Override
protected String doInBackground(String... params) {
UserData userData = UserData.getInstance();
String login = userData.getLogin();
String token = userData.getToken();
Request request =
new Request.Builder()
.url("http://epitech-api.herokuapp.com/user?user="******"&token=" + token)
.get()
.build();
try {
Response response = client.newCall(request).execute();
if (response.code() == 200) {
return response.body().string();
}
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
@Override
public T call() throws Exception {
Request request = null;
Response response = null;
Exception exception = null;
Call call = null;
try {
MNSLog.logD("[call] - ");
if (context.getCancellationHandler().isCancelled()) {
throw new InterruptedIOException("This task is cancelled!");
}
Request.Builder requestBuilder = new Request.Builder();
// build request url
String url = message.buildCanonicalURL();
MNSUtils.signRequest(message);
requestBuilder = requestBuilder.url(url);
// set request headers
for (String key : message.getHeaders().keySet()) {
requestBuilder = requestBuilder.addHeader(key, message.getHeaders().get(key));
}
String contentType = message.getHeaders().get(MNSHeaders.CONTENT_TYPE);
String content = message.getContent();
// set request body
if (message.getContent() != null) {
MNSUtils.assertTrue(contentType != null, "Content type can't be null when send data!");
requestBuilder =
requestBuilder.method(
message.getMethod().toString(),
new ProgressTouchableRequestBody(
message.getContent().getBytes(), contentType, context.getProgressCallback()));
} else {
switch (message.getMethod()) {
case PUT:
requestBuilder =
requestBuilder.method(
message.getMethod().toString(), RequestBody.create(null, new byte[0]));
break;
case GET:
requestBuilder = requestBuilder.get();
break;
case HEAD:
requestBuilder = requestBuilder.head();
break;
case DELETE:
requestBuilder = requestBuilder.delete();
break;
default:
break;
}
}
request = requestBuilder.build();
if (MNSLog.isEnableLog()) {
MNSLog.logD("request url: " + request.url());
Map<String, List<String>> headerMap = request.headers().toMultimap();
for (String key : headerMap.keySet()) {
MNSLog.logD("requestHeader " + key + ": " + headerMap.get(key).get(0));
}
}
call = client.newCall(request);
context.getCancellationHandler().setCall(call);
// send request
response = call.execute();
if (MNSLog.isEnableLog()) {
MNSLog.logD("response code: " + response.code() + " for url: " + request.url());
Map<String, List<String>> headerMap = response.headers().toMultimap();
for (String key : headerMap.keySet()) {
MNSLog.logD("responseHeader " + key + ": " + headerMap.get(key).get(0));
}
}
} catch (Exception e) {
MNSLog.logE("Encounter local execpiton: " + e.toString());
if (MNSLog.isEnableLog()) {
e.printStackTrace();
}
exception = new ClientException(e.getMessage(), e);
}
if (exception == null && (response.code() == 203 || response.code() >= 300)) {
try {
exception = ResponseParsers.parseResponseErrorXML(response);
} catch (IOException e) {
exception = new ClientException(e.getMessage(), e);
}
} else if (exception == null) {
try {
T result = responseParser.parse(response);
if (context.getCompletedCallback() != null) {
context.getCompletedCallback().onSuccess(context.getRequest(), result);
}
return result;
} catch (IOException e) {
exception = new ClientException(e.getMessage(), e);
}
}
// reconstruct exception caused by manually cancelling
if ((call != null && call.isCanceled()) || context.getCancellationHandler().isCancelled()) {
exception = new ClientException("Task is cancelled!", exception.getCause(), true);
}
if (exception instanceof ClientException) {
if (context.getCompletedCallback() != null) {
context
.getCompletedCallback()
.onFailure(context.getRequest(), (ClientException) exception, null);
}
} else {
if (context.getCompletedCallback() != null) {
context
.getCompletedCallback()
.onFailure(context.getRequest(), null, (ServiceException) exception);
}
}
throw exception;
}
/**
* Begin a static scan on FoD
*
* @param uploadRequest zip file to upload
* @return true if the scan succeeded
*/
@SuppressFBWarnings(
value = "REC_CATCH_EXCEPTION",
justification =
"The intent of the catch-all is to make sure that the Jenkins user and logs show the plugin's problem in the build log.")
public boolean startStaticScan(final JobConfigModel uploadRequest) {
PrintStream logger = FodUploaderPlugin.getLogger();
PostStartScanResponse scanStartedResponse = null;
File uploadFile = uploadRequest.getUploadFile();
try (FileInputStream fs = new FileInputStream(uploadFile)) {
byte[] readByteArray = new byte[CHUNK_SIZE];
byte[] sendByteArray;
int fragmentNumber = 0;
int byteCount;
long offset = 0;
if (api.getToken() == null) api.authenticate();
if (!uploadRequest.hasAssessmentTypeId() && !uploadRequest.hasTechnologyStack()) {
logger.println("Missing Assessment Type or Technology Stack.");
return false;
}
// Build 'static' portion of url
String fragUrl =
api.getBaseUrl()
+ "/api/v3/releases/"
+ uploadRequest.getReleaseId()
+ "/static-scans/start-scan?";
fragUrl += "assessmentTypeId=" + uploadRequest.getAssessmentTypeId();
fragUrl += "&technologyStack=" + uploadRequest.getTechnologyStack();
fragUrl += "&entitlementId=" + uploadRequest.getEntitlementId();
fragUrl += "&entitlementFrequencyType=" + uploadRequest.getEntitlementFrequencyTypeId();
if (uploadRequest.hasLanguageLevel())
fragUrl += "&languageLevel=" + uploadRequest.getLanguageLevel();
if (uploadRequest.getIsExpressScan()) fragUrl += "&scanPreferenceType=2";
if (uploadRequest.getIsExpressAudit()) fragUrl += "&auditPreferenceType=2";
if (uploadRequest.getRunOpenSourceAnalysis())
fragUrl += "&doSonatypeScan=" + uploadRequest.getRunOpenSourceAnalysis();
if (uploadRequest.getIsRemediationScan())
fragUrl += "&isRemediationScan=" + uploadRequest.getIsRemediationScan();
if (uploadRequest.getExcludeThirdParty())
fragUrl += "&excludeThirdPartyLibs=" + uploadRequest.getExcludeThirdParty();
Gson gson = new Gson();
// Loop through chunks
logger.println("TOTAL FILE SIZE = " + uploadFile.length());
logger.println("CHUNK_SIZE = " + CHUNK_SIZE);
while ((byteCount = fs.read(readByteArray)) != -1) {
if (byteCount < CHUNK_SIZE) {
sendByteArray = Arrays.copyOf(readByteArray, byteCount);
fragmentNumber = -1;
} else {
sendByteArray = readByteArray;
}
MediaType byteArray = MediaType.parse("application/octet-stream");
Request request =
new Request.Builder()
.addHeader("Authorization", "Bearer " + api.getToken())
.addHeader("Content-Type", "application/octet-stream")
// Add offsets
.url(fragUrl + "&fragNo=" + fragmentNumber++ + "&offset=" + offset)
.post(RequestBody.create(byteArray, sendByteArray))
.build();
// Get the response
Response response = api.getClient().newCall(request).execute();
if (response.code()
== HttpStatus.SC_FORBIDDEN) { // got logged out during polling so log back in
// Re-authenticate
api.authenticate();
// if you had to reauthenticate here, would the loop and request not need to be
// resubmitted?
// possible continue?
}
offset += byteCount;
if (fragmentNumber % 5 == 0) {
logger.println(
"Upload Status - Fragment No: "
+ fragmentNumber
+ ", Bytes sent: "
+ offset
+ " (Response: "
+ response.code()
+ ")");
}
if (response.code() != 202) {
String responseJsonStr = IOUtils.toString(response.body().byteStream(), "utf-8");
// final response has 200, try to deserialize it
if (response.code() == 200) {
scanStartedResponse = gson.fromJson(responseJsonStr, PostStartScanResponse.class);
logger.println(
"Scan "
+ scanStartedResponse.getScanId()
+ " uploaded successfully. Total bytes sent: "
+ offset);
return true;
} else if (!response
.isSuccessful()) { // There was an error along the lines of 'another scan in progress'
// or something
logger.println("An error occurred during the upload.");
GenericErrorResponse errors =
gson.fromJson(responseJsonStr, GenericErrorResponse.class);
if (errors != null)
logger.println(
"Package upload failed for the following reasons: " + errors.toString());
return false; // if there is an error, get out of loop and mark build unstable
}
}
response.body().close();
} // end while
} catch (Exception e) {
e.printStackTrace(logger);
return false;
}
return false;
}
@Override
public void onTrigger(final ProcessContext context, final ProcessSession session)
throws ProcessException {
FlowFile flowFile = null;
if (context.hasIncomingConnection()) {
flowFile = session.get();
// If we have no FlowFile, and all incoming connections are self-loops then we can continue
// on.
// However, if we have no FlowFile and we have connections coming from other Processors, then
// we know that we should run only if we have a FlowFile.
if (flowFile == null && context.hasNonLoopConnection()) {
return;
}
}
OkHttpClient okHttpClient = getClient();
if (flowFile == null) {
flowFile = session.create();
}
final String index =
context.getProperty(INDEX).evaluateAttributeExpressions(flowFile).getValue();
final String docId =
context.getProperty(DOC_ID).evaluateAttributeExpressions(flowFile).getValue();
final String docType =
context.getProperty(TYPE).evaluateAttributeExpressions(flowFile).getValue();
final String fields =
context.getProperty(FIELDS).isSet()
? context.getProperty(FIELDS).evaluateAttributeExpressions(flowFile).getValue()
: null;
// Authentication
final String username =
context.getProperty(USERNAME).evaluateAttributeExpressions(flowFile).getValue();
final String password = context.getProperty(PASSWORD).evaluateAttributeExpressions().getValue();
final ComponentLog logger = getLogger();
Response getResponse = null;
try {
logger.debug("Fetching {}/{}/{} from Elasticsearch", new Object[] {index, docType, docId});
// read the url property from the context
final String urlstr =
StringUtils.trimToEmpty(
context.getProperty(ES_URL).evaluateAttributeExpressions().getValue());
final URL url = buildRequestURL(urlstr, docId, index, docType, fields);
final long startNanos = System.nanoTime();
getResponse = sendRequestToElasticsearch(okHttpClient, url, username, password, "GET", null);
final int statusCode = getResponse.code();
if (isSuccess(statusCode)) {
ResponseBody body = getResponse.body();
final byte[] bodyBytes = body.bytes();
JsonNode responseJson = parseJsonResponse(new ByteArrayInputStream(bodyBytes));
boolean found = responseJson.get("found").asBoolean(false);
String retrievedIndex = responseJson.get("_index").asText();
String retrievedType = responseJson.get("_type").asText();
String retrievedId = responseJson.get("_id").asText();
if (found) {
JsonNode source = responseJson.get("_source");
flowFile = session.putAttribute(flowFile, "filename", retrievedId);
flowFile = session.putAttribute(flowFile, "es.index", retrievedIndex);
flowFile = session.putAttribute(flowFile, "es.type", retrievedType);
if (source != null) {
flowFile =
session.write(
flowFile,
out -> {
out.write(source.toString().getBytes());
});
}
logger.debug("Elasticsearch document " + retrievedId + " fetched, routing to success");
// emit provenance event
final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
if (context.hasNonLoopConnection()) {
session.getProvenanceReporter().fetch(flowFile, url.toExternalForm(), millis);
} else {
session.getProvenanceReporter().receive(flowFile, url.toExternalForm(), millis);
}
session.transfer(flowFile, REL_SUCCESS);
} else {
logger.warn(
"Failed to read {}/{}/{} from Elasticsearch: Document not found",
new Object[] {index, docType, docId});
// We couldn't find the document, so send it to "not found"
session.transfer(flowFile, REL_NOT_FOUND);
}
} else {
if (statusCode == 404) {
logger.warn(
"Failed to read {}/{}/{} from Elasticsearch: Document not found",
new Object[] {index, docType, docId});
// We couldn't find the document, so penalize it and send it to "not found"
session.transfer(flowFile, REL_NOT_FOUND);
} else {
// 5xx -> RETRY, but a server error might last a while, so yield
if (statusCode / 100 == 5) {
logger.warn(
"Elasticsearch returned code {} with message {}, transferring flow file to retry. This is likely a server problem, yielding...",
new Object[] {statusCode, getResponse.message()});
session.transfer(flowFile, REL_RETRY);
context.yield();
} else if (context.hasIncomingConnection()) { // 1xx, 3xx, 4xx -> NO RETRY
logger.warn(
"Elasticsearch returned code {} with message {}, transferring flow file to failure",
new Object[] {statusCode, getResponse.message()});
session.transfer(flowFile, REL_FAILURE);
} else {
logger.warn(
"Elasticsearch returned code {} with message {}",
new Object[] {statusCode, getResponse.message()});
session.remove(flowFile);
}
}
}
} catch (IOException ioe) {
logger.error(
"Failed to read from Elasticsearch due to {}, this may indicate an error in configuration "
+ "(hosts, username/password, etc.). Routing to retry",
new Object[] {ioe.getLocalizedMessage()},
ioe);
if (context.hasIncomingConnection()) {
session.transfer(flowFile, REL_RETRY);
} else {
session.remove(flowFile);
}
context.yield();
} catch (Exception e) {
logger.error(
"Failed to read {} from Elasticsearch due to {}",
new Object[] {flowFile, e.getLocalizedMessage()},
e);
if (context.hasIncomingConnection()) {
session.transfer(flowFile, REL_FAILURE);
} else {
session.remove(flowFile);
}
context.yield();
} finally {
if (getResponse != null) {
getResponse.close();
}
}
}
public static InputStream openUri(Context context, Uri uri, String reqContentTypeSubstring)
throws OpenUriException {
if (uri == null) {
throw new IllegalArgumentException("Uri cannot be empty");
}
String scheme = uri.getScheme();
if (scheme == null) {
throw new OpenUriException(false, new IOException("Uri had no scheme"));
}
InputStream in = null;
if ("content".equals(scheme)) {
try {
in = context.getContentResolver().openInputStream(uri);
} catch (FileNotFoundException | SecurityException e) {
throw new OpenUriException(false, e);
}
} else if ("file".equals(scheme)) {
List<String> segments = uri.getPathSegments();
if (segments != null && segments.size() > 1 && "android_asset".equals(segments.get(0))) {
AssetManager assetManager = context.getAssets();
StringBuilder assetPath = new StringBuilder();
for (int i = 1; i < segments.size(); i++) {
if (i > 1) {
assetPath.append("/");
}
assetPath.append(segments.get(i));
}
try {
in = assetManager.open(assetPath.toString());
} catch (IOException e) {
throw new OpenUriException(false, e);
}
} else {
try {
in = new FileInputStream(new File(uri.getPath()));
} catch (FileNotFoundException e) {
throw new OpenUriException(false, e);
}
}
} else if ("http".equals(scheme) || "https".equals(scheme)) {
OkHttpClient client =
new OkHttpClient.Builder()
.connectTimeout(DEFAULT_CONNECT_TIMEOUT, TimeUnit.SECONDS)
.readTimeout(DEFAULT_READ_TIMEOUT, TimeUnit.SECONDS)
.build();
Request request;
int responseCode = 0;
String responseMessage = null;
try {
request = new Request.Builder().url(new URL(uri.toString())).build();
} catch (MalformedURLException e) {
throw new OpenUriException(false, e);
}
try {
Response response = client.newCall(request).execute();
responseCode = response.code();
responseMessage = response.message();
if (!(responseCode >= 200 && responseCode < 300)) {
throw new IOException("HTTP error response.");
}
if (reqContentTypeSubstring != null) {
String contentType = response.header("Content-Type");
if (contentType == null || !contentType.contains(reqContentTypeSubstring)) {
throw new IOException(
"HTTP content type '"
+ contentType
+ "' didn't match '"
+ reqContentTypeSubstring
+ "'.");
}
}
in = response.body().byteStream();
} catch (IOException e) {
if (responseCode > 0) {
throw new OpenUriException(500 <= responseCode && responseCode < 600, responseMessage, e);
} else {
throw new OpenUriException(false, e);
}
}
}
if (in == null) {
throw new OpenUriException(false, "Null input stream for URI: " + uri, null);
}
return in;
}