public ByteBuffer toBytes() {
byte[] keyBytes = Bytes.toBytes(key);
byte[] keyLengthBytes = Bytes.toBytes(keyBytes.length);
byte[] valueLengthBytes = Bytes.toBytes(value.length);
byte[] oldVersionBytes = Bytes.toBytes(oldVersion);
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
try {
outputStream.write(keyLengthBytes);
outputStream.write(keyBytes);
outputStream.write(valueLengthBytes);
outputStream.write(value);
outputStream.write(oldVersionBytes);
} catch (IOException e) {
log.error("Unexpected error while serializing an option", e);
}
byte[] concatenated = outputStream.toByteArray();
ByteBuffer serialized = ByteBuffer.wrap(concatenated);
try {
outputStream.close();
} catch (IOException ignored) {
}
return serialized;
}
private void createData() throws IOException {
// apache commons' implementation required
ByteArrayOutputStream bstream = new ByteArrayOutputStream();
for (byte b = Byte.MIN_VALUE; b < Byte.MAX_VALUE; b++) bstream.write(b);
bstream.write(Byte.MAX_VALUE);
data = bstream.toByteArray();
bstream.close(); // doesn't really do anything
}
@Test
public void verifyBytesWritten() throws IOException, WriteFileInContainerException {
cont.storeFile(
file, new BufferedInputStream(new ByteArrayInputStream(data)), FileInContainer.NO_TAIL_ID);
// apache commons' implementation required
ByteArrayOutputStream bstream = new ByteArrayOutputStream();
bstream.write(containerHeader);
bstream.write(fileHeader);
bstream.write(data);
bstream.write(checksum);
byte[] expected = bstream.toByteArray();
bstream.close();
byte[] actual = FileUtils.readFileToByteArray(new java.io.File(cont.getFileName()));
assertArrayEquals(expected, actual);
}
public static MD5InputStreamResult generateMD5Result(InputStream toEncode) {
MD5Digest eTag = new MD5Digest();
byte[] resBuf = new byte[eTag.getDigestSize()];
byte[] buffer = new byte[1024];
ByteArrayOutputStream out = new ByteArrayOutputStream();
long length = 0;
int numRead = -1;
try {
do {
numRead = toEncode.read(buffer);
if (numRead > 0) {
length += numRead;
eTag.update(buffer, 0, numRead);
out.write(buffer, 0, numRead);
}
} while (numRead != -1);
} catch (IOException e) {
throw new RuntimeException(e);
} finally {
IOUtils.closeQuietly(out);
IOUtils.closeQuietly(toEncode);
}
eTag.doFinal(resBuf, 0);
return new MD5InputStreamResult(out.toByteArray(), resBuf, length);
}
private ByteArrayOutputStream encodeToBytes() throws IOException {
ByteArrayOutputStream buf = new ByteArrayOutputStream();
ObjectOutputStream oos = new ObjectOutputStream(new GZIPOutputStream(buf));
oos.writeObject(this);
oos.close();
ByteArrayOutputStream buf2 = new ByteArrayOutputStream();
DataOutputStream dos = new DataOutputStream(new Base64OutputStream(buf2, true, -1, null));
buf2.write(PREAMBLE);
dos.writeInt(buf.size());
buf.writeTo(dos);
dos.close();
buf2.write(POSTAMBLE);
return buf2;
}
@Override
public void decompress() {
if (compressed) {
Inflater decompressor = new Inflater();
decompressor.setInput(data);
ByteArrayOutputStream bos = new ByteArrayOutputStream(data.length);
byte[] buf = new byte[1024];
while (!decompressor.finished()) {
try {
int count = decompressor.inflate(buf);
bos.write(buf, 0, count);
} catch (DataFormatException e) {
}
}
try {
bos.close();
} catch (IOException e) {
}
data = bos.toByteArray();
}
}
private void readFiles(InputStream csar_file) throws IOException, NotFoundException {
ZipInputStream zipStream = new ZipInputStream(csar_file);
ZipEntry entry;
this.scripts.clear();
this.folderNames.clear();
this.template = new ByteArrayOutputStream();
this.metadata = new ByteArrayOutputStream();
while ((entry = zipStream.getNextEntry()) != null) {
if (!entry.isDirectory()) {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
int count;
byte[] buffer = new byte[1024];
while ((count = zipStream.read(buffer)) != -1) {
baos.write(buffer, 0, count);
}
String fileName = entry.getName();
if (fileName.toLowerCase().endsWith(".meta")) {
this.metadata = baos;
} else if (fileName.toLowerCase().endsWith(".yaml")) {
if (fileName.toLowerCase().endsWith("metadata.yaml")) {
this.vnfMetadata = baos;
} else {
this.template = baos;
}
} else {
Script script = new Script();
String[] splittedName = fileName.split("/");
if (splittedName.length > 2) {
String scriptName = splittedName[1] + "!_!" + splittedName[splittedName.length - 1];
folderNames.add(splittedName[1]);
script.setName(scriptName);
} else script.setName(splittedName[splittedName.length - 1]);
script.setPayload(baos.toByteArray());
this.scripts.add(script);
}
}
}
if (this.metadata == null) {
throw new NotFoundException("CSARParser: Not found TOSCA.meta");
}
if (this.vnfMetadata == null) {
throw new NotFoundException("CSARParser: Not found Metadata.yaml");
}
if (this.template == null) {
throw new NotFoundException("CSARParser: Not found VNFD / NSD Template");
}
// zipStream.close();
}
@Override
public void write(byte[] b, int off, int len) throws IOException {
size += len;
if (tempMemoryBuffer != null) {
tempMemoryBuffer.write(b, off, len);
} else {
bufOut.write(b, off, len);
}
checkSize();
}
@Override
public void write(int b) throws IOException {
size++;
if (tempMemoryBuffer != null) {
tempMemoryBuffer.write(b);
} else {
bufOut.write(b);
}
checkSize();
}
private static String readInputAsString(InputStream in) throws Exception {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
byte[] buf = new byte[512];
int nRead = 0;
while ((nRead = in.read(buf)) >= 0) {
baos.write(buf, 0, nRead);
}
return new String(baos.toByteArray(), "UTF-8");
}
protected String read(InputStream resource) throws IOException {
int i;
byte[] buffer = new byte[65565];
ByteArrayOutputStream out = new ByteArrayOutputStream();
while ((i = resource.read(buffer, 0, buffer.length)) != -1) {
out.write(buffer, 0, i);
}
out.flush();
out.close();
return new String(out.toByteArray(), "UTF-8");
}
private ICC_Profile buildICCProfile(
final ImageInfo info, final ColorSpace colorSpace, final ByteArrayOutputStream iccStream)
throws IOException {
if (iccStream != null && iccStream.size() > 0) {
if (log.isDebugEnabled()) {
log.debug("Effective ICC profile size: " + iccStream.size());
}
final int alignment = 4;
final int padding = (alignment - iccStream.size() % alignment) % alignment;
if (padding != 0) {
try {
iccStream.write(new byte[padding]);
} catch (final IOException ioe) {
throw new IOException("Error while aligning ICC stream: " + ioe.getMessage());
}
}
ICC_Profile iccProfile = null;
try {
iccProfile = ColorProfileUtil.getICC_Profile(iccStream.toByteArray());
if (log.isDebugEnabled()) {
log.debug("JPEG has an ICC profile: " + iccProfile.toString());
}
} catch (final IllegalArgumentException iae) {
log.warn(
"An ICC profile is present in the JPEG file but it is invalid ("
+ iae.getMessage()
+ "). The color profile will be ignored. ("
+ info.getOriginalURI()
+ ")");
return null;
}
if (iccProfile.getNumComponents() != colorSpace.getNumComponents()) {
log.warn(
"The number of components of the ICC profile ("
+ iccProfile.getNumComponents()
+ ") doesn't match the image ("
+ colorSpace.getNumComponents()
+ "). Ignoring the ICC color profile.");
return null;
} else {
return iccProfile;
}
} else {
return null; // no ICC profile available
}
}
// TODO move to separate thread?
public void compress() {
if (!compressed) {
Deflater deflater = new Deflater();
deflater.setInput(fileData);
deflater.setLevel(Deflater.BEST_COMPRESSION);
deflater.finish();
ByteArrayOutputStream bos = new ByteArrayOutputStream(fileData.length);
byte[] buffer = new byte[1024];
while (!deflater.finished()) {
int bytesCompressed = deflater.deflate(buffer);
bos.write(buffer, 0, bytesCompressed);
}
try {
bos.close();
} catch (IOException e) {
e.printStackTrace();
}
fileData = bos.toByteArray();
compressed = true;
}
}
/**
* Decompresses a compressed byte array.
*
* @param input The byte array to be decompressed.
* @return The byte array in its decompressed, readable form.
*/
public static byte[] decompress(byte[] input) {
try {
final Inflater inflater = new Inflater();
final ByteArrayOutputStream byteOutput = new ByteArrayOutputStream(input.length);
final byte[] buffer = new byte[1024];
inflater.setInput(input);
while (!inflater.finished()) {
final int count = inflater.inflate(buffer);
byteOutput.write(buffer, 0, count);
}
byteOutput.close();
return byteOutput.toByteArray();
} catch (final DataFormatException e) {
RadixCore.getInstance().quitWithException("Error decompressing byte array.", e);
return null;
} catch (final IOException e) {
RadixCore.getInstance().quitWithException("Error decompressing byte array.", e);
return null;
}
}
/**
* Compresses the data in a byte array.
*
* @param input The byte array to be compressed.
* @return The byte array in its compressed form.
*/
public static byte[] compress(byte[] input) {
try {
final Deflater deflater = new Deflater();
deflater.setLevel(Deflater.BEST_COMPRESSION);
deflater.setInput(input);
final ByteArrayOutputStream byteOutput = new ByteArrayOutputStream(input.length);
deflater.finish();
final byte[] buffer = new byte[1024];
while (!deflater.finished()) {
final int count = deflater.deflate(buffer);
byteOutput.write(buffer, 0, count);
}
byteOutput.close();
return byteOutput.toByteArray();
} catch (final IOException e) {
RadixCore.getInstance().quitWithException("Error compressing byte array.", e);
return null;
}
}
/**
* Build, send and process a tracker announce request.
*
* <p>This function first builds an announce request for the specified event with all the required
* parameters. Then, the request is made to the tracker and the response analyzed.
*
* <p>All registered {@link AnnounceResponseListener} objects are then fired with the decoded
* payload.
*
* @param event The announce event type (can be AnnounceEvent.NONE for periodic updates).
* @param inhibitEvents Prevent event listeners from being notified.
*/
@Override
public void announce(AnnounceRequestMessage.RequestEvent event, boolean inhibitEvents)
throws AnnounceException {
logger.info(
"Announcing{} to tracker with {}U/{}D/{}L bytes...",
new Object[] {
this.formatAnnounceEvent(event),
this.torrent.getUploaded(),
this.torrent.getDownloaded(),
this.torrent.getLeft()
});
URL target = null;
try {
HTTPAnnounceRequestMessage request = this.buildAnnounceRequest(event);
target = request.buildAnnounceURL(this.tracker.toURL());
} catch (MalformedURLException mue) {
throw new AnnounceException("Invalid announce URL (" + mue.getMessage() + ")", mue);
} catch (MessageValidationException mve) {
throw new AnnounceException(
"Announce request creation violated " + "expected protocol (" + mve.getMessage() + ")",
mve);
} catch (IOException ioe) {
throw new AnnounceException(
"Error building announce request (" + ioe.getMessage() + ")", ioe);
}
HttpURLConnection conn = null;
InputStream in = null;
try {
conn = (HttpURLConnection) target.openConnection();
in = conn.getInputStream();
} catch (IOException ioe) {
if (conn != null) {
in = conn.getErrorStream();
}
}
// At this point if the input stream is null it means we have neither a
// response body nor an error stream from the server. No point in going
// any further.
if (in == null) {
throw new AnnounceException("No response or unreachable tracker!");
}
try {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
baos.write(in);
// Parse and handle the response
HTTPTrackerMessage message = HTTPTrackerMessage.parse(ByteBuffer.wrap(baos.toByteArray()));
baos.close(); // close stream
this.handleTrackerAnnounceResponse(message, inhibitEvents);
} catch (IOException ioe) {
throw new AnnounceException("Error reading tracker response!", ioe);
} catch (MessageValidationException mve) {
throw new AnnounceException(
"Tracker message violates expected " + "protocol (" + mve.getMessage() + ")", mve);
} finally {
// Make sure we close everything down at the end to avoid resource
// leaks.
try {
in.close();
} catch (IOException ioe) {
logger.warn("Problem ensuring error stream closed!", ioe);
}
// This means trying to close the error stream as well.
InputStream err = conn.getErrorStream();
if (err != null) {
try {
err.close();
} catch (IOException ioe) {
logger.warn("Problem ensuring error stream closed!", ioe);
}
}
}
}
public static InputStream toBufferedInputStream(InputStream input) throws IOException {
ByteArrayOutputStream output = new ByteArrayOutputStream();
output.write(input);
return output.toBufferedInputStream();
}
/**
* Processes a compressed entry from an archive
*
* @param name The name of the entry
* @param isDirectory true if the entry is a directory, false otherwise
* @param is an InputStream for reading the uncompressed data of the entry
* @param filterParam is an additional param for entry filtering function
* @param storeParam is an additional param for entry storing function
* @throws XMLDBException
*/
protected Sequence processCompressedEntry(
String name, boolean isDirectory, InputStream is, Sequence filterParam, Sequence storeParam)
throws IOException, XPathException, XMLDBException {
String dataType = isDirectory ? "folder" : "resource";
// call the entry-filter function
Sequence filterParams[] = new Sequence[3];
filterParams[0] = new StringValue(name);
filterParams[1] = new StringValue(dataType);
filterParams[2] = filterParam;
Sequence entryFilterFunctionResult =
entryFilterFunction.evalFunction(contextSequence, null, filterParams);
if (BooleanValue.FALSE == entryFilterFunctionResult.itemAt(0)) {
return Sequence.EMPTY_SEQUENCE;
} else {
Sequence entryDataFunctionResult;
Sequence uncompressedData = Sequence.EMPTY_SEQUENCE;
// copy the input data
ByteArrayOutputStream baos = new ByteArrayOutputStream();
byte buf[] = new byte[1024];
int read = -1;
while ((read = is.read(buf)) != -1) {
baos.write(buf, 0, read);
}
byte[] entryData = baos.toByteArray();
if (entryDataFunction.getSignature().getArgumentCount() == 3) {
Sequence dataParams[] = new Sequence[3];
System.arraycopy(filterParams, 0, dataParams, 0, 2);
dataParams[2] = storeParam;
entryDataFunctionResult = entryDataFunction.evalFunction(contextSequence, null, dataParams);
String path = entryDataFunctionResult.itemAt(0).getStringValue();
Collection root =
new LocalCollection(
context.getUser(),
context.getBroker().getBrokerPool(),
new AnyURIValue("/db").toXmldbURI(),
context.getAccessContext());
if (isDirectory) {
XMLDBAbstractCollectionManipulator.createCollection(root, path);
} else {
Resource resource;
File file = new File(path);
name = file.getName();
path = file.getParent();
Collection target =
(path == null)
? root
: XMLDBAbstractCollectionManipulator.createCollection(root, path);
MimeType mime = MimeTable.getInstance().getContentTypeFor(name);
try {
NodeValue content =
ModuleUtils.streamToXML(context, new ByteArrayInputStream(baos.toByteArray()));
resource = target.createResource(name, "XMLResource");
ContentHandler handler = ((XMLResource) resource).setContentAsSAX();
handler.startDocument();
content.toSAX(context.getBroker(), handler, null);
handler.endDocument();
} catch (SAXException e) {
resource = target.createResource(name, "BinaryResource");
resource.setContent(baos.toByteArray());
}
if (resource != null) {
if (mime != null) {
((EXistResource) resource).setMimeType(mime.getName());
}
target.storeResource(resource);
}
}
} else {
// try and parse as xml, fall back to binary
try {
uncompressedData = ModuleUtils.streamToXML(context, new ByteArrayInputStream(entryData));
} catch (SAXException saxe) {
if (entryData.length > 0)
uncompressedData =
BinaryValueFromInputStream.getInstance(
context, new Base64BinaryValueType(), new ByteArrayInputStream(entryData));
}
// call the entry-data function
Sequence dataParams[] = new Sequence[4];
System.arraycopy(filterParams, 0, dataParams, 0, 2);
dataParams[2] = uncompressedData;
dataParams[3] = storeParam;
entryDataFunctionResult = entryDataFunction.evalFunction(contextSequence, null, dataParams);
}
return entryDataFunctionResult;
}
}
/** {@inheritDoc} */
@Override
public Image loadImage(final ImageInfo info, final Map hints, final ImageSessionContext session)
throws ImageException, IOException {
if (!MimeConstants.MIME_JPEG.equals(info.getMimeType())) {
throw new IllegalArgumentException(
"ImageInfo must be from a image with MIME type: " + MimeConstants.MIME_JPEG);
}
ColorSpace colorSpace = null;
boolean appeFound = false;
int sofType = 0;
ByteArrayOutputStream iccStream = null;
final Source src = session.needSource(info.getOriginalURI());
final ImageInputStream in = ImageUtil.needImageInputStream(src);
final JPEGFile jpeg = new JPEGFile(in);
in.mark();
try {
outer:
while (true) {
int reclen;
final int segID = jpeg.readMarkerSegment();
if (log.isTraceEnabled()) {
log.trace("Seg Marker: " + Integer.toHexString(segID));
}
switch (segID) {
case EOI:
log.trace("EOI found. Stopping.");
break outer;
case SOS:
log.trace("SOS found. Stopping early."); // TODO Not sure if
// this is safe
break outer;
case SOI:
case NULL:
break;
case SOF0: // baseline
case SOF1: // extended sequential DCT
case SOF2: // progressive (since PDF 1.3)
case SOFA: // progressive (since PDF 1.3)
sofType = segID;
if (log.isTraceEnabled()) {
log.trace("SOF: " + Integer.toHexString(sofType));
}
in.mark();
try {
reclen = jpeg.readSegmentLength();
in.skipBytes(1); // data precision
in.skipBytes(2); // height
in.skipBytes(2); // width
final int numComponents = in.readUnsignedByte();
if (numComponents == 1) {
colorSpace = ColorSpace.getInstance(ColorSpace.CS_GRAY);
} else if (numComponents == 3) {
colorSpace = ColorSpace.getInstance(ColorSpace.CS_LINEAR_RGB);
} else if (numComponents == 4) {
colorSpace = ColorSpaces.getDeviceCMYKColorSpace();
} else {
throw new ImageException(
"Unsupported ColorSpace for image "
+ info
+ ". The number of components supported are 1, 3 and 4.");
}
} finally {
in.reset();
}
in.skipBytes(reclen);
break;
case APP2: // ICC (see ICC1V42.pdf)
in.mark();
try {
reclen = jpeg.readSegmentLength();
// Check for ICC profile
final byte[] iccString = new byte[11];
in.readFully(iccString);
in.skipBytes(1); // string terminator (null byte)
if ("ICC_PROFILE".equals(new String(iccString, "US-ASCII"))) {
in.skipBytes(2); // chunk sequence number and total
// number of chunks
final int payloadSize = reclen - 2 - 12 - 2;
if (ignoreColorProfile(hints)) {
log.debug("Ignoring ICC profile data in JPEG");
in.skipBytes(payloadSize);
} else {
final byte[] buf = new byte[payloadSize];
in.readFully(buf);
if (iccStream == null) {
if (log.isDebugEnabled()) {
log.debug("JPEG has an ICC profile");
final DataInputStream din =
new DataInputStream(new ByteArrayInputStream(buf));
log.debug("Declared ICC profile size: " + din.readInt());
}
// ICC profiles can be split into several
// chunks
// so collect in a byte array output stream
iccStream = new ByteArrayOutputStream();
}
iccStream.write(buf);
}
}
} finally {
in.reset();
}
in.skipBytes(reclen);
break;
case APPE: // Adobe-specific (see 5116.DCT_Filter.pdf)
in.mark();
try {
reclen = jpeg.readSegmentLength();
// Check for Adobe header
final byte[] adobeHeader = new byte[5];
in.readFully(adobeHeader);
if ("Adobe".equals(new String(adobeHeader, "US-ASCII"))) {
// The reason for reading the APPE marker is that
// Adobe Photoshop
// generates CMYK JPEGs with inverted values. The
// correct thing
// to do would be to interpret the values in the
// marker, but for now
// only assume that if APPE marker is present and
// colorspace is CMYK,
// the image is inverted.
appeFound = true;
}
} finally {
in.reset();
}
in.skipBytes(reclen);
break;
default:
jpeg.skipCurrentMarkerSegment();
}
}
} finally {
in.reset();
}
final ICC_Profile iccProfile = buildICCProfile(info, colorSpace, iccStream);
if (iccProfile == null && colorSpace == null) {
throw new ImageException("ColorSpace could not be identified for JPEG image " + info);
}
boolean invertImage = false;
if (appeFound && colorSpace.getType() == ColorSpace.TYPE_CMYK) {
if (log.isDebugEnabled()) {
log.debug(
"JPEG has an Adobe APPE marker. Note: CMYK Image will be inverted. ("
+ info.getOriginalURI()
+ ")");
}
invertImage = true;
}
final ImageRawJPEG rawImage =
new ImageRawJPEG(
info, ImageUtil.needInputStream(src), sofType, colorSpace, iccProfile, invertImage);
return rawImage;
}
public static String downloadURL(String urlIn, String folderOut, String mongoHostIP)
throws IOException {
System.out.println("downloadURL");
String imgHash = null;
byte[] data = null;
// connect to URL and get input stream
URL imageURL = new URL(urlIn);
File localDir = new File(folderOut);
localDir.mkdir();
InputStream inputStream = null;
URLConnection urlConnection = null;
int noOfBytes = 0;
byte[] byteChunk = new byte[4096];
ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream();
urlConnection = imageURL.openConnection();
urlConnection.addRequestProperty(
"User-Agent",
"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.4; en-US; rv:1.9.2.2) Gecko/20100316 Firefox/3.6.2");
urlConnection.connect();
inputStream = urlConnection.getInputStream();
while ((noOfBytes = inputStream.read(byteChunk)) > 0) {
byteOutputStream.write(byteChunk, 0, noOfBytes);
}
// hash creation from image file
try {
System.out.println("Start MD5 Digest");
data = byteOutputStream.toByteArray();
MessageDigest md = MessageDigest.getInstance("MD5");
md.update(data);
byte[] hash = md.digest();
imgHash = String.format("%032x", new java.math.BigInteger(1, hash));
System.out.println("Hash : " + imgHash);
} catch (NoSuchAlgorithmException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
MongoClient mongoclient = new MongoClient(mongoHostIP, 27017);
// System.out.println("mongoHostIP :: " + mongoHostIP);
Morphia morphia = new Morphia();
morphia.map(ForensicReport.class).map(dqReport.class);
Datastore ds = new Morphia().createDatastore(mongoclient, "ForensicDatabase");
ds.ensureCaps();
String baseFolder = folderOut + imgHash + "/";
ForensicReport report = ds.get(ForensicReport.class, imgHash);
// check if hash exist
if (report != null) {
System.out.println("Exists");
} else {
// if hash does not exist in database, then download the image
report = new ForensicReport();
report.id = imgHash;
try {
File writeFolder = new File(baseFolder);
if (!writeFolder.exists()) writeFolder.mkdirs();
File imageFile = new File(baseFolder, "Raw");
OutputStream outputStream = new FileOutputStream(imageFile);
byteOutputStream.writeTo(outputStream);
outputStream.close();
BufferedImage downloadedImage = ImageIO.read(imageFile);
ImageIO.write(downloadedImage, "JPEG", new File(baseFolder, "Display.jpg"));
// store in database image information
report.sourceImage = baseFolder + "Raw";
report.displayImage = baseFolder + "Display.jpg";
report.sourceURL = urlIn;
report.status = "Downloaded";
ds.save(report);
} catch (Exception e) {
e.printStackTrace();
mongoclient.close();
return "URL_ERROR";
}
}
mongoclient.close();
System.out.println("Downloaded.");
return imgHash;
}
