/** Adds byte content into the zip as a file. */
public static void addToZip(ZipOutputStream zos, byte[] content, String path, String comment)
throws IOException {
while (path.length() != 0 && path.charAt(0) == '/') {
path = path.substring(1);
}
if (StringUtil.endsWithChar(path, '/')) {
path = path.substring(0, path.length() - 1);
}
ZipEntry zipEntry = new ZipEntry(path);
zipEntry.setTime(System.currentTimeMillis());
if (comment != null) {
zipEntry.setComment(comment);
}
zos.putNextEntry(zipEntry);
InputStream is = new ByteArrayInputStream(content);
try {
StreamUtil.copy(is, zos);
} finally {
StreamUtil.close(is);
}
zos.closeEntry();
}
private static void write(String name, String comment, String string, ZipOutputStream out)
throws IOException {
ZipEntry entry = new ZipEntry(name);
entry.setComment(comment);
out.putNextEntry(entry);
PrintStream print = new PrintStream(out);
print.println(string);
print.flush();
out.closeEntry();
}
public static void addFolderToZip(ZipOutputStream zos, String path, String comment)
throws IOException {
while (path.length() != 0 && path.charAt(0) == '/') {
path = path.substring(1);
}
// add folder record
if (!StringUtil.endsWithChar(path, '/')) {
path += '/';
}
ZipEntry zipEntry = new ZipEntry(path);
zipEntry.setTime(System.currentTimeMillis());
if (comment != null) {
zipEntry.setComment(comment);
}
zipEntry.setSize(0);
zipEntry.setCrc(0);
zos.putNextEntry(zipEntry);
zos.closeEntry();
}
public void openNewFile(String baseFilename) throws KettleException {
data.writer = null;
ResultFile resultFile = null;
String filename = buildFilename(environmentSubstitute(baseFilename), true);
try {
if (meta.isFileAsCommand()) {
if (log.isDebug()) logDebug("Spawning external process");
if (data.cmdProc != null) {
logError("Previous command not correctly terminated");
setErrors(1);
}
String cmdstr = environmentSubstitute(meta.getFileName());
if (Const.getOS().equals("Windows 95")) {
cmdstr = "command.com /C " + cmdstr;
} else {
if (Const.getOS().startsWith("Windows")) {
cmdstr = "cmd.exe /C " + cmdstr;
}
}
if (log.isDetailed()) logDetailed("Starting: " + cmdstr);
Runtime r = Runtime.getRuntime();
data.cmdProc = r.exec(cmdstr, EnvUtil.getEnvironmentVariablesForRuntimeExec());
data.writer = data.cmdProc.getOutputStream();
StreamLogger stdoutLogger = new StreamLogger(data.cmdProc.getInputStream(), "(stdout)");
StreamLogger stderrLogger = new StreamLogger(data.cmdProc.getErrorStream(), "(stderr)");
new Thread(stdoutLogger).start();
new Thread(stderrLogger).start();
} else {
// Add this to the result file names...
resultFile =
new ResultFile(
ResultFile.FILE_TYPE_GENERAL,
KettleVFS.getFileObject(filename),
getTransMeta().getName(),
getStepname());
resultFile.setComment("This file was created with a text file output step");
addResultFile(resultFile);
OutputStream outputStream;
if (!Const.isEmpty(meta.getFileCompression())
&& !meta.getFileCompression().equals(FILE_COMPRESSION_TYPE_NONE)) {
if (meta.getFileCompression().equals(FILE_COMPRESSION_TYPE_ZIP)) {
if (log.isDetailed())
log.logDetailed(toString(), "Opening output stream in zipped mode");
if (checkPreviouslyOpened(filename)) {
data.fos = KettleVFS.getOutputStream(filename, true);
} else {
data.fos = KettleVFS.getOutputStream(filename, meta.isFileAppended());
}
data.zip = new ZipOutputStream(data.fos);
File entry = new File(filename);
ZipEntry zipentry = new ZipEntry(entry.getName());
zipentry.setComment("Compressed by Kettle");
data.zip.putNextEntry(zipentry);
outputStream = data.zip;
} else if (meta.getFileCompression().equals(FILE_COMPRESSION_TYPE_GZIP)) {
if (log.isDetailed())
log.logDetailed(toString(), "Opening output stream in gzipped mode");
if (checkPreviouslyOpened(filename)) {
data.fos = KettleVFS.getOutputStream(filename, true);
} else {
data.fos = KettleVFS.getOutputStream(filename, meta.isFileAppended());
}
data.gzip = new GZIPOutputStream(data.fos);
outputStream = data.gzip;
} else {
throw new KettleFileException("No compression method specified!");
}
} else {
if (log.isDetailed())
log.logDetailed(toString(), "Opening output stream in nocompress mode");
if (checkPreviouslyOpened(filename)) {
data.fos = KettleVFS.getOutputStream(filename, true);
} else {
data.fos = KettleVFS.getOutputStream(filename, meta.isFileAppended());
}
outputStream = data.fos;
}
if (!Const.isEmpty(meta.getEncoding())) {
if (log.isDetailed())
log.logDetailed(toString(), "Opening output stream in encoding: " + meta.getEncoding());
data.writer = new BufferedOutputStream(outputStream, 5000);
} else {
if (log.isDetailed())
log.logDetailed(toString(), "Opening output stream in default encoding");
data.writer = new BufferedOutputStream(outputStream, 5000);
}
if (log.isDetailed()) logDetailed("Opened new file with name [" + filename + "]");
}
} catch (Exception e) {
throw new KettleException("Error opening new file : " + e.toString());
}
// System.out.println("end of newFile(), splitnr="+splitnr);
data.splitnr++;
if (resultFile != null && meta.isAddToResultFiles()) {
// Add this to the result file names...
addResultFile(resultFile);
}
}
@Override
protected void doGet(final HttpServletRequest req, final HttpServletResponse rsp)
throws IOException {
String keyStr = req.getPathInfo();
// We shouldn't have to do this extra decode pass, but somehow we
// are now receiving our "^1" suffix as "%5E1", which confuses us
// downstream. Other times we get our embedded "," as "%2C", which
// is equally bad. And yet when these happen a "%2F" is left as-is,
// rather than escaped as "%252F", which makes me feel really really
// uncomfortable with a blind decode right here.
//
keyStr = URLDecoder.decode(keyStr, "UTF-8");
if (!keyStr.startsWith("/")) {
rsp.sendError(HttpServletResponse.SC_NOT_FOUND);
return;
}
keyStr = keyStr.substring(1);
final Patch.Key patchKey;
final int side;
{
final int c = keyStr.lastIndexOf('^');
if (c == 0) {
rsp.sendError(HttpServletResponse.SC_NOT_FOUND);
return;
}
if (c < 0) {
side = 0;
} else {
try {
side = Integer.parseInt(keyStr.substring(c + 1));
keyStr = keyStr.substring(0, c);
} catch (NumberFormatException e) {
rsp.sendError(HttpServletResponse.SC_NOT_FOUND);
return;
}
}
try {
patchKey = Patch.Key.parse(keyStr);
} catch (NumberFormatException e) {
rsp.sendError(HttpServletResponse.SC_NOT_FOUND);
return;
}
}
final Change.Id changeId = patchKey.getParentKey().getParentKey();
final Project project;
final PatchSet patchSet;
try {
final ReviewDb db = requestDb.get();
final ChangeControl control = changeControl.validateFor(changeId);
project = control.getProject();
patchSet = db.patchSets().get(patchKey.getParentKey());
if (patchSet == null) {
rsp.sendError(HttpServletResponse.SC_NOT_FOUND);
return;
}
} catch (NoSuchChangeException e) {
rsp.sendError(HttpServletResponse.SC_NOT_FOUND);
return;
} catch (OrmException e) {
getServletContext().log("Cannot query database", e);
rsp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
return;
}
final Repository repo;
try {
repo = repoManager.openRepository(project.getNameKey());
} catch (RepositoryNotFoundException e) {
getServletContext().log("Cannot open repository", e);
rsp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
return;
}
final ObjectLoader blobLoader;
final RevCommit fromCommit;
final String suffix;
final String path = patchKey.getFileName();
try {
final ObjectReader reader = repo.newObjectReader();
try {
final RevWalk rw = new RevWalk(reader);
final RevCommit c;
final TreeWalk tw;
c = rw.parseCommit(ObjectId.fromString(patchSet.getRevision().get()));
if (side == 0) {
fromCommit = c;
suffix = "new";
} else if (1 <= side && side - 1 < c.getParentCount()) {
fromCommit = rw.parseCommit(c.getParent(side - 1));
if (c.getParentCount() == 1) {
suffix = "old";
} else {
suffix = "old" + side;
}
} else {
rsp.sendError(HttpServletResponse.SC_NOT_FOUND);
return;
}
tw = TreeWalk.forPath(reader, path, fromCommit.getTree());
if (tw == null) {
rsp.sendError(HttpServletResponse.SC_NOT_FOUND);
return;
}
if (tw.getFileMode(0).getObjectType() == Constants.OBJ_BLOB) {
blobLoader = reader.open(tw.getObjectId(0), Constants.OBJ_BLOB);
} else {
rsp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
return;
}
} finally {
reader.release();
}
} catch (IOException e) {
getServletContext().log("Cannot read repository", e);
rsp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
return;
} catch (RuntimeException e) {
getServletContext().log("Cannot read repository", e);
rsp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
return;
} finally {
repo.close();
}
final byte[] raw = blobLoader.isLarge() ? null : blobLoader.getCachedBytes();
final long when = fromCommit.getCommitTime() * 1000L;
rsp.setDateHeader("Last-Modified", when);
rsp.setDateHeader("Expires", 0L);
rsp.setHeader("Pragma", "no-cache");
rsp.setHeader("Cache-Control", "no-cache, must-revalidate");
OutputStream out;
ZipOutputStream zo;
final MimeType contentType = registry.getMimeType(path, raw);
if (!registry.isSafeInline(contentType)) {
// The content may not be safe to transmit inline, as a browser might
// interpret it as HTML or JavaScript hosted by this site. Such code
// might then run in the site's security domain, and may be able to use
// the user's cookies to perform unauthorized actions.
//
// Usually, wrapping the content into a ZIP file forces the browser to
// save the content to the local system instead.
//
rsp.setContentType(ZIP.toString());
rsp.setHeader(
"Content-Disposition",
"attachment; filename=\"" + safeFileName(path, suffix) + ".zip" + "\"");
zo = new ZipOutputStream(rsp.getOutputStream());
final ZipEntry e = new ZipEntry(safeFileName(path, rand(req, suffix)));
e.setComment(fromCommit.name() + ":" + path);
e.setSize(blobLoader.getSize());
e.setTime(when);
zo.putNextEntry(e);
out = zo;
} else {
rsp.setContentType(contentType.toString());
rsp.setHeader("Content-Length", "" + blobLoader.getSize());
out = rsp.getOutputStream();
zo = null;
}
if (raw != null) {
out.write(raw);
} else {
blobLoader.copyTo(out);
}
if (zo != null) {
zo.closeEntry();
}
out.close();
}
public boolean openNewFile() {
boolean retval = false;
data.writer = null;
try {
FileObject file = KettleVFS.getFileObject(buildFilename(true), getTransMeta());
if (meta.isAddToResultFiles()) {
// Add this to the result file names...
ResultFile resultFile =
new ResultFile(
ResultFile.FILE_TYPE_GENERAL, file, getTransMeta().getName(), getStepname());
resultFile.setComment("This file was created with a xml output step"); // $NON-NLS-1$
addResultFile(resultFile);
}
OutputStream outputStream;
if (meta.isZipped()) {
OutputStream fos = KettleVFS.getOutputStream(file, false);
data.zip = new ZipOutputStream(fos);
File entry = new File(buildFilename(false));
ZipEntry zipentry = new ZipEntry(entry.getName());
zipentry.setComment("Compressed by Kettle"); // $NON-NLS-1$
data.zip.putNextEntry(zipentry);
outputStream = data.zip;
} else {
OutputStream fos = KettleVFS.getOutputStream(file, false);
outputStream = fos;
}
if (meta.getEncoding() != null && meta.getEncoding().length() > 0) {
logBasic("Opening output stream in encoding: " + meta.getEncoding()); // $NON-NLS-1$
data.writer = new OutputStreamWriter(outputStream, meta.getEncoding());
data.writer.write(XMLHandler.getXMLHeader(meta.getEncoding()).toCharArray());
} else {
logBasic(
"Opening output stream in default encoding : " + Const.XML_ENCODING); // $NON-NLS-1$
data.writer = new OutputStreamWriter(outputStream);
data.writer.write(XMLHandler.getXMLHeader(Const.XML_ENCODING).toCharArray());
}
// Add the name space if defined
StringBuffer nameSpace = new StringBuffer();
if ((meta.getNameSpace() != null) && (!"".equals(meta.getNameSpace()))) { // $NON-NLS-1$
nameSpace.append(" xmlns=\""); // $NON-NLS-1$
nameSpace.append(meta.getNameSpace());
nameSpace.append("\""); // $NON-NLS-1$
}
// OK, write the header & the parent element:
data.writer.write(
("<" + meta.getMainElement() + nameSpace.toString() + ">" + Const.CR)
.toCharArray()); //$NON-NLS-1$//$NON-NLS-2$
retval = true;
} catch (Exception e) {
logError("Error opening new file : " + e.toString()); // $NON-NLS-1$
}
// System.out.println("end of newFile(), splitnr="+splitnr);
data.splitnr++;
return retval;
}
/** Updates an existing jar file. */
boolean update(InputStream in, OutputStream out, InputStream newManifest, JarIndex jarIndex)
throws IOException {
ZipInputStream zis = new ZipInputStream(in);
ZipOutputStream zos = new JarOutputStream(out);
ZipEntry e = null;
boolean foundManifest = false;
boolean updateOk = true;
if (jarIndex != null) {
addIndex(jarIndex, zos);
}
// put the old entries first, replace if necessary
while ((e = zis.getNextEntry()) != null) {
String name = e.getName();
boolean isManifestEntry = equalsIgnoreCase(name, MANIFEST_NAME);
if ((jarIndex != null && equalsIgnoreCase(name, INDEX_NAME)) || (Mflag && isManifestEntry)) {
continue;
} else if (isManifestEntry && ((newManifest != null) || (ename != null) || (pname != null))) {
foundManifest = true;
if (newManifest != null) {
// Don't read from the newManifest InputStream, as we
// might need it below, and we can't re-read the same data
// twice.
FileInputStream fis = new FileInputStream(mname);
boolean ambiguous = isAmbiguousMainClass(new Manifest(fis));
fis.close();
if (ambiguous) {
return false;
}
}
// Update the manifest.
Manifest old = new Manifest(zis);
if (newManifest != null) {
old.read(newManifest);
}
if (!updateManifest(old, zos)) {
return false;
}
} else {
if (!entryMap.containsKey(name)) { // copy the old stuff
// do our own compression
ZipEntry e2 = new ZipEntry(name);
e2.setMethod(e.getMethod());
e2.setTime(e.getTime());
e2.setComment(e.getComment());
e2.setExtra(e.getExtra());
if (e.getMethod() == ZipEntry.STORED) {
e2.setSize(e.getSize());
e2.setCrc(e.getCrc());
}
zos.putNextEntry(e2);
copy(zis, zos);
} else { // replace with the new files
File f = entryMap.get(name);
addFile(zos, f);
entryMap.remove(name);
entries.remove(f);
}
}
}
// add the remaining new files
for (File f : entries) {
addFile(zos, f);
}
if (!foundManifest) {
if (newManifest != null) {
Manifest m = new Manifest(newManifest);
updateOk = !isAmbiguousMainClass(m);
if (updateOk) {
if (!updateManifest(m, zos)) {
updateOk = false;
}
}
} else if (ename != null || pname != null) {
if (!updateManifest(new Manifest(), zos)) {
updateOk = false;
}
}
}
zis.close();
zos.close();
return updateOk;
}
protected void processJarFile(final File file) throws Exception {
if (verbose) {
log("processing " + file.toURI());
}
final File tempFile =
File.createTempFile(file.getName(), null, new File(file.getAbsoluteFile().getParent()));
try {
final ZipInputStream zip = new ZipInputStream(new FileInputStream(file));
try {
final FileOutputStream fout = new FileOutputStream(tempFile);
try {
final ZipOutputStream out = new ZipOutputStream(fout);
ZipEntry entry;
while ((entry = zip.getNextEntry()) != null) {
byte bytes[] = getBytes(zip);
if (!entry.isDirectory()) {
final DataInputStream din = new DataInputStream(new ByteArrayInputStream(bytes));
if (din.readInt() == CLASS_MAGIC) {
bytes = process(bytes);
} else {
if (verbose) {
log("ignoring " + entry.toString());
}
}
}
final ZipEntry outEntry = new ZipEntry(entry.getName());
outEntry.setMethod(entry.getMethod());
outEntry.setComment(entry.getComment());
outEntry.setSize(bytes.length);
if (outEntry.getMethod() == ZipEntry.STORED) {
final CRC32 crc = new CRC32();
crc.update(bytes);
outEntry.setCrc(crc.getValue());
outEntry.setCompressedSize(bytes.length);
}
out.putNextEntry(outEntry);
out.write(bytes);
out.closeEntry();
zip.closeEntry();
}
out.close();
} finally {
fout.close();
}
} finally {
zip.close();
}
if (file.delete()) {
final File newFile = new File(tempFile.getAbsolutePath());
if (!newFile.renameTo(file)) {
throw new IOException("can not rename " + tempFile + " to " + file);
}
} else {
throw new IOException("can not delete " + file);
}
} finally {
tempFile.delete();
}
}
/**
* Adds single entry to ZIP output stream.
*
* @param zos zip output stream
* @param file file or folder to add
* @param path relative path of file entry; if <code>null</code> files name will be used instead
* @param comment optional comment
* @param recursive when set to <code>true</code> content of added folders will be added, too
*/
public static void addToZip(
ZipOutputStream zos, File file, String path, String comment, boolean recursive)
throws IOException {
if (file.exists() == false) {
throw new FileNotFoundException(file.toString());
}
if (path == null) {
path = file.getName();
}
while (path.length() != 0 && path.charAt(0) == '/') {
path = path.substring(1);
}
boolean isDir = file.isDirectory();
if (isDir) {
// add folder record
if (!StringUtil.endsWithChar(path, '/')) {
path += '/';
}
}
ZipEntry zipEntry = new ZipEntry(path);
zipEntry.setTime(file.lastModified());
if (comment != null) {
zipEntry.setComment(comment);
}
if (isDir) {
zipEntry.setSize(0);
zipEntry.setCrc(0);
}
zos.putNextEntry(zipEntry);
if (!isDir) {
InputStream is = new FileInputStream(file);
try {
StreamUtil.copy(is, zos);
} finally {
StreamUtil.close(is);
}
}
zos.closeEntry();
// continue adding
if (recursive && file.isDirectory()) {
boolean noRelativePath = StringUtil.isEmpty(path);
final File[] children = file.listFiles();
if (children != null && children.length != 0) {
for (File child : children) {
String childRelativePath = (noRelativePath ? StringPool.EMPTY : path) + child.getName();
addToZip(zos, child, childRelativePath, comment, recursive);
}
}
}
}