/**
* Creates a staging file with contents from the given DataMatrix.
*
* @param portalMetadata PortalMetadata
* @param cancerStudyMetadata CancerStudyMetadata
* @param datatypeMetadata DatatypeMetadata
* @param dataMatrix DataMatrix
* @throws Exception
*/
@Override
public void writeStagingFile(
PortalMetadata portalMetadata,
CancerStudyMetadata cancerStudyMetadata,
DatatypeMetadata datatypeMetadata,
DataMatrix dataMatrix)
throws Exception {
// staging file
String stagingFilename = datatypeMetadata.getStagingFilename();
stagingFilename =
stagingFilename.replaceAll(
DatatypeMetadata.CANCER_STUDY_TAG, cancerStudyMetadata.toString());
File stagingFile =
org.apache.commons.io.FileUtils.getFile(
portalMetadata.getStagingDirectory(),
cancerStudyMetadata.getStudyPath(),
stagingFilename);
if (LOG.isInfoEnabled()) {
LOG.info("writingStagingFile(), staging file: " + stagingFile);
}
FileOutputStream out = org.apache.commons.io.FileUtils.openOutputStream(stagingFile, false);
dataMatrix.write(out);
IOUtils.closeQuietly(out);
// meta file
if (datatypeMetadata.requiresMetafile()) {
if (LOG.isInfoEnabled()) {
LOG.info("writingStagingFile(), creating metadata file for staging file: " + stagingFile);
}
writeMetadataFile(portalMetadata, cancerStudyMetadata, datatypeMetadata, dataMatrix);
}
}
@Override
public void run() {
File root = new File(Environment.getExternalStorageDirectory(), "library");
try {
File target = new File(root, itemPath);
Adler32 checksum = new Adler32();
InputStream input = prepareConnection();
if (input != null) {
FileOutputStream output = FileUtils.openOutputStream(target);
byte[] buffer = new byte[4 * 1024];
int n;
while (-1 != (n = input.read(buffer))) {
output.write(buffer, 0, n);
checksum.update(buffer, 0, n);
}
output.flush();
output.close();
input.close();
}
tracker.track(config, target, checksum.getValue());
} catch (IOException e) {
Log.e("DOWNLOAD", "Failed to download", e);
}
progressUpdater.increment();
itemsLeftLatch.countDown();
}
public static FileOutputStream openOutputStream(File file) {
try {
return FileUtils.openOutputStream(file);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
@BeforeClass
public static void beforeClass() throws Exception {
File tempSolrHome = createTempDir().toFile();
// Schemaless renames schema.xml -> schema.xml.bak, and creates + modifies conf/managed-schema,
// which violates the test security manager's rules, which disallow writes outside the build
// dir,
// so we copy the example/example-schemaless/solr/ directory to a new temp dir where writes are
// allowed.
FileUtils.copyFileToDirectory(new File(ExternalPaths.SERVER_HOME, "solr.xml"), tempSolrHome);
File collection1Dir = new File(tempSolrHome, "collection1");
FileUtils.forceMkdir(collection1Dir);
FileUtils.copyDirectoryToDirectory(
new File(ExternalPaths.SCHEMALESS_CONFIGSET), collection1Dir);
Properties props = new Properties();
props.setProperty("name", "collection1");
OutputStreamWriter writer = null;
try {
writer =
new OutputStreamWriter(
FileUtils.openOutputStream(new File(collection1Dir, "core.properties")), "UTF-8");
props.store(writer, null);
} finally {
if (writer != null) {
try {
writer.close();
} catch (Exception ignore) {
}
}
}
createJetty(tempSolrHome.getAbsolutePath());
}
@Before
public void setUp() throws IOException {
final File dir = new File("test-results/alignments/last-to-compact/");
dir.mkdirs();
final FileWriter writer1 =
new FileWriter("test-results/alignments/last-to-compact/last-101.maf");
writer1.write(getMafInput1());
writer1.close();
final FileWriter writer2 =
new FileWriter("test-results/alignments/last-to-compact/last-102.maf");
writer2.write(getMafInput2());
writer2.close();
final FileWriter writer3 =
new FileWriter("test-results/alignments/last-to-compact/last-103-variations.maf");
writer3.write(getMafInput3Variations());
writer3.close();
final ReadsWriter referenceWriter =
new ReadsWriterImpl(
FileUtils.openOutputStream(
new File("test-results/alignments/last-to-compact/last-reference.compact-reads")));
referenceWriter.setIdentifier("0");
referenceWriter.appendEntry();
referenceWriter.close();
}
private void runTest(AbstractTest test, String name, byte[] conf) {
if (true) {
// Create the repository directory
File dir = new File(new File("target", "repository"), name + "-" + test);
dir.mkdirs();
try {
// Copy the configuration file into the repository directory
File xml = new File(dir, "repository.xml");
OutputStream output = FileUtils.openOutputStream(xml);
try {
output.write(conf, 0, conf.length);
} finally {
output.close();
}
// Create the repository
RepositoryImpl repository = createRepository(dir, xml);
try {
// Run the test
DescriptiveStatistics statistics = runTest(test, repository);
if (statistics.getN() > 0) {
writeReport(test.toString(), name, statistics);
}
} finally {
repository.shutdown();
}
} catch (Throwable t) {
System.out.println("Unable to run " + test + ": " + t.getMessage());
t.printStackTrace();
} finally {
// FileUtils.deleteQuietly(dir);
}
}
}
/**
* Creates the script file.
*
* @return the string
* @throws Exception the exception
*/
public String createScriptFile() throws Exception {
File tempScriptFile;
if (meta.getGenerateScript()) {
tempScriptFile = File.createTempFile(FilenameUtils.getBaseName(meta.getScriptFileName()), "");
} else {
tempScriptFile =
File.createTempFile(FilenameUtils.getBaseName(meta.getExistingScriptFile()), "");
}
tempScriptFile.deleteOnExit();
try {
scriptFile = FileUtils.openOutputStream(tempScriptFile);
scriptFilePrintStream = new PrintStream(scriptFile);
} catch (IOException e) {
throw new KettleException(
BaseMessages.getString(
PKG, "TeraDataBulkLoaderMeta.Exception.OpenScriptFile", scriptFile),
e);
}
if (meta.getGenerateScript()) {
createGeneratedScriptFile();
} else {
createFromExistingScriptFile();
}
scriptFilePrintStream.close();
IOUtils.closeQuietly(scriptFile);
return tempScriptFile.getAbsolutePath();
}
private File copyServiceToFileSystem(String resourceName, String fileName) throws IOException {
File file =
new File(
System.getProperty("basedir") + File.separator + "target" + File.separator + fileName);
if (file.exists()) {
FileUtils.deleteQuietly(file);
}
FileUtils.touch(file);
OutputStream os = FileUtils.openOutputStream(file);
InputStream is =
new FileInputStream(
FrameworkPathUtil.getSystemResourceLocation()
+ File.separator
+ "artifacts"
+ File.separator
+ "AXIS2"
+ File.separator
+ "config"
+ File.separator
+ resourceName);
if (is != null) {
byte[] data = new byte[1024];
int len;
while ((len = is.read(data)) != -1) {
os.write(data, 0, len);
}
os.flush();
os.close();
is.close();
}
return file;
}
/**
* internal helper to write output file
*
* @param message
* @param logFile
*/
private void writeFile(String message, String logFile) {
try {
File file = new File(logFile);
FileOutputStream fout = FileUtils.openOutputStream(file);
IOUtils.copy(IOUtils.toInputStream(message), fout);
fout.close();
} catch (Exception ex) {
ex.printStackTrace();
}
}
private void copy(IFolder destinationFolder, String fileName) throws IOException {
IFile destFile = destinationFolder.getFile(fileName);
if (!destFile.exists()) {
File destination = destFile.getRawLocation().toFile();
InputStream in = this.getClass().getResourceAsStream(fileName);
OutputStream out = FileUtils.openOutputStream(destination);
IOUtils.copy(in, out);
IOUtils.closeQuietly(out);
}
}
private void createLeiEtAlSolution(Solution solution) throws IOException {
// open the output file
Calendar c = Calendar.getInstance();
SimpleDateFormat sdf = new SimpleDateFormat("YYYY-MM-dd-HH-mm-ss");
String outputDirectory = getOutputDirectory();
if (StringUtils.isNotEmpty(outputDirectory)) {
File outputFile =
new File(
outputDirectory
+ "LeiEtAl-"
+ solution.getVrpProblem().getDescription()
+ "-"
+ sdf.format(c.getTime())
+ ".csv");
outputStream = FileUtils.openOutputStream(outputFile, true);
LeiEtAlHeuristic metaHeuristic = getMetaheuristicOutputStream();
IOUtils.write(
"Iteration; #CustomersChanged; RemovalHeuristic; TimeForRemoval; InsertionHeuristic; "
+ "TimeForInsertion; BestSolution?; ImprovedSolution?; AcceptedSolution?; TotalDistance; RecourseCost;"
+ "PenaltyCost; Anzahl Touren; KFZ Soll; Solution; Weights \n\n",
outputStream);
long startTime = System.nanoTime();
Solution improvedSolutionVehicleMinimizationStage = minimizationStage(solution);
Solution improvedSolutionDistanceMinimizationStage =
metaHeuristic.improve(improvedSolutionVehicleMinimizationStage, configuration);
long endTime = System.nanoTime();
String timeTaken = (endTime - startTime) / 1000 / 1000 + " ms\n";
// create an intermediate --- line, just for easier cutting
IOUtils.write("----------\n", outputStream);
IOUtils.write(timeTaken, outputStream);
IOUtils.write(improvedSolutionDistanceMinimizationStage.getIteration() + "\n", outputStream);
IOUtils.write(improvedSolutionDistanceMinimizationStage.getSolutionAsString(), outputStream);
// IOUtils.closeQuietly(outputStream);
} else {
LeiEtAlHeuristic metaHeuristic = getMetaheuristicConsoleOutput();
long startTime = System.nanoTime();
Solution improvedSolutionVehicleMinimizationStage = minimizationStage(solution);
Solution improvedSolutionDistanceMinimizationStage =
metaHeuristic.improve(improvedSolutionVehicleMinimizationStage, configuration);
long endTime = System.nanoTime();
String timeTaken = (endTime - startTime) / 1000 / 1000 + " ms\n";
System.out.println("----------");
System.out.println(timeTaken);
System.out.println(improvedSolutionDistanceMinimizationStage.getIteration());
System.out.println(improvedSolutionDistanceMinimizationStage.getSolutionAsString());
}
}
private OutputStream getOutputStream() throws IOException {
if (this.logOutputStream == null) {
synchronized (this) {
if (this.logOutputStream == null) {
this.logOutputStream = FileUtils.openOutputStream(getLogFile(), true);
}
}
}
return this.logOutputStream;
}
private static void initFiles() throws IOException {
baseDir = new File(System.getProperty("java.io.tmpdir"), UUID.randomUUID().toString());
File file = new File(baseDir, "test.txt");
sampleFilePath = file.getAbsolutePath();
OutputStream out = FileUtils.openOutputStream(file);
InputStream in = IOUtils.toInputStream("content!");
IOUtils.copy(in, out);
out.close();
}
public String exportAllData() {
Identity.setSecurityEnabled(false);
Collection<ContentItem> items = getContentItemService().getContentItems();
File f = new File(pathToExportFile);
OutputStream out = null;
try {
out = FileUtils.openOutputStream(f);
exporter.exportItems(items, "application/x-kiwi", out);
} catch (IOException ex) {
log.error("Error by creating zip file #0", ex.getMessage());
} finally {
try {
out.close();
} catch (IOException ex) {
log.error("Error by close outputstream #0", ex.getMessage());
}
}
log.info("Create export file #0", f.getAbsolutePath());
try {
InputStream newIs = new FileInputStream(f);
byte[] baBinary = new byte[newIs.available()];
newIs.read(baBinary);
HttpServletResponse response = (HttpServletResponse) extCtx.getResponse();
String contentType = "application/zip";
String fileName = "export.kiwi";
response.setContentType(contentType);
response.addHeader("Content-disposition", "attachment; filename=\"" + fileName + "\"");
ServletOutputStream os = response.getOutputStream();
os.write(baBinary);
os.flush();
os.close();
facesContext.responseComplete();
} catch (Exception ex) {
log.error("Error by downloading file #0", ex.getMessage());
} finally {
try {
out.close();
} catch (IOException ex) {
log.error("Error by close outputstream #0", ex.getMessage());
}
}
return null;
}
/**
* Make a temporary copy of the resource by saving the content to a temp file.
*
* @param resourceInputStream
* @throws IOException
*/
private File stashResourceContent(InputStream resourceInputStream) throws IOException {
File resourceTempFile = null;
OutputStream oStream = null;
try {
resourceTempFile = File.createTempFile("s-ramp-resource", ".tmp");
oStream = FileUtils.openOutputStream(resourceTempFile);
} finally {
IOUtils.copy(resourceInputStream, oStream);
IOUtils.closeQuietly(resourceInputStream);
IOUtils.closeQuietly(oStream);
}
return resourceTempFile;
}
/**
* Method which writes the cancer study metadata file.
*
* @param portalMetadata PortalMetadata
* @param cancerStudyMetadata CancerStudyMetadata
* @param numCases int
* @throws Exception
*/
@Override
public void writeCancerStudyMetadataFile(
PortalMetadata portalMetadata, CancerStudyMetadata cancerStudyMetadata, int numCases)
throws Exception {
File metaFile =
org.apache.commons.io.FileUtils.getFile(
portalMetadata.getStagingDirectory(),
cancerStudyMetadata.getStudyPath(),
cancerStudyMetadata.getCancerStudyMetadataFilename());
if (LOG.isInfoEnabled()) {
LOG.info("writeMetadataFile(), meta file: " + metaFile);
}
PrintWriter writer =
new PrintWriter(org.apache.commons.io.FileUtils.openOutputStream(metaFile, false));
writer.print("type_of_cancer: " + cancerStudyMetadata.getTumorType() + "\n");
writer.print("cancer_study_identifier: " + cancerStudyMetadata + "\n");
String name =
(cancerStudyMetadata.getName().length() > 0)
? cancerStudyMetadata.getName()
: cancerStudyMetadata.getTumorTypeMetadata().getName();
name =
name.replaceAll(
CancerStudyMetadata.TUMOR_TYPE_NAME_TAG,
cancerStudyMetadata.getTumorTypeMetadata().getName());
writer.print("name: " + name + "\n");
String description = cancerStudyMetadata.getDescription();
description =
description.replaceAll(CancerStudyMetadata.NUM_CASES_TAG, Integer.toString(numCases));
description =
description.replaceAll(
CancerStudyMetadata.TUMOR_TYPE_TAG,
cancerStudyMetadata.getTumorTypeMetadata().getType());
description =
description.replaceAll(
CancerStudyMetadata.TUMOR_TYPE_NAME_TAG,
cancerStudyMetadata.getTumorTypeMetadata().getName());
writer.print("description: " + description + "\n");
if (cancerStudyMetadata.getCitation().length() > 0) {
writer.print("citation: " + cancerStudyMetadata.getCitation() + "\n");
}
if (cancerStudyMetadata.getPMID().length() > 0) {
writer.print("pmid: " + cancerStudyMetadata.getPMID() + "\n");
}
writer.flush();
writer.close();
}
protected void dumpCurrentConfigurationIntoDirectory() throws IOException {
String configurationAsString = configuration.getConfigurationAsString();
if (StringUtils.isNotEmpty(configuration.getConfigurationValueString(OUTPUT_DIRECTORY))) {
File outputFile =
new File(
configuration.getConfigurationValueString(OUTPUT_DIRECTORY)
+ "configuration"
+ "-"
+ sdf.format(Calendar.getInstance().getTime()));
outputStream = FileUtils.openOutputStream(outputFile, true);
IOUtils.write(configurationAsString, outputStream);
IOUtils.closeQuietly(outputStream);
} else {
System.out.println(configurationAsString);
}
}
private void setup() throws Exception {
record = new Record();
morphlineContext = new MorphlineContext.Builder().build();
URL morphlineURL = getClass().getResource(morphlineResourceFile);
File morphlineFile =
File.createTempFile(
FilenameUtils.getBaseName(morphlineURL.getFile()),
FilenameUtils.getExtension(morphlineURL.getFile()));
IOUtils.copy(morphlineURL.openStream(), FileUtils.openOutputStream(morphlineFile));
// TODO: file JIRA, Compiler api should accept an InputStream
morphline =
new Compiler()
.compile(morphlineFile, morphlineId, morphlineContext, this /* LastCommand */);
}
/**
* Creates a staging file for mutation data (and meta file) with contents from the given
* DataMatrix. This is called when the mutation file needs to be run through the Oncotator and
* Mutation Assessor Tools.
*
* @param portalMetadata PortalMetadata
* @param cancerStudy CancerStudyMetadata
* @param datatypeMetadata DatatypeMetadata
* @param dataMatrix DataMatrix
* @throws Exception
*/
@Override
public void writeMutationStagingFile(
PortalMetadata portalMetadata,
CancerStudyMetadata cancerStudyMetadata,
DatatypeMetadata datatypeMetadata,
DataMatrix dataMatrix)
throws Exception {
// we only have data matrix at this point, we need to create a temp with its contents
File oncotatorInputFile =
org.apache.commons.io.FileUtils.getFile(
org.apache.commons.io.FileUtils.getTempDirectory(), "oncotatorInputFile");
FileOutputStream out = org.apache.commons.io.FileUtils.openOutputStream(oncotatorInputFile);
dataMatrix.write(out);
IOUtils.closeQuietly(out);
// output should be the path/name of staging file
String stagingFilename = datatypeMetadata.getStagingFilename();
stagingFilename =
stagingFilename.replaceAll(
DatatypeMetadata.CANCER_STUDY_TAG, cancerStudyMetadata.toString());
File stagingFile =
org.apache.commons.io.FileUtils.getFile(
portalMetadata.getStagingDirectory(),
cancerStudyMetadata.getStudyPath(),
stagingFilename);
// call oncotateAF
oncotateMAF(
FileUtils.FILE_URL_PREFIX + oncotatorInputFile.getCanonicalPath(),
FileUtils.FILE_URL_PREFIX + stagingFile.getCanonicalPath());
// clean up
if (oncotatorInputFile.exists()) {
org.apache.commons.io.FileUtils.forceDelete(oncotatorInputFile);
}
// meta file
if (datatypeMetadata.requiresMetafile()) {
if (LOG.isInfoEnabled()) {
LOG.info(
"writingMutationStagingFile(), creating metadata file for staging file: "
+ stagingFile);
}
writeMetadataFile(portalMetadata, cancerStudyMetadata, datatypeMetadata, dataMatrix);
}
}
/**
* Method which writes a metadata file for the given DatatypeMetadata. DataMatrix may be null.
*
* @param portalMetadata PortalMetadata
* @param cancerStudyMetadata CancerStudyMetadata
* @param datatypeMetadata DatatypeMetadata
* @param dataMatrix DataMatrix
* @throws Exception
*/
@Override
public void writeMetadataFile(
PortalMetadata portalMetadata,
CancerStudyMetadata cancerStudyMetadata,
DatatypeMetadata datatypeMetadata,
DataMatrix dataMatrix)
throws Exception {
File metaFile =
org.apache.commons.io.FileUtils.getFile(
portalMetadata.getStagingDirectory(),
cancerStudyMetadata.getStudyPath(),
datatypeMetadata.getMetaFilename());
if (LOG.isInfoEnabled()) {
LOG.info("writeMetadataFile(), meta file: " + metaFile);
}
PrintWriter writer =
new PrintWriter(org.apache.commons.io.FileUtils.openOutputStream(metaFile, false));
writer.print("cancer_study_identifier: " + cancerStudyMetadata + "\n");
writer.print(
"genetic_alteration_type: " + datatypeMetadata.getMetaGeneticAlterationType() + "\n");
String stableID = datatypeMetadata.getMetaStableID();
stableID =
stableID.replaceAll(DatatypeMetadata.CANCER_STUDY_TAG, cancerStudyMetadata.toString());
writer.print("stable_id: " + stableID + "\n");
writer.print(
"show_profile_in_analysis_tab: "
+ datatypeMetadata.getMetaShowProfileInAnalysisTab()
+ "\n");
String profileDescription = datatypeMetadata.getMetaProfileDescription();
if (dataMatrix != null) {
profileDescription =
profileDescription.replaceAll(
DatatypeMetadata.NUM_GENES_TAG, Integer.toString(dataMatrix.getGeneIDs().size()));
profileDescription =
profileDescription.replaceAll(
DatatypeMetadata.NUM_CASES_TAG, Integer.toString(dataMatrix.getCaseIDs().size()));
}
profileDescription =
profileDescription.replaceAll(
DatatypeMetadata.TUMOR_TYPE_TAG, cancerStudyMetadata.getTumorType());
writer.print("profile_description: " + profileDescription + "\n");
writer.print("profile_name: " + datatypeMetadata.getMetaProfileName() + "\n");
writer.flush();
writer.close();
}
/**
* Make a temporary copy of the resource by saving the content to a temp file.
*
* @param resourceInputStream
* @throws IOException
*/
private File stashResourceContent(InputStream resourceInputStream) throws IOException {
File resourceTempFile = null;
OutputStream oStream = null;
try {
resourceTempFile =
File.createTempFile("dtgov-ui-upload", ".tmp"); // $NON-NLS-1$ //$NON-NLS-2$
oStream = FileUtils.openOutputStream(resourceTempFile);
IOUtils.copy(resourceInputStream, oStream);
return resourceTempFile;
} catch (IOException e) {
FileUtils.deleteQuietly(resourceTempFile);
throw e;
} finally {
IOUtils.closeQuietly(resourceInputStream);
IOUtils.closeQuietly(oStream);
}
}
public static void writeFileOfSize(File outputFile, int numBytes) throws IOException {
BufferedOutputStream bos = null;
try {
bos = new BufferedOutputStream(FileUtils.openOutputStream(outputFile));
byte b = (byte) (255 * Math.random());
for (int i = 0; i < numBytes; i++) {
bos.write(b);
}
} catch (IOException ie) {
throw ie;
} finally {
if (bos != null) {
IOUtils.closeQuietly(bos);
}
}
}
/**
* Create a case list file from the given case list metadata file.
*
* @param portalMetadata PortalMetadata
* @param cancerStudyMetadata CancerStudyMetadata
* @param caseListMetadata CaseListMetadata
* @param caseList String[]
* @throws Exception
*/
@Override
public void writeCaseListFile(
PortalMetadata portalMetadata,
CancerStudyMetadata cancerStudyMetadata,
CaseListMetadata caseListMetadata,
String[] caseList)
throws Exception {
File caseListFile =
org.apache.commons.io.FileUtils.getFile(
portalMetadata.getStagingDirectory(),
cancerStudyMetadata.getStudyPath(),
"case_lists",
caseListMetadata.getCaseListFilename());
if (LOG.isInfoEnabled()) {
LOG.info("writeCaseListFile(), case list file: " + caseListFile.getCanonicalPath());
}
PrintWriter writer =
new PrintWriter(org.apache.commons.io.FileUtils.openOutputStream(caseListFile, false));
writer.print("cancer_study_identifier: " + cancerStudyMetadata + "\n");
String stableID = caseListMetadata.getMetaStableID();
stableID =
stableID.replaceAll(DatatypeMetadata.CANCER_STUDY_TAG, cancerStudyMetadata.toString());
writer.print("stable_id: " + stableID + "\n");
writer.print("case_list_name: " + caseListMetadata.getMetaCaseListName() + "\n");
String caseListDescription = caseListMetadata.getMetaCaseListDescription();
caseListDescription =
caseListDescription.replaceAll(
DatatypeMetadata.NUM_CASES_TAG, Integer.toString(caseList.length));
writer.print("case_list_description: " + caseListDescription + "\n");
writer.print("case_list_category: " + caseListMetadata.getMetaCaseListCategory() + "\n");
writer.print("case_list_ids: ");
for (String caseID : caseList) {
writer.print(caseID + Converter.VALUE_DELIMITER);
}
writer.println();
writer.flush();
writer.close();
}
/** Save an image to a specified file. */
protected static void saveImage(BufferedImage image, File file) throws IOException {
String filename = file.getName();
int pos = filename.lastIndexOf('.');
if (pos == -1 || (pos + 1) >= filename.length()) {
throw new IOException("Unknown image file type " + filename);
}
String imageType = filename.substring(pos + 1);
File imageFile = new File(file.getParent(), filename);
// use a FileOutputStream and make sure it gets closed to prevent unclosed file
// errors on some operating systems
FileOutputStream fos = null;
try {
// use the FileUtils utility method to ensure parent directories are created
// if necessary
fos = FileUtils.openOutputStream(imageFile);
boolean result = ImageIO.write(image, imageType, fos);
if (!result) {
throw new IOException("No appropriate writer found when writing image: " + filename);
}
} finally {
IOUtils.closeQuietly(fos);
}
}
public String ftpGet(
String remoteAddr,
int remotePort,
String remotePath,
String remoteFileName,
String localPath,
String username,
String password)
throws SocketException, IOException {
FTPClient ftp = new FTPClient();
FTPClientConfig config = new FTPClientConfig();
ftp.configure(config);
try {
ftp.connect(remoteAddr, remotePort);
log.debug("Connected to " + remoteAddr + ":" + remotePort + ".\n" + ftp.getReplyString());
int reply = ftp.getReplyCode();
if (!FTPReply.isPositiveCompletion(reply)) {
ftp.disconnect();
log.error("FTP server refused connection.");
throw new RuntimeException("FTP server refused connection.");
}
ftp.login(username, password);
reply = ftp.getReplyCode();
if (!FTPReply.isPositiveCompletion(reply)) {
ftp.disconnect();
log.error("FTP server refused connection.");
throw new RuntimeException("FTP server refused connection.");
}
if (!ftp.setFileType(FTP.BINARY_FILE_TYPE)) {
log.error("set BINARY_FILE_TYPE error.");
throw new RuntimeException("FTP server refused connection.");
}
// ftp.setFileTransferMode(FTPClient.STREAM_TRANSFER_MODE);
ftp.enterLocalPassiveMode();
String remote = remotePath + "/" + remoteFileName;
String local = localPath + "/" + remoteFileName;
String returnName = genFileName(local);
File f = new File(returnName);
FileOutputStream fos = FileUtils.openOutputStream(f);
try {
long startTime = System.currentTimeMillis();
if (!ftp.retrieveFile(remote, fos)) {
log.error("get " + remote + " error.");
throw new RuntimeException("get " + remote + " error.");
}
log.debug("get remote file[" + remote + "], local file[" + local + "] .");
long finishedTime = System.currentTimeMillis();
log.debug("remote time :" + (finishedTime - startTime) / 1000f + " sec.");
} finally {
IOUtils.closeQuietly(fos);
}
ftp.logout();
return returnName;
} finally {
if (ftp.isConnected()) {
try {
ftp.disconnect();
} catch (IOException ioe) {
}
}
}
}
@Override
public void write(File file) throws IOException {
OutputStream out = FileUtils.openOutputStream(file);
write(out);
}
/**
* 读取xml定义的模板，创建excel，感觉没啥用
*
* @author David
* @param args
*/
public static void main(String[] args) {
// 获取解析xml文件路径
String path = System.getProperty("user.dir") + "/bin/student2.xml";
File file = new File(path);
SAXBuilder builder = new SAXBuilder();
try {
// 解析xml文件
Document parse = builder.build(file);
// 创建Excel
HSSFWorkbook wb = new HSSFWorkbook();
// 创建sheet
HSSFSheet sheet = wb.createSheet("Sheet0");
// 获取xml文件跟节点
Element root = parse.getRootElement();
// 获取模板名称
String templateName = root.getAttribute("name").getValue();
int rownum = 0;
int column = 0;
// 设置列宽
Element colgroup = root.getChild("colgroup");
setColumnWidth(sheet, colgroup);
// 设置标题
Element title = root.getChild("title");
List<Element> trs = title.getChildren("tr");
for (int i = 0; i < trs.size(); i++) {
Element tr = trs.get(i);
List<Element> tds = tr.getChildren("td");
HSSFRow row = sheet.createRow(rownum);
HSSFCellStyle cellStyle = wb.createCellStyle();
cellStyle.setAlignment(HSSFCellStyle.ALIGN_CENTER);
for (column = 0; column < tds.size(); column++) {
Element td = tds.get(column);
HSSFCell cell = row.createCell(column);
Attribute rowSpan = td.getAttribute("rowspan");
Attribute colSpan = td.getAttribute("colspan");
Attribute value = td.getAttribute("value");
if (value != null) {
String val = value.getValue();
cell.setCellValue(val);
int rspan = rowSpan.getIntValue() - 1;
int cspan = colSpan.getIntValue() - 1;
// 设置字体
HSSFFont font = wb.createFont();
font.setFontName("仿宋_GB2312");
font.setBoldweight(HSSFFont.BOLDWEIGHT_BOLD); // 字体加粗
// font.setFontHeight((short)12);
font.setFontHeightInPoints((short) 12);
cellStyle.setFont(font);
cell.setCellStyle(cellStyle);
// 合并单元格居中
sheet.addMergedRegion(new CellRangeAddress(rspan, rspan, 0, cspan));
}
}
rownum++;
}
// 设置表头
Element thead = root.getChild("thead");
trs = thead.getChildren("tr");
for (int i = 0; i < trs.size(); i++) {
Element tr = trs.get(i);
HSSFRow row = sheet.createRow(rownum);
List<Element> ths = tr.getChildren("th");
for (column = 0; column < ths.size(); column++) {
Element th = ths.get(column);
Attribute valueAttr = th.getAttribute("value");
HSSFCell cell = row.createCell(column);
if (valueAttr != null) {
String value = valueAttr.getValue();
cell.setCellValue(value);
}
}
rownum++;
}
// 设置数据区域样式
Element tbody = root.getChild("tbody");
Element tr = tbody.getChild("tr");
int repeat = tr.getAttribute("repeat").getIntValue();
List<Element> tds = tr.getChildren("td");
for (int i = 0; i < repeat; i++) {
HSSFRow row = sheet.createRow(rownum);
for (column = 0; column < tds.size(); column++) {
Element td = tds.get(column);
HSSFCell cell = row.createCell(column);
setType(wb, cell, td);
}
rownum++;
}
// 生成Excel导入模板
File tempFile = new File("e:/" + templateName + ".xls");
tempFile.delete();
tempFile.createNewFile();
FileOutputStream stream = FileUtils.openOutputStream(tempFile);
wb.write(stream);
stream.close();
} catch (Exception e) {
e.printStackTrace();
}
}
@Override
public void execute() {
try {
// 根据URL地址，获取网页内容
String html = HttpUtils.getHtml(httpclient, url);
if (html == null) {
throw new RuntimeException("无法获取【" + url + "】网址的内容");
}
Topic a = new Topic();
// 设置文章的来源
a.setSource("www.ibm.com");
// 对网页内容进行分析和提取
// 设置文章的标题
MetaTag titleTag = ParseUtils.parseTag(html, MetaTag.class, "name", "title");
a.setTitle(titleTag.getMetaContent());
// 设置文章的关键字
MetaTag keywordTag = ParseUtils.parseTag(html, MetaTag.class, "name", "Keywords");
if (keywordTag.getMetaContent().length() > 255) {
a.setKeyword(keywordTag.getMetaContent().substring(0, 255));
}
// 设置文章的简介
MetaTag introTag = ParseUtils.parseTag(html, MetaTag.class, "name", "Abstract");
a.setSummary(introTag.getMetaContent());
// 设置文章的作者
List<Div> authors = ParseUtils.parseTags(html, Div.class, "class", "author");
String author = "";
for (int i = 0; i < authors.size(); i++) {
if (i != 0) {
author = author + ",";
}
Div div = authors.get(i);
author = author + ParseUtils.parseTag(div.getStringText(), LinkTag.class).getStringText();
}
a.setAuthor(author);
// 设置文章的内容
String content =
StringUtils.substringBetween(html, "<!-- MAIN_COLUMN_CONTENT_BEGIN -->", "<!-- CMA");
// 查询文章的内容中所包含的图片，并下载到upload目录，然后创建Attachment对象，设置到Article对象中
List<ImageTag> imageTags = ParseUtils.parseTags(content, ImageTag.class);
if (imageTags != null) {
for (ImageTag it : imageTags) {
// 得到图片所在的路径目录
String baseUrl = url.substring(0, url.lastIndexOf("/") + 1);
// 这个是<img>标签中的src的值
String imageUrl = it.getImageURL();
// 图片的绝对路径
String absoluteUrl = baseUrl + imageUrl;
// : "文章标题/xxx.jpg"
String imageName =
a.getTitle().replaceAll("/|\\\\|\\:|\\*|\\?|\\||\\<|>", "_") + "/" + imageUrl;
// 把图片保存到upload目录
// 首先确定，保存到本地的图片的路径
String imageLocalFile = ""; // Attachment.ATTACHMENT_DIR + imageName;
// 如果图片已经被下载到本地，则不再下载
if (!new File(imageLocalFile).exists()) {
// 下载图片的信息
byte[] image = HttpUtils.getImage(httpclient, absoluteUrl);
// 直接使用new FileOutputStream(imageLocalFile)这种方式，创建一个
// 文件输出流，存在的问题就是：如果这个文件所在的目录不存在，则创建不了
// 输出流，会抛出异常！
// 所以，使用辅助的工具类来创建一个文件输出流:FileUtils.openOutputStream(new File(imageLocalFile))
// 通过这个方法，当文件所在的父目录不存在的时候，将自动创建其所有的父目录
IOUtils.write(image, FileUtils.openOutputStream(new File(imageLocalFile)));
System.out.println("图片【" + absoluteUrl + "】已下载");
}
// 针对每张图片，创建一个Attachment对象
Attachment attachment = new Attachment();
attachment.setType("image/jpeg");
attachment.setOldName(imageName);
// a.addAttachment(attachment);
}
}
// 修改content中的所有图片的src的值
// 将src的值，加上前缀：upload_image/文章标题/图片.jpg
content =
ParseUtils.modifyImageUrl(
content,
"upload_image/" + a.getTitle().replaceAll("/|\\\\|\\:|\\*|\\?|\\||\\<|>", "_") + "/");
// 删除<hr>和"回首页"的链接标签
content = ParseUtils.reomveTags(content, Div.class, "class", "ibm-alternate-rule");
content =
ParseUtils.reomveTags(
content, ParagraphTag.class, "class", "ibm-ind-link ibm-back-to-top");
a.setContent(content);
// 将文章对象放入HttpContext
List<Topic> articles = new ArrayList<Topic>();
articles.add(a);
context.setAttribute("articles", articles);
} catch (Exception e) {
e.printStackTrace();
}
}
public static void main(String[] args) throws Exception {
if (args.length != 2) {
throw new Exception(
"Expecting 2 args: [directory of segment file directories] [output CSV file]");
}
int argCount = 0;
File inDir = new File(args[argCount++]);
if (!inDir.exists()) {
throw new Exception("Input directory does not exist.");
}
File outFile = new File(args[argCount++]);
if (outFile.exists()) {
throw new Exception("Output file already exists.");
}
// Open a stream to the output file
PrintStream out = null;
try {
out = new PrintStream(FileUtils.openOutputStream(outFile));
// Header
String header = "scene,segment";
for (int rgbHistBinID = 0;
rgbHistBinID < Math.pow(Utils.RGB_HIST_DIM_BINS, 3);
rgbHistBinID++) {
header += ",h" + rgbHistBinID;
}
header += ",label";
out.println(header);
// Each label
for (String labelDirName : inDir.list()) {
// We do not care about the '0' label
if (labelDirName.equals("0")) {
continue;
}
File labelDir = new File(inDir, labelDirName);
// Each object in each label dir
for (String objFileName : labelDir.list()) {
String[] objFileNameSplit = objFileName.split("_");
// Identifying information
String scene = objFileNameSplit[0].replace("scene", "");
String segment = objFileNameSplit[2].replace("s", "").replace(".ply", "");
// Read in the point cloud representing the object
File objFile = new File(labelDir, objFileName);
PointCloud objPointCloud = new PointCloud(objFile);
// Compute a histogram of the RGB values for the segment
int[][][] rgbHist =
new int[Utils.RGB_HIST_DIM_BINS][Utils.RGB_HIST_DIM_BINS][Utils.RGB_HIST_DIM_BINS];
// Everything starts with 1 value (expect computation problems if 0 value in hists)
for (int rBin = 0; rBin < Utils.RGB_HIST_DIM_BINS; rBin++) {
for (int gBin = 0; gBin < Utils.RGB_HIST_DIM_BINS; gBin++) {
for (int bBin = 0; bBin < Utils.RGB_HIST_DIM_BINS; bBin++) {
rgbHist[rBin][gBin][bBin] = 1;
}
}
}
// Accumulate the histogram
for (Point p : objPointCloud.points) {
int rBin = (int) Math.floor(p.r / Utils.RGB_HIST_BID_SIZE);
int gBin = (int) Math.floor(p.g / Utils.RGB_HIST_BID_SIZE);
int bBin = (int) Math.floor(p.b / Utils.RGB_HIST_BID_SIZE);
rgbHist[rBin][gBin][bBin]++;
}
// Write out the row
String row = scene + "," + segment;
for (int rBin = 0; rBin < Utils.RGB_HIST_DIM_BINS; rBin++) {
for (int gBin = 0; gBin < Utils.RGB_HIST_DIM_BINS; gBin++) {
for (int bBin = 0; bBin < Utils.RGB_HIST_DIM_BINS; bBin++) {
row += "," + rgbHist[rBin][gBin][bBin];
}
}
}
row += "," + labelDirName;
out.println(row);
}
}
} finally {
IOUtils.closeQuietly(out);
}
}
public static void upgrade(File wrFile) throws Exception {
System.out.println("Installing/upgrading Myna in '" + wrFile.toString() + "'...");
wrFile.mkdirs();
File web_inf = new File(wrFile.toURI().resolve("WEB-INF"));
boolean isUpgrade = false;
File backupDir = null;
if (web_inf.exists()) {
String dateString =
new java.text.SimpleDateFormat("MM-dd-yyyy_HH.mm.ss.S").format(new Date());
String backupBase = "WEB-INF/upgrade_backups/backup_" + dateString;
backupDir = new File(wrFile.toURI().resolve(backupBase));
backupDir.mkdirs();
isUpgrade = true;
System.out.println("Backups stored in " + backupDir);
// backup entire /myna folder because we're wiping it out
FileUtils.copyDirectory(
new File(wrFile.toURI().resolve("myna")), new File(backupDir.toURI().resolve("myna")));
FileUtils.deleteDirectory(new File(wrFile.toURI().resolve("myna")));
}
if (isJar) {
String jarFilePath = classUrl.substring(classUrl.indexOf(":") + 1, classUrl.indexOf("!"));
File jarFile = new File(new java.net.URL(jarFilePath).toURI());
ZipFile zipFile = new ZipFile(jarFile);
for (ZipEntry entry : java.util.Collections.list(zipFile.entries())) {;
File outputFile =
new File(wrFile.toURI().resolve(java.net.URLEncoder.encode(entry.getName(), "UTF-8")));
File backupFile = null;
if (isUpgrade) {
backupFile =
new File(
backupDir.toURI().resolve(java.net.URLEncoder.encode(entry.getName(), "UTF-8")));
}
if (entry.isDirectory()) {
outputFile.mkdirs();
if (isUpgrade) backupFile.mkdirs();
} else {
if (isUpgrade && outputFile.exists()) {
java.io.InputStream sourceIS = zipFile.getInputStream(entry);
java.io.InputStream targetIS = FileUtils.openInputStream(outputFile);
boolean isSame = IOUtils.contentEquals(sourceIS, targetIS);
sourceIS.close();
targetIS.close();
if (isSame
|| entry.toString().equals("index.html")
|| entry.toString().equals("application.sjs")
|| entry.toString().equals("WEB-INF/classes/general.properties")
|| entry.toString().startsWith("WEB-INF/myna/ds")) {
continue;
} else {
System.out.println("...backing up " + entry);
FileUtils.copyFile(outputFile, backupFile, true);
// outputFile.copyTo(backupFile);
// fusebox.upgradeLog("Backup: " + backupFile);
}
}
java.io.InputStream is = zipFile.getInputStream(entry);
java.io.OutputStream os = FileUtils.openOutputStream(outputFile);
IOUtils.copyLarge(is, os);
is.close();
os.close();
}
}
zipFile.close();
// FileUtils.deleteDirectory()
System.out.println("Done unpacking.");
}
}