/**
* All files containing serialization policy are located during construction of this object.
* Serialization policies are loaded from them (and cached) as needed.
*
* @param servletContext
* @throws IOException
*/
@Autowired(required = false)
public MultiModuleSerializationPolicyProvider(ServletContext servletContext) throws IOException {
for (File rpcPolicyManifest : listRpcPolicyManifestFiles(servletContext.getRealPath("/"))) {
File moduleDir = rpcPolicyManifest.getParentFile().getParentFile();
LineIterator entries = FileUtils.lineIterator(rpcPolicyManifest);
while (entries.hasNext()) {
String line = entries.nextLine();
if (line.startsWith("#") || line.trim().length() == 0) continue;
String[] entry = line.split(",");
assert entry.length == 2 : "Invalid format of file: " + rpcPolicyManifest.getAbsolutePath();
String rpcServiceInterfaceName = entry[0].trim();
String rpcPolicyStrongFileName = entry[1].trim();
if (serializationPolicyFiles.containsKey(rpcServiceInterfaceName)) {
assert serializationPolicyFiles
.get(rpcServiceInterfaceName)
.getName()
.equals(rpcPolicyStrongFileName);
} else {
File serializationPolicyFile = new File(moduleDir, rpcPolicyStrongFileName);
assert serializationPolicyFile.exists();
serializationPolicyFiles.put(rpcServiceInterfaceName, serializationPolicyFile);
}
}
LineIterator.closeQuietly(entries);
}
}
// For example, here is a line from the 5kb chr1 MAPQGE30 raw observed contact matrix
// (GM12878_combined/5kb_resolution_intrachromosomal/chr1/MAPQGE30/chr1_5kb.RAWobserved):
// 40000000 40100000 59.0
private static void processRawContactInformation(
String fileToRead,
double minValue,
ArrayList<DesiredChrContact> contactsToCheck,
boolean intra)
throws IOException {
// Check if sorted version is available
// If not make sorted available.
if (!Gpio.exists(fileToRead + ".sorted")) {
if (intra) {
umcg.genetica.io.chrContacts.SortIntraChrContacts.readNonSortedWriteSorted(
fileToRead, fileToRead + ".sorted");
} else {
umcg.genetica.io.chrContacts.SortInterChrContacts.readNonSortedWriteSorted(
fileToRead, fileToRead + ".sorted");
}
}
int numberToBeMatched = 0;
LineIterator it = FileUtils.lineIterator(new File(fileToRead + ".sorted"), "UTF-8");
try {
while (it.hasNext()) {
String[] parts = StringUtils.split(it.nextLine(), '\t');
int posChr1 = org.apache.commons.lang.math.NumberUtils.createInteger(parts[0]);
int posChr2 = org.apache.commons.lang.math.NumberUtils.createInteger(parts[1]);
while (numberToBeMatched < contactsToCheck.size()) {
if (posChr1 < contactsToCheck.get(numberToBeMatched).getChrLocationSmaller()) {
break;
} else if (posChr1 == contactsToCheck.get(numberToBeMatched).getChrLocationSmaller()) {
if (posChr2 < contactsToCheck.get(numberToBeMatched).getChrLocationLarger()) {
break;
}
if (posChr2 == contactsToCheck.get(numberToBeMatched).getChrLocationLarger()) {
double contact = org.apache.commons.lang.math.NumberUtils.createDouble(parts[2]);
if (contact >= minValue) {
contactsToCheck.get(numberToBeMatched).setContact();
numberToBeMatched++;
} else {
numberToBeMatched++;
}
} else if (posChr2 > contactsToCheck.get(numberToBeMatched).getChrLocationLarger()) {
numberToBeMatched++;
}
} else if (posChr1 > contactsToCheck.get(numberToBeMatched).getChrLocationSmaller()) {
numberToBeMatched++;
}
}
}
} finally {
LineIterator.closeQuietly(it);
}
}
protected List<DiagnosGrupp> getDiagnosGrupperInternal(Resource resource) throws IOException {
LineIterator it = FileUtils.lineIterator(resource.getFile(), "UTF-8");
List<DiagnosGrupp> list = new ArrayList<>();
try {
while (it.hasNext()) {
String line = it.nextLine();
list.add(new DiagnosGrupp(line));
}
} finally {
LineIterator.closeQuietly(it);
}
return list;
}
public static Set<String> getPharmaceuticalCompanies() {
Set<String> result = new HashSet<String>();
// InputStreamReader(JochemCurator.class.getResourceAsStream("pharmaceuticalCompanies.txt")));
File file = new File(FOLDER_PATH + "pharmaceuticalCompanies.txt");
LineIterator it = null;
try {
it = FileUtils.lineIterator(file);
while (it.hasNext()) {
result.add(it.next().trim().toLowerCase());
}
} catch (IOException e) {
e.printStackTrace();
} finally {
LineIterator.closeQuietly(it);
}
return result;
}
public static Set<String> getUndesiredTermsToFilterOut(String filename) {
Set<String> result = new HashSet<String>();
File file = new File(FOLDER_PATH + filename);
LineIterator it = null;
try {
it = FileUtils.lineIterator(file);
while (it.hasNext()) {
result.add(it.next().trim().toLowerCase());
}
} catch (IOException e) {
e.printStackTrace();
} finally {
LineIterator.closeQuietly(it);
}
return result;
}
public static List<String> lines(final File file) {
List<String> lines = new LinkedList<String>();
if (file.exists()) {
LineIterator it = null;
try {
it = FileUtils.lineIterator(file, "UTF-8");
while (it.hasNext()) {
lines.add(it.nextLine());
}
} catch (IOException e) {
logger.warn("I/O error with file " + file.getPath(), e);
} finally {
LineIterator.closeQuietly(it);
}
}
return lines;
}
/**
* Helper function to create DataMatrix.
*
* @param data InputStream
* @return DataMatrix
*/
private DataMatrix getDataMatrix(InputStream data) throws Exception {
// iterate over all lines in byte[]
List<String> columnNames = null;
List<LinkedList<String>> rowData = null;
LineIterator it = IOUtils.lineIterator(data, null);
try {
int count = -1;
while (it.hasNext()) {
// first row is our column heading, create column vector
if (++count == 0) {
columnNames =
new LinkedList(Arrays.asList(it.nextLine().split(Converter.VALUE_DELIMITER, -1)));
}
// all other rows are rows in the table
else {
rowData = (rowData == null) ? new LinkedList<LinkedList<String>>() : rowData;
rowData.add(
new LinkedList(Arrays.asList(it.nextLine().split(Converter.VALUE_DELIMITER, -1))));
}
}
} finally {
LineIterator.closeQuietly(it);
}
// problem reading from data?
if (columnNames == null || rowData == null) {
if (LOG.isInfoEnabled()) {
LOG.info(
"getDataMatrix(), problem creating DataMatrix from file, data file probably missing data, returning null");
}
return null;
}
// made it here, we can create DataMatrix
if (LOG.isInfoEnabled()) {
LOG.info("creating new DataMatrix(), from file data");
}
// outta here
return new DataMatrix(rowData, columnNames);
}
public static Set<Integer> getUndesiredConceptsToFilterOut() {
Set<Integer> things = new HashSet<Integer>();
// InputStreamReader(JochemCurator.class.getResourceAsStream("conceptsToRemove.txt")));
File file = new File(FOLDER_PATH + "conceptsToRemove.txt");
LineIterator it = null;
try {
it = FileUtils.lineIterator(file);
while (it.hasNext()) {
String conceptLine = it.next().trim();
String[] conceptNumbers = conceptLine.split(";");
for (String conceptNumber : conceptNumbers) {
if (conceptNumber.length() != 0) things.add(Integer.parseInt(conceptNumber));
}
}
} catch (IOException e) {
e.printStackTrace();
} finally {
LineIterator.closeQuietly(it);
}
return things;
}
/**
* Reads the precomputed md5 digest out of a .md5 file (firehose). Assume the file only contains
* one line wit checksum.
*
* @param file File
* @return String
* @throws Exception
*/
@Override
public String getPrecomputedMD5Digest(File file) throws Exception {
if (LOG.isInfoEnabled()) {
LOG.info("getPrecomputedMD5Digest(): " + file.getCanonicalPath());
}
String toReturn = "";
LineIterator it = org.apache.commons.io.FileUtils.lineIterator(file);
try {
while (it.hasNext()) {
String content = it.nextLine();
if (content.split(" ").length == 2) {
toReturn = content.split(" ")[0].toUpperCase();
}
}
} finally {
LineIterator.closeQuietly(it);
}
// outta here
return toReturn;
}
private static void processNormalizedIntraContactInformation(
String fileToRead,
String baseName,
String normMethod,
String chrSmaller,
ArrayList<DesiredChrContact> contactsToCheck,
String resolution,
double minValue,
TextFile outWriter)
throws IOException {
// ReadIn normalization chr1
TextFile inputNormChr1 =
new TextFile(
baseName + "\\chr" + chrSmaller + "_" + resolution + "." + normMethod, TextFile.R);
ArrayList<String> normFactorSmallerChr = inputNormChr1.readAsArrayList();
inputNormChr1.close();
// System.out.println("Done reading norm factor 1");
if (!Gpio.exists(fileToRead + ".sorted")) {
umcg.genetica.io.chrContacts.SortIntraChrContacts.readNonSortedWriteSorted(
fileToRead, fileToRead + ".sorted");
}
int numberToBeMatched = 0;
LineIterator it = FileUtils.lineIterator(new File(fileToRead + ".sorted"), "UTF-8");
try {
while (it.hasNext()) {
String[] parts = StringUtils.split(it.nextLine(), '\t');
int posChr1 = org.apache.commons.lang.math.NumberUtils.createInteger(parts[0]);
int posChr2 = org.apache.commons.lang.math.NumberUtils.createInteger(parts[1]);
while (numberToBeMatched < contactsToCheck.size()) {
if (posChr1 < contactsToCheck.get(numberToBeMatched).getChrLocationSmaller()) {
break;
} else if (posChr1 == contactsToCheck.get(numberToBeMatched).getChrLocationSmaller()) {
if (posChr2 < contactsToCheck.get(numberToBeMatched).getChrLocationLarger()) {
break;
}
if (posChr2 == contactsToCheck.get(numberToBeMatched).getChrLocationLarger()) {
String factor1Base =
normFactorSmallerChr.get((posChr1 / getNumericResolution(resolution)) + 1);
String factor2Base =
normFactorSmallerChr.get((posChr2 / getNumericResolution(resolution)) + 1);
double factor1;
double factor2;
if (StringUtils.isNumeric(factor1Base) && StringUtils.isNumeric(factor2Base)) {
factor1 = org.apache.commons.lang.math.NumberUtils.createDouble(factor1Base);
factor2 = org.apache.commons.lang.math.NumberUtils.createDouble(factor2Base);
double contact =
org.apache.commons.lang.math.NumberUtils.createDouble(parts[2])
/ (factor1 * factor2);
if (contact >= minValue) {
outWriter.writeln(
contactsToCheck.get(numberToBeMatched).getSnpName()
+ "\t"
+ contactsToCheck.get(numberToBeMatched).getProbeName()
+ "\t"
+ posChr1
+ "\t"
+ posChr2
+ "\tContact\t"
+ contact
+ "\t"
+ org.apache.commons.lang.math.NumberUtils.createDouble(parts[2]));
numberToBeMatched++;
} else {
outWriter.writeln(
contactsToCheck.get(numberToBeMatched).getSnpName()
+ "\t"
+ contactsToCheck.get(numberToBeMatched).getProbeName()
+ "\t"
+ posChr1
+ "\t"
+ posChr2
+ "\t-\t-\t-");
numberToBeMatched++;
}
} else {
System.out.println("Error in files.");
numberToBeMatched++;
}
} else if (posChr2 > contactsToCheck.get(numberToBeMatched).getChrLocationLarger()) {
outWriter.writeln(
contactsToCheck.get(numberToBeMatched).getSnpName()
+ "\t"
+ contactsToCheck.get(numberToBeMatched).getProbeName()
+ "\t"
+ posChr1
+ "\t"
+ posChr2
+ "\t-\t-\t-");
numberToBeMatched++;
}
} else if (posChr1 > contactsToCheck.get(numberToBeMatched).getChrLocationSmaller()) {
outWriter.writeln(
contactsToCheck.get(numberToBeMatched).getSnpName()
+ "\t"
+ contactsToCheck.get(numberToBeMatched).getProbeName()
+ "\t"
+ posChr1
+ "\t"
+ posChr2
+ "\t-\t-\t-");
numberToBeMatched++;
}
}
}
} finally {
LineIterator.closeQuietly(it);
}
}
public int run() {
int rt = -1;
String step = leadsfile.split("_")[2];
if (!"01".equals(step)) {
logger.info(
"Leads setp :" + step + " skip leadsInsert",
leadsfile,
"",
"2-1",
"",
"名单营销波次为:" + step + ",跳过LeadsInsert.");
return 0;
}
LineIterator it = null;
JDBCExecute jdbcExecute =
new JDBCExecute(
conf.getProperty("leads.insert.db.driverClass"),
conf.getProperty("leads.insert.db.databaseURL"),
new KeyMapResource().get(conf.getProperty("leads.insert.db.databaseUser")),
new KeyMapResource().get(conf.getProperty("leads.insert.db.databasePassword")));
String getCampSQL = conf.getProperty("leads.insert.get.camp.sql");
// JDBCExecute jdbcExecute1 = new JDBCExecute(
// conf.getProperty("leads.insert.dwdb.driverClass"),
// conf.getProperty("leads.insert.dwdb.databaseURL"),
// new KeyMapResource().get(conf.getProperty("leads.insert.dwdb.databaseUser")),
// new
// KeyMapResource().get(conf.getProperty("leads.insert.dwdb.databasePassword"))
// );
// logger.info("LeadsInsert.getCampSQL:"+getCampSQL,
// leadsfile,
// "",
// "2-1",
// "",
// "LeadsInsert.getCampSQL:"+getCampSQL);
List<String[]> sqls = new ArrayList<String[]>();
List<String> file_sqls = new ArrayList<String>();
try {
File readfile = new File(conf.getProperty("leads.insert.readpath") + leadsfile);
File outfile = new File(conf.getProperty("leads.insert.savepath") + insertfile);
String tableName = conf.getProperty("leads.insert.db.table.name");
logger.info(
"Leads Insert Table Name:" + tableName,
leadsfile,
"",
"2-1",
"",
"名单要插入的表为:" + tableName);
// INSERT INTO ABC(C1,C1) VALUES ('1','2');
StringBuffer sqlColumnBuffer = new StringBuffer("INSERT INTO");
sqlColumnBuffer.append(" ");
sqlColumnBuffer.append(tableName);
sqlColumnBuffer.append("(");
StringBuffer sqlValuesBuffer = new StringBuffer("VALUES (");
int i = 1;
String tempStr = conf.getProperty("leads.insert.column." + i);
String[] tempArr;
List<InsertColumn> insertColumns = new ArrayList<InsertColumn>();
InsertColumn ic;
while (tempStr != null) {
ic = new InsertColumn();
tempArr = tempStr.split("\\|", -1);
// System.out.println(tempStr);
if (i != 1) {
sqlColumnBuffer.append(",");
}
ic.setName(tempArr[0]);
ic.setLctype(tempArr[1]);
ic.setExpstr(tempArr[2]);
insertColumns.add(ic);
// System.out.println(tempArr[0]);
sqlColumnBuffer.append(tempArr[0]);
i++;
tempStr = conf.getProperty("leads.insert.column." + i);
// System.out.println(tempArr);
}
sqlColumnBuffer.append(") ");
it = FileUtils.lineIterator(readfile, conf.getProperty("leads.insert.file.encoding"));
String[] sql;
String line;
String[] file_columns;
String[] oneObjects = null;
String value;
String selectSQL = conf.getProperty("leads.insert.check.sql");
String newSelectSQL;
boolean isOneQuery = false;
while (it.hasNext()) {
line = it.nextLine();
file_columns = line.split("\\|", -1);
if (!isOneQuery) { // 文件读取循环开始时，执行一次。
String tempsql = SqlStrUtils.toSQL(getCampSQL, file_columns);
logger.info(
"LeadsInsert.getCampSQL:" + tempsql.replaceAll("\'", ""),
leadsfile,
"",
"2-1",
"",
"LeadsInsert.getCampSQL:" + tempsql.replaceAll("\'", ""));
oneObjects = jdbcExecute.getOneObject(tempsql);
isOneQuery = true;
}
for (int j = 0; j < insertColumns.size(); j++) { // int index : indexStrs){
// indexStrs[j];
if (j != 0) {
sqlValuesBuffer.append(",");
}
value = toValue(oneObjects, file_columns, insertColumns.get(j));
if (value != null
&& !value.equals("")) { // toValue(camp_resultSet,file_columns,insertColumns.get(j))
sqlValuesBuffer.append("'" + value + "'");
} else {
sqlValuesBuffer.append("NULL");
}
}
sqlValuesBuffer.append(")");
newSelectSQL = toExpstrValue(selectSQL, file_columns);
sql = new String[2];
sql[0] = sqlColumnBuffer.toString() + sqlValuesBuffer.toString();
sql[1] = newSelectSQL;
file_sqls.add(sql[0]);
sqls.add(sql);
sqlValuesBuffer.delete(8, sqlValuesBuffer.length());
}
// FileUtils.writeLines(outfile,sqls,conf.getProperty("leads.insert.file.encoding"));
FileUtils.writeLines(outfile, conf.getProperty("leads.insert.file.encoding"), file_sqls);
jdbcExecute.execute(sqls);
logger.info(
"Leads Insert Table[" + tableName + "] success. row count:" + sqls.size(),
leadsfile,
"",
"2-1",
"",
"名单要插入的表[" + tableName + "]完成!插入笔数:" + sqls.size());
logger.info(
"Leads Insert run success!gen .SQL file:" + insertfile + " ",
leadsfile,
"",
"2-1",
"LINST00001",
"LeadsInsert运行成功!生成 " + insertfile + " 完成!");
rt = 0;
// List<String[]> leadsList = new ArrayList<String[]>();
} catch (Exception e) {
if (e instanceof FileNotFoundException) {
rt = 805;
logger.error(
e.getMessage().replaceAll("\'", ""),
leadsfile,
"",
"2-1",
"805",
"没有找到指定文件!" + e.getMessage().replaceAll("\'", ""));
} else if (e instanceof SQLIntegrityConstraintViolationException) {
rt = 815;
logger.error(
e.getMessage().replaceAll("\'", ""),
leadsfile,
"",
"2-1",
"815",
"插入数据库发生异常!" + e.getMessage().replaceAll("\'", ""));
} else {
rt = 809;
e.printStackTrace();
logger.error(
"leadsInsert error:" + e.getMessage().replaceAll("\'", ""),
leadsfile,
"",
"2-1",
"809",
"程序发生异常!" + e.getMessage().replaceAll("\'", ""));
}
} finally {
LineIterator.closeQuietly(it);
jdbcExecute.closeConnection();
}
return rt;
}
/* (non-Javadoc)
* @see org.opennms.upgrade.api.OnmsUpgrade#execute()
*/
@Override
public void execute() throws OnmsUpgradeException {
String jettySSL = getMainProperties().getProperty("org.opennms.netmgt.jetty.https-port", null);
String jettyAJP = getMainProperties().getProperty("org.opennms.netmgt.jetty.ajp-port", null);
boolean sslWasFixed = false;
boolean ajpWasFixed = false;
try {
log("SSL Enabled ? %s\n", jettySSL != null);
log("AJP Enabled ? %s\n", jettyAJP != null);
if (jettySSL != null || jettyAJP != null) {
File jettyXmlExample =
new File(
getHomeDirectory(),
"etc" + File.separator + "examples" + File.separator + "jetty.xml");
File jettyXml = new File(getHomeDirectory(), "etc" + File.separator + "jetty.xml");
if (!jettyXml.exists() && !jettyXmlExample.exists()) {
throw new FileNotFoundException("The required file doesn't exist: " + jettyXmlExample);
}
if (!jettyXml.exists()) {
log("Copying %s into %s\n", jettyXmlExample, jettyXml);
FileUtils.copyFile(jettyXmlExample, jettyXml);
}
log("Creating %s\n", jettyXml);
File tempFile = new File(jettyXml.getAbsoluteFile() + ".tmp");
FileWriter w = new FileWriter(tempFile);
LineIterator it = FileUtils.lineIterator(jettyXmlExample);
boolean startSsl = false;
boolean startAjp = false;
while (it.hasNext()) {
String line = it.next();
if (startAjp) {
if (line.matches("^\\s+[<][!]--\\s*$")) {
continue;
}
if (line.matches("^\\s+--[>]\\s*$")) {
startAjp = false;
ajpWasFixed = true;
continue;
}
}
if (startSsl) {
if (line.matches("^\\s+[<][!]--\\s*$")) {
continue;
}
if (line.matches("^\\s+--[>]\\s*$")) {
startSsl = false;
sslWasFixed = true;
continue;
}
}
w.write(line + "\n");
if (startAjp == false && line.contains("<!-- Add AJP support -->") && jettyAJP != null) {
startAjp = true;
log("Enabling AjpConnector\n");
}
if (startSsl == false
&& line.contains("<!-- Add HTTPS support -->")
&& jettySSL != null) {
startSsl = true;
log("Enabling SslSelectChannelConnector\n");
}
}
LineIterator.closeQuietly(it);
w.close();
FileUtils.copyFile(tempFile, jettyXml);
FileUtils.deleteQuietly(tempFile);
} else {
log("Neither SSL nor AJP are enabled.\n");
}
} catch (Exception e) {
throw new OnmsUpgradeException("Can't fix Jetty configuration because " + e.getMessage(), e);
}
if (jettyAJP != null && !ajpWasFixed) {
throw new OnmsUpgradeException(
"Can't enable APJ, please manually edit jetty.xml and uncomment the section where org.eclipse.jetty.ajp.Ajp13SocketConnector is defined.");
}
if (jettySSL != null && !sslWasFixed) {
throw new OnmsUpgradeException(
"Can't enable SSL, please manually edit jetty.xml and uncomment the section where org.eclipse.jetty.server.ssl.SslSelectChannelConnector is defined.");
}
}