/** csv格式 */
@Test
public void testImportCsv() throws IOException {
long beginTime = System.currentTimeMillis();
File file = new File("D:\\Backup\\test.csv");
InputStream is = new BufferedInputStream(new FileInputStream(file));
String encoding = FileCharset.getCharset(file);
LineIterator iterator = IOUtils.lineIterator(is, encoding);
String separator = ",";
int batchSize = 100; // 批处理大小
int totalSize = 0; // 总大小
final List<ExcelData> dataList = Lists.newArrayList();
if (iterator.hasNext()) {
iterator.nextLine(); // 跳过第一行标题
}
while (iterator.hasNext()) {
totalSize++;
String line = iterator.nextLine();
String[] dataArray = StringUtils.split(line, separator);
ExcelData data = new ExcelData();
data.setId(Long.valueOf(dataArray[0]));
data.setContent(dataArray[1]);
dataList.add(data);
if (totalSize % batchSize == 0) {
try {
doBatchSave(dataList);
} catch (Exception e) {
Long fromId = dataList.get(0).getId();
Long endId = dataList.get(dataList.size() - 1).getId();
log.error("from " + fromId + " to " + endId + ", error", e);
}
dataList.clear();
}
}
IOUtils.closeQuietly(is);
long endTime = System.currentTimeMillis();
log.info("耗时(秒):" + (endTime - beginTime) / 1000);
}
@Test
public void testNginxDockerfileBuilder() throws DockerException, IOException {
File baseDir =
new File(Thread.currentThread().getContextClassLoader().getResource("nginx").getFile());
ClientResponse response = dockerClient.buildImageCmd(baseDir).exec();
StringWriter logwriter = new StringWriter();
try {
LineIterator itr = IOUtils.lineIterator(response.getEntityInputStream(), "UTF-8");
while (itr.hasNext()) {
String line = itr.next();
logwriter.write(line + "\n");
LOG.info(line);
}
} finally {
IOUtils.closeQuietly(response.getEntityInputStream());
}
String fullLog = logwriter.toString();
assertThat(fullLog, containsString("Successfully built"));
String imageId = StringUtils.substringBetween(fullLog, "Successfully built ", "\\n\"}").trim();
ImageInspectResponse imageInspectResponse = dockerClient.inspectImageCmd(imageId).exec();
assertThat(imageInspectResponse, not(nullValue()));
LOG.info("Image Inspect: {}", imageInspectResponse.toString());
tmpImgs.add(imageInspectResponse.getId());
assertThat(
imageInspectResponse.getAuthor(),
equalTo("Guillaume J. Charmes \"[email protected]\""));
}
/**
* All files containing serialization policy are located during construction of this object.
* Serialization policies are loaded from them (and cached) as needed.
*
* @param servletContext
* @throws IOException
*/
@Autowired(required = false)
public MultiModuleSerializationPolicyProvider(ServletContext servletContext) throws IOException {
for (File rpcPolicyManifest : listRpcPolicyManifestFiles(servletContext.getRealPath("/"))) {
File moduleDir = rpcPolicyManifest.getParentFile().getParentFile();
LineIterator entries = FileUtils.lineIterator(rpcPolicyManifest);
while (entries.hasNext()) {
String line = entries.nextLine();
if (line.startsWith("#") || line.trim().length() == 0) continue;
String[] entry = line.split(",");
assert entry.length == 2 : "Invalid format of file: " + rpcPolicyManifest.getAbsolutePath();
String rpcServiceInterfaceName = entry[0].trim();
String rpcPolicyStrongFileName = entry[1].trim();
if (serializationPolicyFiles.containsKey(rpcServiceInterfaceName)) {
assert serializationPolicyFiles
.get(rpcServiceInterfaceName)
.getName()
.equals(rpcPolicyStrongFileName);
} else {
File serializationPolicyFile = new File(moduleDir, rpcPolicyStrongFileName);
assert serializationPolicyFile.exists();
serializationPolicyFiles.put(rpcServiceInterfaceName, serializationPolicyFile);
}
}
LineIterator.closeQuietly(entries);
}
}
public DoubleMatrix getScoreMatrix(File file) {
Counter<String> docWords = new Counter<String>();
try {
LineIterator iter = FileUtils.lineIterator(file);
while (iter.hasNext()) {
Tokenizer t =
tokenizerFactory.create((new InputHomogenization(iter.nextLine()).transform()));
while (t.hasMoreTokens()) {
docWords.incrementCount(t.nextToken(), 1.0);
}
}
iter.close();
} catch (IOException e) {
throw new IllegalStateException("Unable to read file", e);
}
DoubleMatrix ret = new DoubleMatrix(1, currVocab.size());
for (int i = 0; i < currVocab.size(); i++) {
if (docWords.getCount(currVocab.get(i).toString()) > 0) {
ret.put(i, wordScores.getCount(currVocab.get(i).toString()));
}
}
return ret;
}
@Override
protected void setUp() throws Exception {
inMemorySenseiService =
InMemorySenseiService.valueOf(
new File(
InMemoryIndexPerfTest.class
.getClassLoader()
.getResource("test-conf/node1/")
.toURI()));
LineIterator lineIterator =
FileUtils.lineIterator(
new File(
InMemoryIndexPerfTest.class
.getClassLoader()
.getResource("data/test_data.json")
.toURI()));
int i = 0;
docs = new ArrayList<JSONObject>();
while (lineIterator.hasNext() && i < 100) {
String car = lineIterator.next();
if (car != null && car.contains("{")) docs.add(new JSONObject(car));
i++;
}
lineIterator.close();
}
// For example, here is a line from the 5kb chr1 MAPQGE30 raw observed contact matrix
// (GM12878_combined/5kb_resolution_intrachromosomal/chr1/MAPQGE30/chr1_5kb.RAWobserved):
// 40000000 40100000 59.0
private static void processRawContactInformation(
String fileToRead,
double minValue,
ArrayList<DesiredChrContact> contactsToCheck,
boolean intra)
throws IOException {
// Check if sorted version is available
// If not make sorted available.
if (!Gpio.exists(fileToRead + ".sorted")) {
if (intra) {
umcg.genetica.io.chrContacts.SortIntraChrContacts.readNonSortedWriteSorted(
fileToRead, fileToRead + ".sorted");
} else {
umcg.genetica.io.chrContacts.SortInterChrContacts.readNonSortedWriteSorted(
fileToRead, fileToRead + ".sorted");
}
}
int numberToBeMatched = 0;
LineIterator it = FileUtils.lineIterator(new File(fileToRead + ".sorted"), "UTF-8");
try {
while (it.hasNext()) {
String[] parts = StringUtils.split(it.nextLine(), '\t');
int posChr1 = org.apache.commons.lang.math.NumberUtils.createInteger(parts[0]);
int posChr2 = org.apache.commons.lang.math.NumberUtils.createInteger(parts[1]);
while (numberToBeMatched < contactsToCheck.size()) {
if (posChr1 < contactsToCheck.get(numberToBeMatched).getChrLocationSmaller()) {
break;
} else if (posChr1 == contactsToCheck.get(numberToBeMatched).getChrLocationSmaller()) {
if (posChr2 < contactsToCheck.get(numberToBeMatched).getChrLocationLarger()) {
break;
}
if (posChr2 == contactsToCheck.get(numberToBeMatched).getChrLocationLarger()) {
double contact = org.apache.commons.lang.math.NumberUtils.createDouble(parts[2]);
if (contact >= minValue) {
contactsToCheck.get(numberToBeMatched).setContact();
numberToBeMatched++;
} else {
numberToBeMatched++;
}
} else if (posChr2 > contactsToCheck.get(numberToBeMatched).getChrLocationLarger()) {
numberToBeMatched++;
}
} else if (posChr1 > contactsToCheck.get(numberToBeMatched).getChrLocationSmaller()) {
numberToBeMatched++;
}
}
}
} finally {
LineIterator.closeQuietly(it);
}
}
public Map<String, Object> next() {
Map<String, Object> map = null;
do {
String line = delegate.nextLine();
try {
map = mapper.readValue(line, Map.class);
return map;
} catch (IOException e) {
}
} while (map == null && delegate.hasNext());
throw new NoSuchElementException();
}
/**
* Splits the source code into three blocks: the line to highlight and the source code before and
* after this line.
*
* @param sourceFile the source code of the whole file as rendered HTML string
*/
public final void splitSourceFile(final String sourceFile) {
StringBuilder output = new StringBuilder(sourceFile.length());
KloFile kloFile = kloWorkspaceFile.getKloFile();
LineIterator lineIterator = IOUtils.lineIterator(new StringReader(sourceFile));
int lineNumber = 1;
// ---header
while (lineNumber < SOURCE_GENERATOR_OFFSET) {
copyLine(output, lineIterator);
lineNumber++;
}
lineNumber = 1;
// ---iterate before the error line
while (lineNumber < Integer.parseInt(((String) kloFile.get("line")))) {
copyLine(output, lineIterator);
lineNumber++;
}
output.append("</code>\n");
// ---Error message
output.append("</td></tr>\n");
output.append("<tr><td bgcolor=\"");
appendRangeColor(output);
output.append("\">\n");
output.append("<div tooltip=\"");
// AM
// outputEscaped(output, kloFile.getProblemId()+":"+kloFile.getMessage());
outputEscaped(output, kloFile.get("problemID") + ":" + kloFile.get("message"));
output.append("\" nodismiss=\"\">\n");
output.append("<code><b>\n");
// The current line error
copyLine(output, lineIterator);
lineNumber++;
// End of the code
output.append("</b></code>\n");
output.append("</div>\n");
output.append("</td></tr>\n");
output.append("<tr><td>\n");
output.append("<code>\n");
while (lineIterator.hasNext()) {
copyLine(output, lineIterator);
}
output.append("</code>\n");
output.append("</td></tr>\n");
sourceCode = output.toString();
}
@Test
public void testNetCatDockerfileBuilder()
throws DockerException, IOException, InterruptedException {
File baseDir =
new File(Thread.currentThread().getContextClassLoader().getResource("netcat").getFile());
ClientResponse response = dockerClient.buildImageCmd(baseDir).exec();
StringWriter logwriter = new StringWriter();
try {
LineIterator itr = IOUtils.lineIterator(response.getEntityInputStream(), "UTF-8");
while (itr.hasNext()) {
String line = itr.next();
logwriter.write(line + "\n");
LOG.info(line);
}
} finally {
IOUtils.closeQuietly(response.getEntityInputStream());
}
String fullLog = logwriter.toString();
assertThat(fullLog, containsString("Successfully built"));
String imageId = StringUtils.substringBetween(fullLog, "Successfully built ", "\\n\"}").trim();
ImageInspectResponse imageInspectResponse = dockerClient.inspectImageCmd(imageId).exec();
assertThat(imageInspectResponse, not(nullValue()));
LOG.info("Image Inspect: {}", imageInspectResponse.toString());
tmpImgs.add(imageInspectResponse.getId());
ContainerCreateResponse container =
dockerClient.createContainerCmd(imageInspectResponse.getId()).exec();
assertThat(container.getId(), not(isEmptyString()));
dockerClient.startContainerCmd(container.getId()).exec();
tmpContainers.add(container.getId());
ContainerInspectResponse containerInspectResponse =
dockerClient.inspectContainerCmd(container.getId()).exec();
assertThat(containerInspectResponse.getId(), notNullValue());
assertThat(containerInspectResponse.getNetworkSettings().getPorts(), notNullValue());
// No use as such if not running on the server
for (Ports.Port p : containerInspectResponse.getNetworkSettings().getPorts().getAllPorts()) {
int port = Integer.valueOf(p.getHostPort());
LOG.info("Checking port {} is open", port);
assertThat(available(port), is(false));
}
dockerClient.stopContainerCmd(container.getId()).withTimeout(0).exec();
}
public static void main(String[] args) throws IOException {
String workDir = "E:/dev_workspace/tmp/workspace/duc2007";
String idfFilename = "duc2007.idf";
final double TOTAL_PAGE_COUNT = 30000000000.0D;
Map<String, Double> idfValues = new HashMap<String, Double>();
File idfFIle = FileUtils.getFile(workDir + "/" + DIR_IDF_FILE, idfFilename);
log.info("Loading idf value file[" + idfFIle.getAbsolutePath() + "]");
LineIterator lineIterator = null;
try {
lineIterator = FileUtils.lineIterator(idfFIle, DEFAULT_CHARSET.toString());
while (lineIterator.hasNext()) {
String line = lineIterator.nextLine();
String[] strs = line.split("###");
if (strs.length != 2) {
log.warn("Line[" + line + "] format is illegal, ignore it!");
continue;
}
idfValues.put(strs[0].trim(), Long.parseLong(strs[1]) / TOTAL_PAGE_COUNT);
}
log.info("Load idf value file[" + idfFIle.getAbsolutePath() + "] finished!");
} catch (IOException e) {
log.error("Load idf value file[" + idfFIle.getAbsolutePath() + "] error!", e);
throw e;
} finally {
if (lineIterator != null) {
lineIterator.close();
}
}
String question =
"Describe the legal battle between various recording artists and members of the record industry and the Internet music site Napster. What support, or lack thereof, have the litigants received?";
EhCacheUtil ehCacheUtil = new EhCacheUtil("db_cache_vec", "lab");
SummaryBuilderByVector summaryBuilder =
new SummaryBuilderByVector(
workDir, "0", "D0714D.txt", 10, idfValues, question, ehCacheUtil, 1.0f, 1.6f);
ExecutorService es = Executors.newSingleThreadExecutor();
Future<Boolean> future = es.submit(summaryBuilder);
try {
future.get();
} catch (InterruptedException | ExecutionException e) {
e.printStackTrace();
}
es.shutdown();
EhCacheUtil.close();
}
protected List<DiagnosGrupp> getDiagnosGrupperInternal(Resource resource) throws IOException {
LineIterator it = FileUtils.lineIterator(resource.getFile(), "UTF-8");
List<DiagnosGrupp> list = new ArrayList<>();
try {
while (it.hasNext()) {
String line = it.nextLine();
list.add(new DiagnosGrupp(line));
}
} finally {
LineIterator.closeQuietly(it);
}
return list;
}
static {
if (JiraSystemProperties.isXsrfDetectionCheckRequired()) {
log.setLevel(Level.INFO);
}
LineIterator iterator =
IOUtils.lineIterator(
new InputStreamReader(
XsrfVulnerabilityDetectionSQLInterceptor.class.getResourceAsStream(
"/security/xsrf/xsrf-white-list.txt")));
for (; iterator.hasNext(); ) {
String line = ((String) iterator.next()).trim();
if (line.length() > 0 && !line.startsWith("#")) {
actionWhiteList.add(line);
}
}
}
public static Set<String> getUndesiredTermsToFilterOut(String filename) {
Set<String> result = new HashSet<String>();
File file = new File(FOLDER_PATH + filename);
LineIterator it = null;
try {
it = FileUtils.lineIterator(file);
while (it.hasNext()) {
result.add(it.next().trim().toLowerCase());
}
} catch (IOException e) {
e.printStackTrace();
} finally {
LineIterator.closeQuietly(it);
}
return result;
}
public static Set<String> getPharmaceuticalCompanies() {
Set<String> result = new HashSet<String>();
// InputStreamReader(JochemCurator.class.getResourceAsStream("pharmaceuticalCompanies.txt")));
File file = new File(FOLDER_PATH + "pharmaceuticalCompanies.txt");
LineIterator it = null;
try {
it = FileUtils.lineIterator(file);
while (it.hasNext()) {
result.add(it.next().trim().toLowerCase());
}
} catch (IOException e) {
e.printStackTrace();
} finally {
LineIterator.closeQuietly(it);
}
return result;
}
private static String getProcessResult(Process process) {
StringBuilder stringBuilder = new StringBuilder();
try {
stringBuilder.append("-------start\r\n");
InputStream inStream = process.getInputStream();
LineIterator li = IOUtils.lineIterator(inStream, encoding);
while (li.hasNext()) {
String line = li.next();
System.out.println(line);
stringBuilder.append(line + "\r\n");
}
stringBuilder.append("-------end\r\n");
} catch (Exception e) {
e.printStackTrace();
stringBuilder.append(e.getClass().getName() + "," + e.getMessage());
}
return stringBuilder.toString();
}
/**
* Creates a hash code from the source code of the warning line and the surrounding context.
*
* @param fileName the absolute path of the file to read
* @param line the line of the warning
* @param encoding the encoding of the file, if <code>null</code> or empty then the default
* encoding of the platform is used
* @return a has code of the source code
* @throws IOException if the contents of the file could not be read
*/
public int create(final String fileName, final int line, final String encoding)
throws IOException {
LineIterator lineIterator = EncodingValidator.readFile(fileName, encoding);
StringBuilder context = new StringBuilder(1000);
for (int i = 0; lineIterator.hasNext(); i++) {
String currentLine = lineIterator.nextLine();
if (i >= line - 3) {
context.append(currentLine);
}
if (i > line + 3) {
break;
}
}
lineIterator.close();
return context.toString().hashCode();
}
/** 打印帮助信息 */
private static void showHelpInfo() {
String helpfile =
System.getProperty("user.dir") + File.separator + "conf" + File.separator + "help.info";
File f = new File(helpfile);
if (!f.exists()) {
System.out.println("help.info not exists");
} else {
try {
LineIterator itr = FileUtils.lineIterator(f, "UTF-8");
while (itr.hasNext()) {
System.out.println(itr.nextLine());
}
itr.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
/**
* Loads an in memory cache from the given path (sets syn0 and the vocab)
*
* @param vectorsFile the path of the file to load
* @return
* @throws FileNotFoundException
*/
public static Pair<InMemoryLookupTable, VocabCache> loadTxt(File vectorsFile)
throws FileNotFoundException {
BufferedReader write = new BufferedReader(new FileReader(vectorsFile));
VocabCache cache = new InMemoryLookupCache();
InMemoryLookupTable lookupTable;
LineIterator iter = IOUtils.lineIterator(write);
List<INDArray> arrays = new ArrayList<>();
while (iter.hasNext()) {
String line = iter.nextLine();
String[] split = line.split(" ");
String word = split[0];
VocabWord word1 = new VocabWord(1.0, word);
cache.addToken(word1);
cache.addWordToIndex(cache.numWords(), word);
word1.setIndex(cache.numWords());
cache.putVocabWord(word);
INDArray row = Nd4j.create(Nd4j.createBuffer(split.length - 1));
for (int i = 1; i < split.length; i++) {
row.putScalar(i - 1, Float.parseFloat(split[i]));
}
arrays.add(row);
}
INDArray syn = Nd4j.create(new int[] {arrays.size(), arrays.get(0).columns()});
for (int i = 0; i < syn.rows(); i++) {
syn.putRow(i, arrays.get(i));
}
lookupTable =
(InMemoryLookupTable)
new InMemoryLookupTable.Builder()
.vectorLength(arrays.get(0).columns())
.useAdaGrad(false)
.cache(cache)
.build();
Nd4j.clearNans(syn);
lookupTable.setSyn0(syn);
iter.close();
return new Pair<>(lookupTable, cache);
}
@TestData
@Provides
Map<String, String> provideTestData() {
ClassLoader cl = Thread.currentThread().getContextClassLoader();
try (InputStream is = cl.getResourceAsStream("testdata/fibonacci.txt")) {
Map<String, String> result = newHashMapWithExpectedSize(20);
for (LineIterator it = lineIterator(is, "UTF-8"); it.hasNext(); ) {
String line = it.nextLine();
if (line.startsWith("#")) {
continue;
}
String[] columns = line.split(";");
result.put(columns[0], columns[1]);
}
return result;
} catch (IOException ex) {
throw new IllegalStateException("Error reading test data.", ex);
}
}
private InputStream stripLineEnds(InputStream is, String charset, char chartoStrip)
throws IOException {
log.debug("Stripping [ {} ] from the end of lines.", chartoStrip);
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
final PrintStream printStream = new PrintStream(baos);
final LineIterator lineIterator = IOUtils.lineIterator(is, charset);
while (lineIterator.hasNext()) {
String line = StringUtils.stripToNull(lineIterator.next());
if (line != null) {
line = StringUtils.stripEnd(line, String.valueOf(chartoStrip));
printStream.println(line);
}
}
return new ByteArrayInputStream(baos.toByteArray());
}
/**
* Adds a populated terminating field to the ends of CSV entries. If the last entry in a CSV row
* is empty, the CSV library has difficulty understanding that is the end of the row.
*
* @param is the CSV file as an inputstream
* @param separator The field separator
* @param charset The charset
* @return An inputstream that is the same as is, but each line has a populated line termination
* entry
* @throws IOException
*/
public static InputStream terminateLines(
final InputStream is, final char separator, final String charset) throws IOException {
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
final PrintStream printStream = new PrintStream(baos);
final LineIterator lineIterator = IOUtils.lineIterator(is, charset);
while (lineIterator.hasNext()) {
String line = StringUtils.stripToNull(lineIterator.next());
if (line != null) {
line += separator + TERMINATED;
printStream.println(line);
}
}
return new ByteArrayInputStream(baos.toByteArray());
}
public static List<String> lines(final File file) {
List<String> lines = new LinkedList<String>();
if (file.exists()) {
LineIterator it = null;
try {
it = FileUtils.lineIterator(file, "UTF-8");
while (it.hasNext()) {
lines.add(it.nextLine());
}
} catch (IOException e) {
logger.warn("I/O error with file " + file.getPath(), e);
} finally {
LineIterator.closeQuietly(it);
}
}
return lines;
}
public void update(File fPathInfo) {
try {
updateinfo = new Hashtable<String, Long[]>();
LineIterator lineIterator = FileUtils.lineIterator(fPathInfo);
while (lineIterator.hasNext()) {
String line = lineIterator.nextLine();
String name = line;
long crc = 0;
if (line.indexOf("----------") > 0) {
name = line.substring(0, line.indexOf("----------"));
crc = Long.parseLong(line.substring(line.indexOf("----------") + "----------".length()));
}
updateinfo.put(name, new Long[] {-1l, crc});
}
System.out.println("updateinfo " + updateinfo.size());
listDir(new File(home), 2);
System.out.println("updateinfo " + updateinfo.size());
File foUpdate = new File(home + "\\Update.zip");
if (foUpdate.exists()) {
foUpdate.delete();
}
ZipOutputStream out = new ZipOutputStream(new FileOutputStream(home + "\\Update.zip"));
StringBuffer sb = new StringBuffer();
for (String fname : updateinfo.keySet()) {
Long[] status = updateinfo.get(fname);
if (status[0] == -1) {
sb.append("delete " + fname + "\n");
} else if (status[0] == 1) {
out.putNextEntry(new ZipEntry(fname));
IOUtils.copy(new FileInputStream(fname), out);
}
}
IOUtils.closeQuietly(out);
FileUtils.writeStringToFile(new File("del.bat"), sb.toString());
} catch (IOException e) {
e.printStackTrace(System.err);
}
}
private String dockerfileBuild(File baseDir, String expectedText)
throws DockerException, IOException {
// Build image
ClientResponse response = dockerClient.buildImageCmd(baseDir).exec();
StringWriter logwriter = new StringWriter();
try {
LineIterator itr = IOUtils.lineIterator(response.getEntityInputStream(), "UTF-8");
while (itr.hasNext()) {
String line = itr.next();
logwriter.write(line + "\n");
LOG.info(line);
}
} finally {
IOUtils.closeQuietly(response.getEntityInputStream());
}
String fullLog = logwriter.toString();
assertThat(fullLog, containsString("Successfully built"));
String imageId = StringUtils.substringBetween(fullLog, "Successfully built ", "\\n\"}").trim();
// Create container based on image
ContainerCreateResponse container = dockerClient.createContainerCmd(imageId).exec();
LOG.info("Created container: {}", container.toString());
assertThat(container.getId(), not(isEmptyString()));
dockerClient.startContainerCmd(container.getId()).exec();
dockerClient.waitContainerCmd(container.getId()).exec();
tmpContainers.add(container.getId());
// Log container
ClientResponse logResponse = logContainer(container.getId());
assertThat(logResponseStream(logResponse), containsString(expectedText));
return container.getId();
}
/**
* Helper function to create DataMatrix.
*
* @param data InputStream
* @return DataMatrix
*/
private DataMatrix getDataMatrix(InputStream data) throws Exception {
// iterate over all lines in byte[]
List<String> columnNames = null;
List<LinkedList<String>> rowData = null;
LineIterator it = IOUtils.lineIterator(data, null);
try {
int count = -1;
while (it.hasNext()) {
// first row is our column heading, create column vector
if (++count == 0) {
columnNames =
new LinkedList(Arrays.asList(it.nextLine().split(Converter.VALUE_DELIMITER, -1)));
}
// all other rows are rows in the table
else {
rowData = (rowData == null) ? new LinkedList<LinkedList<String>>() : rowData;
rowData.add(
new LinkedList(Arrays.asList(it.nextLine().split(Converter.VALUE_DELIMITER, -1))));
}
}
} finally {
LineIterator.closeQuietly(it);
}
// problem reading from data?
if (columnNames == null || rowData == null) {
if (LOG.isInfoEnabled()) {
LOG.info(
"getDataMatrix(), problem creating DataMatrix from file, data file probably missing data, returning null");
}
return null;
}
// made it here, we can create DataMatrix
if (LOG.isInfoEnabled()) {
LOG.info("creating new DataMatrix(), from file data");
}
// outta here
return new DataMatrix(rowData, columnNames);
}
/**
* Load a look up cache from an input stream delimited by \n
*
* @param from the input stream to read from
* @return the in memory lookup cache
*/
public static InMemoryLookupCache load(InputStream from) {
Reader inputStream = new InputStreamReader(from);
LineIterator iter = IOUtils.lineIterator(inputStream);
String line;
InMemoryLookupCache ret = new InMemoryLookupCache();
int count = 0;
while ((iter.hasNext())) {
line = iter.nextLine();
if (line.isEmpty()) continue;
ret.incrementWordCount(line);
VocabWord word = new VocabWord(1.0, line);
word.setIndex(count);
ret.addToken(word);
ret.addWordToIndex(count, line);
ret.putVocabWord(line);
count++;
}
return ret;
}
/**
* Reads headers from the batch starting from the given position.
*
* <p>Retrieved headers will be added to the map given by target parameter.
*
* @param iterator batch iterator.
* @param target destination of the retrieved headers.
*/
public static void readHeaders(
final ODataBatchLineIterator iterator, final Map<String, Collection<String>> target) {
try {
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
readBatchPart(new ODataBatchController(iterator, null), baos, true);
final LineIterator headers =
IOUtils.lineIterator(new ByteArrayInputStream(baos.toByteArray()), Constants.UTF8);
while (headers.hasNext()) {
final String line = headers.nextLine().trim();
if (StringUtils.isNotBlank(line)) {
addHeaderLine(line, target);
}
}
} catch (Exception e) {
LOG.error("Error retrieving headers", e);
throw new IllegalStateException(e);
}
}
public static Set<Integer> getUndesiredConceptsToFilterOut() {
Set<Integer> things = new HashSet<Integer>();
// InputStreamReader(JochemCurator.class.getResourceAsStream("conceptsToRemove.txt")));
File file = new File(FOLDER_PATH + "conceptsToRemove.txt");
LineIterator it = null;
try {
it = FileUtils.lineIterator(file);
while (it.hasNext()) {
String conceptLine = it.next().trim();
String[] conceptNumbers = conceptLine.split(";");
for (String conceptNumber : conceptNumbers) {
if (conceptNumber.length() != 0) things.add(Integer.parseInt(conceptNumber));
}
}
} catch (IOException e) {
e.printStackTrace();
} finally {
LineIterator.closeQuietly(it);
}
return things;
}
@Override
public List<Route> parseRoutes(File file) throws IOException {
LineIterator it = FileUtils.lineIterator(file, "UTF-8");
List<Route> routes = new LinkedList<Route>();
Route currentRoute = null;
while (it.hasNext()) {
String line = it.nextLine();
if (line.trim().isEmpty()) {
currentRoute = null;
} else if (line.startsWith(" ")) {
loadModelEntryForRoute(currentRoute, line);
} else {
currentRoute = parseLine(line.trim());
if (currentRoute != null) {
routes.add(currentRoute);
}
}
}
return routes;
}
@Override
public void loadFile(String fullnamePath, final List<String> acceptedStocks) throws IOException {
if (this.logger.isInfoEnabled()) {
this.logger.info(String.format("Load data from file name path [%s].", fullnamePath));
}
LineIterator lineIterator = FileUtils.lineIterator(new File(fullnamePath), FILE_FORMAT);
while (lineIterator.hasNext()) {
final String line = lineIterator.next().toLowerCase().trim();
StockRecordBean record = null;
try {
if (line.length() >= RECORD_SIZE) {
record = StockDataLoader.this.parseLine(line);
if ((acceptedStocks == null)
|| acceptedStocks.isEmpty()
|| acceptedStocks.contains(record.getStockName().toLowerCase())) {
if (record.getBdi() == 2) {
StockDataLoader.this.loadRecord(record);
} else {
if (this.logger.isTraceEnabled()) {
this.logger.trace(String.format("Record [%s] with DBI not equals 2.", record));
}
}
}
} else {
if (this.logger.isInfoEnabled()) {
this.logger.info(String.format("First or Last line aren´t loaded [%s].", line));
}
}
} catch (Exception e) {
this.logger.error(
String.format(
"Problem to parse or save the record [%s] and the bean [%s].", line, record));
this.logger.error(e.getMessage(), e);
}
}
}