@Test
public void testImportExcel2007() throws Exception {
long beginTime = System.currentTimeMillis();
String fileName = "D:\\Backup\\test.xlsx";
List<ExcelData> dataList = Lists.newArrayList();
OPCPackage pkg = OPCPackage.open(fileName);
XSSFReader r = new XSSFReader(pkg);
XMLReader parser = XMLReaderFactory.createXMLReader();
ContentHandler handler = new Excel2007SheetHandler(dataList);
parser.setContentHandler(handler);
Iterator<InputStream> sheets = r.getSheetsData();
while (sheets.hasNext()) {
InputStream sheet = sheets.next();
InputSource sheetSource = new InputSource(sheet);
parser.parse(sheetSource);
sheet.close();
}
// 把最后剩下的不足batchSize大小
if (dataList.size() > 0) {
doBatchSave(dataList);
}
long endTime = System.currentTimeMillis();
log.info("耗时(秒):" + (endTime - beginTime) / 1000);
}
@Override
public VPackage parse() {
logger.debug("Starting parsing package: " + xmlFile.getAbsolutePath());
long startParsing = System.currentTimeMillis();
try {
Document document = getDocument();
Element root = document.getDocumentElement();
_package = new VPackage(xmlFile);
Node name = root.getElementsByTagName(EL_NAME).item(0);
_package.setName(name.getTextContent());
Node descr = root.getElementsByTagName(EL_DESCRIPTION).item(0);
_package.setDescription(descr.getTextContent());
NodeList list = root.getElementsByTagName(EL_CLASS);
boolean initPainters = false;
for (int i = 0; i < list.getLength(); i++) {
PackageClass pc = parseClass((Element) list.item(i));
if (pc.getPainterName() != null) {
initPainters = true;
}
}
if (initPainters) {
_package.initPainters();
}
logger.info(
"Parsing the package '{}' finished in {}ms.\n",
_package.getName(),
(System.currentTimeMillis() - startParsing));
} catch (Exception e) {
collector.collectDiagnostic(e.getMessage(), true);
if (RuntimeProperties.isLogDebugEnabled()) {
e.printStackTrace();
}
}
try {
checkProblems("Error parsing package file " + xmlFile.getName());
} catch (Exception e) {
return null;
}
return _package;
}
/** csv格式 */
@Test
public void testImportCsv() throws IOException {
long beginTime = System.currentTimeMillis();
File file = new File("D:\\Backup\\test.csv");
InputStream is = new BufferedInputStream(new FileInputStream(file));
String encoding = FileCharset.getCharset(file);
LineIterator iterator = IOUtils.lineIterator(is, encoding);
String separator = ",";
int batchSize = 100; // 批处理大小
int totalSize = 0; // 总大小
final List<ExcelData> dataList = Lists.newArrayList();
if (iterator.hasNext()) {
iterator.nextLine(); // 跳过第一行标题
}
while (iterator.hasNext()) {
totalSize++;
String line = iterator.nextLine();
String[] dataArray = StringUtils.split(line, separator);
ExcelData data = new ExcelData();
data.setId(Long.valueOf(dataArray[0]));
data.setContent(dataArray[1]);
dataList.add(data);
if (totalSize % batchSize == 0) {
try {
doBatchSave(dataList);
} catch (Exception e) {
Long fromId = dataList.get(0).getId();
Long endId = dataList.get(dataList.size() - 1).getId();
log.error("from " + fromId + " to " + endId + ", error", e);
}
dataList.clear();
}
}
IOUtils.closeQuietly(is);
long endTime = System.currentTimeMillis();
log.info("耗时(秒):" + (endTime - beginTime) / 1000);
}
/**
* Main program.
*
* <p>\u0040param args Program parameters.
*/
public static void main(String[] args) {
// Initialize.
try {
if (!initialize(args)) {
System.exit(1);
}
// Process all files.
long startTime = System.currentTimeMillis();
int filesProcessed = processFiles(args);
long processingTime = (System.currentTimeMillis() - startTime + 999) / 1000;
// Terminate.
terminate(filesProcessed, processingTime);
} catch (Exception e) {
System.out.println(e.getMessage());
}
}
/** 导入 excel 2003 biff格式 如果是xml格式的 可以使用SAX（未测试） */
@Test
public void testImportExcel2003() throws Exception {
long beginTime = System.currentTimeMillis();
String fileName = "D:\\Backup\\Book1.xls";
List<ExcelData> dataList = Lists.newArrayList();
// 输入流
InputStream fis = new BufferedInputStream(new FileInputStream(fileName));
// 创建 org.apache.poi.poifs.filesystem.Filesystem
POIFSFileSystem poifs = new POIFSFileSystem(fis);
// 从输入流 得到 Workbook(excel 部分)流
InputStream din = poifs.createDocumentInputStream("Workbook");
// 构造 HSSFRequest
HSSFRequest req = new HSSFRequest();
// 添加监听器
req.addListenerForAllRecords(new Excel2003Listener(dataList));
// 创建事件工厂
HSSFEventFactory factory = new HSSFEventFactory();
// 根据文档输入流处理事件
factory.processEvents(req, din);
// 关闭输入流
fis.close();
// 关闭文档流
din.close();
System.out.println(dataList.size());
// 把最后剩下的不足batchSize大小
if (dataList.size() > 0) {
doBatchSave(dataList);
}
long endTime = System.currentTimeMillis();
log.info("耗时(秒):" + (endTime - beginTime) / 1000);
}
public void saveGlyphFile(OutputStream a_output) {
try {
Transformer transformer = getTransformer();
Document document = m_glyph.makeDocument();
DOMSource source = new DOMSource(document);
StreamResult result = new StreamResult(a_output);
transformer.transform(source, result);
} catch (ParserConfigurationException | TransformerException e) {
e.printStackTrace();
}
m_savedTime = System.currentTimeMillis();
m_modifiedTime = m_savedTime;
}
public Memento createMemento(String a_description) {
m_modifiedTime = System.currentTimeMillis();
byte[] bytes = m_glyph.makeTextDocument().getBytes();
return new Memento(a_description, bytes);
}