コード例 #1
0
ファイル: SpiderContext.java プロジェクト: ayumiono/lmspider
 public SpiderContext build() throws Exception {
   SpiderContext _context = new SpiderContext();
   _context.setParameter(SpiderContext.JETTY_PORT, jettyPort);
   _context.setParameter(SpiderContext.FILE_SERVER_PORT, fileServerPort);
   _context.setParameter(SpiderContext.LOG_DIR, logDir);
   _context.setParameter(SpiderContext.PERSISTENCE_METHOD, persistenceMethod);
   _context.setParameter(SpiderContext.BIND_IP, bindIp);
   _context.setParameter(SpiderContext.NODE_LEVEL, level);
   try {
     logger.info("SpiderContext: initialize jdbc...");
     JdbcTemplate jdbcTemplate = new JdbcTemplate();
     BasicDataSource dataSource = new BasicDataSource();
     dataSource.setDriverClassName(SpiderGlobalConfig.getValue("jdbc.driverClassName"));
     dataSource.setUrl(SpiderGlobalConfig.getValue("jdbc.url"));
     dataSource.setUsername(SpiderGlobalConfig.getValue("jdbc.username"));
     dataSource.setPassword(SpiderGlobalConfig.getValue("jdbc.password"));
     dataSource.setMaxActive(100);
     dataSource.setMaxIdle(30);
     dataSource.setMaxWait(5000);
     dataSource.setTestOnBorrow(true);
     dataSource.setTestWhileIdle(true);
     dataSource.setValidationQuery("select 1 from dual");
     _context.setParameter(SpiderContext.DATASOURCE, dataSource);
     jdbcTemplate.setDataSource(dataSource);
     _context.setParameter(SpiderContext.JDBCTEMPLATE, jdbcTemplate);
     DataSourceTransactionManager transactionManager =
         new DataSourceTransactionManager(dataSource);
     _context.setParameter(SpiderContext.TRANSACTION_MANAGER, transactionManager);
     logger.info("SpiderContext: jdbc initialize finished.");
   } catch (Exception e) {
     logger.error("SpiderContext: jdbc initialize failed.", e);
     throw new Exception("SpiderContext: jdbc initialize failed.", e);
   }
   return _context;
 }
コード例 #2
0
ファイル: SpiderContext.java プロジェクト: ayumiono/lmspider
  public static void main(String[] args) {
    final BlockingQueue<Map<String, String>> recordCache =
        new LinkedBlockingQueue<Map<String, String>>();
    for (int i = 10001; i >= 0; i--) {
      Map<String, String> values = new HashMap<String, String>();
      values.put("fingerprint", "fingerprint" + i);
      values.put("url", "url" + i);
      values.put("model", "model" + i);
      values.put("tag", "tag" + i);
      recordCache.add(values);
    }
    final String sqlStr = "insert into lmdna_carbiz(fingerprint,url,model,tag) values(?,?,?,?)";
    JdbcTemplate jdbcTemplate = new JdbcTemplate();
    BasicDataSource dataSource = new BasicDataSource();
    dataSource.setDriverClassName(SpiderGlobalConfig.getValue("jdbc.driverClassName"));
    dataSource.setUrl(SpiderGlobalConfig.getValue("jdbc.url"));
    dataSource.setUsername(SpiderGlobalConfig.getValue("jdbc.username"));
    dataSource.setPassword(SpiderGlobalConfig.getValue("jdbc.password"));
    dataSource.setMaxActive(100);
    dataSource.setMaxIdle(30);
    dataSource.setMaxWait(5000);
    dataSource.setTestOnBorrow(true);
    dataSource.setTestWhileIdle(true);
    dataSource.setValidationQuery("select 1 from dual");
    dataSource.setDefaultAutoCommit(false);
    DataSourceTransactionManager transactionManager = new DataSourceTransactionManager(dataSource);
    DefaultTransactionDefinition def = new DefaultTransactionDefinition();
    TransactionStatus status = transactionManager.getTransaction(def);
    jdbcTemplate.setDataSource(dataSource);
    final List<Map<String, String>> ds = new ArrayList<Map<String, String>>();
    ds.addAll(recordCache);
    try {
      jdbcTemplate.batchUpdate(
          sqlStr,
          new BatchPreparedStatementSetter() {
            @Override
            public void setValues(PreparedStatement ps, int i) throws SQLException {
              ps.setString(1, ds.get(i).get("fingerprint"));
              ps.setString(2, ds.get(i).get("url"));
              ps.setString(3, ds.get(i).get("model"));
              ps.setString(4, ds.get(i).get("tag"));
            }

            @Override
            public int getBatchSize() {
              return ds.size();
            }
          });
      transactionManager.commit(status);
    } catch (Exception e) {
      LoggerUtil.error("批量插入出错", e);
    }
  }