示例#1
0
 public void dealFile(String fileName, byte[] bs) {
   try {
     bs = convertCharset(bs);
   } catch (UnsupportedEncodingException e) {
     e.printStackTrace();
   }
   fileName = fileName.substring("File:".length());
   this.task.setCurrentInfo("正在导入站点文件:" + fileName);
   String root =
       Config.getContextRealPath()
           + Config.getValue("Statical.TargetDir")
           + "/"
           + this.map.getString("Alias");
   root = root + "/";
   fileName = root + fileName;
   fileName = fileName.replaceAll("[\\\\/]+", "/");
   String dirName = fileName.substring(0, fileName.lastIndexOf('/'));
   File dir = new File(dirName);
   if (!dir.exists()) {
     dir.mkdirs();
   }
   FileUtil.writeByte(fileName, bs);
 }
示例#2
0
   public DocumentList crawl() {
     prepareList();
     try {
       this.list.setCrawler(this);
       this.fileDownloader
         .setDenyExtension(".gif.jpg.jpeg.swf.bmp.png.js.wmv.css.ico.avi.mpg.mpeg.mp3.mp4.wma.rm.rmvb.exe.tar.gz.zip.rar");
       this.fileDownloader.setThreadCount(this.config.getThreadCount());
       this.fileDownloader.setTimeout(this.config.getTimeout() * 1000);
 
       if (this.config.isProxyFlag()) {
         this.fileDownloader.setProxyFlag(this.config.isProxyFlag());
         this.fileDownloader.setProxyHost(this.config.getProxyHost());
         this.fileDownloader.setProxyPassword(this.config.getProxyPassword());
         this.fileDownloader.setProxyUserName(this.config.getProxyUserName());
         this.fileDownloader.setProxyPort(this.config.getProxyPort());
       } else if ("Y".equalsIgnoreCase(Config.getValue("Proxy.IsUseProxy"))) {
         this.fileDownloader.setProxyFlag(true);
         this.fileDownloader.setProxyHost(Config.getValue("Proxy.Host"));
         this.fileDownloader.setProxyPassword(Config.getValue("Proxy.Password"));
         this.fileDownloader.setProxyUserName(Config.getValue("Proxy.UserName"));
         this.fileDownloader.setProxyPort(Integer.parseInt(Config.getValue("Proxy.Port")));
       }
 
       prepareScript();
       for (int i = 0; (i < this.config.getUrlLevels().length) && (i <= this.config.getMaxLevel()); ++i) {
         try {
           if ((i >= this.startLevel) || (this.task.checkStop()))
           {
             this.task.setCurrentInfo("正在处理第" + (i + 1) + "层级URL");
             this.currentLevel = i;
             dealOneLevel();
           }
         } catch (Throwable e) {
           e.printStackTrace();
         }
       }
       if (this.task.checkStop())
         break label457;
       if (this.config.isCopyImageFlag()) {
         int maxPage = this.config.getMaxPageCount();
         this.config.setMaxPageCount(-1);
         this.fileDownloader
           .setDenyExtension(".html.htm.jsp.php.asp.shtml.swf.js.css.ico.avi.mpg.mpeg.mp3.mp4.wma.wmv.rm.rmvb.exe.tar.gz.zip.rar");
         this.currentLevel += 1;
         this.task.setCurrentInfo("正在处理第" + (this.currentLevel + 1) + "层级URL");
         String[] urls = this.config.getUrlLevels();
         urls = (String[])
           ArrayUtils.add(urls, 
           "${A}.gif\n${A}.jpg\n${A}.jpeg\n${A}.png\n${A}.bmp\n${A}.GIF\n${A}.JPG\n${A}.JPEG\n${A}.PNG\n${A}.BMP");
         this.config.setUrlLevels(urls);
         dealOneLevel();
         this.config.setMaxPageCount(maxPage);
         this.fileDownloader
           .setDenyExtension(".gif.jpg.jpeg.swf.bmp.png.js.wmv.css.ico.avi.mpg.mpeg.mp3.mp4.wma.rm.rmvb.exe.tar.gz.zip.rar");
       }
       retryWithFilter();
       label457: writeArticle();
     }
     catch (Exception e1) {
       e1.printStackTrace();
       return null;
     } finally {
       this.list.save();
       this.list.close();
     }
     return this.list;
   }
示例#3
0
  public void dealZCSite(DataRow dr) {
    if (this.isNewSite) {
      if ((LicenseInfo.getName().equals("TrailUser"))
          && (new QueryBuilder("select count(*) from ZCSite").executeInt() >= 1)) {
        throw new RuntimeException("站点数超出限制,请联系泽元软件更换License!");
      }
      this.newSiteID = NoUtil.getMaxID("SiteID");
      ZCSiteSchema site = new ZCSiteSchema();
      site.setValue(dr);
      site.setID(this.newSiteID);
      site.setName(this.map.getString("Name"));
      site.setAlias(this.map.getString("Alias"));
      site.setURL(this.map.getString("URL"));
      site.setHitCount(0L);
      site.setChannelCount(0L);
      site.setSpecialCount(0L);
      site.setMagzineCount(0L);
      site.setArticleCount(0L);
      site.setImageLibCount(1L);
      site.setVideoLibCount(1L);
      site.setAudioLibCount(1L);
      site.setAttachmentLibCount(1L);
      site.setBranchInnerCode(User.getBranchInnerCode());
      site.setAddTime(new Date());
      site.setAddUser(User.getUserName());
      site.setConfigXML(ConfigImageLib.imageLibConfigDefault);
      this.da.insert(site);
      Transaction trans = new Transaction();
      Site.addDefaultPriv(this.newSiteID, trans);
      trans.setDataAccess(this.da);
      trans.commit(false);
      addIDMapping("ZCSite", String.valueOf(this.siteID), String.valueOf(this.newSiteID));
    } else {
      ZCSiteSchema site = new ZCSiteSchema();
      site.setID(this.map.getString("ID"));
      this.newSiteID = site.getID();
      site.fill();

      this.task.setCurrentInfo("正在备份数据,可能需要较长时间,请耐心等待");

      SiteExporter se = new SiteExporter(site.getID());
      se.exportSite(
          Config.getContextRealPath()
              + "WEB-INF/data/backup/"
              + site.getAlias()
              + "_"
              + System.currentTimeMillis()
              + ".dat");

      addIDMapping("ZCSite", dr.getString("ID"), String.valueOf(site.getID()));
      this.da.deleteAndBackup(site);
      BlockingTransaction tran = new BlockingTransaction(this.da);
      Site.delSiteRela(site.getID(), tran);

      String sitePath =
          Config.getContextRealPath()
              + Config.getValue("Statical.TemplateDir")
              + "/"
              + site.getAlias();
      FileUtil.delete(sitePath);

      site = new ZCSiteSchema();
      site.setValue(dr);

      this.da.insert(site);
    }
  }