Ejemplo n.º 1
0
 @Override
 public void updateConfig(XMLConfiguration config) {
   nsTableId = config.getInt("storage_table", 511);
   dataExpireTime = config.getInt("data_expiretime", 7 * 24 * 3600);
   topNum = config.getInt("top_num", 30);
   debug = config.getBoolean("debug", false);
 }
  @Override
  public void loadFromXML(Reader in) {
    XMLConfiguration xml = ConfigurationUtil.newXMLConfiguration(in);
    setSourceReferenceField(xml.getString("sourceReferenceField", sourceReferenceField));
    setKeepSourceReferenceField(
        xml.getBoolean("sourceReferenceField[@keep]", keepSourceReferenceField));
    setTargetReferenceField(xml.getString("targetReferenceField", targetReferenceField));
    setSourceContentField(xml.getString("sourceContentField", sourceContentField));
    setKeepSourceContentField(xml.getBoolean("sourceContentField[@keep]", keepSourceContentField));
    setTargetContentField(xml.getString("targetContentField", targetContentField));
    setQueueDir(xml.getString("queueDir", DEFAULT_QUEUE_DIR));
    setQueueSize(xml.getInt("queueSize", AbstractBatchCommitter.DEFAULT_QUEUE_SIZE));
    setCommitBatchSize(
        xml.getInt("commitBatchSize", AbstractBatchCommitter.DEFAULT_COMMIT_BATCH_SIZE));
    setMaxRetries(xml.getInt("maxRetries", 0));
    setMaxRetryWait(xml.getInt("maxRetryWait", 0));

    loadFromXml(xml);
  }
Ejemplo n.º 3
0
 public static boolean loadConfig(String configFilePath) {
   String tmp = System.getProperty("os.name").toLowerCase();
   if (tmp.startsWith("windows")) {
     g.os = g.OS_WINDOWS;
   } else if (tmp.startsWith("linux")) {
     g.os = g.OS_LINUX;
   } else {
     g.os = g.OS_OTHER;
   }
   tmp = System.getProperty("sun.arch.data.model");
   if (tmp.equals("64")) {
     g.arch = g.ARCH_64;
   } else {
     g.arch = g.ARCH_32;
   }
   try {
     if (configFilePath == null) {
       config = new XMLConfiguration(configPath + "config.xml");
     } else {
       config = new XMLConfiguration(configFilePath);
     }
     dbConfig = config.getString("db", "").toLowerCase();
     siteUrl = config.getString("serverInfo.siteUrl");
     siteName = config.getString("serverInfo.siteName");
     uploadTemp = rootPath + config.getString("serverInfo.uploadTemp");
     uploadSizeMax = config.getLong("serverInfo.uploadSizeMax", 0);
     sessionExpiredTime = config.getInt("serverInfo.sessionExpiredTime", 0);
     sessionIdName = config.getString("serverInfo.sessionIdName", "ycsid");
     String _s = g.getConfig().getString("startup");
     if ("".equals(_s)) {
       startup = null;
     } else {
       _s = _s.replaceAll("\\s", "");
       startup = _s.split(";");
     }
     _s = null;
     // 加载全局参数
     List<HierarchicalConfiguration> fields = config.configurationsAt("globalVars.var");
     if (fields != null && fields.size() > 0) {
       vars = new String[fields.size()][2];
       HierarchicalConfiguration sub;
       for (int i = 0; i < fields.size(); i++) {
         sub = fields.get(i);
         vars[i][0] = sub.getString("[@name]");
         vars[i][1] = sub.getString("[@value]");
       }
       sub = null;
     }
     return true;
   } catch (ConfigurationException e) {
     return false;
   }
 }
  /**
   * The constructor is {@code private} to have strict control what instances exist at any time.
   * Instead of the constructor the {@code public} <i>static factory method</i> {@link
   * #getInstance(XMLConfiguration, CoreService, URL, URL, DefaultHttpClient)} returns the instances
   * of the class.
   *
   * @param xmlConfiguration The {@code XMLConfiguration} object (of the {@code StorageService})
   * @param coreService The corresponding {@code CoreService}
   * @param apiBaseURL The base {@code URL} of the API
   * @param serviceURL The service {@code URL}
   * @param httpClient The <i>Apache</i> {@code DefaultHttpClient}
   */
  private DownloadManager(
      XMLConfiguration xmlConfiguration,
      CoreService coreService,
      URL apiBaseURL,
      URL serviceURL,
      DefaultHttpClient httpClient) {
    this.coreService = coreService;
    this.apiBaseURL = apiBaseURL;
    this.serviceURL = serviceURL;
    this.httpClient = httpClient;

    executorService =
        Executors.newFixedThreadPool(xmlConfiguration.getInt("download/max-parallel-downloads"));
  }
Ejemplo n.º 5
0
  @SuppressWarnings("unchecked")
  public BasicDataSource getDataSource(String dataSourceName, String dbName)
      throws SecurityException, ClassNotFoundException, IllegalArgumentException,
          InstantiationException, IllegalAccessException, InvocationTargetException {
    Configuration dbConf = confDbs.configurationAt(dbName);
    String dbClass = dbConf.getString("class");

    Constructor[] a = Class.forName(dbClass).getConstructors();
    BasicDataSource ds = (BasicDataSource) a[0].newInstance();
    ds.configDriver(confDbs, dbName);
    ds.configDataSource(confDatasources, dataSourceName);
    if (confDatasources.getProperty(dataSourceName + ".fillRandom") != null) {
      int fillRandomValues = confDatasources.getInt(dataSourceName + ".fillRandom");

      if (fillRandomValues == 1) ds.fillRandomValues();
    }
    ds.setName(dataSourceName + dbName);
    return ds;
  }
Ejemplo n.º 6
0
  @SuppressWarnings("unchecked")
  public void test() {
    try {
      confMonoTests.setFileName("confMonoTests.xml");
      confDbs.setFileName("confDbs.xml");
      confDatasources.setFileName("confDatasources.xml");

      confMonoTests.load();
      confDbs.load();
      confDatasources.load();

      List<String> runs = confMonoTests.getList("run");
      log.info("Start of testing");
      // runs = new TestRun[confRuns.getInt("threads")];
      int numRuns = confMonoTests.getInt("threads");
      String resultsFile = "";
      semRuns = new Semaphore(numRuns);
      // Iterate through runs
      for (String run : runs) {
        int dbId = 0;
        run = "runs." + run;
        // Iterate through dbs
        while (confMonoTests.getProperty(run + ".dbs.db(" + dbId + ").name") != null) {
          String dbName = confMonoTests.getString(run + ".dbs.db(" + dbId + ").name");
          List<String> datasources =
              confMonoTests.getList(run + ".dbs.db(" + dbId + ").datasources");
          int datasourceId = 0;
          // Iterate through datasources
          for (String datasourceName : datasources) {
            int methodId = 0;
            // Load possible normalizers
            List<String> numericalNorms =
                confMonoTests.getList(run + ".normalizers.numericalNormalizer");
            List<String> nominalNormalizers =
                confMonoTests.getList(run + ".normalizers.nominalNormalizer");
            List<String> listNormalizer =
                confMonoTests.getList(run + ".normalizers.listNormalizer");
            // Iterate through tests
            if (confMonoTests.getProperty(run + ".test.path") != null) {
              synchronized (semRuns) {
                // Waiting for a free slot;
                semRuns.acquire();

                System.gc();
                BasicDataSource trainDataSource = getDataSource(datasourceName, dbName);
                trainDataSource.configDriver(confMonoTests, run + ".dbs.db(" + dbId + ")");
                trainDataSource.configDataSource(
                    confMonoTests, run + ".dbs.db(" + dbId + ").datasources(" + datasourceId + ")");
                BasicDataSource testDataSource = getDataSource(datasourceName, dbName);
                testDataSource.configDriver(confMonoTests, run + ".dbs.db(" + dbId + ")");
                testDataSource.configDataSource(
                    confMonoTests, run + ".dbs.db(" + dbId + ").datasources(" + datasourceId + ")");

                MonotonicityTest mt = new MonotonicityTest();
                mt.configTest(confMonoTests, run + ".test");
                log.info("Testing datasource " + trainDataSource.getName());
                // Making new thread for a new test.
                new TestRun(
                        this,
                        mt,
                        trainDataSource,
                        testDataSource,
                        getNorms(nominalNormalizers),
                        getNorms(numericalNorms),
                        getNorms(listNormalizer))
                    .start();
                // Wait 1 second for avoid of the colision
                // in writing of results.
                Thread.sleep(1000);
              }
            }
            datasourceId++;
          }
          dbId++;
        }
      }

      // Waiting for all other threads to finish.
      for (int i = 0; i < numRuns; i++) semRuns.acquire();

      log.info("End of testing");

      Process p = Runtime.getRuntime().exec("cmd ", null, null);
      p.getOutputStream().write(("loadResults.bat " + resultsFile + "\n").getBytes());
      p.getOutputStream().flush();
      p.getOutputStream().write("exit\n".getBytes());
      p.getOutputStream().flush();
      BufferedReader stdOut = new BufferedReader(new InputStreamReader(p.getInputStream()));
      BufferedReader stdErr = new BufferedReader(new InputStreamReader(p.getErrorStream()));
      while (true) {
        try {
          p.exitValue();
          break;
        } catch (Exception e) {
          while (stdOut.ready() || stdErr.ready()) {
            if (stdOut.ready()) stdOut.readLine();
            else stdErr.readLine();
          }
        }
      }
      p.waitFor();

    } catch (Exception e) {
      e.printStackTrace();
    }
  }
 public static int getInt(String key, int defaultValue) {
   return pc.getInt(key, defaultValue);
 }
Ejemplo n.º 8
0
  @SuppressWarnings("unchecked")
  public void test() {
    try {
      confRuns.setFileName("confRuns.xml");
      confDbs.setFileName("confDbs.xml");
      confDatasources.setFileName("confDatasources.xml");
      confMethods.setFileName("confMethods.xml");

      confRuns.load();
      confDbs.load();
      confDatasources.load();
      confMethods.load();

      List<String> runs = confRuns.getList("run");
      log.info("Start of testing");
      // runs = new TestRun[confRuns.getInt("threads")];
      int numRuns = confRuns.getInt("threads");
      String resultsFile = "";
      semRuns = new Semaphore(numRuns);
      // Iterate through runs
      for (String run : runs) {
        int dbId = 0;
        run = "runs." + run;
        // Iterate through dbs
        while (confRuns.getProperty(run + ".dbs.db(" + dbId + ").name") != null) {
          String dbName = confRuns.getString(run + ".dbs.db(" + dbId + ").name");
          List<String> datasources = confRuns.getList(run + ".dbs.db(" + dbId + ").datasources");
          int datasourceId = 0;
          // Iterate through datasources
          for (String datasourceName : datasources) {
            int methodId = 0;
            // Iterate through methods
            while (confRuns.getProperty(run + ".methods.method(" + methodId + ").name") != null) {
              String methodName =
                  confRuns.getString(run + ".methods.method(" + methodId + ").name");
              int testId = 0;
              // Iterate through tests
              while (confRuns.getProperty(run + ".tests.test(" + testId + ").class") != null) {
                synchronized (semRuns) {
                  // Waiting for a free slot;
                  semRuns.acquire();

                  String testClass = confRuns.getString(run + ".tests.test(" + testId + ").class");
                  Constructor[] a = Class.forName(testClass).getConstructors();
                  log.info("Datasource " + datasourceName + ", method " + methodName);
                  System.gc();
                  BasicDataSource trainDataSource = getDataSource(datasourceName, dbName);
                  trainDataSource.configDriver(confRuns, run + ".dbs.db(" + dbId + ")");
                  trainDataSource.configDataSource(
                      confRuns, run + ".dbs.db(" + dbId + ").datasources(" + datasourceId + ")");
                  BasicDataSource testDataSource = getDataSource(datasourceName, dbName);
                  testDataSource.configDriver(confRuns, run + ".dbs.db(" + dbId + ")");
                  testDataSource.configDataSource(
                      confRuns, run + ".dbs.db(" + dbId + ").datasources(" + datasourceId + ")");

                  // ExportAsTHSource.exportAsTHSource(trainDataSource, "C:\\data\\",
                  // trainDataSource.getName());
                  // writeRecords(trainDataSource);
                  // writeRecords(testDataSource);
                  // DataSourceStatistics b = new DataSourceStatistics(trainDataSource.getName());
                  // b.getStatistics(trainDataSource);
                  // semRuns.release();
                  InductiveMethod method = getMethod(methodName);
                  method.configClassifier(confRuns, run + ".methods.method(" + methodId + ")");

                  Test test = (Test) a[0].newInstance();
                  test.configTest(confRuns, run + ".tests.test(" + testId + ")");
                  resultsFile = test.getResultsInterpreter().getFilePrefix() + ".csv";

                  log.info("Testing datasource " + trainDataSource.getName());
                  // Making new thread for a new test.
                  new TestRun(this, trainDataSource, testDataSource, method, test).start();
                  // Wait 1 second for avoid of the colision
                  // in writing of results.
                  Thread.sleep(1000);
                }
                testId++;
              }
              methodId++;
            }
            datasourceId++;
          }
          dbId++;
        }
      }
      // Waiting for all other threads to finish.
      for (int i = 0; i < numRuns; i++) semRuns.acquire();

      log.info("End of testing");

      Process p = Runtime.getRuntime().exec("cmd ", null, null);
      p.getOutputStream().write(("loadResults.bat " + resultsFile + "\n").getBytes());
      p.getOutputStream().flush();
      p.getOutputStream().write("exit\n".getBytes());
      p.getOutputStream().flush();
      BufferedReader stdOut = new BufferedReader(new InputStreamReader(p.getInputStream()));
      BufferedReader stdErr = new BufferedReader(new InputStreamReader(p.getErrorStream()));
      while (true) {
        try {
          p.exitValue();
          break;
        } catch (Exception e) {
          while (stdOut.ready() || stdErr.ready()) {
            if (stdOut.ready()) stdOut.readLine();
            else stdErr.readLine();
          }
        }
      }
      p.waitFor();

    } catch (Exception e) {
      e.printStackTrace();
    }
  }