Пример #1
0
  public static CompilerConfiguration generateCompilerConfigurationFromOptions(CommandLine cli)
      throws IOException {
    //
    // Setup the configuration data

    CompilerConfiguration configuration = new CompilerConfiguration();

    if (cli.hasOption("classpath")) {
      configuration.setClasspath(cli.getOptionValue("classpath"));
    }

    if (cli.hasOption('d')) {
      configuration.setTargetDirectory(cli.getOptionValue('d'));
    }

    if (cli.hasOption("encoding")) {
      configuration.setSourceEncoding(cli.getOptionValue("encoding"));
    }

    if (cli.hasOption("basescript")) {
      configuration.setScriptBaseClass(cli.getOptionValue("basescript"));
    }

    // joint compilation parameters
    if (cli.hasOption('j')) {
      Map<String, Object> compilerOptions = new HashMap<String, Object>();

      String[] opts = cli.getOptionValues("J");
      compilerOptions.put("namedValues", opts);

      opts = cli.getOptionValues("F");
      compilerOptions.put("flags", opts);

      configuration.setJointCompilationOptions(compilerOptions);
    }

    if (cli.hasOption("indy")) {
      configuration.getOptimizationOptions().put("int", false);
      configuration.getOptimizationOptions().put("indy", true);
    }

    if (cli.hasOption("configscript")) {
      String path = cli.getOptionValue("configscript");
      File groovyConfigurator = new File(path);
      Binding binding = new Binding();
      binding.setVariable("configuration", configuration);

      CompilerConfiguration configuratorConfig = new CompilerConfiguration();
      ImportCustomizer customizer = new ImportCustomizer();
      customizer.addStaticStars(
          "org.codehaus.groovy.control.customizers.builder.CompilerCustomizationBuilder");
      configuratorConfig.addCompilationCustomizers(customizer);

      GroovyShell shell = new GroovyShell(binding, configuratorConfig);
      shell.evaluate(groovyConfigurator);
    }

    return configuration;
  }
 private static Cassandra.Client createThriftClient(
     String host, int port, String user, String passwd, ITransportFactory transportFactory)
     throws Exception {
   TTransport trans = transportFactory.openTransport(host, port);
   TProtocol protocol = new TBinaryProtocol(trans);
   Cassandra.Client client = new Cassandra.Client(protocol);
   if (user != null && passwd != null) {
     Map<String, String> credentials = new HashMap<>();
     credentials.put(IAuthenticator.USERNAME_KEY, user);
     credentials.put(IAuthenticator.PASSWORD_KEY, passwd);
     AuthenticationRequest authenticationRequest = new AuthenticationRequest(credentials);
     client.login(authenticationRequest);
   }
   return client;
 }
  private static Map<String, String> createAuthorizationAttributeMap(
      String snaaName, Properties props) {
    Map<String, String> attributes = new HashMap<String, String>();

    List<String> keys = new LinkedList<String>();
    // getting keys from properties
    for (Object o : props.keySet()) {
      if (((String) o).startsWith(snaaName + authorizationAtt)
          && ((String) o).endsWith(authorizationKeyAtt)) {
        keys.add((String) o);
      }
    }

    for (String k : keys) {
      String key = props.getProperty(k);
      // getting plain key-number from properties
      String plainKeyProperty = k.replaceAll(snaaName + authorizationAtt + ".", "");
      plainKeyProperty = plainKeyProperty.replaceAll(authorizationKeyAtt, "");

      String keyPrefix = snaaName + authorizationAtt + "." + plainKeyProperty;

      // building value-property-string
      String value = props.getProperty(keyPrefix + authorizationValAtt);

      // finally put key and values
      attributes.put(key, value);
    }

    return attributes;
  }
Пример #4
0
  private Map<String, List<String>> loadHistoSectors(String sectorFile) {
    FileReader input;
    Map<String, List<String>> sectors = new HashMap<String, List<String>>();
    try {
      input = new FileReader(sectorFile);
      BufferedReader bufRead = new BufferedReader(input);
      String line;

      int i = 1;
      while ((line = bufRead.readLine()) != null) {
        Bin sectorRecord = Utils.readBin(line);
        List<String> stockList = sectorRecord.symbols;
        String startEnd = formatter.format(sectorRecord.end);
        String key = intgerFormaterr.format(i) + ":" + startEnd;
        sectors.put(key, stockList);
        for (String s : stockList) {
          invertedSectors.put(s, key);
        }
        i++;
      }
    } catch (IOException e) {
      throw new RuntimeException("Failed to load sector file", e);
    }
    return sectors;
  }
Пример #5
0
  private Map<String, Integer> loadFixedClasses(String file) {
    FileReader input;
    try {
      Map<Integer, List<String>> fixedClaszzes = new HashMap<Integer, List<String>>();
      Map<String, Integer> invertedFixedClasses = new HashMap<String, Integer>();
      File f = new File(file);
      if (!f.exists()) {
        System.out.println("Extra classes file doesn't exist: " + fixedClassesFile);
        return invertedFixedClasses;
      }
      input = new FileReader(f);
      BufferedReader bufRead = new BufferedReader(input);
      String line;
      while ((line = bufRead.readLine()) != null) {
        String parts[] = line.split(",");
        int clazz = Integer.parseInt(parts[0]);
        List<String> symbols = new ArrayList<String>();
        symbols.addAll(Arrays.asList(parts).subList(1, parts.length));
        fixedClaszzes.put(clazz, symbols);
      }

      for (Map.Entry<Integer, List<String>> e : fixedClaszzes.entrySet()) {
        for (String s : e.getValue()) {
          invertedFixedClasses.put(s, e.getKey());
        }
      }
      return invertedFixedClasses;
    } catch (IOException e) {
      e.printStackTrace();
    }
    return null;
  }
Пример #6
0
 public void changeClassLabels() {
   System.out.println("Reading cluster file: " + cluserInputFile);
   Clusters clusters = loadClusters(cluserInputFile);
   if (clusters == null) {
     System.out.println("Not clusters found to change");
     return;
   }
   Map<Integer, String> classToSector = new HashMap<Integer, String>();
   for (Map.Entry<String, Integer> e : sectorToClazz.entrySet()) {
     classToSector.put(e.getValue(), e.getKey());
   }
   for (Cluster c : clusters.getCluster()) {
     // find the cluster label
     // this is the label
     String key = classToSector.get(c.getKey());
     if (key != null) {
       System.out.println("Setting label: " + key + " to cluster: " + c.getKey());
       c.setLabel(key);
     }
   }
   try {
     System.out.println("Writing cluster file: " + clusterOutputFile);
     saveClusters(clusterOutputFile, clusters);
   } catch (FileNotFoundException | JAXBException e) {
     throw new RuntimeException("Failed to write clusters", e);
   }
 }
Пример #7
0
 private void init() {
   permNoToSymbol = Utils.loadMapping(originalStockFile);
   Map<String, Integer> symbolToPerm = new HashMap<String, Integer>();
   for (Map.Entry<Integer, String> entry : permNoToSymbol.entrySet()) {
     symbolToPerm.put(entry.getValue(), entry.getKey());
   }
 }
    private static void configureTransportFactory(
        ITransportFactory transportFactory, LoaderOptions opts) {
      Map<String, String> options = new HashMap<>();
      // If the supplied factory supports the same set of options as our SSL impl, set those
      if (transportFactory.supportedOptions().contains(SSLTransportFactory.TRUSTSTORE))
        options.put(SSLTransportFactory.TRUSTSTORE, opts.encOptions.truststore);
      if (transportFactory.supportedOptions().contains(SSLTransportFactory.TRUSTSTORE_PASSWORD))
        options.put(SSLTransportFactory.TRUSTSTORE_PASSWORD, opts.encOptions.truststore_password);
      if (transportFactory.supportedOptions().contains(SSLTransportFactory.PROTOCOL))
        options.put(SSLTransportFactory.PROTOCOL, opts.encOptions.protocol);
      if (transportFactory.supportedOptions().contains(SSLTransportFactory.CIPHER_SUITES))
        options.put(
            SSLTransportFactory.CIPHER_SUITES, Joiner.on(',').join(opts.encOptions.cipher_suites));

      if (transportFactory.supportedOptions().contains(SSLTransportFactory.KEYSTORE)
          && opts.encOptions.require_client_auth)
        options.put(SSLTransportFactory.KEYSTORE, opts.encOptions.keystore);
      if (transportFactory.supportedOptions().contains(SSLTransportFactory.KEYSTORE_PASSWORD)
          && opts.encOptions.require_client_auth)
        options.put(SSLTransportFactory.KEYSTORE_PASSWORD, opts.encOptions.keystore_password);

      // Now check if any of the factory's supported options are set as system properties
      for (String optionKey : transportFactory.supportedOptions())
        if (System.getProperty(optionKey) != null)
          options.put(optionKey, System.getProperty(optionKey));

      transportFactory.setOptions(options);
    }
Пример #9
0
 private Map<String, Integer> convertSectorsToClazz(Map<String, List<String>> sectors) {
   List<String> sectorNames = new ArrayList<String>(sectors.keySet());
   Collections.sort(sectorNames);
   Map<String, Integer> sectorsToClazz = new HashMap<String, Integer>();
   for (int i = 0; i < sectorNames.size(); i++) {
     sectorsToClazz.put(sectorNames.get(i), i + 1);
     System.out.println(sectorNames.get(i) + ": " + (i + 1));
   }
   return sectorsToClazz;
 }
Пример #10
0
    public void handleStreamEvent(StreamEvent event) {
      if (event.eventType == StreamEvent.Type.STREAM_PREPARED) {
        SessionInfo session = ((StreamEvent.SessionPreparedEvent) event).session;
        sessionsByHost.put(session.peer, session);
      } else if (event.eventType == StreamEvent.Type.FILE_PROGRESS) {
        ProgressInfo progressInfo = ((StreamEvent.ProgressEvent) event).progress;

        // update progress
        Set<ProgressInfo> progresses = progressByHost.get(progressInfo.peer);
        if (progresses == null) {
          progresses = Sets.newSetFromMap(new ConcurrentHashMap<ProgressInfo, Boolean>());
          progressByHost.put(progressInfo.peer, progresses);
        }
        if (progresses.contains(progressInfo)) progresses.remove(progressInfo);
        progresses.add(progressInfo);

        StringBuilder sb = new StringBuilder();
        sb.append("\rprogress: ");

        long totalProgress = 0;
        long totalSize = 0;
        for (Map.Entry<InetAddress, Set<ProgressInfo>> entry : progressByHost.entrySet()) {
          SessionInfo session = sessionsByHost.get(entry.getKey());

          long size = session.getTotalSizeToSend();
          long current = 0;
          int completed = 0;
          for (ProgressInfo progress : entry.getValue()) {
            if (progress.currentBytes == progress.totalBytes) completed++;
            current += progress.currentBytes;
          }
          totalProgress += current;
          totalSize += size;
          sb.append("[").append(entry.getKey());
          sb.append(" ").append(completed).append("/").append(session.getTotalFilesToSend());
          sb.append(" (").append(size == 0 ? 100L : current * 100L / size).append("%)] ");
        }
        long time = System.nanoTime();
        long deltaTime = Math.max(1L, TimeUnit.NANOSECONDS.toMillis(time - lastTime));
        lastTime = time;
        long deltaProgress = totalProgress - lastProgress;
        lastProgress = totalProgress;

        sb.append("[total: ")
            .append(totalSize == 0 ? 100L : totalProgress * 100L / totalSize)
            .append("% - ");
        sb.append(mbPerSec(deltaProgress, deltaTime)).append("MB/s");
        sb.append(" (avg: ")
            .append(mbPerSec(totalProgress, TimeUnit.NANOSECONDS.toMillis(time - start)))
            .append("MB/s)]");

        System.out.print(sb.toString());
      }
    }
Пример #11
0
 private boolean register(File f) {
   boolean rc = false;
   if (f.isDirectory()) {
     File file[] = f.listFiles(this);
     for (int i = 0; i < file.length; i++) {
       if (register(file[i])) rc = true;
     }
   } else if (dirMap.get(f) == null) {
     dirMap.put(f, new QEntry());
     rc = true;
   }
   return rc;
 }
  private Map<Integer, String> readTraining(File f) throws IOException {
    Map<Integer, String> map = new TreeMap<Integer, String>();

    LineNumberReader lr =
        new LineNumberReader(new InputStreamReader(new FileInputStream(f), "UTF-8"));
    String line = null;
    StringBuilder sb = new StringBuilder();
    int id = -1;
    while ((line = lr.readLine()) != null) {
      String[] s = line.split("\t");
      if (s.length > 0 && s[0].length() > 0 && !s[0].equals("sid")) {
        sb.append(line + "\n");
        id = Integer.parseInt(s[0]);
      } else {
        if (sb.length() > 0) {
          map.put(id, sb.toString());
        }

        sb = new StringBuilder();
      }
    }
    map.put(id, sb.toString());
    return map;
  }
  private Map<String, String[]> readRoles(File f) throws IOException {
    Map<String, String[]> map = new TreeMap<String, String[]>();

    LineNumberReader lr =
        new LineNumberReader(new InputStreamReader(new FileInputStream(f), "UTF-8"));
    String line = null;
    StringBuilder sb = new StringBuilder();

    while ((line = lr.readLine()) != null) {
      String[] s = line.toLowerCase().split("\t");
      if (s.length > 2) {
        logger.trace(Arrays.toString(s));
        map.put(s[1], s);
      }
    }

    return map;
  }
Пример #14
0
    @Override
    public void init(String keyspace) {
      Iterator<InetAddress> hostiter = hosts.iterator();
      while (hostiter.hasNext()) {
        try {
          // Query endpoint to ranges map and schemas from thrift
          InetAddress host = hostiter.next();
          Cassandra.Client client =
              createThriftClient(
                  host.getHostAddress(), rpcPort, this.user, this.passwd, this.transportFactory);

          setPartitioner(client.describe_partitioner());
          Token.TokenFactory tkFactory = getPartitioner().getTokenFactory();

          for (TokenRange tr : client.describe_ring(keyspace)) {
            Range<Token> range =
                new Range<>(
                    tkFactory.fromString(tr.start_token),
                    tkFactory.fromString(tr.end_token),
                    getPartitioner());
            for (String ep : tr.endpoints) {
              addRangeForEndpoint(range, InetAddress.getByName(ep));
            }
          }

          String query =
              String.format(
                  "SELECT * FROM %s.%s WHERE keyspace_name = '%s'",
                  Keyspace.SYSTEM_KS, SystemKeyspace.SCHEMA_COLUMNFAMILIES_CF, keyspace);
          CqlResult result =
              client.execute_cql3_query(
                  ByteBufferUtil.bytes(query), Compression.NONE, ConsistencyLevel.ONE);
          for (CqlRow row : result.rows) {
            CFMetaData metadata = CFMetaData.fromThriftCqlRow(row);
            knownCfs.put(metadata.cfName, metadata);
          }
          break;
        } catch (Exception e) {
          if (!hostiter.hasNext())
            throw new RuntimeException("Could not retrieve endpoint ranges: ", e);
        }
      }
    }
Пример #15
0
  private Map<String, List<String>> loadStockSectors(String sectorFile) {
    FileReader input;
    Map<String, List<String>> sectors = new HashMap<String, List<String>>();
    try {
      input = new FileReader(sectorFile);
      BufferedReader bufRead = new BufferedReader(input);
      String line;

      while ((line = bufRead.readLine()) != null) {
        SectorRecord sectorRecord = Utils.readSectorRecord(line);
        List<String> stockList = sectors.get(sectorRecord.getSector());
        if (stockList == null) {
          stockList = new ArrayList<String>();
          sectors.put(sectorRecord.getSector(), stockList);
        }
        stockList.add(sectorRecord.getSymbol());

        invertedSectors.put(sectorRecord.getSymbol(), sectorRecord.getSector());
      }
    } catch (IOException e) {
      throw new RuntimeException("Failed to load sector file", e);
    }
    return sectors;
  }
Пример #16
0
  private static long getTermCount(
      SolrServer server,
      Map<String, Integer> singleCountMap,
      SearchTermAndList searchTermAndList,
      Map<String, String> filterGrayList,
      Map<String, String> keepGrayList) {
    List<String[]> searchTerms = searchTermAndList.asListArray();
    Integer searchTermCountObject =
        searchTerms.size() != 1 ? null : singleCountMap.get(join(searchTerms.get(0), ","));
    long searchTermCount = 0;
    if (searchTermCountObject == null) {
      // didn't find it in map, so need to go get count
      SolrQuery query = new SolrQuery();
      query.setQuery("+text:(*:*)");
      query.addFilterQuery("+pub_date_year:[1993 TO 2013]");
      query.setParam("fl", "pmid");

      for (int i = 0; i < searchTerms.size(); i++) {
        String term1 = "";
        for (String aTermArray : searchTerms.get(i)) {
          if (filterGrayList.containsKey(aTermArray.toLowerCase())) {
            String filterTerms = filterGrayList.get(aTermArray.toLowerCase());
            String[] splitFilterTerms = filterTerms.split(",");
            term1 = term1 + "(+\"" + aTermArray + "\" -(";

            for (String splitFilterTerm : splitFilterTerms) {
              term1 = term1 + "\"" + splitFilterTerm + "\" ";
            }
            term1 = term1 + ")) ";
          } else if (keepGrayList.containsKey(aTermArray.toLowerCase())) {
            String keepTerms = keepGrayList.get(aTermArray.toLowerCase());
            String[] splitKeepTerms = keepTerms.split(",");
            term1 = term1 + "(+\"" + aTermArray + "\" +(";

            for (String splitKeepTerm : splitKeepTerms) {
              term1 = term1 + "\"" + splitKeepTerm + "\" ";
            }
            term1 = term1 + ")) ";
          } else {
            term1 = term1 + "\"" + aTermArray + "\" ";
          }
        }

        query.addFilterQuery("+text:(" + term1 + ")");
      }

      try {
        QueryResponse rsp = server.query(query);
        searchTermCount = rsp.getResults().getNumFound();
        singleCountMap.put(
            join(searchTerms.get(0), ","), Integer.parseInt(Long.toString(searchTermCount)));
      } catch (SolrServerException e) {
        // exit out if there is an error
        log.warning(e.getMessage());
        System.exit(1);
      }
    } else {
      searchTermCount = searchTermCountObject;
    }
    return searchTermCount;
  }