private Map<String, Integer> loadFixedClasses(String file) {
    FileReader input;
    try {
      Map<Integer, List<String>> fixedClaszzes = new HashMap<Integer, List<String>>();
      Map<String, Integer> invertedFixedClasses = new HashMap<String, Integer>();
      File f = new File(file);
      if (!f.exists()) {
        System.out.println("Extra classes file doesn't exist: " + fixedClassesFile);
        return invertedFixedClasses;
      }
      input = new FileReader(f);
      BufferedReader bufRead = new BufferedReader(input);
      String line;
      while ((line = bufRead.readLine()) != null) {
        String parts[] = line.split(",");
        int clazz = Integer.parseInt(parts[0]);
        List<String> symbols = new ArrayList<String>();
        symbols.addAll(Arrays.asList(parts).subList(1, parts.length));
        fixedClaszzes.put(clazz, symbols);
      }

      for (Map.Entry<Integer, List<String>> e : fixedClaszzes.entrySet()) {
        for (String s : e.getValue()) {
          invertedFixedClasses.put(s, e.getKey());
        }
      }
      return invertedFixedClasses;
    } catch (IOException e) {
      e.printStackTrace();
    }
    return null;
  }
 public void changeClassLabels() {
   System.out.println("Reading cluster file: " + cluserInputFile);
   Clusters clusters = loadClusters(cluserInputFile);
   if (clusters == null) {
     System.out.println("Not clusters found to change");
     return;
   }
   Map<Integer, String> classToSector = new HashMap<Integer, String>();
   for (Map.Entry<String, Integer> e : sectorToClazz.entrySet()) {
     classToSector.put(e.getValue(), e.getKey());
   }
   for (Cluster c : clusters.getCluster()) {
     // find the cluster label
     // this is the label
     String key = classToSector.get(c.getKey());
     if (key != null) {
       System.out.println("Setting label: " + key + " to cluster: " + c.getKey());
       c.setLabel(key);
     }
   }
   try {
     System.out.println("Writing cluster file: " + clusterOutputFile);
     saveClusters(clusterOutputFile, clusters);
   } catch (FileNotFoundException | JAXBException e) {
     throw new RuntimeException("Failed to write clusters", e);
   }
 }
  private static Map<String, String> createAuthorizationAttributeMap(
      String snaaName, Properties props) {
    Map<String, String> attributes = new HashMap<String, String>();

    List<String> keys = new LinkedList<String>();
    // getting keys from properties
    for (Object o : props.keySet()) {
      if (((String) o).startsWith(snaaName + authorizationAtt)
          && ((String) o).endsWith(authorizationKeyAtt)) {
        keys.add((String) o);
      }
    }

    for (String k : keys) {
      String key = props.getProperty(k);
      // getting plain key-number from properties
      String plainKeyProperty = k.replaceAll(snaaName + authorizationAtt + ".", "");
      plainKeyProperty = plainKeyProperty.replaceAll(authorizationKeyAtt, "");

      String keyPrefix = snaaName + authorizationAtt + "." + plainKeyProperty;

      // building value-property-string
      String value = props.getProperty(keyPrefix + authorizationValAtt);

      // finally put key and values
      attributes.put(key, value);
    }

    return attributes;
  }
Esempio n. 4
0
    public String createCombinedTerm(String[] termArray) {
      String term1Combined = "";

      for (String aTermArray : termArray) {
        if (filterGrayList.containsKey(aTermArray.toLowerCase())) {
          String filterTerms = filterGrayList.get(aTermArray.toLowerCase());
          String[] splitFilterTerms = filterTerms.split(",");

          term1Combined = term1Combined + "(+\"" + aTermArray + "\" -(";
          for (String splitFilterTerm : splitFilterTerms) {
            term1Combined = term1Combined + "\"" + splitFilterTerm + "\" ";
          }
          term1Combined = term1Combined + ")) ";
        } else if (keepGrayList.containsKey(aTermArray.toLowerCase())) {
          String keepTerms = keepGrayList.get(aTermArray.toLowerCase());
          String[] splitKeepTerms = keepTerms.split(",");

          term1Combined = term1Combined + "(+\"" + aTermArray + "\" +(";
          for (String splitKeepTerm : splitKeepTerms) {
            term1Combined = term1Combined + "\"" + splitKeepTerm + "\" ";
          }
          term1Combined = term1Combined + ")) ";
        } else {
          term1Combined = term1Combined + "\"" + aTermArray + "\" ";
        }
      }

      return term1Combined;
    }
 private void init() {
   permNoToSymbol = Utils.loadMapping(originalStockFile);
   Map<String, Integer> symbolToPerm = new HashMap<String, Integer>();
   for (Map.Entry<Integer, String> entry : permNoToSymbol.entrySet()) {
     symbolToPerm.put(entry.getValue(), entry.getKey());
   }
 }
Esempio n. 6
0
  public static CompilerConfiguration generateCompilerConfigurationFromOptions(CommandLine cli)
      throws IOException {
    //
    // Setup the configuration data

    CompilerConfiguration configuration = new CompilerConfiguration();

    if (cli.hasOption("classpath")) {
      configuration.setClasspath(cli.getOptionValue("classpath"));
    }

    if (cli.hasOption('d')) {
      configuration.setTargetDirectory(cli.getOptionValue('d'));
    }

    if (cli.hasOption("encoding")) {
      configuration.setSourceEncoding(cli.getOptionValue("encoding"));
    }

    if (cli.hasOption("basescript")) {
      configuration.setScriptBaseClass(cli.getOptionValue("basescript"));
    }

    // joint compilation parameters
    if (cli.hasOption('j')) {
      Map<String, Object> compilerOptions = new HashMap<String, Object>();

      String[] opts = cli.getOptionValues("J");
      compilerOptions.put("namedValues", opts);

      opts = cli.getOptionValues("F");
      compilerOptions.put("flags", opts);

      configuration.setJointCompilationOptions(compilerOptions);
    }

    if (cli.hasOption("indy")) {
      configuration.getOptimizationOptions().put("int", false);
      configuration.getOptimizationOptions().put("indy", true);
    }

    if (cli.hasOption("configscript")) {
      String path = cli.getOptionValue("configscript");
      File groovyConfigurator = new File(path);
      Binding binding = new Binding();
      binding.setVariable("configuration", configuration);

      CompilerConfiguration configuratorConfig = new CompilerConfiguration();
      ImportCustomizer customizer = new ImportCustomizer();
      customizer.addStaticStars(
          "org.codehaus.groovy.control.customizers.builder.CompilerCustomizationBuilder");
      configuratorConfig.addCompilationCustomizers(customizer);

      GroovyShell shell = new GroovyShell(binding, configuratorConfig);
      shell.evaluate(groovyConfigurator);
    }

    return configuration;
  }
 private Map<String, Integer> convertSectorsToClazz(Map<String, List<String>> sectors) {
   List<String> sectorNames = new ArrayList<String>(sectors.keySet());
   Collections.sort(sectorNames);
   Map<String, Integer> sectorsToClazz = new HashMap<String, Integer>();
   for (int i = 0; i < sectorNames.size(); i++) {
     sectorsToClazz.put(sectorNames.get(i), i + 1);
     System.out.println(sectorNames.get(i) + ": " + (i + 1));
   }
   return sectorsToClazz;
 }
    public void handleStreamEvent(StreamEvent event) {
      if (event.eventType == StreamEvent.Type.STREAM_PREPARED) {
        SessionInfo session = ((StreamEvent.SessionPreparedEvent) event).session;
        sessionsByHost.put(session.peer, session);
      } else if (event.eventType == StreamEvent.Type.FILE_PROGRESS) {
        ProgressInfo progressInfo = ((StreamEvent.ProgressEvent) event).progress;

        // update progress
        Set<ProgressInfo> progresses = progressByHost.get(progressInfo.peer);
        if (progresses == null) {
          progresses = Sets.newSetFromMap(new ConcurrentHashMap<ProgressInfo, Boolean>());
          progressByHost.put(progressInfo.peer, progresses);
        }
        if (progresses.contains(progressInfo)) progresses.remove(progressInfo);
        progresses.add(progressInfo);

        StringBuilder sb = new StringBuilder();
        sb.append("\rprogress: ");

        long totalProgress = 0;
        long totalSize = 0;
        for (Map.Entry<InetAddress, Set<ProgressInfo>> entry : progressByHost.entrySet()) {
          SessionInfo session = sessionsByHost.get(entry.getKey());

          long size = session.getTotalSizeToSend();
          long current = 0;
          int completed = 0;
          for (ProgressInfo progress : entry.getValue()) {
            if (progress.currentBytes == progress.totalBytes) completed++;
            current += progress.currentBytes;
          }
          totalProgress += current;
          totalSize += size;
          sb.append("[").append(entry.getKey());
          sb.append(" ").append(completed).append("/").append(session.getTotalFilesToSend());
          sb.append(" (").append(size == 0 ? 100L : current * 100L / size).append("%)] ");
        }
        long time = System.nanoTime();
        long deltaTime = Math.max(1L, TimeUnit.NANOSECONDS.toMillis(time - lastTime));
        lastTime = time;
        long deltaProgress = totalProgress - lastProgress;
        lastProgress = totalProgress;

        sb.append("[total: ")
            .append(totalSize == 0 ? 100L : totalProgress * 100L / totalSize)
            .append("% - ");
        sb.append(mbPerSec(deltaProgress, deltaTime)).append("MB/s");
        sb.append(" (avg: ")
            .append(mbPerSec(totalProgress, TimeUnit.NANOSECONDS.toMillis(time - start)))
            .append("MB/s)]");

        System.out.print(sb.toString());
      }
    }
Esempio n. 9
0
 private boolean register(File f) {
   boolean rc = false;
   if (f.isDirectory()) {
     File file[] = f.listFiles(this);
     for (int i = 0; i < file.length; i++) {
       if (register(file[i])) rc = true;
     }
   } else if (dirMap.get(f) == null) {
     dirMap.put(f, new QEntry());
     rc = true;
   }
   return rc;
 }
Esempio n. 10
0
  private void applyLabel(String inPointsFile, String outPointsFile, List<String> symbols) {
    System.out.println("Applying labels for points file: " + inPointsFile);
    FileReader input;
    BufferedWriter bufWriter = null;
    try {
      FileOutputStream fos = new FileOutputStream(outPointsFile);
      bufWriter = new BufferedWriter(new OutputStreamWriter(fos));

      File inFile = new File(inPointsFile);
      if (!inFile.exists()) {
        System.out.println("ERROR: In file doens't exist");
        return;
      }
      input = new FileReader(inPointsFile);
      BufferedReader bufRead = new BufferedReader(input);
      String inputLine;
      int index = 0;
      while ((inputLine = bufRead.readLine()) != null && index < symbols.size()) {
        Point p = Utils.readPoint(inputLine);
        String symbol = symbols.get(index);
        int clazz = 0;
        if (this.invertedFixedClases.containsKey(symbol)) {
          clazz = this.invertedFixedClases.get(symbol);
        } else {
          // get the corresponding symbol
          // get the class for this one
          String sector = invertedSectors.get(symbol);
          if (sector != null) {
            clazz = sectorToClazz.get(sector);
          } else {
            //                    System.out.println("No sector: " + symbol);
          }
        }
        p.setClazz(clazz);
        String s = p.serialize();
        bufWriter.write(s);
        bufWriter.newLine();
        index++;
      }
      System.out.println("Read lines: " + index);
    } catch (Exception e) {
      throw new RuntimeException("Failed to read/write file", e);
    } finally {
      if (bufWriter != null) {
        try {
          bufWriter.close();
        } catch (IOException ignore) {
        }
      }
    }
  }
    private static void configureTransportFactory(
        ITransportFactory transportFactory, LoaderOptions opts) {
      Map<String, String> options = new HashMap<>();
      // If the supplied factory supports the same set of options as our SSL impl, set those
      if (transportFactory.supportedOptions().contains(SSLTransportFactory.TRUSTSTORE))
        options.put(SSLTransportFactory.TRUSTSTORE, opts.encOptions.truststore);
      if (transportFactory.supportedOptions().contains(SSLTransportFactory.TRUSTSTORE_PASSWORD))
        options.put(SSLTransportFactory.TRUSTSTORE_PASSWORD, opts.encOptions.truststore_password);
      if (transportFactory.supportedOptions().contains(SSLTransportFactory.PROTOCOL))
        options.put(SSLTransportFactory.PROTOCOL, opts.encOptions.protocol);
      if (transportFactory.supportedOptions().contains(SSLTransportFactory.CIPHER_SUITES))
        options.put(
            SSLTransportFactory.CIPHER_SUITES, Joiner.on(',').join(opts.encOptions.cipher_suites));

      if (transportFactory.supportedOptions().contains(SSLTransportFactory.KEYSTORE)
          && opts.encOptions.require_client_auth)
        options.put(SSLTransportFactory.KEYSTORE, opts.encOptions.keystore);
      if (transportFactory.supportedOptions().contains(SSLTransportFactory.KEYSTORE_PASSWORD)
          && opts.encOptions.require_client_auth)
        options.put(SSLTransportFactory.KEYSTORE_PASSWORD, opts.encOptions.keystore_password);

      // Now check if any of the factory's supported options are set as system properties
      for (String optionKey : transportFactory.supportedOptions())
        if (System.getProperty(optionKey) != null)
          options.put(optionKey, System.getProperty(optionKey));

      transportFactory.setOptions(options);
    }
 private static Cassandra.Client createThriftClient(
     String host, int port, String user, String passwd, ITransportFactory transportFactory)
     throws Exception {
   TTransport trans = transportFactory.openTransport(host, port);
   TProtocol protocol = new TBinaryProtocol(trans);
   Cassandra.Client client = new Cassandra.Client(protocol);
   if (user != null && passwd != null) {
     Map<String, String> credentials = new HashMap<>();
     credentials.put(IAuthenticator.USERNAME_KEY, user);
     credentials.put(IAuthenticator.PASSWORD_KEY, passwd);
     AuthenticationRequest authenticationRequest = new AuthenticationRequest(credentials);
     client.login(authenticationRequest);
   }
   return client;
 }
Esempio n. 13
0
  private Map<String, List<String>> loadHistoSectors(String sectorFile) {
    FileReader input;
    Map<String, List<String>> sectors = new HashMap<String, List<String>>();
    try {
      input = new FileReader(sectorFile);
      BufferedReader bufRead = new BufferedReader(input);
      String line;

      int i = 1;
      while ((line = bufRead.readLine()) != null) {
        Bin sectorRecord = Utils.readBin(line);
        List<String> stockList = sectorRecord.symbols;
        String startEnd = formatter.format(sectorRecord.end);
        String key = intgerFormaterr.format(i) + ":" + startEnd;
        sectors.put(key, stockList);
        for (String s : stockList) {
          invertedSectors.put(s, key);
        }
        i++;
      }
    } catch (IOException e) {
      throw new RuntimeException("Failed to load sector file", e);
    }
    return sectors;
  }
Esempio n. 14
0
 private void deploy() {
   List startList = new ArrayList();
   Iterator iter = dirMap.entrySet().iterator();
   try {
     while (iter.hasNext() && !shutdown) {
       Map.Entry entry = (Map.Entry) iter.next();
       File f = (File) entry.getKey();
       QEntry qentry = (QEntry) entry.getValue();
       long deployed = qentry.getDeployed();
       if (deployed == 0) {
         if (deploy(f)) {
           if (qentry.isQBean()) startList.add(qentry.getInstance());
           qentry.setDeployed(f.lastModified());
         } else {
           // deploy failed, clean up.
           iter.remove();
         }
       } else if (deployed != f.lastModified()) {
         undeploy(f);
         iter.remove();
         loader.forceNewClassLoaderOnNextScan();
       }
     }
     iter = startList.iterator();
     while (iter.hasNext()) {
       start((ObjectInstance) iter.next());
     }
   } catch (Exception e) {
     log.error("deploy", e);
   }
 }
  /**
   * Construct table schema from info stored in SSTable's Stats.db
   *
   * @param desc SSTable's descriptor
   * @return Restored CFMetaData
   * @throws IOException when Stats.db cannot be read
   */
  public static CFMetaData metadataFromSSTable(Descriptor desc) throws IOException {
    if (!desc.version.storeRows()) throw new IOException("pre-3.0 SSTable is not supported.");

    EnumSet<MetadataType> types =
        EnumSet.of(MetadataType.VALIDATION, MetadataType.STATS, MetadataType.HEADER);
    Map<MetadataType, MetadataComponent> sstableMetadata =
        desc.getMetadataSerializer().deserialize(desc, types);
    ValidationMetadata validationMetadata =
        (ValidationMetadata) sstableMetadata.get(MetadataType.VALIDATION);
    SerializationHeader.Component header =
        (SerializationHeader.Component) sstableMetadata.get(MetadataType.HEADER);

    IPartitioner partitioner =
        SecondaryIndexManager.isIndexColumnFamily(desc.cfname)
            ? new LocalPartitioner(header.getKeyType())
            : FBUtilities.newPartitioner(validationMetadata.partitioner);

    CFMetaData.Builder builder =
        CFMetaData.Builder.create("keyspace", "table").withPartitioner(partitioner);
    header
        .getStaticColumns()
        .entrySet()
        .stream()
        .forEach(
            entry -> {
              ColumnIdentifier ident =
                  ColumnIdentifier.getInterned(UTF8Type.instance.getString(entry.getKey()), true);
              builder.addStaticColumn(ident, entry.getValue());
            });
    header
        .getRegularColumns()
        .entrySet()
        .stream()
        .forEach(
            entry -> {
              ColumnIdentifier ident =
                  ColumnIdentifier.getInterned(UTF8Type.instance.getString(entry.getKey()), true);
              builder.addRegularColumn(ident, entry.getValue());
            });
    builder.addPartitionKey("PartitionKey", header.getKeyType());
    for (int i = 0; i < header.getClusteringTypes().size(); i++) {
      builder.addClusteringColumn(
          "clustering" + (i > 0 ? i : ""), header.getClusteringTypes().get(i));
    }
    return builder.build();
  }
Esempio n. 16
0
  public String getPath() {
    Map<String, String> environment = new ProcessBuilder().environment();

    String path = environment.get("Path");
    if (path == null) {
      path = environment.get("PATH");
    }
    if (path == null) {
      path = environment.get("path");
    }
    if (path == null) {
      throw new UnexpectedLiquibaseException(
          "Cannot find path variable in environment. Possible variables are "
              + StringUtils.join(environment.keySet(), ","));
    }

    return path;
  }
  private Map<String, String[]> readRoles(File f) throws IOException {
    Map<String, String[]> map = new TreeMap<String, String[]>();

    LineNumberReader lr =
        new LineNumberReader(new InputStreamReader(new FileInputStream(f), "UTF-8"));
    String line = null;
    StringBuilder sb = new StringBuilder();

    while ((line = lr.readLine()) != null) {
      String[] s = line.toLowerCase().split("\t");
      if (s.length > 2) {
        logger.trace(Arrays.toString(s));
        map.put(s[1], s);
      }
    }

    return map;
  }
Esempio n. 18
0
  private void undeploy() {
    Object[] set = dirMap.entrySet().toArray();
    int l = set.length;

    while (l-- > 0) {
      Map.Entry entry = (Map.Entry) set[l];
      File f = (File) entry.getKey();
      undeploy(f);
    }
  }
Esempio n. 19
0
  public void process() {
    File inFolder = new File(vectorFolder);
    if (!inFolder.isDirectory()) {
      System.out.println("In should be a folder: " + vectorFolder);
      return;
    }
    boolean clusterSaved = false;
    this.invertedFixedClases = loadFixedClasses(fixedClassesFile);
    if (!histogram) {
      Map<String, List<String>> sectors = loadStockSectors(sectorFile);
      sectorToClazz = convertSectorsToClazz(sectors);
      if (!clusterSaved) {
        changeClassLabels();
        clusterSaved = true;
      }
      for (Map.Entry<String, Integer> entry : sectorToClazz.entrySet()) {
        System.out.println(entry.getKey() + " : " + entry.getValue());
      }
    }

    for (File inFile : inFolder.listFiles()) {
      String fileName = inFile.getName();
      String fileNameWithOutExt = FilenameUtils.removeExtension(fileName);
      if (histogram) {
        sectorToClazz.clear();
        invertedSectors.clear();

        Map<String, List<String>> sectors =
            loadHistoSectors(sectorFile + "/" + fileNameWithOutExt + ".csv");
        sectorToClazz = convertSectorsToClazz(sectors);
        if (!clusterSaved) {
          changeClassLabels();
          clusterSaved = true;
        }
        for (Map.Entry<String, Integer> entry : sectorToClazz.entrySet()) {
          System.out.println(entry.getKey() + " : " + entry.getValue());
        }
      }
      processFile(fileNameWithOutExt);
    }
  }
Esempio n. 20
0
  private void undeploy(File f) {
    QEntry qentry = (QEntry) dirMap.get(f);
    try {
      if (log != null) log.trace("undeploying:" + f.getCanonicalPath());

      Object obj = qentry.getObject();
      ObjectName name = qentry.getObjectName();
      factory.destroyQBean(this, name, obj);
      if (log != null) log.info("undeployed:" + f.getCanonicalPath());
    } catch (Exception e) {
      getLog().warn("undeploy", e);
    }
  }
Esempio n. 21
0
 // load symbols for each point in file
 private List<String> loadSymbols(String vectorFile) {
   System.out.println("Loading symbols from vector file: " + vectorFile);
   File vf = new File(vectorFile);
   List<VectorPoint> vectorPoints = Utils.readVectors(vf, 0, 7000);
   List<String> symbols = new ArrayList<String>();
   for (int i = 0; i < vectorPoints.size(); i++) {
     VectorPoint v = vectorPoints.get(i);
     String symbol = permNoToSymbol.get(v.getKey());
     symbols.add(symbol);
   }
   System.out.println("No of symbols for point: " + symbols.size());
   return symbols;
 }
Esempio n. 22
0
  private boolean deploy(File f) {
    try {
      if (log != null) log.info("deploy:" + f.getCanonicalPath());
      QEntry qentry = (QEntry) dirMap.get(f);
      SAXBuilder builder = createSAXBuilder();
      Document doc;
      if (decorator != null && !f.getName().equals(LOGGER_CONFIG)) {
        doc = decrypt(builder.build(new StringReader(decorator.decorateFile(f))));
      } else {
        doc = decrypt(builder.build(f));
      }

      Element rootElement = doc.getRootElement();
      String iuuid = rootElement.getAttributeValue("instance");
      if (iuuid != null) {
        UUID uuid = UUID.fromString(iuuid);
        if (!uuid.equals(getInstanceId())) {
          deleteFile(f, iuuid);
          return false;
        }
      }
      Object obj = factory.instantiate(this, rootElement);
      qentry.setObject(obj);

      ObjectInstance instance = factory.createQBean(this, doc.getRootElement(), obj);
      qentry.setInstance(instance);
    } catch (InstanceAlreadyExistsException e) {
      /*
       * Ok, the file we tried to deploy, holds an object
       *  that already has been deployed.
       *
       * Rename it out of the way.
       *
       */
      tidyFileAway(f, DUPLICATE_EXTENSION);
      getLog().warn("deploy", e);
      return false;
    } catch (Exception e) {
      getLog().warn("deploy", e);
      tidyFileAway(f, ERROR_EXTENSION);
      // This will also save deploy error repeats...
      return false;
    } catch (Error e) {
      getLog().warn("deploy", e);
      tidyFileAway(f, ENV_EXTENSION);
      // This will also save deploy error repeats...
      return false;
    }
    return true;
  }
Esempio n. 23
0
 private void checkModified() {
   Iterator iter = dirMap.entrySet().iterator();
   while (iter.hasNext()) {
     Map.Entry entry = (Map.Entry) iter.next();
     File f = (File) entry.getKey();
     QEntry qentry = (QEntry) entry.getValue();
     if (qentry.isQBean() && qentry.isQPersist()) {
       ObjectName name = qentry.getObjectName();
       if (getState(name) == QBean.STARTED && isModified(name)) {
         qentry.setDeployed(persist(f, name));
       }
     }
   }
 }
  private Map<Integer, String> readTraining(File f) throws IOException {
    Map<Integer, String> map = new TreeMap<Integer, String>();

    LineNumberReader lr =
        new LineNumberReader(new InputStreamReader(new FileInputStream(f), "UTF-8"));
    String line = null;
    StringBuilder sb = new StringBuilder();
    int id = -1;
    while ((line = lr.readLine()) != null) {
      String[] s = line.split("\t");
      if (s.length > 0 && s[0].length() > 0 && !s[0].equals("sid")) {
        sb.append(line + "\n");
        id = Integer.parseInt(s[0]);
      } else {
        if (sb.length() > 0) {
          map.put(id, sb.toString());
        }

        sb = new StringBuilder();
      }
    }
    map.put(id, sb.toString());
    return map;
  }
    @Override
    public void init(String keyspace) {
      Iterator<InetAddress> hostiter = hosts.iterator();
      while (hostiter.hasNext()) {
        try {
          // Query endpoint to ranges map and schemas from thrift
          InetAddress host = hostiter.next();
          Cassandra.Client client =
              createThriftClient(
                  host.getHostAddress(), rpcPort, this.user, this.passwd, this.transportFactory);

          setPartitioner(client.describe_partitioner());
          Token.TokenFactory tkFactory = getPartitioner().getTokenFactory();

          for (TokenRange tr : client.describe_ring(keyspace)) {
            Range<Token> range =
                new Range<>(
                    tkFactory.fromString(tr.start_token),
                    tkFactory.fromString(tr.end_token),
                    getPartitioner());
            for (String ep : tr.endpoints) {
              addRangeForEndpoint(range, InetAddress.getByName(ep));
            }
          }

          String query =
              String.format(
                  "SELECT * FROM %s.%s WHERE keyspace_name = '%s'",
                  Keyspace.SYSTEM_KS, SystemKeyspace.SCHEMA_COLUMNFAMILIES_CF, keyspace);
          CqlResult result =
              client.execute_cql3_query(
                  ByteBufferUtil.bytes(query), Compression.NONE, ConsistencyLevel.ONE);
          for (CqlRow row : result.rows) {
            CFMetaData metadata = CFMetaData.fromThriftCqlRow(row);
            knownCfs.put(metadata.cfName, metadata);
          }
          break;
        } catch (Exception e) {
          if (!hostiter.hasNext())
            throw new RuntimeException("Could not retrieve endpoint ranges: ", e);
        }
      }
    }
Esempio n. 26
0
  private Map<String, List<String>> loadStockSectors(String sectorFile) {
    FileReader input;
    Map<String, List<String>> sectors = new HashMap<String, List<String>>();
    try {
      input = new FileReader(sectorFile);
      BufferedReader bufRead = new BufferedReader(input);
      String line;

      while ((line = bufRead.readLine()) != null) {
        SectorRecord sectorRecord = Utils.readSectorRecord(line);
        List<String> stockList = sectors.get(sectorRecord.getSector());
        if (stockList == null) {
          stockList = new ArrayList<String>();
          sectors.put(sectorRecord.getSector(), stockList);
        }
        stockList.add(sectorRecord.getSymbol());

        invertedSectors.put(sectorRecord.getSymbol(), sectorRecord.getSector());
      }
    } catch (IOException e) {
      throw new RuntimeException("Failed to load sector file", e);
    }
    return sectors;
  }
  public AnnotationMigration(File labelled, File unlabelled, File roles, File output, int start)
      throws IOException {
    pw =
        new PrintWriter(
            new BufferedWriter(new OutputStreamWriter(new FileOutputStream(output), "UTF-8")));
    tokenier = new HardTokenizer();
    Map<Integer, String> labelledMap = readTraining(labelled);
    Map<Integer, String> unlabelledMap = readTraining(unlabelled);
    roleMap = readRoles(roles);
    // logger.info(labelledMap);
    if (labelledMap.size() != unlabelledMap.size()) {
      logger.warn(labelledMap.size() + " != " + unlabelledMap.size());
    }

    for (int i = start; i < labelledMap.size(); i++) {
      logger.info(i);

      logger.info("\n" + labelledMap.get(i));
      logger.info("\n" + unlabelledMap.get(i));

      process(labelledMap.get(i), unlabelledMap.get(i));
      logger.info("***\n");
    }

    /*Iterator<Integer> it = unlabelledMap.keySet().iterator();
    while(it.hasNext()) {
    	int id = it.next();
    	if (labelledMap.get(id)==null){
    		logger.warn(id);
    		logger.warn(unlabelledMap.get(id));

    	}
    }*/
    pw.close();
  }
 @Override
 public CFMetaData getCFMetaData(String keyspace, String cfName) {
   return knownCfs.get(cfName);
 }
Esempio n. 29
0
  private static long getTermCount(
      SolrServer server,
      Map<String, Integer> singleCountMap,
      SearchTermAndList searchTermAndList,
      Map<String, String> filterGrayList,
      Map<String, String> keepGrayList) {
    List<String[]> searchTerms = searchTermAndList.asListArray();
    Integer searchTermCountObject =
        searchTerms.size() != 1 ? null : singleCountMap.get(join(searchTerms.get(0), ","));
    long searchTermCount = 0;
    if (searchTermCountObject == null) {
      // didn't find it in map, so need to go get count
      SolrQuery query = new SolrQuery();
      query.setQuery("+text:(*:*)");
      query.addFilterQuery("+pub_date_year:[1993 TO 2013]");
      query.setParam("fl", "pmid");

      for (int i = 0; i < searchTerms.size(); i++) {
        String term1 = "";
        for (String aTermArray : searchTerms.get(i)) {
          if (filterGrayList.containsKey(aTermArray.toLowerCase())) {
            String filterTerms = filterGrayList.get(aTermArray.toLowerCase());
            String[] splitFilterTerms = filterTerms.split(",");
            term1 = term1 + "(+\"" + aTermArray + "\" -(";

            for (String splitFilterTerm : splitFilterTerms) {
              term1 = term1 + "\"" + splitFilterTerm + "\" ";
            }
            term1 = term1 + ")) ";
          } else if (keepGrayList.containsKey(aTermArray.toLowerCase())) {
            String keepTerms = keepGrayList.get(aTermArray.toLowerCase());
            String[] splitKeepTerms = keepTerms.split(",");
            term1 = term1 + "(+\"" + aTermArray + "\" +(";

            for (String splitKeepTerm : splitKeepTerms) {
              term1 = term1 + "\"" + splitKeepTerm + "\" ";
            }
            term1 = term1 + ")) ";
          } else {
            term1 = term1 + "\"" + aTermArray + "\" ";
          }
        }

        query.addFilterQuery("+text:(" + term1 + ")");
      }

      try {
        QueryResponse rsp = server.query(query);
        searchTermCount = rsp.getResults().getNumFound();
        singleCountMap.put(
            join(searchTerms.get(0), ","), Integer.parseInt(Long.toString(searchTermCount)));
      } catch (SolrServerException e) {
        // exit out if there is an error
        log.warning(e.getMessage());
        System.exit(1);
      }
    } else {
      searchTermCount = searchTermCountObject;
    }
    return searchTermCount;
  }
  private void process(String labelled, String unlabelled) throws IOException {
    if (labelled == null || unlabelled == null) {
      return;
    }
    List<String[]> examples = extractExamples(unlabelled);
    List<String> roles = extractRoles(labelled);
    boolean[] found = new boolean[roles.size()];

    logger.debug("roles: " + roles);
    for (int i = 0; i < roles.size(); i++) {

      String[] t = roles.get(i).split("\t");
      String[] translatedRole = roleMap.get(t[0]);
      if (translatedRole != null) {
        for (int k = 2; k < translatedRole.length; k++) {
          logger.debug(i + "\t" + roles.get(i) + "\t" + translatedRole[k]);
          if (translatedRole[k] != null) {
            Boundary boundary = matcher(translatedRole[k], examples);
            if (boundary != null) {
              logger.info("///");
              logger.debug(boundary + "\t" + translatedRole[k] + ":");
              logger.debug(boundary.begin + "\t" + Arrays.toString(examples.get(boundary.begin)));
              examples.get(boundary.begin)[5] = t[1];
              examples.get(boundary.begin)[6] = t[2];
              for (int j = boundary.begin + 1; j <= boundary.end; j++) {
                logger.debug(j + "\t" + Arrays.toString(examples.get(j)));
                examples.get(j)[5] = t[1];
                examples.get(j)[6] = t[2].replaceAll("B-", "I-");
              }
              found[i] = true;
              logger.info("\\\\\\ break at " + k + "/" + translatedRole.length);
              break;
            }
          }
        }
      }
    }

    logger.info("===");
    boolean b = false;
    for (int i = 0; i < found.length; i++) {
      if (!found[i]) {
        String[] t = roles.get(i).split("\t");
        logger.info("# Missing " + t[0] + "\t" + t[1] + "\t" + t[2]);

        pw.println("# Missing " + t[0] + "\t" + t[1] + "\t" + t[2]);
        b = true;
      }
    }
    if (b) {
      StringBuilder sb = new StringBuilder();
      for (int j = 0; j < examples.size(); j++) {
        if (j > 0) {
          sb.append(" ");
        }
        sb.append(examples.get(j)[2]);
      }
      logger.info("# " + sb.toString());
    }

    for (int i = 0; i < examples.size(); i++) {
      String[] example = examples.get(i);
      logger.info(
          example[0]
              + "\t"
              + example[1]
              + "\t"
              + example[2]
              + "\t"
              + example[3]
              + "\t"
              + example[4]
              + "\t"
              + example[5]
              + "\t"
              + example[6]);
      pw.println(
          example[0]
              + "\t"
              + example[1]
              + "\t"
              + example[2]
              + "\t"
              + example[3]
              + "\t"
              + example[4]
              + "\t"
              + example[5]
              + "\t"
              + example[6]);
    }
    pw.print("\n");
    logger.info("***");
  }