Example #1
0
  /** Extracts document and cluster lists before serialization. */
  @Persist
  private void beforeSerialization() {
    /*
     * See http://issues.carrot2.org/browse/CARROT-693; this monitor does not save us
     * in multi-threaded environment anyway. A better solution would be to prepare
     * this eagerly in the constructor, but we try to balance overhead and full
     * correctness here.
     */
    synchronized (this) {
      query = (String) attributes.get(AttributeNames.QUERY);

      if (getDocuments() != null) {
        documents = Lists.newArrayList(getDocuments());
      } else {
        documents = null;
      }

      if (getClusters() != null) {
        clusters = Lists.newArrayList(getClusters());
      } else {
        clusters = null;
      }

      otherAttributesForSerialization = MapUtils.asHashMap(SimpleXmlWrappers.wrap(attributes));
      otherAttributesForSerialization.remove(AttributeNames.QUERY);
      otherAttributesForSerialization.remove(AttributeNames.CLUSTERS);
      otherAttributesForSerialization.remove(AttributeNames.DOCUMENTS);
      if (otherAttributesForSerialization.isEmpty()) {
        otherAttributesForSerialization = null;
      }
    }
  }
Example #2
0
  private static List<String> loadLines(InputStream stream, Charset charset) throws IOException {
    BufferedReader reader = new BufferedReader(new InputStreamReader(stream, charset));

    boolean threw = true;
    try {
      List<String> strings = Lists.newArrayList();

      String line = reader.readLine();
      while (line != null) {
        strings.add(line);
        line = reader.readLine();
      }

      threw = false;
      return strings;

    } finally {
      try {
        reader.close();
      } catch (IOException e) {
        if (!threw) {
          throw e; // if there was an initial exception, don't hide it
        }
      }
    }
  }
Example #3
0
 @Override
 public void readFields(DataInput input) throws IOException {
   byte[] tableNameBytes = Bytes.readByteArray(input);
   PName tableName = new PNameImpl(tableNameBytes);
   PTableType tableType = PTableType.values()[WritableUtils.readVInt(input)];
   long sequenceNumber = WritableUtils.readVLong(input);
   long timeStamp = input.readLong();
   byte[] pkNameBytes = Bytes.readByteArray(input);
   String pkName = pkNameBytes.length == 0 ? null : Bytes.toString(pkNameBytes);
   int nColumns = WritableUtils.readVInt(input);
   List<PColumn> columns = Lists.newArrayListWithExpectedSize(nColumns);
   for (int i = 0; i < nColumns; i++) {
     PColumn column = new PColumnImpl();
     column.readFields(input);
     columns.add(column);
   }
   Map<String, byte[][]> guidePosts = new HashMap<String, byte[][]>();
   int size = WritableUtils.readVInt(input);
   for (int i = 0; i < size; i++) {
     String key = WritableUtils.readString(input);
     int valueSize = WritableUtils.readVInt(input);
     byte[][] value = new byte[valueSize][];
     for (int j = 0; j < valueSize; j++) {
       value[j] = Bytes.readByteArray(input);
     }
     guidePosts.put(key, value);
   }
   PTableStats stats = new PTableStatsImpl(guidePosts);
   init(tableName, tableType, timeStamp, sequenceNumber, pkName, columns, stats);
 }
  /**
   * 将原始数据切割后装入ThreadInfo,并以Key=WaitID,Value= ThreadInfo实体放入到Multimap<String, ThreadInfo>集合中
   * ps:Multimap 类似于Map<key,collection>, key:value-> 1:n
   *
   * @param rawDatas
   * @return
   */
  public Multimap<String, ThreadInfo> getThreadInfo(List<String[]> rawDatas) {
    Multimap<String, ThreadInfo> w_IdMap = HashMultimap.create();
    List<ThreadInfo> threadsList = Lists.newArrayList();
    for (String[] rawData : rawDatas) {
      ThreadInfo threadInfo = new ThreadInfo();
      Pattern t_id = Pattern.compile("tid=(0x[\\d\\w]+)");
      Pattern t_name = Pattern.compile("\"([\\d\\D]*)\"");
      Pattern w_Id = Pattern.compile("\\[(0x[\\d\\w]+)\\]");
      Matcher tIdMatcher = t_id.matcher(rawData[0]);
      Matcher nameMatcher = t_name.matcher(rawData[0]);
      Matcher w_IdMatcher = w_Id.matcher(rawData[0]);
      if (tIdMatcher.find()) {
        threadInfo.setThreadId(tIdMatcher.group(1));
      }

      if (nameMatcher.find()) {
        threadInfo.setThreadName(nameMatcher.group(1));
      }

      if (w_IdMatcher.find()) {
        threadInfo.setWaitThreadId(w_IdMatcher.group(1));
      }
      threadInfo.setThreadCondition(rawData[1]);
      w_IdMap.put(threadInfo.getWaitThreadId(), threadInfo);
    }
    return w_IdMap;
  }
  /**
   * 清除掉没有处于waiting on condition状态的线程
   *
   * @param rawDatas
   * @return
   */
  public List<String[]> remove(List<String[]> rawDatas) {
    List<String[]> pickedData = Lists.newArrayList();
    for (String[] temp : rawDatas) {
      if (temp[0].lastIndexOf("waiting on condition") > -1) pickedData.add(temp);
    }

    return pickedData;
  }
  public static List<NameValuePair> convertAttributesToNameValuePair(
      Map<String, String> aAttributes) {
    List<NameValuePair> myNameValuePairs = Lists.newArrayList();

    for (Map.Entry<String, String> myEntry : aAttributes.entrySet()) {
      myNameValuePairs.add(new BasicNameValuePair(myEntry.getKey(), myEntry.getValue()));
    }
    return myNameValuePairs;
  }
Example #7
0
 @Override
 public Iterable<DataKey> getSubKeys() {
   final Tag tag = this.findLastTag(this.path, false);
   if (!(tag instanceof CompoundTag)) {
     return (Iterable<DataKey>) Collections.emptyList();
   }
   final List<DataKey> subKeys = (List<DataKey>) Lists.newArrayList();
   for (final String name : ((CompoundTag) tag).getValue().keySet()) {
     subKeys.add(new NBTKey(this.createRelativeKey(name)));
   }
   return subKeys;
 }
Example #8
0
 @GET
 @Path("/files")
 public List<ToDo> files() {
   List<ToDo> response = Lists.newArrayList();
   File[] files = new File("/lib").listFiles();
   if (files != null) {
     for (File file : files) {
       if (file.isFile())
         response.add(
             new ToDo(
                 ++id,
                 file.getName(),
                 file.isFile() ? file.length() : -1,
                 System.currentTimeMillis()));
     }
   }
   return response;
 }
  protected void scan() {
    if (configuration.getUrls() == null || configuration.getUrls().isEmpty()) {
      if (log != null) log.warn("given scan urls are empty. set urls in the configuration");
      return;
    }

    if (log != null && log.isDebugEnabled()) {
      log.debug("going to scan these urls:\n" + Joiner.on("\n").join(configuration.getUrls()));
    }

    long time = System.currentTimeMillis();
    int scannedUrls = 0;
    ExecutorService executorService = configuration.getExecutorService();
    List<Future<?>> futures = Lists.newArrayList();

    for (final URL url : configuration.getUrls()) {
      try {
        if (executorService != null) {
          futures.add(
              executorService.submit(
                  new Runnable() {
                    public void run() {
                      if (log != null && log.isDebugEnabled())
                        log.debug("[" + Thread.currentThread().toString() + "] scanning " + url);
                      scan(url);
                    }
                  }));
        } else {
          scan(url);
        }
        scannedUrls++;
      } catch (ReflectionsException e) {
        if (log != null && log.isWarnEnabled())
          log.warn("could not create Vfs.Dir from url. ignoring the exception and continuing", e);
      }
    }

    // todo use CompletionService
    if (executorService != null) {
      for (Future future : futures) {
        try {
          future.get();
        } catch (Exception e) {
          throw new RuntimeException(e);
        }
      }
    }

    time = System.currentTimeMillis() - time;

    // gracefully shutdown the parallel scanner executor service.
    if (executorService != null) {
      executorService.shutdown();
    }

    if (log != null) {
      int keys = 0;
      int values = 0;
      for (String index : store.keySet()) {
        keys += store.get(index).keySet().size();
        values += store.get(index).size();
      }

      log.info(
          format(
              "Reflections took %d ms to scan %d urls, producing %d keys and %d values %s",
              time,
              scannedUrls,
              keys,
              values,
              executorService != null && executorService instanceof ThreadPoolExecutor
                  ? format(
                      "[using %d cores]",
                      ((ThreadPoolExecutor) executorService).getMaximumPoolSize())
                  : ""));
    }
  }
Example #10
0
  private void init(
      PName name,
      PTableType type,
      long timeStamp,
      long sequenceNumber,
      String pkName,
      List<PColumn> columns,
      PTableStats stats) {
    this.name = name;
    this.type = type;
    this.timeStamp = timeStamp;
    this.sequenceNumber = sequenceNumber;
    this.columnsByName = ArrayListMultimap.create(columns.size(), 1);
    this.pkName = pkName;
    List<PColumn> pkColumns = Lists.newArrayListWithExpectedSize(columns.size() - 1);
    PColumn[] allColumns = new PColumn[columns.size()];
    RowKeySchemaBuilder builder = new RowKeySchemaBuilder();
    for (int i = 0; i < allColumns.length; i++) {
      PColumn column = columns.get(i);
      allColumns[column.getPosition()] = column;
      PName familyName = column.getFamilyName();
      if (familyName == null) {
        pkColumns.add(column);
        builder.addField(column);
      }
      columnsByName.put(column.getName().getString(), column);
    }
    this.pkColumns = ImmutableList.copyOf(pkColumns);
    this.rowKeySchema = builder.setMinNullable(pkColumns.size()).build();
    this.allColumns = ImmutableList.copyOf(allColumns);

    // Two pass so that column order in column families matches overall column order
    // and to ensure that column family order is constant
    int maxExpectedSize = allColumns.length - pkColumns.size();
    // Maintain iteration order so that column families are ordered as they are listed
    Map<PName, List<PColumn>> familyMap = Maps.newLinkedHashMap();
    for (PColumn column : allColumns) {
      PName familyName = column.getFamilyName();
      if (familyName != null) {
        List<PColumn> columnsInFamily = familyMap.get(familyName);
        if (columnsInFamily == null) {
          columnsInFamily = Lists.newArrayListWithExpectedSize(maxExpectedSize);
          familyMap.put(familyName, columnsInFamily);
        }
        columnsInFamily.add(column);
      }
    }

    Iterator<Map.Entry<PName, List<PColumn>>> iterator = familyMap.entrySet().iterator();
    PColumnFamily[] families = new PColumnFamily[familyMap.size()];
    ImmutableMap.Builder<String, PColumnFamily> familyByString = ImmutableMap.builder();
    ImmutableSortedMap.Builder<byte[], PColumnFamily> familyByBytes =
        ImmutableSortedMap.orderedBy(Bytes.BYTES_COMPARATOR);
    for (int i = 0; i < families.length; i++) {
      Map.Entry<PName, List<PColumn>> entry = iterator.next();
      PColumnFamily family = new PColumnFamilyImpl(entry.getKey(), entry.getValue());
      families[i] = family;
      familyByString.put(family.getName().getString(), family);
      familyByBytes.put(family.getName().getBytes(), family);
    }
    this.families = ImmutableList.copyOf(families);
    this.familyByBytes = familyByBytes.build();
    this.familyByString = familyByString.build();
    this.stats = stats;
  }