public static TabletLocationState createTabletLocationState(Key k, Value v)
      throws IOException, BadLocationStateException {
    final SortedMap<Key, Value> decodedRow = WholeRowIterator.decodeRow(k, v);
    KeyExtent extent = null;
    TServerInstance future = null;
    TServerInstance current = null;
    TServerInstance last = null;
    long lastTimestamp = 0;
    List<Collection<String>> walogs = new ArrayList<Collection<String>>();
    boolean chopped = false;

    for (Entry<Key, Value> entry : decodedRow.entrySet()) {
      Key key = entry.getKey();
      Text row = key.getRow();
      Text cf = key.getColumnFamily();
      Text cq = key.getColumnQualifier();

      if (cf.compareTo(TabletsSection.FutureLocationColumnFamily.NAME) == 0) {
        TServerInstance location = new TServerInstance(entry.getValue(), cq);
        if (future != null) {
          throw new BadLocationStateException(
              "found two assignments for the same extent "
                  + key.getRow()
                  + ": "
                  + future
                  + " and "
                  + location);
        }
        future = location;
      } else if (cf.compareTo(TabletsSection.CurrentLocationColumnFamily.NAME) == 0) {
        TServerInstance location = new TServerInstance(entry.getValue(), cq);
        if (current != null) {
          throw new BadLocationStateException(
              "found two locations for the same extent "
                  + key.getRow()
                  + ": "
                  + current
                  + " and "
                  + location);
        }
        current = location;
      } else if (cf.compareTo(LogColumnFamily.NAME) == 0) {
        String[] split = entry.getValue().toString().split("\\|")[0].split(";");
        walogs.add(Arrays.asList(split));
      } else if (cf.compareTo(TabletsSection.LastLocationColumnFamily.NAME) == 0) {
        if (lastTimestamp < entry.getKey().getTimestamp())
          last = new TServerInstance(entry.getValue(), cq);
      } else if (cf.compareTo(ChoppedColumnFamily.NAME) == 0) {
        chopped = true;
      } else if (TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.equals(cf, cq)) {
        extent = new KeyExtent(row, entry.getValue());
      }
    }
    if (extent == null) {
      log.warn("No prev-row for key extent: " + decodedRow);
      return null;
    }
    return new TabletLocationState(extent, future, current, last, walogs, chopped);
  }
Beispiel #2
0
 @Override
 public int compareTo(Pair p) {
   int compare = element.compareTo(p.element);
   if (compare != 0) return compare;
   if (neighbour.toString() == "*") return -1;
   else if (p.neighbour.toString() == "*") return 1;
   else return neighbour.compareTo(p.neighbour);
 }
 @Override
 public int compareTo(TextPair tp) {
   int cmp = first.compareTo(tp.first);
   if (cmp != 0) {
     return cmp;
   }
   return second.compareTo(tp.second);
 }
Beispiel #4
0
  public static void splitDatafiles(
      Text table,
      Text midRow,
      double splitRatio,
      Map<FileRef, FileUtil.FileInfo> firstAndLastRows,
      SortedMap<FileRef, DataFileValue> datafiles,
      SortedMap<FileRef, DataFileValue> lowDatafileSizes,
      SortedMap<FileRef, DataFileValue> highDatafileSizes,
      List<FileRef> highDatafilesToRemove) {

    for (Entry<FileRef, DataFileValue> entry : datafiles.entrySet()) {

      Text firstRow = null;
      Text lastRow = null;

      boolean rowsKnown = false;

      FileUtil.FileInfo mfi = firstAndLastRows.get(entry.getKey());

      if (mfi != null) {
        firstRow = mfi.getFirstRow();
        lastRow = mfi.getLastRow();
        rowsKnown = true;
      }

      if (rowsKnown && firstRow.compareTo(midRow) > 0) {
        // only in high
        long highSize = entry.getValue().getSize();
        long highEntries = entry.getValue().getNumEntries();
        highDatafileSizes.put(
            entry.getKey(), new DataFileValue(highSize, highEntries, entry.getValue().getTime()));
      } else if (rowsKnown && lastRow.compareTo(midRow) <= 0) {
        // only in low
        long lowSize = entry.getValue().getSize();
        long lowEntries = entry.getValue().getNumEntries();
        lowDatafileSizes.put(
            entry.getKey(), new DataFileValue(lowSize, lowEntries, entry.getValue().getTime()));

        highDatafilesToRemove.add(entry.getKey());
      } else {
        long lowSize = (long) Math.floor((entry.getValue().getSize() * splitRatio));
        long lowEntries = (long) Math.floor((entry.getValue().getNumEntries() * splitRatio));
        lowDatafileSizes.put(
            entry.getKey(), new DataFileValue(lowSize, lowEntries, entry.getValue().getTime()));

        long highSize = (long) Math.ceil((entry.getValue().getSize() * (1.0 - splitRatio)));
        long highEntries =
            (long) Math.ceil((entry.getValue().getNumEntries() * (1.0 - splitRatio)));
        highDatafileSizes.put(
            entry.getKey(), new DataFileValue(highSize, highEntries, entry.getValue().getTime()));
      }
    }
  }
Beispiel #5
0
 @Override
 public int compareTo(TextLong other) {
   int cmp = first.compareTo(other.getFirst());
   if (0 == cmp) {
     cmp = second.compareTo(other.getSecond());
   }
   return cmp;
 }
  @Override
  public int compareTo(TermDocIdWritable tuple) {
    int cmp = __term.compareTo(tuple.getTerm());
    if (cmp != 0) {
      return cmp;
    }

    return __documentId.compareTo(tuple.getDocumentId());
  }
 public int compareTo(BigramWritable bw) {
   int cmp = leftBigram.compareTo(bw.leftBigram);
   if (cmp != 0) return cmp;
   else return rightBigram.compareTo(bw.rightBigram);
 }
 public int compare(int i, int j) {
   Text left = records.get(i);
   Text right = records.get(j);
   return left.compareTo(right);
 }
Beispiel #9
0
 public int baseCompareTo(TextLong other) {
   int cmp = first.compareTo(other.getFirst());
   return cmp;
 }
Beispiel #10
0
  /** Delete the dst files/dirs which do not exist in src */
  private static void deleteNonexisting(
      FileSystem dstfs,
      FileStatus dstroot,
      Path dstsorted,
      FileSystem jobfs,
      Path jobdir,
      JobConf jobconf,
      Configuration conf)
      throws IOException {
    if (!dstroot.isDir()) {
      throw new IOException(
          "dst must be a directory when option "
              + Options.DELETE.cmd
              + " is set, but dst (= "
              + dstroot.getPath()
              + ") is not a directory.");
    }

    // write dst lsr results
    final Path dstlsr = new Path(jobdir, "_distcp_dst_lsr");
    final SequenceFile.Writer writer =
        SequenceFile.createWriter(
            jobfs,
            jobconf,
            dstlsr,
            Text.class,
            FileStatus.class,
            SequenceFile.CompressionType.NONE);
    try {
      // do lsr to get all file statuses in dstroot
      final Stack<FileStatus> lsrstack = new Stack<FileStatus>();
      for (lsrstack.push(dstroot); !lsrstack.isEmpty(); ) {
        final FileStatus status = lsrstack.pop();
        if (status.isDir()) {
          for (FileStatus child : dstfs.listStatus(status.getPath())) {
            String relative = makeRelative(dstroot.getPath(), child.getPath());
            writer.append(new Text(relative), child);
            lsrstack.push(child);
          }
        }
      }
    } finally {
      checkAndClose(writer);
    }

    // sort lsr results
    final Path sortedlsr = new Path(jobdir, "_distcp_dst_lsr_sorted");
    SequenceFile.Sorter sorter =
        new SequenceFile.Sorter(
            jobfs, new Text.Comparator(), Text.class, FileStatus.class, jobconf);
    sorter.sort(dstlsr, sortedlsr);

    // compare lsr list and dst list
    SequenceFile.Reader lsrin = null;
    SequenceFile.Reader dstin = null;
    try {
      lsrin = new SequenceFile.Reader(jobfs, sortedlsr, jobconf);
      dstin = new SequenceFile.Reader(jobfs, dstsorted, jobconf);

      // compare sorted lsr list and sorted dst list
      final Text lsrpath = new Text();
      final FileStatus lsrstatus = new FileStatus();
      final Text dstpath = new Text();
      final Text dstfrom = new Text();
      final FsShell shell = new FsShell(conf);
      final String[] shellargs = {"-rmr", null};

      boolean hasnext = dstin.next(dstpath, dstfrom);
      for (; lsrin.next(lsrpath, lsrstatus); ) {
        int dst_cmp_lsr = dstpath.compareTo(lsrpath);
        for (; hasnext && dst_cmp_lsr < 0; ) {
          hasnext = dstin.next(dstpath, dstfrom);
          dst_cmp_lsr = dstpath.compareTo(lsrpath);
        }

        if (dst_cmp_lsr == 0) {
          // lsrpath exists in dst, skip it
          hasnext = dstin.next(dstpath, dstfrom);
        } else {
          // lsrpath does not exist, delete it
          String s = new Path(dstroot.getPath(), lsrpath.toString()).toString();
          if (shellargs[1] == null || !isAncestorPath(shellargs[1], s)) {
            shellargs[1] = s;
            int r = 0;
            try {
              r = shell.run(shellargs);
            } catch (Exception e) {
              throw new IOException("Exception from shell.", e);
            }
            if (r != 0) {
              throw new IOException(
                  "\"" + shellargs[0] + " " + shellargs[1] + "\" returns non-zero value " + r);
            }
          }
        }
      }
    } finally {
      checkAndClose(lsrin);
      checkAndClose(dstin);
    }
  }
  /** Compare two objects with their respective ObjectInspectors. */
  public static int compare(
      Object o1,
      ObjectInspector oi1,
      Object o2,
      ObjectInspector oi2,
      MapEqualComparer mapEqualComparer) {
    if (oi1.getCategory() != oi2.getCategory()) {
      return oi1.getCategory().compareTo(oi2.getCategory());
    }

    if (o1 == null) {
      return o2 == null ? 0 : -1;
    } else if (o2 == null) {
      return 1;
    }

    switch (oi1.getCategory()) {
      case PRIMITIVE:
        {
          PrimitiveObjectInspector poi1 = ((PrimitiveObjectInspector) oi1);
          PrimitiveObjectInspector poi2 = ((PrimitiveObjectInspector) oi2);
          if (poi1.getPrimitiveCategory() != poi2.getPrimitiveCategory()) {
            return poi1.getPrimitiveCategory().compareTo(poi2.getPrimitiveCategory());
          }
          switch (poi1.getPrimitiveCategory()) {
            case VOID:
              return 0;
            case BOOLEAN:
              {
                int v1 = ((BooleanObjectInspector) poi1).get(o1) ? 1 : 0;
                int v2 = ((BooleanObjectInspector) poi2).get(o2) ? 1 : 0;
                return v1 - v2;
              }
            case BYTE:
              {
                int v1 = ((ByteObjectInspector) poi1).get(o1);
                int v2 = ((ByteObjectInspector) poi2).get(o2);
                return v1 - v2;
              }
            case SHORT:
              {
                int v1 = ((ShortObjectInspector) poi1).get(o1);
                int v2 = ((ShortObjectInspector) poi2).get(o2);
                return v1 - v2;
              }
            case INT:
              {
                int v1 = ((IntObjectInspector) poi1).get(o1);
                int v2 = ((IntObjectInspector) poi2).get(o2);
                return v1 > v2 ? 1 : (v1 < v2 ? -1 : 0);
              }
            case LONG:
              {
                long v1 = ((LongObjectInspector) poi1).get(o1);
                long v2 = ((LongObjectInspector) poi2).get(o2);
                return v1 > v2 ? 1 : (v1 < v2 ? -1 : 0);
              }
            case FLOAT:
              {
                float v1 = ((FloatObjectInspector) poi1).get(o1);
                float v2 = ((FloatObjectInspector) poi2).get(o2);
                return Float.compare(v1, v2);
              }
            case DOUBLE:
              {
                double v1 = ((DoubleObjectInspector) poi1).get(o1);
                double v2 = ((DoubleObjectInspector) poi2).get(o2);
                return Double.compare(v1, v2);
              }
            case STRING:
              {
                if (poi1.preferWritable() || poi2.preferWritable()) {
                  Text t1 = (Text) poi1.getPrimitiveWritableObject(o1);
                  Text t2 = (Text) poi2.getPrimitiveWritableObject(o2);
                  return t1 == null ? (t2 == null ? 0 : -1) : (t2 == null ? 1 : t1.compareTo(t2));
                } else {
                  String s1 = (String) poi1.getPrimitiveJavaObject(o1);
                  String s2 = (String) poi2.getPrimitiveJavaObject(o2);
                  return s1 == null ? (s2 == null ? 0 : -1) : (s2 == null ? 1 : s1.compareTo(s2));
                }
              }
            case CHAR:
              {
                HiveCharWritable t1 =
                    ((HiveCharObjectInspector) poi1).getPrimitiveWritableObject(o1);
                HiveCharWritable t2 =
                    ((HiveCharObjectInspector) poi2).getPrimitiveWritableObject(o2);
                return t1.compareTo(t2);
              }
            case VARCHAR:
              {
                HiveVarcharWritable t1 =
                    ((HiveVarcharObjectInspector) poi1).getPrimitiveWritableObject(o1);
                HiveVarcharWritable t2 =
                    ((HiveVarcharObjectInspector) poi2).getPrimitiveWritableObject(o2);
                return t1.compareTo(t2);
              }
            case BINARY:
              {
                BytesWritable bw1 = ((BinaryObjectInspector) poi1).getPrimitiveWritableObject(o1);
                BytesWritable bw2 = ((BinaryObjectInspector) poi2).getPrimitiveWritableObject(o2);
                return bw1.compareTo(bw2);
              }

            case DATE:
              {
                DateWritable d1 = ((DateObjectInspector) poi1).getPrimitiveWritableObject(o1);
                DateWritable d2 = ((DateObjectInspector) poi2).getPrimitiveWritableObject(o2);
                return d1.compareTo(d2);
              }
            case TIMESTAMP:
              {
                TimestampWritable t1 =
                    ((TimestampObjectInspector) poi1).getPrimitiveWritableObject(o1);
                TimestampWritable t2 =
                    ((TimestampObjectInspector) poi2).getPrimitiveWritableObject(o2);
                return t1.compareTo(t2);
              }
            case INTERVAL_YEAR_MONTH:
              {
                HiveIntervalYearMonthWritable i1 =
                    ((HiveIntervalYearMonthObjectInspector) poi1).getPrimitiveWritableObject(o1);
                HiveIntervalYearMonthWritable i2 =
                    ((HiveIntervalYearMonthObjectInspector) poi2).getPrimitiveWritableObject(o2);
                return i1.compareTo(i2);
              }
            case INTERVAL_DAY_TIME:
              {
                HiveIntervalDayTimeWritable i1 =
                    ((HiveIntervalDayTimeObjectInspector) poi1).getPrimitiveWritableObject(o1);
                HiveIntervalDayTimeWritable i2 =
                    ((HiveIntervalDayTimeObjectInspector) poi2).getPrimitiveWritableObject(o2);
                return i1.compareTo(i2);
              }
            case DECIMAL:
              {
                HiveDecimalWritable t1 =
                    ((HiveDecimalObjectInspector) poi1).getPrimitiveWritableObject(o1);
                HiveDecimalWritable t2 =
                    ((HiveDecimalObjectInspector) poi2).getPrimitiveWritableObject(o2);
                return t1.compareTo(t2);
              }
            default:
              {
                throw new RuntimeException("Unknown type: " + poi1.getPrimitiveCategory());
              }
          }
        }
      case STRUCT:
        {
          StructObjectInspector soi1 = (StructObjectInspector) oi1;
          StructObjectInspector soi2 = (StructObjectInspector) oi2;
          List<? extends StructField> fields1 = soi1.getAllStructFieldRefs();
          List<? extends StructField> fields2 = soi2.getAllStructFieldRefs();
          int minimum = Math.min(fields1.size(), fields2.size());
          for (int i = 0; i < minimum; i++) {
            int r =
                compare(
                    soi1.getStructFieldData(o1, fields1.get(i)),
                    fields1.get(i).getFieldObjectInspector(),
                    soi2.getStructFieldData(o2, fields2.get(i)),
                    fields2.get(i).getFieldObjectInspector(),
                    mapEqualComparer);
            if (r != 0) {
              return r;
            }
          }
          return fields1.size() - fields2.size();
        }
      case LIST:
        {
          ListObjectInspector loi1 = (ListObjectInspector) oi1;
          ListObjectInspector loi2 = (ListObjectInspector) oi2;
          int minimum = Math.min(loi1.getListLength(o1), loi2.getListLength(o2));
          for (int i = 0; i < minimum; i++) {
            int r =
                compare(
                    loi1.getListElement(o1, i),
                    loi1.getListElementObjectInspector(),
                    loi2.getListElement(o2, i),
                    loi2.getListElementObjectInspector(),
                    mapEqualComparer);
            if (r != 0) {
              return r;
            }
          }
          return loi1.getListLength(o1) - loi2.getListLength(o2);
        }
      case MAP:
        {
          if (mapEqualComparer == null) {
            throw new RuntimeException("Compare on map type not supported!");
          } else {
            return mapEqualComparer.compare(
                o1, (MapObjectInspector) oi1, o2, (MapObjectInspector) oi2);
          }
        }
      case UNION:
        {
          UnionObjectInspector uoi1 = (UnionObjectInspector) oi1;
          UnionObjectInspector uoi2 = (UnionObjectInspector) oi2;
          byte tag1 = uoi1.getTag(o1);
          byte tag2 = uoi2.getTag(o2);
          if (tag1 != tag2) {
            return tag1 - tag2;
          }
          return compare(
              uoi1.getField(o1),
              uoi1.getObjectInspectors().get(tag1),
              uoi2.getField(o2),
              uoi2.getObjectInspectors().get(tag2),
              mapEqualComparer);
        }
      default:
        throw new RuntimeException("Compare on unknown type: " + oi1.getCategory());
    }
  }
 @Override
 public int compareTo(RowColumn other) {
   int result = row.compareTo(other.row);
   if (result != 0) return result;
   return column.compareTo(other.column);
 }
 public int compare(Text a, Text b) {
   return -a.compareTo(b);
 }