public static TabletLocationState createTabletLocationState(Key k, Value v)
throws IOException, BadLocationStateException {
final SortedMap<Key, Value> decodedRow = WholeRowIterator.decodeRow(k, v);
KeyExtent extent = null;
TServerInstance future = null;
TServerInstance current = null;
TServerInstance last = null;
long lastTimestamp = 0;
List<Collection<String>> walogs = new ArrayList<Collection<String>>();
boolean chopped = false;
for (Entry<Key, Value> entry : decodedRow.entrySet()) {
Key key = entry.getKey();
Text row = key.getRow();
Text cf = key.getColumnFamily();
Text cq = key.getColumnQualifier();
if (cf.compareTo(TabletsSection.FutureLocationColumnFamily.NAME) == 0) {
TServerInstance location = new TServerInstance(entry.getValue(), cq);
if (future != null) {
throw new BadLocationStateException(
"found two assignments for the same extent "
+ key.getRow()
+ ": "
+ future
+ " and "
+ location);
}
future = location;
} else if (cf.compareTo(TabletsSection.CurrentLocationColumnFamily.NAME) == 0) {
TServerInstance location = new TServerInstance(entry.getValue(), cq);
if (current != null) {
throw new BadLocationStateException(
"found two locations for the same extent "
+ key.getRow()
+ ": "
+ current
+ " and "
+ location);
}
current = location;
} else if (cf.compareTo(LogColumnFamily.NAME) == 0) {
String[] split = entry.getValue().toString().split("\\|")[0].split(";");
walogs.add(Arrays.asList(split));
} else if (cf.compareTo(TabletsSection.LastLocationColumnFamily.NAME) == 0) {
if (lastTimestamp < entry.getKey().getTimestamp())
last = new TServerInstance(entry.getValue(), cq);
} else if (cf.compareTo(ChoppedColumnFamily.NAME) == 0) {
chopped = true;
} else if (TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.equals(cf, cq)) {
extent = new KeyExtent(row, entry.getValue());
}
}
if (extent == null) {
log.warn("No prev-row for key extent: " + decodedRow);
return null;
}
return new TabletLocationState(extent, future, current, last, walogs, chopped);
}
@Override
public int compareTo(Pair p) {
int compare = element.compareTo(p.element);
if (compare != 0) return compare;
if (neighbour.toString() == "*") return -1;
else if (p.neighbour.toString() == "*") return 1;
else return neighbour.compareTo(p.neighbour);
}
@Override
public int compareTo(TextPair tp) {
int cmp = first.compareTo(tp.first);
if (cmp != 0) {
return cmp;
}
return second.compareTo(tp.second);
}
public static void splitDatafiles(
Text table,
Text midRow,
double splitRatio,
Map<FileRef, FileUtil.FileInfo> firstAndLastRows,
SortedMap<FileRef, DataFileValue> datafiles,
SortedMap<FileRef, DataFileValue> lowDatafileSizes,
SortedMap<FileRef, DataFileValue> highDatafileSizes,
List<FileRef> highDatafilesToRemove) {
for (Entry<FileRef, DataFileValue> entry : datafiles.entrySet()) {
Text firstRow = null;
Text lastRow = null;
boolean rowsKnown = false;
FileUtil.FileInfo mfi = firstAndLastRows.get(entry.getKey());
if (mfi != null) {
firstRow = mfi.getFirstRow();
lastRow = mfi.getLastRow();
rowsKnown = true;
}
if (rowsKnown && firstRow.compareTo(midRow) > 0) {
// only in high
long highSize = entry.getValue().getSize();
long highEntries = entry.getValue().getNumEntries();
highDatafileSizes.put(
entry.getKey(), new DataFileValue(highSize, highEntries, entry.getValue().getTime()));
} else if (rowsKnown && lastRow.compareTo(midRow) <= 0) {
// only in low
long lowSize = entry.getValue().getSize();
long lowEntries = entry.getValue().getNumEntries();
lowDatafileSizes.put(
entry.getKey(), new DataFileValue(lowSize, lowEntries, entry.getValue().getTime()));
highDatafilesToRemove.add(entry.getKey());
} else {
long lowSize = (long) Math.floor((entry.getValue().getSize() * splitRatio));
long lowEntries = (long) Math.floor((entry.getValue().getNumEntries() * splitRatio));
lowDatafileSizes.put(
entry.getKey(), new DataFileValue(lowSize, lowEntries, entry.getValue().getTime()));
long highSize = (long) Math.ceil((entry.getValue().getSize() * (1.0 - splitRatio)));
long highEntries =
(long) Math.ceil((entry.getValue().getNumEntries() * (1.0 - splitRatio)));
highDatafileSizes.put(
entry.getKey(), new DataFileValue(highSize, highEntries, entry.getValue().getTime()));
}
}
}
@Override
public int compareTo(TextLong other) {
int cmp = first.compareTo(other.getFirst());
if (0 == cmp) {
cmp = second.compareTo(other.getSecond());
}
return cmp;
}
@Override
public int compareTo(TermDocIdWritable tuple) {
int cmp = __term.compareTo(tuple.getTerm());
if (cmp != 0) {
return cmp;
}
return __documentId.compareTo(tuple.getDocumentId());
}
public int compareTo(BigramWritable bw) {
int cmp = leftBigram.compareTo(bw.leftBigram);
if (cmp != 0) return cmp;
else return rightBigram.compareTo(bw.rightBigram);
}
public int compare(int i, int j) {
Text left = records.get(i);
Text right = records.get(j);
return left.compareTo(right);
}
public int baseCompareTo(TextLong other) {
int cmp = first.compareTo(other.getFirst());
return cmp;
}
/** Delete the dst files/dirs which do not exist in src */
private static void deleteNonexisting(
FileSystem dstfs,
FileStatus dstroot,
Path dstsorted,
FileSystem jobfs,
Path jobdir,
JobConf jobconf,
Configuration conf)
throws IOException {
if (!dstroot.isDir()) {
throw new IOException(
"dst must be a directory when option "
+ Options.DELETE.cmd
+ " is set, but dst (= "
+ dstroot.getPath()
+ ") is not a directory.");
}
// write dst lsr results
final Path dstlsr = new Path(jobdir, "_distcp_dst_lsr");
final SequenceFile.Writer writer =
SequenceFile.createWriter(
jobfs,
jobconf,
dstlsr,
Text.class,
FileStatus.class,
SequenceFile.CompressionType.NONE);
try {
// do lsr to get all file statuses in dstroot
final Stack<FileStatus> lsrstack = new Stack<FileStatus>();
for (lsrstack.push(dstroot); !lsrstack.isEmpty(); ) {
final FileStatus status = lsrstack.pop();
if (status.isDir()) {
for (FileStatus child : dstfs.listStatus(status.getPath())) {
String relative = makeRelative(dstroot.getPath(), child.getPath());
writer.append(new Text(relative), child);
lsrstack.push(child);
}
}
}
} finally {
checkAndClose(writer);
}
// sort lsr results
final Path sortedlsr = new Path(jobdir, "_distcp_dst_lsr_sorted");
SequenceFile.Sorter sorter =
new SequenceFile.Sorter(
jobfs, new Text.Comparator(), Text.class, FileStatus.class, jobconf);
sorter.sort(dstlsr, sortedlsr);
// compare lsr list and dst list
SequenceFile.Reader lsrin = null;
SequenceFile.Reader dstin = null;
try {
lsrin = new SequenceFile.Reader(jobfs, sortedlsr, jobconf);
dstin = new SequenceFile.Reader(jobfs, dstsorted, jobconf);
// compare sorted lsr list and sorted dst list
final Text lsrpath = new Text();
final FileStatus lsrstatus = new FileStatus();
final Text dstpath = new Text();
final Text dstfrom = new Text();
final FsShell shell = new FsShell(conf);
final String[] shellargs = {"-rmr", null};
boolean hasnext = dstin.next(dstpath, dstfrom);
for (; lsrin.next(lsrpath, lsrstatus); ) {
int dst_cmp_lsr = dstpath.compareTo(lsrpath);
for (; hasnext && dst_cmp_lsr < 0; ) {
hasnext = dstin.next(dstpath, dstfrom);
dst_cmp_lsr = dstpath.compareTo(lsrpath);
}
if (dst_cmp_lsr == 0) {
// lsrpath exists in dst, skip it
hasnext = dstin.next(dstpath, dstfrom);
} else {
// lsrpath does not exist, delete it
String s = new Path(dstroot.getPath(), lsrpath.toString()).toString();
if (shellargs[1] == null || !isAncestorPath(shellargs[1], s)) {
shellargs[1] = s;
int r = 0;
try {
r = shell.run(shellargs);
} catch (Exception e) {
throw new IOException("Exception from shell.", e);
}
if (r != 0) {
throw new IOException(
"\"" + shellargs[0] + " " + shellargs[1] + "\" returns non-zero value " + r);
}
}
}
}
} finally {
checkAndClose(lsrin);
checkAndClose(dstin);
}
}
/** Compare two objects with their respective ObjectInspectors. */
public static int compare(
Object o1,
ObjectInspector oi1,
Object o2,
ObjectInspector oi2,
MapEqualComparer mapEqualComparer) {
if (oi1.getCategory() != oi2.getCategory()) {
return oi1.getCategory().compareTo(oi2.getCategory());
}
if (o1 == null) {
return o2 == null ? 0 : -1;
} else if (o2 == null) {
return 1;
}
switch (oi1.getCategory()) {
case PRIMITIVE:
{
PrimitiveObjectInspector poi1 = ((PrimitiveObjectInspector) oi1);
PrimitiveObjectInspector poi2 = ((PrimitiveObjectInspector) oi2);
if (poi1.getPrimitiveCategory() != poi2.getPrimitiveCategory()) {
return poi1.getPrimitiveCategory().compareTo(poi2.getPrimitiveCategory());
}
switch (poi1.getPrimitiveCategory()) {
case VOID:
return 0;
case BOOLEAN:
{
int v1 = ((BooleanObjectInspector) poi1).get(o1) ? 1 : 0;
int v2 = ((BooleanObjectInspector) poi2).get(o2) ? 1 : 0;
return v1 - v2;
}
case BYTE:
{
int v1 = ((ByteObjectInspector) poi1).get(o1);
int v2 = ((ByteObjectInspector) poi2).get(o2);
return v1 - v2;
}
case SHORT:
{
int v1 = ((ShortObjectInspector) poi1).get(o1);
int v2 = ((ShortObjectInspector) poi2).get(o2);
return v1 - v2;
}
case INT:
{
int v1 = ((IntObjectInspector) poi1).get(o1);
int v2 = ((IntObjectInspector) poi2).get(o2);
return v1 > v2 ? 1 : (v1 < v2 ? -1 : 0);
}
case LONG:
{
long v1 = ((LongObjectInspector) poi1).get(o1);
long v2 = ((LongObjectInspector) poi2).get(o2);
return v1 > v2 ? 1 : (v1 < v2 ? -1 : 0);
}
case FLOAT:
{
float v1 = ((FloatObjectInspector) poi1).get(o1);
float v2 = ((FloatObjectInspector) poi2).get(o2);
return Float.compare(v1, v2);
}
case DOUBLE:
{
double v1 = ((DoubleObjectInspector) poi1).get(o1);
double v2 = ((DoubleObjectInspector) poi2).get(o2);
return Double.compare(v1, v2);
}
case STRING:
{
if (poi1.preferWritable() || poi2.preferWritable()) {
Text t1 = (Text) poi1.getPrimitiveWritableObject(o1);
Text t2 = (Text) poi2.getPrimitiveWritableObject(o2);
return t1 == null ? (t2 == null ? 0 : -1) : (t2 == null ? 1 : t1.compareTo(t2));
} else {
String s1 = (String) poi1.getPrimitiveJavaObject(o1);
String s2 = (String) poi2.getPrimitiveJavaObject(o2);
return s1 == null ? (s2 == null ? 0 : -1) : (s2 == null ? 1 : s1.compareTo(s2));
}
}
case CHAR:
{
HiveCharWritable t1 =
((HiveCharObjectInspector) poi1).getPrimitiveWritableObject(o1);
HiveCharWritable t2 =
((HiveCharObjectInspector) poi2).getPrimitiveWritableObject(o2);
return t1.compareTo(t2);
}
case VARCHAR:
{
HiveVarcharWritable t1 =
((HiveVarcharObjectInspector) poi1).getPrimitiveWritableObject(o1);
HiveVarcharWritable t2 =
((HiveVarcharObjectInspector) poi2).getPrimitiveWritableObject(o2);
return t1.compareTo(t2);
}
case BINARY:
{
BytesWritable bw1 = ((BinaryObjectInspector) poi1).getPrimitiveWritableObject(o1);
BytesWritable bw2 = ((BinaryObjectInspector) poi2).getPrimitiveWritableObject(o2);
return bw1.compareTo(bw2);
}
case DATE:
{
DateWritable d1 = ((DateObjectInspector) poi1).getPrimitiveWritableObject(o1);
DateWritable d2 = ((DateObjectInspector) poi2).getPrimitiveWritableObject(o2);
return d1.compareTo(d2);
}
case TIMESTAMP:
{
TimestampWritable t1 =
((TimestampObjectInspector) poi1).getPrimitiveWritableObject(o1);
TimestampWritable t2 =
((TimestampObjectInspector) poi2).getPrimitiveWritableObject(o2);
return t1.compareTo(t2);
}
case INTERVAL_YEAR_MONTH:
{
HiveIntervalYearMonthWritable i1 =
((HiveIntervalYearMonthObjectInspector) poi1).getPrimitiveWritableObject(o1);
HiveIntervalYearMonthWritable i2 =
((HiveIntervalYearMonthObjectInspector) poi2).getPrimitiveWritableObject(o2);
return i1.compareTo(i2);
}
case INTERVAL_DAY_TIME:
{
HiveIntervalDayTimeWritable i1 =
((HiveIntervalDayTimeObjectInspector) poi1).getPrimitiveWritableObject(o1);
HiveIntervalDayTimeWritable i2 =
((HiveIntervalDayTimeObjectInspector) poi2).getPrimitiveWritableObject(o2);
return i1.compareTo(i2);
}
case DECIMAL:
{
HiveDecimalWritable t1 =
((HiveDecimalObjectInspector) poi1).getPrimitiveWritableObject(o1);
HiveDecimalWritable t2 =
((HiveDecimalObjectInspector) poi2).getPrimitiveWritableObject(o2);
return t1.compareTo(t2);
}
default:
{
throw new RuntimeException("Unknown type: " + poi1.getPrimitiveCategory());
}
}
}
case STRUCT:
{
StructObjectInspector soi1 = (StructObjectInspector) oi1;
StructObjectInspector soi2 = (StructObjectInspector) oi2;
List<? extends StructField> fields1 = soi1.getAllStructFieldRefs();
List<? extends StructField> fields2 = soi2.getAllStructFieldRefs();
int minimum = Math.min(fields1.size(), fields2.size());
for (int i = 0; i < minimum; i++) {
int r =
compare(
soi1.getStructFieldData(o1, fields1.get(i)),
fields1.get(i).getFieldObjectInspector(),
soi2.getStructFieldData(o2, fields2.get(i)),
fields2.get(i).getFieldObjectInspector(),
mapEqualComparer);
if (r != 0) {
return r;
}
}
return fields1.size() - fields2.size();
}
case LIST:
{
ListObjectInspector loi1 = (ListObjectInspector) oi1;
ListObjectInspector loi2 = (ListObjectInspector) oi2;
int minimum = Math.min(loi1.getListLength(o1), loi2.getListLength(o2));
for (int i = 0; i < minimum; i++) {
int r =
compare(
loi1.getListElement(o1, i),
loi1.getListElementObjectInspector(),
loi2.getListElement(o2, i),
loi2.getListElementObjectInspector(),
mapEqualComparer);
if (r != 0) {
return r;
}
}
return loi1.getListLength(o1) - loi2.getListLength(o2);
}
case MAP:
{
if (mapEqualComparer == null) {
throw new RuntimeException("Compare on map type not supported!");
} else {
return mapEqualComparer.compare(
o1, (MapObjectInspector) oi1, o2, (MapObjectInspector) oi2);
}
}
case UNION:
{
UnionObjectInspector uoi1 = (UnionObjectInspector) oi1;
UnionObjectInspector uoi2 = (UnionObjectInspector) oi2;
byte tag1 = uoi1.getTag(o1);
byte tag2 = uoi2.getTag(o2);
if (tag1 != tag2) {
return tag1 - tag2;
}
return compare(
uoi1.getField(o1),
uoi1.getObjectInspectors().get(tag1),
uoi2.getField(o2),
uoi2.getObjectInspectors().get(tag2),
mapEqualComparer);
}
default:
throw new RuntimeException("Compare on unknown type: " + oi1.getCategory());
}
}
@Override
public int compareTo(RowColumn other) {
int result = row.compareTo(other.row);
if (result != 0) return result;
return column.compareTo(other.column);
}
public int compare(Text a, Text b) {
return -a.compareTo(b);
}