/**
* Flushes the snapshot of the memstore. Flushes the mob data to the mob files, and flushes the
* name of these mob files to HBase.
*
* @param snapshot The snapshot of the memstore.
* @throws IOException
*/
private void internalFlushCache(final MemStoreSnapshot snapshot) throws IOException {
if (snapshot.getCellsCount() == 0) {
return;
}
// generate the files into a temp directory.
String tempPathString = context.getConfiguration().get(SweepJob.WORKING_FILES_DIR_KEY);
StoreFile.Writer mobFileWriter =
MobUtils.createWriter(
conf,
fs,
hcd,
partitionId.getDate(),
new Path(tempPathString),
snapshot.getCellsCount(),
hcd.getCompactionCompression(),
partitionId.getStartKey(),
cacheConfig,
cryptoContext);
String relativePath = mobFileWriter.getPath().getName();
LOG.info("Create files under a temp directory " + mobFileWriter.getPath().toString());
byte[] referenceValue = Bytes.toBytes(relativePath);
KeyValueScanner scanner = snapshot.getScanner();
Cell cell = null;
while (null != (cell = scanner.next())) {
mobFileWriter.append(cell);
}
scanner.close();
// Write out the log sequence number that corresponds to this output
// hfile. The hfile is current up to and including logCacheFlushId.
mobFileWriter.appendMetadata(Long.MAX_VALUE, false, snapshot.getCellsCount());
mobFileWriter.close();
MobUtils.commitFile(conf, fs, mobFileWriter.getPath(), mobFamilyDir, cacheConfig);
context.getCounter(SweepCounter.FILE_AFTER_MERGE_OR_CLEAN).increment(1);
// write reference/fileName back to the store files of HBase.
scanner = snapshot.getScanner();
scanner.seek(KeyValueUtil.createFirstOnRow(HConstants.EMPTY_START_ROW));
cell = null;
Tag tableNameTag =
new ArrayBackedTag(
TagType.MOB_TABLE_NAME_TAG_TYPE, Bytes.toBytes(this.table.getName().toString()));
long updatedCount = 0;
while (null != (cell = scanner.next())) {
KeyValue reference = MobUtils.createMobRefKeyValue(cell, referenceValue, tableNameTag);
Put put =
new Put(reference.getRowArray(), reference.getRowOffset(), reference.getRowLength());
put.add(reference);
table.mutate(put);
updatedCount++;
}
table.flush();
context.getCounter(SweepCounter.RECORDS_UPDATED).increment(updatedCount);
scanner.close();
}
private static void DeleteEvent(BufferedMutator VTEvent, String iMO_str, VesselEvent after_VE)
throws IOException {
// TODO Auto-generated method stub
byte[] rowkey =
Bytes.toBytes(
iMO_str
+ LpadNum(Long.MAX_VALUE - after_VE.entrytime, 19)
+ LpadNum(after_VE.polygonid, 10));
VTEvent.mutate(new Delete(rowkey));
}
/**
* Update a record in the database. Any field/value pairs in the specified values HashMap will be
* written into the record with the specified record key, overwriting any existing values with the
* same field name.
*
* @param table The name of the table
* @param key The record key of the record to write
* @param values A HashMap of field/value pairs to update in the record
* @return Zero on success, a non-zero error code on error
*/
@Override
public Status update(String table, String key, HashMap<String, ByteIterator> values) {
// if this is a "new" table, init HTable object. Else, use existing one
if (!tableName.equals(table)) {
currentTable = null;
try {
getHTable(table);
tableName = table;
} catch (IOException e) {
System.err.println("Error accessing HBase table: " + e);
return Status.ERROR;
}
}
if (debug) {
System.out.println("Setting up put for key: " + key);
}
Put p = new Put(Bytes.toBytes(key));
p.setDurability(durability);
for (Map.Entry<String, ByteIterator> entry : values.entrySet()) {
byte[] value = entry.getValue().toArray();
if (debug) {
System.out.println(
"Adding field/value "
+ entry.getKey()
+ "/"
+ Bytes.toStringBinary(value)
+ " to put request");
}
p.addColumn(columnFamilyBytes, Bytes.toBytes(entry.getKey()), value);
}
try {
if (clientSideBuffering) {
Preconditions.checkNotNull(bufferedMutator);
bufferedMutator.mutate(p);
} else {
currentTable.put(p);
}
} catch (IOException e) {
if (debug) {
System.err.println("Error doing put: " + e);
}
return Status.ERROR;
} catch (ConcurrentModificationException e) {
// do nothing for now...hope this is rare
return Status.ERROR;
}
return Status.OK;
}
public MemStoreWrapper(
Context context,
FileSystem fs,
BufferedMutator table,
HColumnDescriptor hcd,
MemStore memstore,
CacheConfig cacheConfig)
throws IOException {
this.memstore = memstore;
this.context = context;
this.fs = fs;
this.table = table;
this.hcd = hcd;
this.conf = context.getConfiguration();
this.cacheConfig = cacheConfig;
flushSize =
this.conf.getLong(
MobConstants.MOB_SWEEP_TOOL_COMPACTION_MEMSTORE_FLUSH_SIZE,
MobConstants.DEFAULT_MOB_SWEEP_TOOL_COMPACTION_MEMSTORE_FLUSH_SIZE);
mobFamilyDir = MobUtils.getMobFamilyPath(conf, table.getName(), hcd.getNameAsString());
cryptoContext = EncryptionUtil.createEncryptionContext(conf, hcd);
}
/**
* Cleanup any state for this DB. Called once per DB instance; there is one DB instance per client
* thread.
*/
@Override
public void cleanup() throws DBException {
// Get the measurements instance as this is the only client that should
// count clean up time like an update if client-side buffering is
// enabled.
Measurements measurements = Measurements.getMeasurements();
try {
long st = System.nanoTime();
if (bufferedMutator != null) {
bufferedMutator.close();
}
if (currentTable != null) {
currentTable.close();
}
long en = System.nanoTime();
final String type = clientSideBuffering ? "UPDATE" : "CLEANUP";
measurements.measure(type, (int) ((en - st) / 1000));
connection.close();
} catch (IOException e) {
throw new DBException(e);
}
}
/**
* Delete a record from the database.
*
* @param table The name of the table
* @param key The record key of the record to delete.
* @return Zero on success, a non-zero error code on error
*/
@Override
public Status delete(String table, String key) {
// if this is a "new" table, init HTable object. Else, use existing one
if (!tableName.equals(table)) {
currentTable = null;
try {
getHTable(table);
tableName = table;
} catch (IOException e) {
System.err.println("Error accessing HBase table: " + e);
return Status.ERROR;
}
}
if (debug) {
System.out.println("Doing delete for key: " + key);
}
final Delete d = new Delete(Bytes.toBytes(key));
d.setDurability(durability);
try {
if (clientSideBuffering) {
Preconditions.checkNotNull(bufferedMutator);
bufferedMutator.mutate(d);
} else {
currentTable.delete(d);
}
} catch (IOException e) {
if (debug) {
System.err.println("Error doing delete: " + e);
}
return Status.ERROR;
}
return Status.OK;
}
public static void DeleteEvents(BufferedMutator VTEvent, List<Delete> deletes)
throws IOException {
VTEvent.mutate(deletes);
}
@Override
protected void reduce(
Key_IMOAndRecordTime key, Iterable<TextArrayWritable> LocationList, Context context)
throws IOException, InterruptedException {
try {
context.getCounter(Counters.VESSEL_PROCESSED).increment(1);
String IMO_str = LpadNum(key.getIMO().get(), 7);
long first_pos_time = key.getRecordTime().get();
/////////////////////////////////////////////////////////////////////////////////
// Populate newPoints with new locations
List<VesselLocation> newPoints = new ArrayList<VesselLocation>();
for (TextArrayWritable rowcontent : LocationList) {
// population location
context.getCounter(Counters.LOCATION_ROWS).increment(1);
VesselLocation newlocation = new VesselLocation();
try {
Writable[] content = rowcontent.get();
String Latitude = content[16].toString().trim();
String Longitude = content[15].toString().trim();
String Coordinates = Latitude + "," + Longitude;
String Speed = content[18].toString().trim();
String Destination = content[9].toString().trim();
String Timestamp = content[21].toString().trim().substring(0, 19);
long record_time = DateTime.parse(Timestamp, rawformatter).getMillis();
newlocation.coordinates = Coordinates;
newlocation.recordtime = record_time;
newlocation.speed = Speed;
newlocation.destination = Destination;
context.getCounter(Counters.LOCATION_VALID).increment(1);
} catch (Exception e) {
e.printStackTrace();
context.getCounter(Counters.LOCATION_ERROR).increment(1);
continue;
}
newPoints.add(newlocation);
}
/////////////////////////////////////////////////////////////////////////////////
// Get last new post time
long last_pos_time = newPoints.get(newPoints.size() - 1).recordtime;
////////////////////////////////////////////////////////////////////////////////
// Get Existing trackinfo
VesselTrackInfo VTI = getTrackInfo(TrackInfo_Table, IMO_str);
List<VesselLocation> AllBetweenPoints = new ArrayList<VesselLocation>();
String BeforeRowKey = null;
String AfterRowKey = null;
// //////////////////////////////////////////////////////////////////////////////
// Retrieve all the existing locations between the first new location and the last new
// location.
if ((VTI.FirstRecordTime != null) && (VTI.LastRecordTime != null)) {
if (last_pos_time < VTI.FirstRecordTime) {
AfterRowKey = IMO_str + LpadNum(Long.MAX_VALUE - VTI.FirstRecordTime, 19);
} else if (first_pos_time > VTI.LastRecordTime) {
BeforeRowKey = IMO_str + LpadNum(Long.MAX_VALUE - VTI.LastRecordTime, 19);
} else {
AllBetweenPoints =
ImportReducer.getLocationsBetween(
VTLocation_Table, IMO_str, first_pos_time, last_pos_time);
if (AllBetweenPoints.size() == 0) {
// Search for the first DB point before the first new point
VesselLocation BeforeLocation =
getLocationBefore(VTLocation_Table, IMO_str, first_pos_time);
BeforeRowKey = IMO_str + LpadNum(Long.MAX_VALUE - BeforeLocation.recordtime, 19);
AfterRowKey = BeforeLocation.nextlocation;
} else {
java.util.Collections.sort(AllBetweenPoints);
BeforeRowKey = AllBetweenPoints.get(0).previouslocation;
AfterRowKey = AllBetweenPoints.get(AllBetweenPoints.size() - 1).nextlocation;
}
List<Delete> deletes =
ImportReducer.GetDeleteEventsBetween(
VTEvent_Table, IMO_str, first_pos_time, last_pos_time);
ImportReducer.DeleteEvents(VTEvent, deletes);
VTEvent.flush();
}
}
// Find out the location before the first new location in
VesselLocation BeforeLocation = getLocation(VTLocation_Table, BeforeRowKey);
// Find out the location after the last new location in
VesselLocation AfterLocation = getLocation(VTLocation_Table, AfterRowKey);
Map<Integer, VesselEvent> PreviousZoneEvents = new HashMap<Integer, VesselEvent>();
;
Map<Integer, VesselEvent> AfterZoneEvents = new HashMap<Integer, VesselEvent>();
if (BeforeLocation != null) {
// Get all events with exit at last location
PreviousZoneEvents =
getAllEventsStartBeforeEndAfterBeforeLocation(VTEvent_Table, IMO_str, BeforeLocation);
}
////////////////////////////////////////////////////
// Analyze and calculate previous and next location
for (VesselLocation newlocation : newPoints) {
int index = AllBetweenPoints.indexOf(newlocation);
if (index != -1) {
VesselLocation dblocation = AllBetweenPoints.get(index);
dblocation.coordinates = newlocation.coordinates;
dblocation.destination = newlocation.destination;
dblocation.speed = newlocation.speed;
} else {
AllBetweenPoints.add(newlocation);
}
}
java.util.Collections.sort(AllBetweenPoints);
String previousRowKey = null;
for (VesselLocation location : AllBetweenPoints) {
location.previouslocation = previousRowKey;
previousRowKey = IMO_str + LpadNum(Long.MAX_VALUE - location.recordtime, 19);
}
String NextRowKey = null;
for (int i = (AllBetweenPoints.size() - 1); i >= 0; i--) {
VesselLocation location = AllBetweenPoints.get(i);
location.nextlocation = NextRowKey;
NextRowKey = IMO_str + LpadNum(Long.MAX_VALUE - location.recordtime, 19);
}
AllBetweenPoints.get(0).previouslocation = BeforeRowKey;
AllBetweenPoints.get(AllBetweenPoints.size() - 1).nextlocation = AfterRowKey;
////////////////////////////////////////////////////
// Upsert all locations
for (VesselLocation location : AllBetweenPoints) {
// population location
try {
byte[] rowkey =
Bytes.toBytes(IMO_str + LpadNum(Long.MAX_VALUE - location.recordtime, 19));
Put put = new Put(rowkey);
put.addColumn(details, speed, Bytes.toBytes(location.speed));
put.addColumn(details, destination, Bytes.toBytes(location.destination));
put.addColumn(details, coordinates, Bytes.toBytes(location.coordinates));
put.addColumn(
details,
timestamp,
Bytes.toBytes(new DateTime(location.recordtime).toString(rawformatter)));
if (location.previouslocation != null) {
put.addColumn(details, previouslocation, Bytes.toBytes(location.previouslocation));
}
if (location.nextlocation != null) {
put.addColumn(details, nextlocation, Bytes.toBytes(location.nextlocation));
}
VTLocation.mutate(put);
} catch (Exception e) {
e.printStackTrace();
context.getCounter(Counters.LOCATION_ERROR).increment(1);
continue;
}
}
// update before next location and after previous location
if (BeforeRowKey != null) {
Put BeforeLocationPut = new Put(Bytes.toBytes(BeforeRowKey));
BeforeLocationPut.addColumn(
details,
nextlocation,
Bytes.toBytes(
IMO_str + LpadNum(Long.MAX_VALUE - AllBetweenPoints.get(0).recordtime, 19)));
VTLocation.mutate(BeforeLocationPut);
}
if (AfterRowKey != null) {
Put AfterLocationPut = new Put(Bytes.toBytes(AfterRowKey));
AfterLocationPut.addColumn(
details,
previouslocation,
Bytes.toBytes(
IMO_str
+ LpadNum(
Long.MAX_VALUE
- AllBetweenPoints.get(AllBetweenPoints.size() - 1).recordtime,
19)));
VTLocation.mutate(AfterLocationPut);
}
VTLocation.flush();
/////////////////////////////////////////////////////////////////////
// Store latest location
// rowkey: global zone id (4)+ longlat22
// ((long11(sign(1)+integer(3)+digit(7)))(lat11(sign(1)+integer(3)+(7))))+imo(7)+recordtime(19)
/////////////////////////////////////////////////////////////////////
Put vessel_track_info = new Put(Bytes.toBytes(IMO_str));
if (AfterLocation == null) {
// Get the last location
VesselLocation lastLocation = AllBetweenPoints.get(AllBetweenPoints.size() - 1);
// update the last location
String[] longlat = lastLocation.coordinates.split(",");
GeometryFactory geometryFactory = JTSFactoryFinder.getGeometryFactory(null);
Coordinate coord =
new Coordinate(Double.parseDouble(longlat[1]), Double.parseDouble(longlat[0]));
Point point = geometryFactory.createPoint(coord);
Integer BelongedGlobalZoneIndex = null;
for (int i = 0; i < VesselZone.GlobalZones.length; i++) {
if (VesselZone.GlobalZones[i].covers(point)) {
BelongedGlobalZoneIndex = i;
break;
}
}
if (VTI.LastLocation != null) {
LastLocation_BM.mutate(new Delete(VTI.LastLocation));
}
byte[] lastlocationrowkey =
Bytes.toBytes(
LpadNum(BelongedGlobalZoneIndex, 4)
+ ConvertCoordinatesToStr(longlat[1])
+ ConvertCoordinatesToStr(longlat[0]));
Put lastlocation_put = new Put(lastlocationrowkey);
lastlocation_put.addColumn(details, imo, Bytes.toBytes(IMO_str));
lastlocation_put.addColumn(
details,
timestamp,
Bytes.toBytes(new DateTime(lastLocation.recordtime).toString(rawformatter)));
LastLocation_BM.mutate(lastlocation_put);
LastLocation_BM.flush();
vessel_track_info.addColumn(details, lastlocation, lastlocationrowkey);
vessel_track_info.addColumn(
details,
lastrecordtime,
Bytes.toBytes(new DateTime(lastLocation.recordtime).toString(rawformatter)));
} else {
// Get events that start before last new location and end after last new location
AfterZoneEvents =
getAllEventsStartBeforeEndAfter(VTEvent_Table, IMO_str, AfterLocation.recordtime);
}
// update firstrecordtime and lastrecordtime
if (BeforeLocation == null) {
vessel_track_info.addColumn(
details,
firstrecordtime,
Bytes.toBytes(
new DateTime(AllBetweenPoints.get(0).recordtime).toString(rawformatter)));
}
if (!vessel_track_info.isEmpty()) {
TrackInfo_BM.mutate(vessel_track_info);
TrackInfo_BM.flush();
}
////////////////////////////////////////////////////////////////////
ArrayList<VesselEvent> DerivedEventList = new ArrayList<VesselEvent>();
///////////////////////////////////////////////////////////
// Get Vessel
String VesselType = getVesselType(Vessel_Table, IMO_str);
if (VesselType == null) {
context.getCounter(Counters.VESSEL_WITHOUTTYPE).increment(1);
return;
}
// calculating event
for (VesselLocation VL : AllBetweenPoints) {
ArrayList<Integer> CurrentZones = LocateCurrentZone(VL.coordinates, VesselType, Zonemap);
Iterator<Map.Entry<Integer, VesselEvent>> it = PreviousZoneEvents.entrySet().iterator();
while (it.hasNext()) {
Map.Entry<Integer, VesselEvent> thisEntry = it.next();
int Zone_Axsmarine_id = thisEntry.getKey();
if (!CurrentZones.contains(Zone_Axsmarine_id)) {
VesselEvent PreviousEvent = thisEntry.getValue();
if (!DerivedEventList.contains(PreviousEvent)) {
DerivedEventList.add(PreviousEvent);
}
// remove close event from PreviousZoneEvents;
it.remove();
}
}
for (Integer thisZone_Axsmarine_id : CurrentZones) {
if (PreviousZoneEvents.containsKey(thisZone_Axsmarine_id)) {
//////////////////////////////////////////////////
// For current zones which both previous and current locations belong to, update exit
// point of previous open events with current locations.
//////////////////////////////////////////////////
VesselEvent PreviousEvent = PreviousZoneEvents.get(thisZone_Axsmarine_id);
PreviousEvent.exitcoordinates = VL.coordinates;
PreviousEvent.exittime = VL.recordtime;
PreviousEvent.destination = VL.destination;
if (!DerivedEventList.contains(PreviousEvent)) {
DerivedEventList.add(PreviousEvent);
}
} else {
//////////////////////////////////////////////////
// For current zones which only current locations belong to, fire new open events
//////////////////////////////////////////////////
VesselEvent NewEvent = new VesselEvent();
NewEvent.entrycoordinates = VL.coordinates;
NewEvent.entrytime = VL.recordtime;
NewEvent.exitcoordinates = VL.coordinates;
NewEvent.exittime = VL.recordtime;
NewEvent.destination = VL.destination;
NewEvent.polygonid = thisZone_Axsmarine_id;
PreviousZoneEvents.put(thisZone_Axsmarine_id, NewEvent);
DerivedEventList.add(NewEvent);
}
}
}
///////////////////////////////////////////////////////////////////////////////////////
// Merge with PreviousZoneEvents with AfterZoneEvents
Iterator<Map.Entry<Integer, VesselEvent>> it = AfterZoneEvents.entrySet().iterator();
while (it.hasNext()) {
Map.Entry<Integer, VesselEvent> thisEntry = it.next();
int Zone_Axsmarine_id = thisEntry.getKey();
VesselEvent After_VE = thisEntry.getValue();
VesselEvent Previous_VE = PreviousZoneEvents.get(Zone_Axsmarine_id);
if (Previous_VE != null) {
Previous_VE.exitcoordinates = After_VE.exitcoordinates;
Previous_VE.exittime = After_VE.exittime;
Previous_VE.destination = After_VE.destination;
if (!DerivedEventList.contains(Previous_VE)) {
DerivedEventList.add(Previous_VE);
}
} else {
VesselEvent NewEvent = new VesselEvent();
NewEvent.entrycoordinates = AfterLocation.coordinates;
NewEvent.entrytime = AfterLocation.recordtime;
NewEvent.exitcoordinates = After_VE.exitcoordinates;
NewEvent.exittime = After_VE.exittime;
NewEvent.destination = After_VE.destination;
NewEvent.polygonid = Zone_Axsmarine_id;
DerivedEventList.add(NewEvent);
}
// Delete This Event from HBase
DeleteEvent(VTEvent, IMO_str, After_VE);
}
VTEvent.flush();
// pupulate Derived Events into Hbase
for (VesselEvent newEvent : DerivedEventList) {
// rowkey: IMO(7)+timestamp(19 desc)+polygonid(8)
// qualifier:entrytime,entrycoordinates,exittime,exitcoordinates,destination
context.getCounter(Counters.EVENT_UPSERTS).increment(1);
byte[] rowkey =
Bytes.toBytes(
IMO_str
+ LpadNum(Long.MAX_VALUE - newEvent.entrytime, 19)
+ LpadNum(newEvent.polygonid, 10));
Put put = new Put(rowkey);
put.addColumn(
details,
entrytime,
Bytes.toBytes(new DateTime(newEvent.entrytime).toString(rawformatter)));
put.addColumn(details, entrycoordinates, Bytes.toBytes(newEvent.entrycoordinates));
put.addColumn(
details,
exittime,
Bytes.toBytes(new DateTime(newEvent.exittime).toString(rawformatter)));
put.addColumn(details, exitcoordinates, Bytes.toBytes(newEvent.exitcoordinates));
put.addColumn(details, destination, Bytes.toBytes(newEvent.destination));
VTEvent.mutate(put);
context.getCounter(Counters.EVENT_VALID).increment(1);
}
// VTLocation.flush(); Moved to the first step
VTEvent.flush();
} catch (RuntimeException e) {
// TODO Auto-generated catch block
System.out.println("Exception occured while loading data for:" + key.getIMO());
throw e;
}
}
@Test
public void testRun() throws Exception {
TableName tn = TableName.valueOf(tableName);
byte[] mobValueBytes = new byte[100];
// get the path where mob files lie in
Path mobFamilyPath = MobUtils.getMobFamilyPath(TEST_UTIL.getConfiguration(), tn, family);
Put put = new Put(Bytes.toBytes(row));
put.addColumn(Bytes.toBytes(family), Bytes.toBytes(qf), 1, mobValueBytes);
Put put2 = new Put(Bytes.toBytes(row + "ignore"));
put2.addColumn(Bytes.toBytes(family), Bytes.toBytes(qf), 1, mobValueBytes);
table.mutate(put);
table.mutate(put2);
table.flush();
admin.flush(tn);
FileStatus[] fileStatuses = TEST_UTIL.getTestFileSystem().listStatus(mobFamilyPath);
// check the generation of a mob file
assertEquals(1, fileStatuses.length);
String mobFile1 = fileStatuses[0].getPath().getName();
Configuration configuration = new Configuration(TEST_UTIL.getConfiguration());
configuration.setFloat(MobConstants.MOB_SWEEP_TOOL_COMPACTION_RATIO, 0.6f);
configuration.setStrings(TableInputFormat.INPUT_TABLE, tableName);
configuration.setStrings(TableInputFormat.SCAN_COLUMN_FAMILY, family);
configuration.setStrings(SweepJob.WORKING_VISITED_DIR_KEY, "jobWorkingNamesDir");
configuration.setStrings(SweepJob.WORKING_FILES_DIR_KEY, "compactionFileDir");
configuration.setStrings(
CommonConfigurationKeys.IO_SERIALIZATIONS_KEY, JavaSerialization.class.getName());
configuration.set(SweepJob.WORKING_VISITED_DIR_KEY, "compactionVisitedDir");
configuration.setLong(
MobConstants.MOB_SWEEP_TOOL_COMPACTION_START_DATE,
System.currentTimeMillis() + 24 * 3600 * 1000);
ZooKeeperWatcher zkw = new ZooKeeperWatcher(configuration, "1", new DummyMobAbortable());
TableName lockName = MobUtils.getTableLockName(tn);
String znode = ZKUtil.joinZNode(zkw.tableLockZNode, lockName.getNameAsString());
configuration.set(SweepJob.SWEEP_JOB_ID, "1");
configuration.set(SweepJob.SWEEP_JOB_TABLE_NODE, znode);
ServerName serverName = SweepJob.getCurrentServerName(configuration);
configuration.set(SweepJob.SWEEP_JOB_SERVERNAME, serverName.toString());
TableLockManager tableLockManager =
TableLockManager.createTableLockManager(configuration, zkw, serverName);
TableLock lock = tableLockManager.writeLock(lockName, "Run sweep tool");
lock.acquire();
try {
// use the same counter when mocking
Counter counter = new GenericCounter();
Reducer<Text, KeyValue, Writable, Writable>.Context ctx = mock(Reducer.Context.class);
when(ctx.getConfiguration()).thenReturn(configuration);
when(ctx.getCounter(Matchers.any(SweepCounter.class))).thenReturn(counter);
when(ctx.nextKey()).thenReturn(true).thenReturn(false);
when(ctx.getCurrentKey()).thenReturn(new Text(mobFile1));
byte[] refBytes = Bytes.toBytes(mobFile1);
long valueLength = refBytes.length;
byte[] newValue = Bytes.add(Bytes.toBytes(valueLength), refBytes);
KeyValue kv2 =
new KeyValue(
Bytes.toBytes(row),
Bytes.toBytes(family),
Bytes.toBytes(qf),
1,
KeyValue.Type.Put,
newValue);
List<KeyValue> list = new ArrayList<KeyValue>();
list.add(kv2);
when(ctx.getValues()).thenReturn(list);
SweepReducer reducer = new SweepReducer();
reducer.run(ctx);
} finally {
lock.release();
}
FileStatus[] filsStatuses2 = TEST_UTIL.getTestFileSystem().listStatus(mobFamilyPath);
String mobFile2 = filsStatuses2[0].getPath().getName();
// new mob file is generated, old one has been archived
assertEquals(1, filsStatuses2.length);
assertEquals(false, mobFile2.equalsIgnoreCase(mobFile1));
// test sequence file
String workingPath = configuration.get(SweepJob.WORKING_VISITED_DIR_KEY);
FileStatus[] statuses = TEST_UTIL.getTestFileSystem().listStatus(new Path(workingPath));
Set<String> files = new TreeSet<String>();
for (FileStatus st : statuses) {
files.addAll(
getKeyFromSequenceFile(TEST_UTIL.getTestFileSystem(), st.getPath(), configuration));
}
assertEquals(1, files.size());
assertEquals(true, files.contains(mobFile1));
}
