public int compare(Segment<K, V> o1, Segment<K, V> o2) {
if (o1.getLength() == o2.getLength()) {
return 0;
}
return o1.getLength() < o2.getLength() ? -1 : 1;
}
/**
* Write as eger.
*
* @param dos the dos
* @param type the type
* @throws IOException Signals that an I/O exception has occurred.
*/
public void writeAsEGER(OutputStreamWriter dos, String type) throws IOException {
String line;
String nameEN;
for (Segment segment : segmentSet) {
if (type.equals("writing")) {
EntitySet entitySet = segment.getTranscription().getEntitySet();
for (Entity entity : entitySet) {
// logger.info("WRITING entity : "+entity.getListOfWords());
if (entity.isPerson()) {
nameEN = SpeakerNameUtils.normalizeSpeakerName(entity.getListOfWords());
line = segment.getShowName();
line += " " + segment.getStartInSecond();
line += " " + segment.getLastInSecond();
line += " written";
line += " " + nameEN + "\n";
if (segment.getLength() > 0) {
dos.write(line);
}
}
}
} else {
line = segment.getShowName();
line += " " + segment.getStartInSecond();
line += " " + segment.getLastInSecond();
line += " " + type;
line += " " + name + "\n";
if (segment.getLength() > 0) {
dos.write(line);
}
}
}
}
/**
* Write the cluster and the segments to a CTL \e *FILE.
*
* @param dos the DataOutPutstream to write to
* @throws IOException Signals that an I/O exception has occurred.
*/
public void writeAsCTL(OutputStreamWriter dos) throws IOException {
DecimalFormatSymbols dfs = new DecimalFormatSymbols();
dfs.setDecimalSeparator('.');
DecimalFormat df = new DecimalFormat();
df.setDecimalFormatSymbols(dfs);
df.setMaximumFractionDigits(2);
df.setMinimumFractionDigits(2);
df.setGroupingUsed(false);
for (Segment seg : segmentSet) {
float start = seg.getStartInSecond();
float end = seg.getEndInSecond();
String band = Segment.bandwidthNistStrings[0];
if (seg.getBandwidth() == Segment.bandwidthStrings[1]) {
band = Segment.bandwidthNistStrings[1];
} else if (seg.getBandwidth() == Segment.bandwidthStrings[2]) {
band = Segment.bandwidthNistStrings[2];
}
String line =
seg.getShowName() + " " + seg.getStart() + " " + (seg.getStart() + seg.getLength());
line += " " + seg.getShowName() + "-" + df.format(start) + "-" + df.format(end) + "-" + band;
line += "-" + this.gender + "-" + name + "\n";
dos.write(line);
}
}
/**
* Gets the sum of the segment lengths.
*
* @return the length sum
*/
public int getLength() {
int length = 0;
for (Segment segment : segmentSet) {
length += segment.getLength();
}
return length;
}
public double getLength() {
double length = 0;
for (Segment s : getSegments()) {
length += s.getLength();
}
return length;
}
/**
* Write the cluster and the segments to a \e *FILE.
*
* @param dos the DataOutPutstream to write to
* @throws IOException Signals that an I/O exception has occurred.
*/
public void writeAsSeg(OutputStreamWriter dos) throws IOException {
Set<Entry<String, Object>> set = getInformation().entrySet();
String info = "";
for (Entry<String, Object> entry : set) {
info += "[ " + entry.getKey() + " = " + entry.getValue().toString() + " ] ";
}
dos.write(";; cluster " + name + " " + info + "\n");
String line;
/*
* String line = getInformations(); if (line != "") { line = ";; cluster:" + name + getInformations() + "\n"; dos.write(line); }
*/
for (Segment segment : segmentSet) {
line = segment.getShowName() + " " + segment.getChannel() + " " + segment.getStart();
line += " " + segment.getLength() + " " + gender + " " + segment.getBandwidth();
line += " " + segment.getEnvironement() + " " + name + segment.getInformations() + "\n";
if (segment.getLength() > 0) {
dos.write(line);
}
}
}
/**
* Collapse the near segments.
*
* @param delay the possible delay between segment
* @return the array list
*/
public ArrayList<Integer> collapse(int delay) {
Iterator<Segment> segmentIterator = iterator();
ArrayList<Integer> list = new ArrayList<Integer>();
Segment previous, current;
if (segmentIterator.hasNext()) {
previous = segmentIterator.next();
while (segmentIterator.hasNext()) {
current = segmentIterator.next();
if (current.getShowName().compareTo(previous.getShowName()) == 0) {
int previousStart = previous.getStart();
int previousEnd = previousStart + previous.getLength();
int currentStart = current.getStart();
int currentLength = current.getLength();
list.add(currentLength);
if ((previousEnd + delay) >= currentStart) {
previous.setLength((currentStart - previousStart) + currentLength);
LinkSet linkSetSave = previous.getLinkSet();
EntitySet entitiesSave = previous.getEntities();
previous.getInformation().putAll(current.getInformation());
previous.setLinkSet(linkSetSave);
previous.setEntities(entitiesSave);
int sizeLink = previous.getLinkSet().size();
for (Link link : current.getLinkSet()) {
link.setId(sizeLink++);
previous.getLinkSet().add(link);
}
for (Entity entity : current.getEntities()) {
previous.getEntities().add(entity);
}
segmentIterator.remove();
} else {
previous = current;
}
} else {
previous = current;
}
}
}
return list;
}
public double getLengthFrom(Way w) {
double length = Double.NEGATIVE_INFINITY;
for (Segment s : getSegments()) {
length += s.getLength();
if (w.equals(s.getWay())) {
length = 0;
}
}
if (length < 0) {
throw new IllegalArgumentException("Way must be part of the route.");
}
return length;
}
/**
* Cluster to frames convertor.
*
* @return the tree map< integer, segment> containing the segment sorted by show first and start
* time
*/
public TreeMap<Integer, Segment> clusterToFrames() {
TreeMap<Integer, Segment> segmentTreeMapResult = new TreeMap<Integer, Segment>();
TreeSet<Segment> segmentList = getSegments();
for (Segment segment : segmentList) {
int start = segment.getStart();
int length = segment.getLength();
for (int i = start; i < (start + length); i++) {
Segment newSegment = (segment.clone());
newSegment.setStart(i);
newSegment.setLength(1);
newSegment.setCluster(this);
// Plus utile le genre porte sur le cluster
// newSegment.setGender(getGender());
segmentTreeMapResult.put(i, newSegment);
}
}
return segmentTreeMapResult;
}
/**
* Write the cluster and the segments to a \e *FILE.
*
* @param dos the DataOutPutstream to write to
* @throws IOException Signals that an I/O exception has occurred.
*/
public void writeAsXSeg(OutputStreamWriter dos) throws IOException {
String line = getInformations();
if (line != "") {
line = "audio cluster " + name + " " + getInformations() + "\n";
dos.write(line);
}
for (Segment segment : segmentSet) {
line =
"audio segment "
+ segment.getShowName()
+ " "
+ segment.getChannel()
+ " "
+ segment.getStart();
line += " " + segment.getLength() + " " + gender + " " + segment.getBandwidth();
line += " " + segment.getEnvironement() + " " + name + segment.getInformations() + "\n";
dos.write(line);
}
}
@Override
public int compareTo(Segment segment) {
return segment.getLength() - this.length;
}
RawKeyValueIterator merge(
Class<K> keyClass, Class<V> valueClass, int factor, int inMem, Path tmpDir)
throws IOException {
LOG.info("Merging " + segments.size() + " sorted segments");
// create the MergeStreams from the sorted map created in the constructor
// and dump the final output to a file
int numSegments = segments.size();
int origFactor = factor;
int passNo = 1;
do {
// get the factor for this pass of merge. We assume in-memory segments
// are the first entries in the segment list and that the pass factor
// doesn't apply to them
factor = getPassFactor(factor, passNo, numSegments - inMem);
if (1 == passNo) {
factor += inMem;
}
List<Segment<K, V>> segmentsToMerge = new ArrayList<Segment<K, V>>();
int segmentsConsidered = 0;
int numSegmentsToConsider = factor;
while (true) {
// extract the smallest 'factor' number of segments
// Call cleanup on the empty segments (no key/value data)
List<Segment<K, V>> mStream = getSegmentDescriptors(numSegmentsToConsider);
for (Segment<K, V> segment : mStream) {
// Initialize the segment at the last possible moment;
// this helps in ensuring we don't use buffers until we need them
segment.init();
long startPos = segment.getPosition();
boolean hasNext = segment.next();
long endPos = segment.getPosition();
totalBytesProcessed += endPos - startPos;
mergeProgress.set(totalBytesProcessed * progPerByte);
if (hasNext) {
segmentsToMerge.add(segment);
segmentsConsidered++;
} else {
segment.close();
numSegments--; // we ignore this segment for the merge
}
}
// if we have the desired number of segments
// or looked at all available segments, we break
if (segmentsConsidered == factor || segments.size() == 0) {
break;
}
numSegmentsToConsider = factor - segmentsConsidered;
}
// feed the streams to the priority queue
initialize(segmentsToMerge.size());
clear();
for (Segment<K, V> segment : segmentsToMerge) {
put(segment);
}
// if we have lesser number of segments remaining, then just return the
// iterator, else do another single level merge
if (numSegments <= factor) {
// calculate the length of the remaining segments. Required for
// calculating the merge progress
long totalBytes = 0;
for (int i = 0; i < segmentsToMerge.size(); i++) {
totalBytes += segmentsToMerge.get(i).getLength();
}
if (totalBytes != 0) // being paranoid
progPerByte = 1.0f / (float) totalBytes;
if (totalBytes != 0) mergeProgress.set(totalBytesProcessed * progPerByte);
else mergeProgress.set(1.0f); // Last pass and no segments left - we're done
LOG.info(
"Down to the last merge-pass, with "
+ numSegments
+ " segments left of total size: "
+ totalBytes
+ " bytes");
return this;
} else {
LOG.info(
"Merging "
+ segmentsToMerge.size()
+ " intermediate segments out of a total of "
+ (segments.size() + segmentsToMerge.size()));
// we want to spread the creation of temp files on multiple disks if
// available under the space constraints
long approxOutputSize = 0;
for (Segment<K, V> s : segmentsToMerge) {
approxOutputSize +=
s.getLength() + ChecksumFileSystem.getApproxChkSumLength(s.getLength());
}
Path tmpFilename = new Path(tmpDir, "intermediate").suffix("." + passNo);
Path outputFile =
lDirAlloc.getLocalPathForWrite(tmpFilename.toString(), approxOutputSize, conf);
Writer<K, V> writer = new Writer<K, V>(conf, fs, outputFile, keyClass, valueClass, codec);
writeFile(this, writer, reporter);
writer.close();
// we finished one single level merge; now clean up the priority
// queue
this.close();
// Add the newly create segment to the list of segments to be merged
Segment<K, V> tempSegment = new Segment<K, V>(conf, fs, outputFile, codec, false);
segments.add(tempSegment);
numSegments = segments.size();
Collections.sort(segments, segmentComparator);
passNo++;
}
// we are worried about only the first pass merge factor. So reset the
// factor to what it originally was
factor = origFactor;
} while (true);
}
