/** Reconstruct the statement. */
protected For_c reconstruct(List inits, Expr cond, List iters, Stmt body) {
if (!CollectionUtil.equals(inits, this.inits)
|| cond != this.cond
|| !CollectionUtil.equals(iters, this.iters)
|| body != this.body) {
For_c n = (For_c) copy();
n.inits = TypedList.copyAndCheck(inits, ForInit.class, true);
n.cond = cond;
n.iters = TypedList.copyAndCheck(iters, ForUpdate.class, true);
n.body = body;
return n;
}
return this;
}
@Test
public void testReverseOrderRandomIntegers() {
Comparator<Integer> naturalOrder = new NaturalOrder<Integer>();
Comparator<Integer> reverse = CollectionUtil.reverseOrder(naturalOrder);
Random random = new Random(243249878l); // Stable "random" sequence
for (int i = 0; i < 65536; i++) {
// Verified to be ~ 50/50 lt/gt
int integer = random.nextInt();
int integerToo = random.nextInt();
assertEquals(0, reverse.compare(integer, integer));
assertEquals(0, reverse.compare(integerToo, integerToo));
int natural = naturalOrder.compare(integer, integerToo);
if (natural == 0) {
// Actually never hits, but eq case is tested above
assertEquals(0, reverse.compare(integer, integerToo));
} else if (natural < 0) {
assertTrue(reverse.compare(integer, integerToo) > 0);
} else {
assertTrue(reverse.compare(integer, integerToo) < 0);
}
}
}
@Test
public void testArrayIterator() {
Iterator<String> iterator = CollectionUtil.iterator(new String[] {"foo", "bar", "baz"});
int count = 0;
for (Object stringObject : stringObjects) {
assertTrue(iterator.hasNext());
assertEquals(stringObject, iterator.next());
try {
iterator.remove();
fail("Array have fixed length, iterator.remove() must throw exception");
} catch (UnsupportedOperationException expected) {
}
count++;
}
assertEquals(3, count);
assertFalse(iterator.hasNext());
try {
iterator.next();
fail("Iterator has more elements than array");
} catch (NoSuchElementException expected) {
}
}
@Test
public void testEnumIterator() {
@SuppressWarnings("unchecked")
Iterator<String> iterator =
CollectionUtil.iterator((Enumeration) new StringTokenizer("foo, bar, baz", ", "));
int count = 0;
for (Object stringObject : stringObjects) {
assertTrue(iterator.hasNext());
assertEquals(stringObject, iterator.next());
try {
iterator.remove();
fail("Enumeration has no remove method, iterator.remove() must throw exception");
} catch (UnsupportedOperationException expected) {
}
count++;
}
assertEquals(3, count);
assertFalse(iterator.hasNext());
try {
iterator.next();
fail("Iterator has more elements than enumeration");
} catch (NoSuchElementException expected) {
}
}
@Test(expected = ArrayStoreException.class)
public void testMergeArraysNativeIllegalType() {
char[] chars = {'a', 'b', 'c'};
int[] integers = {1, 2, 3}; // Integer not assignable to String
CollectionUtil.mergeArrays(chars, integers);
}
@Test(expected = ArrayStoreException.class)
public void testMergeArraysObjectIllegalType() {
String[] strings = {"foo", "bar", "baz"};
Integer[] integers = {1, 2, 3}; // Integer not assignable to String
CollectionUtil.mergeArrays(strings, integers);
}
@Test
public void testMergeArraysObjectAssignable() {
Integer[] integers = {1, 2, 3}; // Integer assignable to Object
Object[] merged = (Object[]) CollectionUtil.mergeArrays(stringObjects, integers);
assertArrayEquals(new Object[] {"foo", "bar", "baz", 1, 2, 3}, merged);
}
@Ignore("For development only")
@Test
@SuppressWarnings({"UnusedDeclaration"})
public void testGenerify() {
List list = Collections.singletonList("foo");
@SuppressWarnings({"unchecked"})
Set set = new HashSet(list);
List<String> strs0 = CollectionUtil.generify(list, String.class);
List<Object> objs0 = CollectionUtil.generify(list, String.class);
// List<String> strs01 = CollectionUtil.generify(list, Object.class); // Not okay
try {
List<String> strs1 =
CollectionUtil.generify(set, String.class); // Not ok, runtime CCE unless set is null
} catch (RuntimeException e) {
e.printStackTrace();
}
try {
ArrayList<String> strs01 =
CollectionUtil.generify(list, String.class); // Not ok, runtime CCE unless list is null
} catch (RuntimeException e) {
e.printStackTrace();
}
Set<String> setstr1 = CollectionUtil.generify(set, String.class);
Set<Object> setobj1 = CollectionUtil.generify(set, String.class);
try {
Set<Object> setobj44 =
CollectionUtil.generify(list, String.class); // Not ok, runtime CCE unless list is null
} catch (RuntimeException e) {
e.printStackTrace();
}
List<String> strs2 = CollectionUtil.<List<String>, String>generify2(list);
List<Object> objs2 = CollectionUtil.<List<Object>, String>generify2(list);
// List<String> morestrs = CollectionUtil.<List<Object>, String>generify2(list); // Not
// ok
try {
List<String> strs3 =
CollectionUtil.<List<String>, String>generify2(
set); // Not ok, runtime CCE unless set is null
} catch (RuntimeException e) {
e.printStackTrace();
}
}
@Test
public void testMergeArraysNative() {
char[] chars = {'a', 'b', 'c'};
char[] more = {'x', 'y', 'z'};
char[] merged = (char[]) CollectionUtil.mergeArrays(chars, more);
assertArrayEquals(new char[] {'a', 'b', 'c', 'x', 'y', 'z'}, merged);
}
public static void main(String[] args) {
HashSet<String> set = new HashSet<String>();
set.add("This");
set.add("is");
set.add("is");
set.add("a");
set.add("a");
set.add("test");
CollectionUtil.display(set);
}
@Test
public void testSelectRandomSubset() {
ArrayList<Integer> list = new ArrayList<Integer>();
for (int i = 0; i < 100; i++) {
list.add(i);
}
for (int i = 0; i < 10; i++) {
List<Integer> subset = CollectionUtil.selectRandomSubset(list, 10);
// System.out.println(StringUtil.toString(subset));
assertTrue("length == 10", subset.size() == 10);
}
}
public void testNamePollTally() throws Exception {
V1NamePoll np;
// test a name poll we won
np = makeCompletedNamePoll(4, 1, 0);
assertEquals(5, np.m_tally.numAgree);
assertEquals(1, np.m_tally.numDisagree);
assertEquals(Tallier.RESULT_WON, np.m_tally.getTallyResult());
// test a name poll we lost with a dissenting vote
np = makeCompletedNamePoll(1, 8, 1);
assertEquals(2, np.m_tally.numAgree);
assertEquals(9, np.m_tally.numDisagree);
// build a master list
np.buildPollLists(np.m_tally.pollVotes.iterator());
// these should be different since we lost the poll
assertFalse(CollectionUtil.isIsomorphic(np.m_tally.localEntries, np.m_tally.votedEntries));
// the expected "correct" set is in our disagree msg
assertTrue(CollectionUtil.isIsomorphic(disagree_entries, np.m_tally.votedEntries));
}
/** 回收 */
private void recycling() {
List<Entry<String, Double>> list = CollectionUtil.sortMapByValue(mc.get(), -1);
Set<String> targetSet = x2mat.keySet();
String word;
for (int i = 0; i < recyclingCount; i++) {
word = list.get(i).getKey();
allFreq -= mc.get().remove(word); // 从全局中移除数字
for (String target : targetSet) {
Double r2 = x2mat.get(target).get().remove(word);
if (r2 != null) {
x2mc.add(target, -r2);
}
}
}
}
@Test
public void testReverseOrder() {
Comparator<String> naturalOrder = new NaturalOrder<String>();
Comparator<String> reverse = CollectionUtil.reverseOrder(naturalOrder);
assertNotNull(reverse);
assertEquals(0, naturalOrder.compare("foo", "foo"));
assertEquals(0, reverse.compare("foo", "foo"));
assertTrue(naturalOrder.compare("bar", "baz") < 0);
assertTrue(reverse.compare("bar", "baz") > 0);
assertTrue(naturalOrder.compare("baz", "bar") > 0);
assertTrue(reverse.compare("baz", "bar") < 0);
}
// method calls with spread-dot operator are not rewritten, hence this method doesn't have to care
// about spread-dot
public static void verifyMethodCondition(
@Nullable ErrorCollector errorCollector,
@Nullable ValueRecorder recorder,
@Nullable String text,
int line,
int column,
@Nullable Object message,
Object target,
String method,
Object[] args,
boolean safe,
boolean explicit,
int lastVariableNum) {
MatcherCondition matcherCondition = MatcherCondition.parse(target, method, args, safe);
if (matcherCondition != null) {
matcherCondition.verify(
errorCollector, getValues(recorder), text, line, column, messageToString(message));
return;
}
if (recorder != null) {
recorder.startRecordingValue(lastVariableNum);
}
Object result =
safe
? GroovyRuntimeUtil.invokeMethodNullSafe(target, method, args)
: GroovyRuntimeUtil.invokeMethod(target, method, args);
if (!explicit && result == null && GroovyRuntimeUtil.isVoidMethod(target, method, args)) return;
if (!GroovyRuntimeUtil.isTruthy(result)) {
List<Object> values = getValues(recorder);
if (values != null) CollectionUtil.setLastElement(values, result);
final ConditionNotSatisfiedError conditionNotSatisfiedError =
new ConditionNotSatisfiedError(
new Condition(
values,
text,
TextPosition.create(line, column),
messageToString(message),
null,
null));
errorCollector.collectOrThrow(conditionNotSatisfiedError);
}
}
void verify(
@Nullable ErrorCollector errorCollector,
@Nullable List<Object> values,
@Nullable String text,
int line,
int column,
@Nullable String message) {
if (HamcrestFacade.matches(matcher, actual)) return;
if (values != null) {
CollectionUtil.setLastElement(values, shortSyntax ? actual : false);
replaceMatcherValues(values);
}
String description = HamcrestFacade.getFailureDescription(matcher, actual, message);
Condition condition =
new Condition(values, text, TextPosition.create(line, column), description, null, null);
errorCollector.collectOrThrow(new ConditionNotSatisfiedError(condition));
}
public void lockssRun() {
setPriority(PRIORITY_PARAM_SIZE_CALC, PRIORITY_DEFAULT_SIZE_CALC);
startWDog(WDOG_PARAM_SIZE_CALC, WDOG_DEFAULT_SIZE_CALC);
triggerWDogOnExit(true);
nowRunning();
while (goOn) {
try {
pokeWDog();
if (sizeCalcQueue.isEmpty()) {
Deadline timeout = Deadline.in(Constants.HOUR);
sizeCalcSem.take(timeout);
}
RepositoryNode node;
synchronized (sizeCalcQueue) {
node = (RepositoryNode) CollectionUtil.getAnElement(sizeCalcQueue);
}
if (node != null) {
long start = TimeBase.nowMs();
log.debug2("CalcSize start: " + node);
long dur = 0;
try {
doSizeCalc(node);
dur = TimeBase.nowMs() - start;
log.debug2("CalcSize finish (" + StringUtil.timeIntervalToString(dur) + "): " + node);
} catch (RuntimeException e) {
log.warning("doSizeCalc: " + node, e);
}
synchronized (sizeCalcQueue) {
sizeCalcQueue.remove(node);
}
pokeWDog();
long sleep = sleepTimeToAchieveLoad(dur, sizeCalcMaxLoad);
Deadline.in(sleep).sleep();
}
} catch (InterruptedException e) {
// just wakeup and check for exit
}
}
if (!goOn) {
triggerWDogOnExit(false);
}
}
/**
* 按步长从一个ArrayList<T>中取出值,成为一个新的ArrayList<T> <br>
* 注意,第一个无论如何都会选取第一个元素
*
* @param arrayList 原始ArrayList<T>
* @param stepLength 每隔多少取一个值
* <pre>
* 例如:
* 原始ArrayList<T>：[yinshi,nichao,nileader,test,name,abc]
* stepLength = 2,那么返回
* [yinshi,nileader,name]
* </pre>
*/
public static <T> ArrayList<T> select(ArrayList<T> arrayList, int stepLength) {
ArrayList<T> arrayListNew = new ArrayList<T>();
if (CollectionUtil.isBlank(arrayList)) {
return arrayListNew;
}
stepLength = IntegerUtil.defaultIfSmallerThan0(stepLength, 1);
if (1 == stepLength) return arrayList;
int index = 0;
while (index + stepLength < arrayList.size()) {
arrayListNew.add(arrayList.get(index));
index += stepLength;
}
arrayListNew.add(arrayList.get(index));
return arrayListNew;
}
public static String convertToStringWithSeparator(
List<?> list, String separator, boolean withSpacing) {
String result = "";
if (CollectionUtil.isEmpty(list)) {
return "";
}
for (Iterator<?> iterator = list.iterator(); iterator.hasNext(); ) {
Object o = iterator.next();
String s = o.toString();
result += s;
if (iterator.hasNext()) {
if (withSpacing) result += ", ";
else result += ",";
}
}
return result;
}
/**
* This function returns all the words present in String parameter passed to it in String array.
* If null is passed the default word separator used in function is white space.
*
* <p>
*
* @param text {@link java.lang.String} The text from which words would get separated.
* @param wordSeperator {@link java.lang.String} The word seperator that is used to separate words
* in text from which words would get separated.
* @return Array of {@link java.lang.String}[] containing all words separated by white space from
* the text.
*/
public static String[] getWordsSeperated(String text, String wordSeperator) {
synchronized (text) {
if (isNotEmpty(text)) {
text = text.trim().replaceAll("[\\.]|[!]|[\\?]|[,]", " ");
String[] words = null;
if (null != wordSeperator) {
words = text.split(wordSeperator);
} else {
words = text.split("[\\s]");
}
List<String> result = new ArrayList<String>();
for (String word : words) {
if (isNotEmpty(word)) {
result.add(word);
}
}
return CollectionUtil.toArray(result, String.class);
}
return new String[] {text};
}
}
@Test
public void testArrayIteratorRange() {
Iterator<String> iterator =
CollectionUtil.iterator(new String[] {"foo", "bar", "baz", "xyzzy"}, 1, 2);
for (int i = 1; i < 3; i++) {
assertTrue(iterator.hasNext());
assertEquals(stringObjects[i], iterator.next());
try {
iterator.remove();
fail("Array has no remove method, iterator.remove() must throw exception");
} catch (UnsupportedOperationException expected) {
}
}
assertFalse(iterator.hasNext());
try {
iterator.next();
fail("Iterator has more elements than array range");
} catch (NoSuchElementException expected) {
}
}
/**
* Tool for lifting over a VCF to another genome build and producing a properly header'd, sorted and
* indexed VCF in one go.
*
* @author Tim Fennell
*/
@CommandLineProgramProperties(
summary =
"Lifts a VCF over from one genome build to another using UCSC liftover. The output file will be sorted "
+ "and indexed. Records may be rejected because they cannot be lifted over or because post-liftover the "
+ "reference allele mismatches the target genome build. Rejected records will be emitted with filters "
+ "to the REJECT file, on the source genome.",
oneLineSummary = "Lifts a VCF between genome builds",
programGroup = VariantProgramGroup.class)
public final class LiftOverVcf extends PicardCommandLineProgram {
@Argument(
fullName = StandardArgumentDefinitions.INPUT_LONG_NAME,
shortName = StandardArgumentDefinitions.INPUT_SHORT_NAME,
doc = "The input VCF file to be lifted over.")
public File INPUT;
@Argument(
fullName = StandardArgumentDefinitions.OUTPUT_LONG_NAME,
shortName = StandardArgumentDefinitions.OUTPUT_SHORT_NAME,
doc = "The output location to write the lifted over VCF to.")
public File OUTPUT;
@Argument(
shortName = "C",
doc =
"The liftover chain file. See https://genome.ucsc.edu/goldenPath/help/chain.html for a description"
+ " of chain files. See http://hgdownload.soe.ucsc.edu/downloads.html#terms for where to download chain files.")
public File CHAIN;
@Argument(doc = "File to which to write rejected records.")
public File REJECT;
/** Filter name to use when a target cannot be lifted over. */
public static final String FILTER_CANNOT_LIFTOVER = "FailedLiftover";
/**
* Filter name to use when a target is lifted over, but the reference allele doens't match the new
* reference.
*/
public static final String FILTER_MISMATCHING_REF_ALLELE = "MismatchedRefAllele";
/** Filters to be added to the REJECT file. */
private static final List<VCFFilterHeaderLine> FILTERS =
CollectionUtil.makeList(
new VCFFilterHeaderLine(
FILTER_CANNOT_LIFTOVER, "Variant could not be lifted between genome builds."),
new VCFFilterHeaderLine(
FILTER_MISMATCHING_REF_ALLELE,
"Reference allele does not match reference genome sequence after liftover."));
@Override
protected Object doWork() {
IOUtil.assertFileIsReadable(INPUT);
IOUtil.assertFileIsReadable(REFERENCE_SEQUENCE);
IOUtil.assertFileIsReadable(CHAIN);
IOUtil.assertFileIsWritable(OUTPUT);
IOUtil.assertFileIsWritable(REJECT);
////////////////////////////////////////////////////////////////////////
// Setup the inputs
////////////////////////////////////////////////////////////////////////
final LiftOver liftOver = new LiftOver(CHAIN);
final VCFFileReader in = new VCFFileReader(INPUT, false);
logger.info("Loading up the target reference genome.");
final ReferenceSequenceFileWalker walker = new ReferenceSequenceFileWalker(REFERENCE_SEQUENCE);
final Map<String, byte[]> refSeqs = new HashMap<>();
for (final SAMSequenceRecord rec : walker.getSequenceDictionary().getSequences()) {
refSeqs.put(rec.getSequenceName(), walker.get(rec.getSequenceIndex()).getBases());
}
CloserUtil.close(walker);
////////////////////////////////////////////////////////////////////////
// Setup the outputs
////////////////////////////////////////////////////////////////////////
final VCFHeader inHeader = in.getFileHeader();
final VCFHeader outHeader = new VCFHeader(inHeader);
outHeader.setSequenceDictionary(walker.getSequenceDictionary());
final VariantContextWriter out =
new VariantContextWriterBuilder()
.setOption(Options.INDEX_ON_THE_FLY)
.setOutputFile(OUTPUT)
.setReferenceDictionary(walker.getSequenceDictionary())
.build();
out.writeHeader(outHeader);
final VariantContextWriter rejects =
new VariantContextWriterBuilder()
.setOutputFile(REJECT)
.unsetOption(Options.INDEX_ON_THE_FLY)
.build();
final VCFHeader rejectHeader = new VCFHeader(in.getFileHeader());
for (final VCFFilterHeaderLine line : FILTERS) rejectHeader.addMetaDataLine(line);
rejects.writeHeader(rejectHeader);
////////////////////////////////////////////////////////////////////////
// Read the input VCF, lift the records over and write to the sorting
// collection.
////////////////////////////////////////////////////////////////////////
long failedLiftover = 0, failedAlleleCheck = 0, total = 0;
logger.info("Lifting variants over and sorting.");
final SortingCollection<VariantContext> sorter =
SortingCollection.newInstance(
VariantContext.class,
new VCFRecordCodec(outHeader),
outHeader.getVCFRecordComparator(),
MAX_RECORDS_IN_RAM,
TMP_DIR);
ProgressLogger progress = new ProgressLogger(logger, 1000000, "read");
for (final VariantContext ctx : in) {
++total;
final Interval source =
new Interval(
ctx.getContig(),
ctx.getStart(),
ctx.getEnd(),
false,
ctx.getContig() + ":" + ctx.getStart() + "-" + ctx.getEnd());
final Interval target = liftOver.liftOver(source, 1.0);
if (target == null) {
rejects.add(new VariantContextBuilder(ctx).filter(FILTER_CANNOT_LIFTOVER).make());
failedLiftover++;
} else {
// Fix the alleles if we went from positive to negative strand
final List<Allele> alleles = new ArrayList<>();
for (final Allele oldAllele : ctx.getAlleles()) {
if (target.isPositiveStrand() || oldAllele.isSymbolic()) {
alleles.add(oldAllele);
} else {
alleles.add(
Allele.create(
SequenceUtil.reverseComplement(oldAllele.getBaseString()),
oldAllele.isReference()));
}
}
// Build the new variant context
final VariantContextBuilder builder =
new VariantContextBuilder(
ctx.getSource(), target.getContig(), target.getStart(), target.getEnd(), alleles);
builder.id(ctx.getID());
builder.attributes(ctx.getAttributes());
builder.genotypes(ctx.getGenotypes());
builder.filters(ctx.getFilters());
builder.log10PError(ctx.getLog10PError());
// Check that the reference allele still agrees with the reference sequence
boolean mismatchesReference = false;
for (final Allele allele : builder.getAlleles()) {
if (allele.isReference()) {
final byte[] ref = refSeqs.get(target.getContig());
final String refString =
StringUtil.bytesToString(ref, target.getStart() - 1, target.length());
if (!refString.equalsIgnoreCase(allele.getBaseString())) {
mismatchesReference = true;
}
break;
}
}
if (mismatchesReference) {
rejects.add(new VariantContextBuilder(ctx).filter(FILTER_MISMATCHING_REF_ALLELE).make());
failedAlleleCheck++;
} else {
sorter.add(builder.make());
}
}
progress.record(ctx.getContig(), ctx.getStart());
}
final NumberFormat pfmt = new DecimalFormat("0.0000%");
final String pct = pfmt.format((failedLiftover + failedAlleleCheck) / (double) total);
logger.info("Processed ", total, " variants.");
logger.info(Long.toString(failedLiftover), " variants failed to liftover.");
logger.info(
Long.toString(failedAlleleCheck),
" variants lifted over but had mismatching reference alleles after lift over.");
logger.info(pct, " of variants were not successfully lifted over and written to the output.");
rejects.close();
in.close();
////////////////////////////////////////////////////////////////////////
// Write the sorted outputs to the final output file
////////////////////////////////////////////////////////////////////////
sorter.doneAdding();
progress = new ProgressLogger(logger, 1000000, "written");
logger.info("Writing out sorted records to final VCF.");
for (final VariantContext ctx : sorter) {
out.add(ctx);
progress.record(ctx.getContig(), ctx.getStart());
}
out.close();
sorter.cleanup();
return null;
}
}
@Test(expected = IndexOutOfBoundsException.class)
public void testMergeArraysObjectNegativeSecondLength() {
CollectionUtil.mergeArrays(stringObjects, 1, 2, integerObjects, 2, -1);
}
public static Number median(Collection<? extends Number> col) {
Number[] n = CollectionUtil.toArray(col, Number.class);
Arrays.sort(n);
return n[n.length / 2];
}
/**
* @param args
* @throws Exception
*/
@SuppressWarnings("unchecked")
public void process(String[] args, String... required) throws Exception {
final boolean debug = LOG.isDebugEnabled();
if (debug) LOG.debug("Processing " + args.length + " parameters...");
final Pattern p = Pattern.compile("=");
for (int i = 0, cnt = args.length; i < cnt; i++) {
final String arg = args[i];
final String[] parts = p.split(arg, 2);
if (parts[0].startsWith("-")) parts[0] = parts[0].substring(1);
if (parts.length == 1) {
if (parts[0].startsWith("${") == false) this.opt_params.add(parts[0]);
continue;
} else if (parts[0].equalsIgnoreCase("tag")) {
continue;
} else if (parts[1].startsWith("${") || parts[0].startsWith("#")) {
continue;
}
if (debug) LOG.debug(String.format("%-35s = %s", parts[0], parts[1]));
// DesignerHints Override
if (parts[0].startsWith(PARAM_DESIGNER_HINTS_PREFIX)) {
String param = parts[0].replace(PARAM_DESIGNER_HINTS_PREFIX, "").toLowerCase();
try {
Field f = DesignerHints.class.getField(param);
this.hints_params.put(f.getName(), parts[1]);
if (debug) LOG.debug(String.format("DesignerHints.%s = %s", param, parts[1]));
} catch (NoSuchFieldException ex) {
throw new Exception("Unknown DesignerHints parameter: " + param, ex);
}
}
// HStoreConf Parameter
else if (HStoreConf.isConfParameter(parts[0])) {
this.conf_params.put(parts[0].toLowerCase(), parts[1]);
}
// ArgumentsParser Parameter
else if (PARAMS.contains(parts[0].toLowerCase())) {
this.params.put(parts[0].toLowerCase(), parts[1]);
}
// Invalid!
else {
String suggestions = "";
i = 0;
String end = CollectionUtil.last(parts[0].split("\\."));
for (String param : PARAMS) {
String param_end = CollectionUtil.last(param.split("\\."));
if (param.startsWith(parts[0])
|| (end != null && param.endsWith(end))
|| (end != null && param_end != null && param_end.startsWith(end))) {
if (suggestions.isEmpty()) suggestions = ". Possible Matches:";
suggestions += String.format("\n [%02d] %s", ++i, param);
}
} // FOR
throw new Exception("Unknown parameter '" + parts[0] + "'" + suggestions);
}
} // FOR
// -------------------------------------------------------
// CATALOGS
// -------------------------------------------------------
// Text File
if (this.params.containsKey(PARAM_CATALOG)) {
String path = this.params.get(PARAM_CATALOG);
if (debug) LOG.debug("Loading catalog from file '" + path + "'");
Catalog catalog = CatalogUtil.loadCatalog(path);
if (catalog == null)
throw new Exception("Failed to load catalog object from file '" + path + "'");
this.updateCatalog(catalog, new File(path));
}
// Jar File
else if (this.params.containsKey(PARAM_CATALOG_JAR)) {
String path = this.params.get(PARAM_CATALOG_JAR);
this.params.put(PARAM_CATALOG, path);
File jar_file = new File(path);
Catalog catalog = CatalogUtil.loadCatalogFromJar(path);
if (catalog == null)
throw new Exception("Failed to load catalog object from jar file '" + path + "'");
if (debug) LOG.debug("Loaded catalog from jar file '" + path + "'");
this.updateCatalog(catalog, jar_file);
if (!this.params.containsKey(PARAM_CATALOG_TYPE)) {
String jar_name = jar_file.getName();
int jar_idx = jar_name.lastIndexOf(".jar");
if (jar_idx != -1) {
ProjectType type = ProjectType.get(jar_name.substring(0, jar_idx));
if (type != null) {
if (debug) LOG.debug("Set catalog type '" + type + "' from catalog jar file name");
this.catalog_type = type;
this.params.put(PARAM_CATALOG_TYPE, this.catalog_type.toString());
}
}
}
}
// Schema File
else if (this.params.containsKey(PARAM_CATALOG_SCHEMA)) {
String path = this.params.get(PARAM_CATALOG_SCHEMA);
Catalog catalog = CompilerUtil.compileCatalog(path);
if (catalog == null) throw new Exception("Failed to load schema from '" + path + "'");
if (debug) LOG.debug("Loaded catalog from schema file '" + path + "'");
this.updateCatalog(catalog, new File(path));
}
// Catalog Type
if (this.params.containsKey(PARAM_CATALOG_TYPE)) {
String catalog_type = this.params.get(PARAM_CATALOG_TYPE);
ProjectType type = ProjectType.get(catalog_type);
if (type == null) {
throw new Exception("Unknown catalog type '" + catalog_type + "'");
}
this.catalog_type = type;
}
// Update Cluster Configuration
if (this.params.containsKey(ArgumentsParser.PARAM_CATALOG_HOSTS)) {
ClusterConfiguration cc =
new ClusterConfiguration(this.getParam(ArgumentsParser.PARAM_CATALOG_HOSTS));
this.updateCatalog(FixCatalog.addHostInfo(this.catalog, cc), null);
}
// Check the requirements after loading the catalog, because some of the
// above parameters will set the catalog one
if (required != null && required.length > 0) this.require(required);
// -------------------------------------------------------
// PHYSICAL DESIGN COMPONENTS
// -------------------------------------------------------
if (this.params.containsKey(PARAM_PARTITION_PLAN)) {
assert (this.catalog_db != null);
File path = new File(this.params.get(PARAM_PARTITION_PLAN));
boolean ignoreMissing =
this.getBooleanParam(ArgumentsParser.PARAM_PARTITION_PLAN_IGNORE_MISSING, false);
if (path.exists() || (path.exists() == false && ignoreMissing == false)) {
if (debug) LOG.debug("Loading in partition plan from '" + path + "'");
this.pplan = new PartitionPlan();
this.pplan.load(path.getAbsolutePath(), this.catalog_db);
// Apply!
if (this.params.containsKey(PARAM_PARTITION_PLAN_APPLY)
&& this.getBooleanParam(PARAM_PARTITION_PLAN_APPLY)) {
boolean secondaryIndexes =
this.getBooleanParam(PARAM_PARTITION_PLAN_NO_SECONDARY, false) == false;
LOG.info(
String.format(
"Applying PartitionPlan '%s' to catalog [enableSecondary=%s]",
path.getName(), secondaryIndexes));
this.pplan.apply(this.catalog_db, secondaryIndexes);
}
}
}
// -------------------------------------------------------
// DESIGNER COMPONENTS
// -------------------------------------------------------
if (this.params.containsKey(PARAM_DESIGNER_THREADS)) {
this.max_concurrent = Integer.valueOf(this.params.get(PARAM_DESIGNER_THREADS));
}
if (this.params.containsKey(PARAM_DESIGNER_INTERVALS)) {
this.num_intervals = Integer.valueOf(this.params.get(PARAM_DESIGNER_INTERVALS));
}
if (this.params.containsKey(PARAM_DESIGNER_HINTS)) {
String path = this.params.get(PARAM_DESIGNER_HINTS);
if (debug)
LOG.debug(
"Loading in designer hints from '"
+ path
+ "'.\nForced Values:\n"
+ StringUtil.formatMaps(this.hints_params));
this.designer_hints.load(path, catalog_db, this.hints_params);
}
if (this.params.containsKey(PARAM_DESIGNER_CHECKPOINT)) {
this.designer_checkpoint = new File(this.params.get(PARAM_DESIGNER_CHECKPOINT));
}
String designer_attributes[] = {
PARAM_DESIGNER_PARTITIONER,
PARAM_DESIGNER_MAPPER,
PARAM_DESIGNER_INDEXER,
PARAM_DESIGNER_COSTMODEL
};
ClassLoader loader = ClassLoader.getSystemClassLoader();
for (String key : designer_attributes) {
if (this.params.containsKey(key)) {
String target_name = this.params.get(key);
Class<?> target_class = loader.loadClass(target_name);
assert (target_class != null);
if (debug) LOG.debug("Set " + key + " class to " + target_class.getName());
if (key.equals(PARAM_DESIGNER_PARTITIONER)) {
this.partitioner_class = (Class<? extends AbstractPartitioner>) target_class;
} else if (key.equals(PARAM_DESIGNER_MAPPER)) {
this.mapper_class = (Class<? extends AbstractMapper>) target_class;
} else if (key.equals(PARAM_DESIGNER_INDEXER)) {
this.indexer_class = (Class<? extends AbstractIndexSelector>) target_class;
} else if (key.equals(PARAM_DESIGNER_COSTMODEL)) {
this.costmodel_class = (Class<? extends AbstractCostModel>) target_class;
// Special Case: TimeIntervalCostModel
if (target_name.endsWith(TimeIntervalCostModel.class.getSimpleName())) {
this.costmodel =
new TimeIntervalCostModel<SingleSitedCostModel>(
this.catalog_db, SingleSitedCostModel.class, this.num_intervals);
} else {
this.costmodel =
ClassUtil.newInstance(
this.costmodel_class,
new Object[] {this.catalog_db},
new Class[] {Database.class});
}
} else {
assert (false) : "Invalid key '" + key + "'";
}
}
} // FOR
// -------------------------------------------------------
// TRANSACTION ESTIMATION COMPONENTS
// -------------------------------------------------------
if (this.params.containsKey(PARAM_MAPPINGS)) {
assert (this.catalog_db != null);
File path = new File(this.params.get(PARAM_MAPPINGS));
if (path.exists()) {
this.param_mappings.load(path.getAbsolutePath(), this.catalog_db);
} else {
LOG.warn("The ParameterMappings file '" + path + "' does not exist");
}
}
if (this.params.containsKey(PARAM_MARKOV_THRESHOLDS_VALUE)) {
assert (this.catalog_db != null);
float defaultValue = this.getDoubleParam(PARAM_MARKOV_THRESHOLDS_VALUE).floatValue();
this.thresholds = new EstimationThresholds(defaultValue);
this.params.put(PARAM_MARKOV_THRESHOLDS, this.thresholds.toString());
LOG.debug("CREATED THRESHOLDS: " + this.thresholds);
} else if (this.params.containsKey(PARAM_MARKOV_THRESHOLDS)) {
assert (this.catalog_db != null);
this.thresholds = new EstimationThresholds();
File path = new File(this.params.get(PARAM_MARKOV_THRESHOLDS));
if (path.exists()) {
this.thresholds.load(path.getAbsolutePath(), this.catalog_db);
} else {
LOG.warn("The estimation thresholds file '" + path + "' does not exist");
}
LOG.debug("LOADED THRESHOLDS: " + this.thresholds);
}
// -------------------------------------------------------
// HASHER
// -------------------------------------------------------
if (this.catalog != null) {
if (this.params.containsKey(PARAM_HASHER_CLASS)) {
String hasherClassName = this.params.get(PARAM_HASHER_CLASS);
this.hasher_class = (Class<? extends AbstractHasher>) loader.loadClass(hasherClassName);
}
Constructor<? extends AbstractHasher> constructor =
this.hasher_class.getConstructor(new Class[] {Database.class, Integer.class});
int num_partitions = CatalogUtil.getNumberOfPartitions(this.catalog_db);
this.hasher = constructor.newInstance(new Object[] {this.catalog_db, num_partitions});
if (!(this.hasher instanceof DefaultHasher))
LOG.debug("Loaded hasher " + this.hasher.getClass());
if (this.params.containsKey(PARAM_HASHER_PROFILE)) {
this.hasher.load(this.params.get(PARAM_HASHER_PROFILE), null);
}
}
// -------------------------------------------------------
// SAMPLE WORKLOAD TRACE
// -------------------------------------------------------
this.loadWorkload();
}
public static <T> String join(String delimiter, final Iterator<T> items) {
return (join("", delimiter, CollectionUtil.iterable(items)));
}
@Test
public void testMergeArraysObjectOffset() {
Object[] merged =
(Object[]) CollectionUtil.mergeArrays(stringObjects, 1, 2, integerObjects, 2, 1);
assertArrayEquals(new Object[] {"bar", "baz", 3}, merged);
}
/**
* Find a type object in the context of the class.
*
* @param name The name to search for.
*/
public Named find(Matcher<Named> matcher, Context context) throws SemanticException {
Name name = matcher.name();
if (Report.should_report(TOPICS, 2)) Report.report(2, "Looking for " + name + " in " + this);
if (!(type instanceof ClassType)) {
throw new NoClassException(name.toString(), type);
}
ClassType type = (ClassType) this.type;
Named m = null;
QName fullName = null;
QName rawName = null;
if (type.isGloballyAccessible()) {
fullName = QName.make(type.fullName(), name);
QName q = ts.getTransformedClassName(type.def());
rawName = QName.make(q.qualifier(), Name.make(q.name() + "$" + name));
}
if (fullName != null) {
// First check the system resolver.
m = ts.systemResolver().check(fullName);
// Try the raw class file name.
if (m == null) {
m = ts.systemResolver().check(rawName);
}
if (m == null) {
// Go to disk, but only if there is no job for the type.
// If there is a job, all members should be in the resolver
// already.
boolean useLoadedResolver = true;
if (type instanceof ParsedTypeObject) {
ParsedTypeObject pto = (ParsedTypeObject) type;
if (pto.job() != null) {
useLoadedResolver = false;
}
}
if (useLoadedResolver) {
try {
m = ts.systemResolver().find(rawName);
} catch (SemanticException e) {
// Not found; will fall through to error handling code
}
}
}
// If we found something, verify that it matches.
if (m != null) {
try {
m = matcher.instantiate(m);
} catch (SemanticException e) {
// Doesn't match; try again.
m = null;
}
}
}
// Check if the member was explicitly declared.
if (m == null) {
m = type.memberTypeMatching(matcher);
}
// If we found something, make sure it's accessible.
if (m instanceof ClassType) {
ClassType mt = (ClassType) m;
if (!mt.isMember()) {
throw new SemanticException(
"Class " + mt + " is not a member class, " + " but was found in " + type + ".");
}
if (!mt.outer().equals((Object) type)) {
throw new SemanticException(
"Class " + mt + " is not a member class " + " of " + type + ".");
}
return mt;
}
if (m instanceof MemberInstance) {
MemberInstance<?> mi = (MemberInstance<?>) m;
if (!mi.container().equals((Object) type)) {
throw new SemanticException("Type " + mi + " is not a member " + " of " + type + ".");
}
}
if (m != null) {
if (!canAccess(m, context.currentClassDef(), context)) {
throw new SemanticException("Cannot access member type \"" + m + "\".");
}
return m;
}
// If we struck out, try the super types.
// Collect all members of the super types.
// Use a Set to eliminate duplicates.
Set<Named> acceptable = new HashSet<Named>();
if (type.superClass() != null) {
Type sup = type.superClass();
if (sup instanceof ClassType) {
Resolver r = ts.classContextResolver((ClassType) sup, context);
try {
Named n = r.find(matcher);
acceptable.add(n);
} catch (SemanticException e) {
}
}
}
for (Iterator<Type> i = type.interfaces().iterator(); i.hasNext(); ) {
Type sup = (Type) i.next();
if (sup instanceof ClassType) {
Resolver r = ts.classContextResolver((ClassType) sup, context);
try {
Named n = r.find(matcher);
acceptable.add(n);
} catch (SemanticException e) {
}
}
}
if (acceptable.size() == 0) {
throw new NoClassException(name.toString(), type);
} else if (acceptable.size() > 1) {
Set<Type> containers = new HashSet<Type>(acceptable.size());
for (Named n : acceptable) {
if (n instanceof MemberInstance) {
MemberInstance<?> mi = (MemberInstance<?>) n;
containers.add(mi.container());
}
}
if (containers.size() == 2) {
Iterator<Type> i = containers.iterator();
Type t1 = (Type) i.next();
Type t2 = (Type) i.next();
throw new SemanticException(
"Member \""
+ name
+ "\" of "
+ type
+ " is ambiguous; it is defined in both "
+ t1
+ " and "
+ t2
+ ".");
} else if (containers.size() == 0) {
throw new SemanticException("Member \"" + name + "\" of " + type + " is ambiguous.");
} else {
throw new SemanticException(
"Member \""
+ name
+ "\" of "
+ type
+ " is ambiguous; it is defined in "
+ CollectionUtil.listToString(new ArrayList<Type>(containers))
+ ".");
}
}
assert acceptable.size() == 1;
Named t = acceptable.iterator().next();
if (Report.should_report(TOPICS, 2)) Report.report(2, "Found member type " + t);
return t;
}
/**
* Return key/value maps into a nicely formatted table The maps are displayed in order from first
* to last, and there will be a spacer created between each map. The format for each record is:
*
* <p><KEY><DELIMITER><SPACING><VALUE>
*
* <p>If the delimiter is an equal sign, then the format is:
*
* <p><KEY><SPACING><DELIMITER><VALUE>
*
* @param delimiter
* @param upper Upper-case all keys
* @param box Box results
* @param border_top TODO
* @param border_bottom TODO
* @param recursive TODO
* @param first_element_title TODO
* @param maps
* @return
*/
@SuppressWarnings("unchecked")
public static String formatMaps(
String delimiter,
boolean upper,
boolean box,
boolean border_top,
boolean border_bottom,
boolean recursive,
boolean first_element_title,
Map<?, ?>... maps) {
boolean need_divider = (maps.length > 1 || border_bottom || border_top);
// Figure out the largest key size so we can get spacing right
int max_key_size = 0;
int max_title_size = 0;
final Map<Object, String[]> map_keys[] = (Map<Object, String[]>[]) new Map[maps.length];
final boolean map_titles[] = new boolean[maps.length];
for (int i = 0; i < maps.length; i++) {
Map<?, ?> m = maps[i];
if (m == null) continue;
Map<Object, String[]> keys = new HashMap<Object, String[]>();
boolean first = true;
for (Object k : m.keySet()) {
String k_str[] = LINE_SPLIT.split(k != null ? k.toString() : "");
keys.put(k, k_str);
// If the first element has a null value, then we can let it be the title for this map
// It's length doesn't affect the other keys, but will affect the total size of the map
if (first && first_element_title && m.get(k) == null) {
for (String line : k_str) {
max_title_size = Math.max(max_title_size, line.length());
} // FOR
map_titles[i] = true;
} else {
for (String line : k_str) {
max_key_size = Math.max(max_key_size, line.length());
} // FOR
if (first) map_titles[i] = false;
}
first = false;
} // FOR
map_keys[i] = keys;
} // FOR
boolean equalsDelimiter = delimiter.equals("=");
final String f =
"%-"
+ (max_key_size + delimiter.length() + 1)
+ "s"
+ (equalsDelimiter ? "= " : "")
+ "%s\n";
// Now make StringBuilder blocks for each map
// We do it in this way so that we can get the max length of the values
int max_value_size = 0;
StringBuilder blocks[] = new StringBuilder[maps.length];
for (int map_i = 0; map_i < maps.length; map_i++) {
blocks[map_i] = new StringBuilder();
Map<?, ?> m = maps[map_i];
if (m == null) continue;
Map<Object, String[]> keys = map_keys[map_i];
boolean first = true;
for (Entry<?, ?> e : m.entrySet()) {
String key[] = keys.get(e.getKey());
if (first && map_titles[map_i]) {
blocks[map_i].append(StringUtil.join("\n", key));
if (CollectionUtil.last(key).endsWith("\n") == false) blocks[map_i].append("\n");
} else {
Object v_obj = e.getValue();
String v = null;
if (recursive && v_obj instanceof Map<?, ?>) {
v =
formatMaps(
delimiter,
upper,
box,
border_top,
border_bottom,
recursive,
first_element_title,
(Map<?, ?>) v_obj)
.trim();
} else if (key.length == 1 && key[0].trim().isEmpty() && v_obj == null) {
blocks[map_i].append("\n");
continue;
} else if (v_obj == null) {
v = "null";
} else {
v = v_obj.toString();
}
// If the key or value is multiple lines, format them nicely!
String value[] = LINE_SPLIT.split(v);
int total_lines = Math.max(key.length, value.length);
for (int line_i = 0; line_i < total_lines; line_i++) {
String k_line = (line_i < key.length ? key[line_i] : "");
if (upper) k_line = k_line.toUpperCase();
String v_line = (line_i < value.length ? value[line_i] : "");
if (line_i == (key.length - 1)
&& (first == false || (first && v_line.isEmpty() == false))) {
if (equalsDelimiter == false && k_line.trim().isEmpty() == false) k_line += ":";
}
blocks[map_i].append(String.format(f, k_line, v_line));
if (need_divider) max_value_size = Math.max(max_value_size, v_line.length());
} // FOR
if (v.endsWith("\n")) blocks[map_i].append("\n");
}
first = false;
}
} // FOR
// Put it all together!
// System.err.println("max_title_size=" + max_title_size + ", max_key_size=" +
// max_key_size + ", max_value_size=" + max_value_size + ", delimiter=" + delimiter.length());
int total_width =
Math.max(max_title_size, (max_key_size + max_value_size + delimiter.length())) + 1;
String dividing_line = (need_divider ? repeat("-", total_width) : "");
StringBuilder sb = null;
if (maps.length == 1) {
sb = blocks[0];
} else {
sb = new StringBuilder();
for (int i = 0; i < maps.length; i++) {
if (blocks[i].length() == 0) continue;
if (i != 0 && maps[i].size() > 0) sb.append(dividing_line).append("\n");
sb.append(blocks[i]);
} // FOR
}
return (box
? StringUtil.box(sb.toString())
: (border_top ? dividing_line + "\n" : "")
+ sb.toString()
+ (border_bottom ? dividing_line : ""));
}
@Test(expected = IndexOutOfBoundsException.class)
public void testMergeArraysObjectBadSecondOffset() {
CollectionUtil.mergeArrays(stringObjects, 1, 2, integerObjects, 4, 1);
}
