private final SnippetVector checkValue(
SnippetVector currentVector,
Iterator<SnippetVector> vectorIterator,
int startOffset,
Fragment fragment) {
if (currentVector == null) return null;
StringBuilder result = new StringBuilder();
String originalText = fragment.getOriginalText();
int originalTextLength = originalText.length();
int endOffset = startOffset + originalTextLength;
int pos = 0;
while (currentVector != null) {
int end = currentVector.end - fragment.vectorOffset;
if (end > endOffset) break;
int start = currentVector.start - fragment.vectorOffset;
if (start >= startOffset) {
appendSubString(originalText, pos, start - startOffset, result);
if (tags != null) result.append(tags[0]);
appendSubString(originalText, start - startOffset, end - startOffset, result);
if (tags != null) result.append(tags[1]);
pos = end - startOffset;
}
currentVector = vectorIterator.hasNext() ? vectorIterator.next() : null;
}
if (result.length() == 0) return currentVector;
if (pos < originalTextLength) appendSubString(originalText, pos, originalTextLength, result);
fragment.setHighlightedText(result.toString());
return currentVector;
}
/**
* Overlays this fragment list by some {@link Fragment}. That means that parts that overlap are
* taken from the fragment passed.
*
* @param fragment The fragment.
* @return A fragment list storing the overlay result.
*/
public FragmentList overlayBy(final Fragment fragment) {
final FragmentList result = new FragmentList();
try {
result.addFragment(fragment);
} catch (final IncompatibleFragmentException e) {
throw new Error(e);
}
final PositionInText posTo = fragment.getTo();
final ListIterator<Fragment> it = this.fragments.listIterator();
while (it.hasNext()) {
final Fragment oldFragment = it.next();
if (posTo.lessThan(oldFragment.getFrom())) {
break;
}
final ListIterator<Fragment> fIt = result.fragments.listIterator();
while (fIt.hasNext()) {
final Fragment f = fIt.next();
final FragmentList rest = f.subtract(oldFragment);
fIt.remove();
for (final Fragment newFragment : rest.fragments) {
fIt.add(newFragment);
}
}
}
try {
result.addFragmentList(this);
} catch (final IncompatibleFragmentException e) {
throw new Error(e);
}
return result;
}
public void add(Fragment f, String path) {
if (!this.contains(f)) {
localRoute.put(f.toInt(), path);
} else {
localRoute.remove(f.toInt());
localRoute.put(f.toInt(), path);
}
}
/**
* Subtracts some {@link Fragment} from this fragment list.
*
* @param fragment The fragment to subtract.
* @return A list of remaining fragments.
*/
public FragmentList subtract(final Fragment fragment) {
final FragmentList result = new FragmentList();
try {
for (final Fragment oldFragment : this.fragments) {
if (oldFragment.overlaps(fragment)) {
result.addFragmentList(oldFragment.subtract(fragment));
} else {
result.addFragment(oldFragment);
}
}
} catch (final IncompatibleFragmentException e) {
throw new Error(e);
}
return result;
}
/**
* Adds a fragment to the fragment list.
*
* @param fragment The fragment to add.
* @throws IncompatibleFragmentException if the fragment overlaps some other fragment in this
* fragment list.
*/
public void addFragment(final Fragment fragment) throws IncompatibleFragmentException {
final PositionInText posTo = fragment.getTo();
final ListIterator<Fragment> it = this.fragments.listIterator();
while (it.hasNext()) {
final Fragment oldFragment = it.next();
if (oldFragment.overlaps(fragment)) {
throw new IncompatibleFragmentException();
} else if (posTo.compareTo(oldFragment.getFrom()) < 0) {
it.previous();
it.add(fragment);
return;
}
}
this.fragments.add(fragment);
}
/**
* @param request HTTP request
* @param response HTTP response
* @return rendered HTML output
*/
public String renderFragment(HttpRequest request, HttpResponse response) {
String uriWithoutContextPath = request.getUriWithoutContextPath();
String uriPart = uriWithoutContextPath.substring(UriUtils.FRAGMENTS_URI_PREFIX.length());
String fragmentName = NameUtils.getFullyQualifiedName(rootComponent.getName(), uriPart);
// When building the dependency tree, all fragments are accumulated into the rootComponent.
Fragment fragment = lookup.getAllFragments().get(fragmentName);
if (fragment == null) {
throw new FragmentNotFoundException(
"Requested fragment '" + fragmentName + "' does not exists.");
}
Model model = new MapModel(request.getQueryParams());
RequestLookup requestLookup = createRequestLookup(request, response);
API api = new API(sessionRegistry, requestLookup);
return fragment.render(model, lookup, requestLookup, api);
}
public Object get(Fragment f) {
if (this.contains(f)) {
return localRoute.get(f.toInt());
} else {
return null;
}
}
@Override
public void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
if (getArguments() != null) {
url = getArguments().getString("url");
} else {
Log.e("eMMa", "No URL defined.");
}
}
public void test() throws Exception {
Bundle bundle = _bundleManager.getBundle(SCRIPTS_JS);
List<Fragment> allFragment = bundle.getAllFragment();
assertNotNull(allFragment);
assertEquals(1, allFragment.size());
Fragment fragment = allFragment.get(0);
fragment.getFile();
String uncompressed = FileUtils.readFileToString(fragment.getFile());
assertNotNull(uncompressed);
s_logger.debug("uncompressed length {}", uncompressed.length());
String compressed = _compressor.getBundle(SCRIPTS_JS);
assertNotNull(compressed);
s_logger.debug("compressed length {}", compressed.length());
assertTrue(uncompressed.length() > compressed.length());
}
@Test
public void testBond() {
Fragment frag = new Fragment(mock(Element.class));
Atom a1 = new Atom(1, ChemEl.C, frag);
Atom a2 = new Atom(2, ChemEl.C, frag);
frag.addAtom(a1);
frag.addAtom(a2);
Bond bond = new Bond(a1, a2, 1);
assertNotNull("Got bond", bond);
assertEquals("From = 1", 1, bond.getFrom());
assertEquals("To = 2", 2, bond.getTo());
assertEquals("Order = 1", 1, bond.getOrder());
assertEquals(a1, bond.getFromAtom());
assertEquals(a2, bond.getToAtom());
assertEquals(a2, bond.getOtherAtom(a1));
assertEquals(a1, bond.getOtherAtom(a2));
assertEquals(null, bond.getBondStereo());
assertEquals(null, bond.getSmilesStereochemistry());
}
@Test
public void testBondMutation() {
Fragment frag = new Fragment(mock(Element.class));
Atom a1 = new Atom(1, ChemEl.C, frag);
Atom a2 = new Atom(2, ChemEl.C, frag);
Atom a3 = new Atom(3, ChemEl.C, frag);
Atom a4 = new Atom(4, ChemEl.C, frag);
frag.addAtom(a1);
frag.addAtom(a2);
frag.addAtom(a3);
frag.addAtom(a4);
Bond bond = new Bond(a2, a3, 1);
bond.setOrder(2);
assertEquals("Order = 2", 2, bond.getOrder());
BondStereo bondStereo = new BondStereo(new Atom[] {a1, a2, a3, a4}, BondStereoValue.TRANS);
bond.setBondStereo(bondStereo);
assertEquals(bondStereo, bond.getBondStereo());
bond.setSmilesStereochemistry(SMILES_BOND_DIRECTION.LSLASH);
assertEquals(SMILES_BOND_DIRECTION.LSLASH, bond.getSmilesStereochemistry());
}
/** Merges adjacent fragments. */
public void coalesce() {
final ListIterator<Fragment> it = this.fragments.listIterator();
Fragment fragment = null;
while (it.hasNext()) {
final Fragment oldFragment = it.next();
if (fragment == null) {
fragment = oldFragment;
it.remove();
} else if (oldFragment.isAdjacentTo(fragment)) {
it.remove();
fragment = oldFragment.adjoin(fragment);
} else {
it.remove();
it.add(fragment);
fragment = oldFragment;
}
}
if (fragment != null) {
this.fragments.add(fragment);
}
}
protected void emit(IToken token, int offset, int length) {
log.debug("Emitting ({}, offset: {}, length: {})", o(token.getData(), offset, length));
if (length == 0) {
log.debug(
"Empty token emitted zero length, data: {}, offset: {}, length: {}",
o(token.getData(), offset, length));
} else if (length < 0) {
log.error(
"Invalid token emitted negative length, data: {}, offset: {}, length: {}",
o(token.getData(), offset, length));
} else {
log.debug(
"Token emitted, data: {}, offset: {}, length: {}, content: <{}>",
o(token.getData(), offset, length, getContentForLog(offset, length)));
}
// can we merge previous one?
if (!fragments.isEmpty()) {
Fragment previous = getLastFragment();
// check no hole
int requiredOffset = previous.offset + previous.length;
if (offset != requiredOffset) {
log.debug(
"**hole completion**, offset: {} (vs required: {}), length: {}; previous offset: {}, length: {}",
o(offset, requiredOffset, length, previous.offset, previous.length));
emit(getDefaultToken(), requiredOffset, offset - requiredOffset);
previous = getLastFragment();
}
if (previous.token == token) {
previous.length += length;
log.debug("Token merged, offset: {}, length: {}", o(previous.offset, previous.length));
return;
}
}
Fragment fragment = new Fragment(token, offset, length);
addFragment(fragment);
}
@Override
public SkyValue compute(SkyKey skyKey, Environment env)
throws InterruptedException, SkyFunctionException {
BuildConfigurationValue.Key key = (BuildConfigurationValue.Key) skyKey.argument();
Set<Fragment> fragments;
try {
fragments = getConfigurationFragments(key, env);
} catch (InvalidConfigurationException e) {
throw new BuildConfigurationFunctionException(e);
}
if (fragments == null) {
return null;
}
ClassToInstanceMap<Fragment> fragmentsMap = MutableClassToInstanceMap.create();
for (Fragment fragment : fragments) {
fragmentsMap.put(fragment.getClass(), fragment);
}
return new BuildConfigurationValue(
new BuildConfiguration(
directories, fragmentsMap, key.getBuildOptions(), !key.actionsEnabled()));
}
private static void visit(PlanningSet planningSet, Fragment n) {
Preconditions.checkNotNull(planningSet);
Preconditions.checkNotNull(n);
Wrapper wrapper = planningSet.get(n);
n.getRoot().accept(opStatCollector, wrapper);
// logger.debug("Set stats to {}", wrapper.getStats());
// receivers...
for (ExchangeFragmentPair child : n) {
// get the fragment node that feeds this node.
Fragment childNode = child.getNode();
visit(planningSet, childNode);
}
}
public boolean add(Fragment f, BF bf) {
if (!this.contains(f)) {
ContainerLocal c = new ContainerLocal(limit);
c.add(bf);
localRoute.put(f.toInt(), c);
return true;
} else {
if ((((this.get(f)).getClass()).getName()).equals("java.lang.String")) {
return false;
} else {
return ((ContainerLocal) this.get(f)).add(bf);
}
}
}
@Override
protected boolean hasUnclosableRings(Fragment frag) {
/* --- check for uncloseable rings */
int i, k, n; /* loop variable, edge counter */
Graph graph; /* the fragment as a graph */
Node node; /* to traverse the nodes */
graph = frag.getAsGraph(); /* get the fragment as a graph */
for (i = frag.src; --i >= 0; ) {
node = graph.nodes[i]; /* traverse the unextendable nodes */
for (n = 0, k = node.deg; --k >= 0; ) if (node.edges[k].isInRing()) n++;
if (n == 1) return true; /* if there is a single ring edge, */
} /* a ring cannot be closed anymore, */
return false; /* else all rings may be closable */
} /* hasUnclosableRings() */
public Fragment removeFragment(Fragment fragment) {
getFragments().remove(fragment);
fragment.setIonType(null);
return fragment;
}
public Fragment addFragment(Fragment fragment) {
getFragments().add(fragment);
fragment.setIonType(this);
return fragment;
}
@Override
public void onSaveInstanceState(Bundle outState) {
outState.putString("url", url);
super.onSaveInstanceState(outState);
}
protected QueryWorkUnit generateWorkUnit(
OptionList options,
DrillbitEndpoint foremanNode,
QueryId queryId,
PhysicalPlanReader reader,
Fragment rootNode,
PlanningSet planningSet,
UserSession session,
QueryContextInformation queryContextInfo)
throws ExecutionSetupException {
List<PlanFragment> fragments = Lists.newArrayList();
PlanFragment rootFragment = null;
FragmentRoot rootOperator = null;
// now we generate all the individual plan fragments and associated assignments. Note, we need
// all endpoints
// assigned before we can materialize, so we start a new loop here rather than utilizing the
// previous one.
for (Wrapper wrapper : planningSet) {
Fragment node = wrapper.getNode();
final PhysicalOperator physicalOperatorRoot = node.getRoot();
boolean isRootNode = rootNode == node;
if (isRootNode && wrapper.getWidth() != 1) {
throw new ForemanSetupException(
String.format(
"Failure while trying to setup fragment. "
+ "The root fragment must always have parallelization one. In the current case, the width was set to %d.",
wrapper.getWidth()));
}
// a fragment is self driven if it doesn't rely on any other exchanges.
boolean isLeafFragment = node.getReceivingExchangePairs().size() == 0;
// Create a minorFragment for each major fragment.
for (int minorFragmentId = 0; minorFragmentId < wrapper.getWidth(); minorFragmentId++) {
IndexedFragmentNode iNode = new IndexedFragmentNode(minorFragmentId, wrapper);
wrapper.resetAllocation();
PhysicalOperator op = physicalOperatorRoot.accept(Materializer.INSTANCE, iNode);
Preconditions.checkArgument(op instanceof FragmentRoot);
FragmentRoot root = (FragmentRoot) op;
// get plan as JSON
String plan;
String optionsData;
try {
plan = reader.writeJson(root);
optionsData = reader.writeJson(options);
} catch (JsonProcessingException e) {
throw new ForemanSetupException("Failure while trying to convert fragment into json.", e);
}
FragmentHandle handle =
FragmentHandle //
.newBuilder() //
.setMajorFragmentId(wrapper.getMajorFragmentId()) //
.setMinorFragmentId(minorFragmentId) //
.setQueryId(queryId) //
.build();
PlanFragment fragment =
PlanFragment.newBuilder() //
.setForeman(foremanNode) //
.setFragmentJson(plan) //
.setHandle(handle) //
.setAssignment(wrapper.getAssignedEndpoint(minorFragmentId)) //
.setLeafFragment(isLeafFragment) //
.setContext(queryContextInfo)
.setMemInitial(wrapper.getInitialAllocation()) //
.setMemMax(wrapper.getMaxAllocation())
.setOptionsJson(optionsData)
.setCredentials(session.getCredentials())
.addAllCollector(CountRequiredFragments.getCollectors(root))
.build();
if (isRootNode) {
logger.debug("Root fragment:\n {}", DrillStringUtils.unescapeJava(fragment.toString()));
rootFragment = fragment;
rootOperator = root;
} else {
logger.debug("Remote fragment:\n {}", DrillStringUtils.unescapeJava(fragment.toString()));
fragments.add(fragment);
}
}
}
return new QueryWorkUnit(rootOperator, rootFragment, fragments);
}
public boolean getSnippets(
final int docId,
final ReaderInterface reader,
final List<FieldValueItem> values,
final List<FieldValueItem> snippets,
final Timer parentTimer)
throws IOException, ParseException, SyntaxError, SearchLibException {
if (values == null) return false;
final Timer timer = new Timer(parentTimer, "SnippetField " + this.name);
final long halfTimeExpiration =
this.timeLimit == 0 ? 0 : timer.getStartOffset(this.timeLimit / 2);
final long expiration = this.timeLimit == 0 ? 0 : timer.getStartOffset(this.timeLimit);
FragmenterAbstract fragmenter = fragmenterTemplate.newInstance();
SnippetVector currentVector = null;
Timer t = new Timer(timer, "extractTermVectorIterator");
Iterator<SnippetVector> vectorIterator =
SnippetVectors.extractTermVectorIterator(
docId, reader, snippetQueries, name, values, indexAnalyzer, t, halfTimeExpiration);
if (vectorIterator != null)
currentVector = vectorIterator.hasNext() ? vectorIterator.next() : null;
t.end(null);
t = new Timer(timer, "getFraments");
int startOffset = 0;
FragmentList fragments = new FragmentList();
int vectorOffset = 0;
for (FieldValueItem valueItem : values) {
String value = valueItem.getValue();
if (value != null) {
// VectorOffset++ depends of EndOffset bug #patch Lucene 579 and
// 1458
fragmenter.getFragments(value, fragments, vectorOffset++);
}
}
t.end(null);
if (fragments.size() == 0) {
timer.end(null);
return false;
}
t = new Timer(timer, "checkValue");
Fragment fragment = fragments.first();
while (fragment != null) {
currentVector = checkValue(currentVector, vectorIterator, startOffset, fragment);
startOffset += fragment.getOriginalText().length();
fragment = fragment.next();
}
t.end(null);
Timer sbTimer = new Timer(timer, "snippetBuilder");
boolean result = false;
int snippetCounter = maxSnippetNumber;
int scoredFragment = 0;
while (snippetCounter-- != 0) {
Fragment bestScoreFragment = null;
fragment = Fragment.findNextHighlightedFragment(fragments.first());
List<Fragment> scoreFragments = new ArrayList<Fragment>(0);
double maxSearchScore = 0;
t = new Timer(sbTimer, "fragmentScore");
boolean expired = false;
while (fragment != null) {
double sc = fragment.searchScore(name, queryAnalyzer, query);
if (sc > maxSearchScore) maxSearchScore = sc;
scoreFragments.add(fragment);
fragment = Fragment.findNextHighlightedFragment(fragment.next());
scoredFragment++;
if (expiration != 0) {
if (System.currentTimeMillis() > expiration) {
expired = true;
break;
}
}
}
t.end("fragmentScore " + scoredFragment + " " + expired);
for (Fragment frag : scoreFragments)
bestScoreFragment =
Fragment.bestScore(bestScoreFragment, frag, maxSearchScore, maxSnippetSize);
if (bestScoreFragment != null) {
SnippetBuilder snippetBuilder =
new SnippetBuilder(maxSnippetSize, unescapedSeparator, tags, bestScoreFragment);
if (snippetBuilder.length() > 0)
snippets.add(new FieldValueItem(FieldValueOriginEnum.SNIPPET, snippetBuilder.toString()));
fragments.remove(snippetBuilder.getFragments());
result = true;
continue;
}
if (fragments.first() == null) break;
SnippetBuilder snippetBuilder =
new SnippetBuilder(maxSnippetSize, unescapedSeparator, tags, fragments.first());
if (snippetBuilder.length() > 0) {
snippets.add(new FieldValueItem(FieldValueOriginEnum.SNIPPET, snippetBuilder.toString()));
fragments.remove(snippetBuilder.getFragments());
}
}
sbTimer.end(null);
timer.end(null);
return result;
}
public boolean contains(Fragment f) {
return localRoute.containsKey(f.toInt());
}
public void remove(Fragment f) {
this.localRoute.remove(f.toInt());
}
@Override
public Fragment getItem(int position) {
TabInfo info = mTabs.get(position);
return Fragment.instantiate(mContext, info.clss.getName(), info.args);
}
private QueryWorkUnit generateWorkUnit(
DrillbitEndpoint foremanNode,
QueryId queryId,
PhysicalPlanReader reader,
Fragment rootNode,
PlanningSet planningSet)
throws ExecutionSetupException {
List<PlanFragment> fragments = Lists.newArrayList();
PlanFragment rootFragment = null;
FragmentRoot rootOperator = null;
long queryStartTime = System.currentTimeMillis();
// now we generate all the individual plan fragments and associated assignments. Note, we need
// all endpoints
// assigned before we can materialize, so we start a new loop here rather than utilizing the
// previous one.
for (Wrapper wrapper : planningSet) {
Fragment node = wrapper.getNode();
Stats stats = node.getStats();
final PhysicalOperator physicalOperatorRoot = node.getRoot();
boolean isRootNode = rootNode == node;
if (isRootNode && wrapper.getWidth() != 1)
throw new FragmentSetupException(
String.format(
"Failure while trying to setup fragment. The root fragment must always have parallelization one. In the current case, the width was set to %d.",
wrapper.getWidth()));
// a fragment is self driven if it doesn't rely on any other exchanges.
boolean isLeafFragment = node.getReceivingExchangePairs().size() == 0;
// Create a minorFragment for each major fragment.
for (int minorFragmentId = 0; minorFragmentId < wrapper.getWidth(); minorFragmentId++) {
IndexedFragmentNode iNode = new IndexedFragmentNode(minorFragmentId, wrapper);
PhysicalOperator op = physicalOperatorRoot.accept(materializer, iNode);
Preconditions.checkArgument(op instanceof FragmentRoot);
FragmentRoot root = (FragmentRoot) op;
// get plan as JSON
String plan;
try {
plan = reader.writeJson(root);
} catch (JsonProcessingException e) {
throw new FragmentSetupException(
"Failure while trying to convert fragment into json.", e);
}
FragmentHandle handle =
FragmentHandle //
.newBuilder() //
.setMajorFragmentId(wrapper.getMajorFragmentId()) //
.setMinorFragmentId(minorFragmentId) //
.setQueryId(queryId) //
.build();
PlanFragment fragment =
PlanFragment.newBuilder() //
.setCpuCost(stats.getCpuCost()) //
.setDiskCost(stats.getDiskCost()) //
.setForeman(foremanNode) //
.setMemoryCost(stats.getMemoryCost()) //
.setNetworkCost(stats.getNetworkCost()) //
.setFragmentJson(plan) //
.setHandle(handle) //
.setAssignment(wrapper.getAssignedEndpoint(minorFragmentId)) //
.setLeafFragment(isLeafFragment) //
.setQueryStartTime(queryStartTime)
.build();
if (isRootNode) {
logger.debug("Root fragment:\n {}", fragment);
rootFragment = fragment;
rootOperator = root;
} else {
logger.debug("Remote fragment:\n {}", fragment);
fragments.add(fragment);
}
}
}
return new QueryWorkUnit(rootOperator, rootFragment, fragments);
}
@Override
public void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
}