// depends on testRunPushReplication
private void testRunPullReplication() throws Throwable {
URL remoteDbURL = new URL(syncGatewayUrl + pushThenPullDBName);
Database db = startDatabase();
Log.i(TAG, "Pulling...");
Replication repl = db.getPullReplication(remoteDbURL);
runReplication(repl);
assertNull(repl.getLastError());
Log.i(TAG, "Verifying documents...");
for (int i = 1; i <= kNDocuments; i++) {
Document doc = db.getDocument("doc-" + i);
assertEquals(i, doc.getProperty("index"));
assertEquals(false, doc.getProperty("bar"));
}
}
private void updateBoard(DocumentEvent e) throws BadLocationException {
Document doc = e.getDocument();
int index = (int) doc.getProperty("index");
String valueString = doc.getText(0, doc.getLength());
if (doc.getLength() == 0) valueString = "0";
int value = Integer.parseInt(valueString);
gameBoard.changeCellAt(index, value);
// gameBoard.out();
if (gameBoard.checkGameOver()) {
JOptionPane.showMessageDialog(frame, "NUMBRIX COMPLETED!!!");
}
}
@Override
public void read(Reader in, Document doc, int pos) throws IOException, BadLocationException {
BufferedReader br = new BufferedReader(in);
br.mark(1024);
char[] buffer = new char[80];
br.read(buffer);
br.reset();
Language language =
languageManager.getLanguage(
new String(buffer), (File) doc.getProperty(Document.StreamDescriptionProperty));
SourceDocument document = (SourceDocument) doc;
document.setLanguage(language);
int offset = 0;
String line;
while ((line = br.readLine()) != null) {
document.insertString(offset, line + "\n", null);
offset += line.length() + 1;
}
}
/**
* Writes out all empty elements (all tags that have no corresponding end tag).
*
* @param elem an Element
* @exception IOException on any I/O error
* @exception BadLocationException if pos represents an invalid location within the document.
*/
protected void emptyTag(Element elem) throws BadLocationException, IOException {
if (!inContent && !inPre) {
indent();
}
AttributeSet attr = elem.getAttributes();
closeOutUnwantedEmbeddedTags(attr);
writeEmbeddedTags(attr);
if (matchNameAttribute(attr, HTML.Tag.CONTENT)) {
inContent = true;
text(elem);
} else if (matchNameAttribute(attr, HTML.Tag.COMMENT)) {
comment(elem);
} else {
boolean isBlock = isBlockTag(elem.getAttributes());
if (inContent && isBlock) {
writeLineSeparator();
indent();
}
Object nameTag = (attr != null) ? attr.getAttribute(StyleConstants.NameAttribute) : null;
Object endTag = (attr != null) ? attr.getAttribute(HTML.Attribute.ENDTAG) : null;
boolean outputEndTag = false;
// If an instance of an UNKNOWN Tag, or an instance of a
// tag that is only visible during editing
//
if (nameTag != null
&& endTag != null
&& (endTag instanceof String)
&& ((String) endTag).equals("true")) {
outputEndTag = true;
}
if (completeDoc && matchNameAttribute(attr, HTML.Tag.HEAD)) {
if (outputEndTag) {
// Write out any styles.
writeStyles(((HTMLDocument) getDocument()).getStyleSheet());
}
wroteHead = true;
}
write('<');
if (outputEndTag) {
write('/');
}
write(elem.getName());
writeAttributes(attr);
write('>');
if (matchNameAttribute(attr, HTML.Tag.TITLE) && !outputEndTag) {
Document doc = elem.getDocument();
String title = (String) doc.getProperty(Document.TitleProperty);
write(title);
} else if (!inContent || isBlock) {
writeLineSeparator();
if (isBlock && inContent) {
indent();
}
}
}
}
// TODO if this class extends BasicIndexer, then this method could be inherited
public void createDirectIndex(Collection[] collections) {
logger.info(
"BlockIndexer creating direct index"
+ (Boolean.parseBoolean(
ApplicationSetup.getProperty("block.delimiters.enabled", "false"))
? " delimited-block indexing enabled"
: ""));
currentIndex = Index.createNewIndex(path, prefix);
lexiconBuilder =
FieldScore.FIELDS_COUNT > 0
? new LexiconBuilder(
currentIndex,
"lexicon",
new BlockFieldLexiconMap(FieldScore.FIELDS_COUNT),
FieldLexiconEntry.class.getName())
: new LexiconBuilder(
currentIndex, "lexicon", new BlockLexiconMap(), BlockLexiconEntry.class.getName());
// lexiconBuilder = new BlockLexiconBuilder(currentIndex, "lexicon");
try {
directIndexBuilder =
FieldScore.FIELDS_COUNT > 0
? new BlockFieldDirectInvertedOutputStream(
currentIndex.getPath()
+ ApplicationSetup.FILE_SEPARATOR
+ currentIndex.getPrefix()
+ "."
+ "direct"
+ BitIn.USUAL_EXTENSION)
: new BlockDirectInvertedOutputStream(
currentIndex.getPath()
+ ApplicationSetup.FILE_SEPARATOR
+ currentIndex.getPrefix()
+ "."
+ "direct"
+ BitIn.USUAL_EXTENSION);
} catch (IOException ioe) {
logger.error("Cannot make DirectInvertedOutputStream:", ioe);
}
docIndexBuilder = new DocumentIndexBuilder(currentIndex, "document");
metaBuilder = createMetaIndexBuilder();
emptyDocIndexEntry =
(FieldScore.FIELDS_COUNT > 0)
? new FieldDocumentIndexEntry(FieldScore.FIELDS_COUNT)
: new BasicDocumentIndexEntry();
// int LexiconCount = 0;
int numberOfDocuments = 0;
// int numberOfTokens = 0;
// long startBunchOfDocuments = System.currentTimeMillis();
final boolean boundaryDocsEnabled = BUILDER_BOUNDARY_DOCUMENTS.size() > 0;
boolean stopIndexing = false;
for (int collectionNo = 0; !stopIndexing && collectionNo < collections.length; collectionNo++) {
Collection collection = collections[collectionNo];
long startCollection = System.currentTimeMillis();
boolean notLastDoc = false;
// while(notLastDoc = collection.hasNext()) {
while ((notLastDoc = collection.nextDocument())) {
// get the next document from the collection
// String docid = collection.getDocid();
// Document doc = collection.next();
Document doc = collection.getDocument();
if (doc == null) continue;
numberOfDocuments++;
// setup for parsing
createDocumentPostings();
String term;
numOfTokensInDocument = 0;
numOfTokensInBlock = 0;
blockId = 0;
// get each term in the document
while (!doc.endOfDocument()) {
if ((term = doc.getNextTerm()) != null && !term.equals("")) {
termFields = doc.getFields();
// pass term into TermPipeline (stop, stem etc)
pipeline_first.processTerm(term);
// the term pipeline will eventually add the term to this
// object.
}
if (MAX_TOKENS_IN_DOCUMENT > 0 && numOfTokensInDocument > MAX_TOKENS_IN_DOCUMENT) break;
}
// if we didn't index all tokens from document,
// we need to get to the end of the document.
while (!doc.endOfDocument()) doc.getNextTerm();
// we now have all terms in the DocumentTree
pipeline_first.reset();
// process DocumentTree (tree of terms)
try {
if (termsInDocument.getDocumentLength() == 0) {
// this document is empty, add the
// minimum to the document index
indexEmpty(doc.getAllProperties());
} else {
/* index this docuent */
// numberOfTokens += numOfTokensInDocument;
indexDocument(doc.getAllProperties(), termsInDocument);
}
} catch (Exception ioe) {
logger.error("Failed to index " + doc.getProperty("docno"), ioe);
}
if (MAX_DOCS_PER_BUILDER > 0 && numberOfDocuments >= MAX_DOCS_PER_BUILDER) {
stopIndexing = true;
break;
}
if (boundaryDocsEnabled && BUILDER_BOUNDARY_DOCUMENTS.contains(doc.getProperty("docno"))) {
stopIndexing = true;
break;
}
}
long endCollection = System.currentTimeMillis();
long secs = ((endCollection - startCollection) / 1000);
logger.info(
"Collection #"
+ collectionNo
+ " took "
+ secs
+ "seconds to index "
+ "("
+ numberOfDocuments
+ " documents)\n");
if (secs > 3600)
logger.info(
"Rate: " + ((double) numberOfDocuments / ((double) secs / 3600.0d)) + " docs/hour");
if (!notLastDoc) {
try {
collection.close();
} catch (IOException e) {
logger.warn("Couldnt close collection", e);
}
}
}
/* end of the collection has been reached */
finishedDirectIndexBuild();
currentIndex.addIndexStructure(
"direct",
"org.terrier.structures.BlockDirectIndex",
"org.terrier.structures.Index,java.lang.String,java.lang.Class",
"index,structureName,"
+ (FieldScore.FIELDS_COUNT > 0
? fieldDirectIndexPostingIteratorClass
: basicDirectIndexPostingIteratorClass));
currentIndex.addIndexStructureInputStream(
"direct",
"org.terrier.structures.BlockDirectIndexInputStream",
"org.terrier.structures.Index,java.lang.String,java.lang.Class",
"index,structureName,"
+ (FieldScore.FIELDS_COUNT > 0
? fieldDirectIndexPostingIteratorClass
: basicDirectIndexPostingIteratorClass));
currentIndex.setIndexProperty("index.direct.fields.count", "" + FieldScore.FIELDS_COUNT);
currentIndex.setIndexProperty(
"index.direct.fields.names", ArrayUtils.join(FieldScore.FIELD_NAMES, ","));
if (FieldScore.FIELDS_COUNT > 0) {
currentIndex.addIndexStructure(
"document-factory",
FieldDocumentIndexEntry.Factory.class.getName(),
"java.lang.String",
"${index.direct.fields.count}");
} else {
currentIndex.addIndexStructure(
"document-factory", BasicDocumentIndexEntry.Factory.class.getName(), "", "");
}
/* flush the index buffers */
directIndexBuilder.close();
docIndexBuilder.finishedCollections();
/* and then merge all the temporary lexicons */
lexiconBuilder.finishedDirectIndexBuild();
try {
metaBuilder.close();
} catch (IOException ioe) {
logger.error("Could not finish MetaIndexBuilder: ", ioe);
}
if (FieldScore.FIELDS_COUNT > 0) {
currentIndex.addIndexStructure(
"lexicon-valuefactory",
FieldLexiconEntry.Factory.class.getName(),
"java.lang.String",
"${index.direct.fields.count}");
}
/* reset the in-memory mapping of terms to term codes.*/
TermCodes.reset();
System.gc();
try {
currentIndex.flush();
} catch (IOException ioe) {
logger.error("Could not flush index properties: ", ioe);
}
}
