@BeforeClass
public static void setUpClass() throws Exception {
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
assertTrue("No ctags available", env.validateExuberantCtags());
repository = new TestRepository();
repository.create(IndexDatabase.class.getResourceAsStream("source.zip"));
env.setSourceRoot(repository.getSourceRoot());
env.setDataRoot(repository.getDataRoot());
Indexer indexer = Indexer.getInstance();
indexer.prepareIndexer(
env,
true,
true,
"/c",
null,
false,
false,
false,
null,
null,
new ArrayList<String>(),
false);
indexer.doIndexerExecution(true, 1, null, null);
}
@Override
public void onSetUpBeforeTransaction() throws Exception {
super.onSetUpBeforeTransaction();
Session session = sessionFactory.openSession();
SessionHolder.setSession(session);
Indexer indexer = new Indexer();
session.beginTransaction();
try {
indexer.indexWithHibernate();
} catch (Exception e) {
logger.error(e);
logger.debug("Rolling back the database transaction");
session.getTransaction().rollback();
} finally {
try {
if (session != null) {
session.close();
}
} catch (Exception e) {
/* do nothing */
logger.info("Can't close the session! (ignore it)");
}
}
// Lucene index
}
public void testSaveLoadRoundtrip() throws IOException {
val baos = new ByteArrayOutputStream(3200);
val dos = new DataOutputStream(baos);
avengers.save(dos);
val dis = new DataInputStream(new ByteArrayInputStream(baos.toByteArray()));
val otherAvengers = Indexer.load(dis);
assertEquals(avengers, otherAvengers);
}
/////////////////////////////////////////////////////
// 샘플 도큐먼트에 대한 인덱싱을 수행하는 함수.
private void createIndex() throws IOException, SAXException, TikaException {
indexer = new Indexer(indexDir);
int numIndexed;
long startTime = System.currentTimeMillis();
numIndexed = indexer.createIndex(dataDir, new TextFileFilter());
long endTime = System.currentTimeMillis();
indexer.close();
System.out.println(numIndexed + "File indexed, time taken: " + (endTime - startTime) + " ms");
}
// escidoc:2087580 item without component in status pending
// has no reference
@Test
public void testPendingItem() throws Exception {
indexer.indexItemsStart(new File(TEST_RESOURCES_OBJECTS + "escidoc_2087580"));
indexer.finalizeIndex();
validator = new Validator(indexer);
Map<String, Set<Fieldable>> fieldMap = validator.getFieldsOfDocument();
switch (indexer.getCurrentIndexMode()) {
case LATEST_RELEASE:
assertTrue(
"Expected 0 Found " + indexer.getIndexingReport().getFilesIndexingDone(),
indexer.getIndexingReport().getFilesIndexingDone() == 0);
assertTrue(indexer.getIndexingReport().getFilesErrorOccured() == 0);
assertTrue(indexer.getIndexingReport().getFilesSkippedBecauseOfTime() == 0);
assertTrue(fieldMap == null);
break;
case LATEST_VERSION:
assertTrue(
"Expected 1 Found " + indexer.getIndexingReport().getFilesIndexingDone(),
indexer.getIndexingReport().getFilesIndexingDone() == 1);
assertTrue(indexer.getIndexingReport().getFilesErrorOccured() == 0);
assertTrue(indexer.getIndexingReport().getFilesSkippedBecauseOfTime() == 0);
assertTrue(fieldMap != null);
assertTrue(fieldMap.get("/properties/latest-release/id") == null);
assertTrue(fieldMap.get("sort/properties/latest-release/id") == null);
break;
}
}
@Before
public void setUp() throws Exception {
indexer = new Indexer(new File(TEST_RESOURCES_OBJECTS));
indexer.init();
indexer.setCreateIndex(true);
indexer.prepareIndex();
indexer.getIndexingReport().clear();
LocationHelper.getLocation("escidoc_persistent22");
}
@Override
public void index(Document document) {
if (log.isTraceEnabled()) {
log.trace(
"Indexing "
+ sql
+ " id="
+ JdbcIndexDefinition.this.identifier
+ ", key = "
+ JdbcIndexDefinition.this.key);
}
String id = getIdentifier();
if (id != null) {
document.add(
new Field(
"builder",
"VIRTUAL BUILDER",
Field.Store.YES,
Field.Index.NOT_ANALYZED)); // keyword
document.add(
new Field(
"number", getIdentifier(), Field.Store.YES, Field.Index.NOT_ANALYZED)); // keyword
}
try {
for (int i = 1; i <= meta.getColumnCount(); i++) {
String value = org.mmbase.util.Casting.toString(results.getString(i));
if (log.isTraceEnabled()) {
log.trace(
"Indexing " + value + " for " + meta.getColumnName(i) + " on " + getIdentifier());
}
String fieldName = meta.getColumnName(i);
if (keyWords.contains(fieldName)) {
Indexer.addField(
document,
new Field(fieldName, value, Field.Store.YES, Field.Index.NOT_ANALYZED),
nonDefaultMultiples.get(fieldName)); // keyword
} else {
Field field = new Field(fieldName, value, Field.Store.YES, Field.Index.ANALYZED);
Float boost = boosts.get(fieldName);
if (boost != null) {
field.setBoost(boost);
}
Indexer.addField(document, field, nonDefaultMultiples.get(fieldName));
Field fullText = new Field("fulltext", value, Field.Store.YES, Field.Index.ANALYZED);
if (boost != null) {
fullText.setBoost(boost);
}
document.add(fullText);
}
}
} catch (SQLException sqe) {
log.error(sqe.getMessage(), sqe);
}
}
public void testIndexer() {
assertFalse(avengers.isEmpty());
assertTrue(avengers.size() == 3);
assertTrue(avengers.indexOf("cap") == 0);
assertTrue(avengers.lastIndexOf("cap") == 0);
assertTrue(avengers.get(0).equals("cap"));
assertTrue(avengers.contains("cap"));
assertFalse(avengers.contains("made-up"));
Object[] arr = avengers.toArray();
assertEquals(arr, new String[] {"cap", "iron-man", "hulk"});
assertEquals(avengers, avengers.subList(0, avengers.size()));
}
public static String[] getEntryClassNames() {
Set<String> assetEntryClassNames = new HashSet<String>();
for (Indexer indexer : IndexerRegistryUtil.getIndexers()) {
for (String className : indexer.getClassNames()) {
if (!_excludedEntryClassNames.contains(className)) {
assetEntryClassNames.add(className);
}
}
}
return assetEntryClassNames.toArray(new String[assetEntryClassNames.size()]);
}
private static String _getSearchEngineId(Document document) {
String entryClassName = document.get("entryClassName");
Indexer indexer = IndexerRegistryUtil.getIndexer(entryClassName);
String searchEngineId = indexer.getSearchEngineId();
if (_log.isDebugEnabled()) {
_log.debug("Search engine ID for " + indexer.getClass() + " is " + searchEngineId);
}
return searchEngineId;
}
public void search() {
try {
ArrayList<Pair> searchQuery = indexer.searchResults(searchTextField.getText().split(" "));
String URL = Indexer.docIdToURL(searchQuery.get(0).getKey().toString());
searchTextField.setText(URL);
label.setText("Status: FOUND PAGE");
label.setTextFill(javafx.scene.paint.Paint.valueOf("Green"));
webEngine.load(URL);
} catch (Exception e) {
label.setText("Status: NOTHING FOUND");
label.setTextFill(javafx.scene.paint.Paint.valueOf("Red"));
}
}
// @Override
public void addOrUpdateDocuments(String documents) {
// TODO Auto-generated method stub
// System.out.println("the string json "+ documents);
SearchItemClass items = Json2Doc(documents);
// System.out.println("the string json "+items.content.get(0).title);
try {
indexer.indexItems(items);
indexer.close();
} catch (Exception e) {
e.printStackTrace();
}
}
private static void startServing() throws IOException, ClassNotFoundException {
// Create the handler and its associated indexer.
Indexer indexer = Indexer.Factory.getIndexerByOption(SearchEngine.OPTIONS);
Check(indexer != null, "Indexer " + SearchEngine.OPTIONS._indexerType + " not found!");
indexer.loadIndex();
QueryHandler handler = new QueryHandler(SearchEngine.OPTIONS, indexer);
// Establish the serving environment
InetSocketAddress addr = new InetSocketAddress(SearchEngine.PORT);
HttpServer server = HttpServer.create(addr, -1);
server.createContext("/", handler);
server.setExecutor(Executors.newCachedThreadPool());
server.start();
System.out.println("Listening on port: " + Integer.toString(SearchEngine.PORT));
}
/**
* Starts processing of every file provided by IngestManager. Checks if it is time to commit and
* run search
*
* @param abstractFile file/unallocated file/directory to process
* @return ProcessResult.OK in most cases and ERROR only if error in the pipeline, otherwise does
* not advice to stop the pipeline
*/
@Override
public ProcessResult process(AbstractFile abstractFile) {
if (initialized == false) // error initializing indexing/Solr
{
return ProcessResult.OK;
}
// check if we should skip this file according to HashDb service
// if so do not index it, also postpone indexing and keyword search threads to later
IngestServiceAbstractFile.ProcessResult hashDBResult =
managerProxy.getAbstractFileServiceResult(hashDBServiceName);
// logger.log(Level.INFO, "hashdb result: " + hashDBResult + "file: " + AbstractFile.getName());
if (hashDBResult == IngestServiceAbstractFile.ProcessResult.COND_STOP && skipKnown) {
return ProcessResult.OK;
} else if (hashDBResult == IngestServiceAbstractFile.ProcessResult.ERROR) {
// notify depending service that keyword search (would) encountered error for this file
return ProcessResult.ERROR;
}
if (processedFiles == false) {
processedFiles = true;
}
checkRunCommitSearch();
indexer.indexFile(abstractFile);
return ProcessResult.OK;
}
private void scanFile(File source, Indexer indexer) throws FileNotFoundException, IOException {
if (source.isDirectory()) {
File[] children = source.listFiles();
if (children == null)
throw new FileNotFoundException("Source directory disappeared: " + source);
for (File child : children) scanFile(child, indexer);
return;
}
if (!source.getName().endsWith(".class")) return;
FileInputStream input = new FileInputStream(source);
try {
ClassInfo info = indexer.index(input);
if (verbose && info != null) printIndexEntryInfo(info);
} catch (Exception e) {
String message = e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage();
System.err.println("ERROR: Could not index " + source.getName() + ": " + message);
if (verbose) e.printStackTrace(System.err);
} finally {
safeClose(input);
}
}
private Result indexDirectory(File source, Indexer indexer)
throws FileNotFoundException, IOException {
File outputFile = this.outputFile;
scanFile(source, indexer);
if (modify) {
new File(source, "META-INF").mkdirs();
outputFile = new File(source, "META-INF/jandex.idx");
}
if (outputFile == null) {
outputFile = new File(source.getName().replace('.', '-') + ".idx");
}
FileOutputStream out = new FileOutputStream(outputFile);
IndexWriter writer = new IndexWriter(out);
try {
Index index = indexer.complete();
int bytes = writer.write(index);
return new Result(index, outputFile.getPath(), bytes, outputFile);
} finally {
out.flush();
out.close();
}
}
@Override
public void actionPerformed(ActionEvent e) {
Object src = e.getSource();
if (src == alpha) {
valueIndex.alpha = ((Integer) alpha.getSelectedItem()).intValue();
map.getMainMap().paintImmediately(map.getMainMap().getBounds());
}
if (src == delete) {
map.deleteLayer(this);
map.getMainMap().paintImmediately(map.getMainMap().getBounds());
}
if (src == Sites) {
this.valueIndex.mask = this.Mask.isSelected();
if (map.sameScale.isSelected()) setMaxMinOverall();
else setMaxMin();
map.updateColorsAndNumbers();
map.updateCanvasScale();
map.getMainMap().paintImmediately(map.getMainMap().getBounds());
}
if (src == Mask) {
this.valueIndex.mask = this.Mask.isSelected();
if (map.sameScale.isSelected()) setMaxMinOverall();
else setMaxMin();
map.updateColorsAndNumbers();
map.updateCanvasScale();
map.getMainMap().paintImmediately(map.getMainMap().getBounds());
}
}
@Test
public void testReindex() throws Exception {
mockContent("", true, null);
when(indexer.getIndexerType()).thenReturn(Indexer.IndexerType.RDF);
indexerGroup.reindex();
verify(indexer, atLeastOnce()).update(eq(repoUrl), any());
}
private static void processFile(String filePath) throws IOException {
FileReader fr = new FileReader(filePath);
BufferedReader bf = new BufferedReader(fr);
List<Game> games = new ArrayList<Game>();
String line;
while ((line = bf.readLine()) != null) {
games.add(parseLineToGame(line));
if (games.size() == 1000) {
Indexer.indexData(TYPE_NAME, games, Game.class);
games = new ArrayList<Game>();
}
}
Indexer.indexData(TYPE_NAME, games, Game.class);
bf.close();
}
// escidoc:2146780 item with 1 component (escidoc:2147085 internal-managed, visibility private)
// has no reference
@Test
public void testItemWithPrivateComponent() throws Exception {
indexer.indexItemsStart(new File(TEST_RESOURCES_OBJECTS + "escidoc_2146780"));
indexer.finalizeIndex();
assertTrue(
"Expected 1 Found " + indexer.getIndexingReport().getFilesIndexingDone(),
indexer.getIndexingReport().getFilesIndexingDone() == 1);
assertTrue(indexer.getIndexingReport().getFilesErrorOccured() == 0);
assertTrue(indexer.getIndexingReport().getFilesIndexingDone() == 1);
assertTrue(indexer.getIndexingReport().getFilesSkippedBecauseOfTime() == 0);
validator = new Validator(indexer);
Map<String, Set<Fieldable>> fieldMap = validator.getFieldsOfDocument();
Set<Fieldable> fields = fieldMap.get(getFieldNameFor("stored_filename1"));
assertTrue(fields == null);
fields = fieldMap.get(getFieldNameFor("stored_fulltext"));
switch (indexer.getCurrentIndexMode()) {
case LATEST_RELEASE:
assertTrue(fields == null);
break;
case LATEST_VERSION:
assertTrue(fields != null);
break;
}
}
// escidoc:2110495 released item (1 locator escidoc:2110494)
// has reference
@Test
public void testReleasedItem_2110495() throws Exception {
indexer.indexItemsStart(new File(TEST_RESOURCES_OBJECTS + "escidoc_2110495"));
indexer.finalizeIndex();
assertTrue(
"Expected 1 found " + indexer.getIndexingReport().getFilesIndexingDone(),
indexer.getIndexingReport().getFilesIndexingDone() == 1);
assertTrue(indexer.getIndexingReport().getFilesErrorOccured() == 0);
assertTrue(indexer.getIndexingReport().getFilesSkippedBecauseOfTime() == 0);
validator = new Validator(indexer);
Map<String, Set<Fieldable>> fieldMap = validator.getFieldsOfDocument();
assertTrue(fieldMap != null);
Set<Fieldable> fields = fieldMap.get(getFieldNameFor("stored_filename1"));
assertTrue(fields == null);
assertTrue(fieldMap.get(getFieldNameFor("stored_filename1")) == null);
assertTrue(fieldMap.get(getFieldNameFor("stored_fulltext1")) == null);
assertTrue(fieldMap.get(getFieldNameFor("stored_filename")) == null);
assertTrue(fieldMap.get(getFieldNameFor("stored_fulltext")) == null);
// assertTrue(fieldMap.get("escidoc.property.created-by.name").equals("Nadine Schröder"));
validator.compareToReferenceIndex();
assertTrue(
Arrays.toString(indexer.getIndexingReport().getErrorList().toArray()),
indexer.getIndexingReport().getErrorList().size() == 0);
}
@Test
public void testRDFIndexablePropertyUpdateMessage() throws Exception {
when(indexer.getIndexerType()).thenReturn(Indexer.IndexerType.RDF);
String id = "/test/dc:title";
indexerGroup.onMessage(
createIndexablePropertyMessage(
REPOSITORY_NAMESPACE + EventType.valueOf(PROPERTY_CHANGED).toString(), id));
verify(indexer, atLeastOnce()).update(anyString(), any());
}
@Test
public void testNamedFieldsIndexableObjectUpdateMessage() throws Exception {
when(indexer.getIndexerType()).thenReturn(Indexer.IndexerType.NAMEDFIELDS);
String id = "/test";
indexerGroup.onMessage(
createIndexableMessage(
REPOSITORY_NAMESPACE + EventType.valueOf(NODE_ADDED).toString(), id));
verify(indexer, atLeastOnce()).update(anyString(), any());
}
public DisposableIterator<Map<String, Object>> find(
final String keySpace,
final String columnFamily,
Map<String, Object> properties,
DirectCacheAccess cachingManager)
throws StorageClientException {
checkClosed();
return indexer.find(keySpace, columnFamily, properties, cachingManager);
}
private String getFieldNameFor(String name) {
switch (indexer.getCurrentIndexMode()) {
case LATEST_RELEASE:
return name;
case LATEST_VERSION:
return "aa_" + name;
}
return null;
}
public NodesStressTest build(Settings settings) throws Exception {
settings =
settingsBuilder()
// .put("index.engine.robin.refreshInterval", 1, TimeUnit.SECONDS)
.put(SETTING_NUMBER_OF_SHARDS, 5)
.put(SETTING_NUMBER_OF_REPLICAS, 1)
.put(settings)
.build();
nodes = new Node[numberOfNodes];
clients = new Client[numberOfNodes];
for (int i = 0; i < numberOfNodes; i++) {
nodes[i] =
nodeBuilder().settings(settingsBuilder().put(settings).put("name", "node" + i)).node();
clients[i] = nodes[i].client();
}
for (int i = 0; i < searcherThreads.length; i++) {
searcherThreads[i] = new Searcher(i);
}
for (int i = 0; i < indexThreads.length; i++) {
indexThreads[i] = new Indexer(i);
}
latch = new CountDownLatch(1);
barrier1 = new CyclicBarrier(2);
barrier2 = new CyclicBarrier(2);
// warmup
StopWatch stopWatch = new StopWatch().start();
Indexer warmup = new Indexer(-1).max(10000);
warmup.start();
barrier1.await();
barrier2.await();
latch.await();
stopWatch.stop();
System.out.println("Done Warmup, took [" + stopWatch.totalTime() + "]");
latch = new CountDownLatch(searcherThreads.length + indexThreads.length);
barrier1 = new CyclicBarrier(searcherThreads.length + indexThreads.length + 1);
barrier2 = new CyclicBarrier(searcherThreads.length + indexThreads.length + 1);
return this;
}
public String dumpOut(String indent) {
String ind = indent + " ";
String toDumpOut = ind + "FullTextSearch:\n";
toDumpOut += ind + fullTextSearch.dumpOut(ind) + "\n";
toDumpOut += ind + contentHandlerPlugin.dumpOut(ind) + "\n";
toDumpOut += ind + indexer.dumpOut(ind) + "\n";
toDumpOut += ind + queryBuilder.dumpOut(ind) + "\n";
return toDumpOut;
}
/** Called when the activity is first created. */
@Override
public void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
Bundle extras = this.getIntent().getExtras();
Indexer indexer = new Indexer(getApplicationContext().getFilesDir());
String url = extras.getString("url");
Source source = null;
try {
source = new Source(new URL(url));
indexer.index(source.toString());
} catch (MalformedURLException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
mWebView = new WebView(this);
mWebView.getSettings().setJavaScriptEnabled(true);
mWebView.loadUrl(url);
setContentView(mWebView);
}
// escidoc:2148921 released item with components escidoc:2149144, 2149275, 2149276, 2149277,
// 2149278
// has no reference
@Test
public void testReleasedItemWithManyComponents() throws Exception {
indexer.indexItemsStart(new File(TEST_RESOURCES_OBJECTS + "escidoc_2148921"));
indexer.finalizeIndex();
validator = new Validator(indexer);
Map<String, Set<Fieldable>> fieldMap = validator.getFieldsOfDocument();
assertTrue(
"Expected 1 Found " + indexer.getIndexingReport().getFilesIndexingDone(),
indexer.getIndexingReport().getFilesIndexingDone() == 1);
assertTrue(indexer.getIndexingReport().getFilesErrorOccured() == 0);
assertTrue(indexer.getIndexingReport().getFilesSkippedBecauseOfTime() == 0);
assertTrue(fieldMap != null);
switch (indexer.getCurrentIndexMode()) {
case LATEST_RELEASE:
break;
case LATEST_VERSION:
assertTrue(fieldMap.get("/properties/latest-release/id") != null);
assertTrue(fieldMap.get("sort/properties/latest-release/id") == null);
assertTrue(
fieldMap
.get("/components/component/properties/creation-date")
.iterator()
.next()
.stringValue()
.startsWith("2015"));
break;
}
}
public Map<String, Integer> getLexiconCounts(Indexer indexer, boolean originalContentOnly) {
Map<String, Integer> map = new LinkedHashMap<String, Integer>();
String[] captions =
captionToExpandedQuery.keySet().toArray(new String[captionToExpandedQuery.size()]);
for (String caption : captions) {
String query = captionToExpandedQuery.get(caption);
if (query == null) {
log.warn("Skipping unknown caption '" + caption + "'");
continue;
}
Integer cnt = 0;
try {
if (originalContentOnly)
cnt = indexer.getTotalHits(query, false, Indexer.QueryType.ORIGINAL);
else cnt = indexer.getTotalHits(query, false, Indexer.QueryType.FULL);
} catch (Exception e) {
Util.print_exception("Exception while collecting lexicon counts", e, log);
}
map.put(caption, cnt);
}
return map;
}
