/** 加载量词词典 */
private void loadQuantifierDict() {
// 建立一个量词典实例
_QuantifierDict = new DictSegment((char) 0);
// 读取量词词典文件
File file = new File(configuration.getDictRoot(), Dictionary.PATH_DIC_QUANTIFIER);
InputStream is = null;
try {
is = new FileInputStream(file);
} catch (FileNotFoundException e) {
logger.error("ik-analyzer", e);
}
try {
BufferedReader br = new BufferedReader(new InputStreamReader(is, "UTF-8"), 512);
String theWord = null;
do {
theWord = br.readLine();
if (theWord != null && !"".equals(theWord.trim())) {
_QuantifierDict.fillSegment(theWord.trim().toCharArray());
}
} while (theWord != null);
} catch (IOException ioe) {
logger.error("Quantifier Dictionary loading exception.");
} finally {
try {
if (is != null) {
is.close();
is = null;
}
} catch (IOException e) {
logger.error("ik-analyzer", e);
}
}
}
@Override
public void run() {
if (logger.isDebugEnabled()) {
logger.debug("Create task manager thread.");
}
do {
logger.info("TaskManager: current task index: " + Integer.toString(currentTaskIndex));
try {
String output = currentTask.Run();
logger.info("Task {[]} output: {[]}", currentTask.id(), output);
logger.info("Task [" + currentTask.id() + "] output: " + output);
} catch (IOException ex) {
logger.error("TaskManager: IOException");
} catch (InterruptedException ex) {
logger.error("TaskManager: Interrupted Exception");
}
currentTask = GetNextTask();
} while (null != currentTask);
DeleteMappingRequest req = new DeleteMappingRequest("_river");
req.type(river.riverName().name());
DeleteMappingResponse resp = client.admin().indices().deleteMapping(req).actionGet();
logger.info("TaskManager: delete request: " + resp.toString());
}
/**
* Queries the {@link #rdfEndpoint(String)} with each of the {@link #rdfQueries} and harvests the
* results of the query.
*/
private void harvestFromEndpoint() {
Query query;
QueryExecution qExec;
for (String rdfQuery : rdfQueries) {
if (closed) break;
logger.info(
"Harvesting with query: [{}] on index [{}] and type [{}]", rdfQuery, indexName, typeName);
try {
query = QueryFactory.create(rdfQuery);
} catch (QueryParseException qpe) {
logger.error("Could not parse [{}]. Please provide a relevant query. {}", rdfQuery, qpe);
continue;
}
qExec = QueryExecutionFactory.sparqlService(rdfEndpoint, query);
try {
harvest(qExec);
} catch (Exception e) {
logger.error("Exception [{}] occurred while harvesting", e.getLocalizedMessage());
} finally {
qExec.close();
}
}
}
private void loadPrepDict() {
_PrepDict = new DictSegment((char) 0);
File file = new File(configuration.getDictRoot(), Dictionary.PATH_DIC_PREP);
InputStream is = null;
try {
is = new FileInputStream(file);
} catch (FileNotFoundException e) {
logger.error("ik-analyzer", e);
}
if (is == null) {
throw new RuntimeException("Preposition Dictionary not found!!!");
}
try {
BufferedReader br = new BufferedReader(new InputStreamReader(is, "UTF-8"), 512);
String theWord;
do {
theWord = br.readLine();
if (theWord != null && !"".equals(theWord.trim())) {
_PrepDict.fillSegment(theWord.trim().toCharArray());
}
} while (theWord != null);
} catch (IOException e) {
logger.error("ik-analyzer", e);
} finally {
try {
is.close();
is = null;
} catch (IOException e) {
logger.error("ik-analyzer", e);
}
}
}
public static void removeAMB(String content) {
try {
File file =
new File(AnsjElasticConfigurator.environment.configFile().toFile(), "ansj/ambiguity.dic");
removeFile(content, file, true);
} catch (FileNotFoundException e) {
logger.error("file not found $ES_HOME/config/ansj/user/ext.dic", e);
e.printStackTrace();
} catch (IOException e) {
logger.error("read exception", e);
e.printStackTrace();
}
}
@After
public void stopNodes() {
try {
closeNodes();
} catch (Exception e) {
logger.error("can not close nodes", e);
} finally {
try {
deleteFiles();
logger.info("data files wiped");
} catch (IOException e) {
logger.error(e.getMessage(), e);
}
}
}
/**
* Query SPARQL endpoint with a SELECT query
*
* @param qExec QueryExecution encapsulating the query
* @return model retrieved by querying the endpoint
*/
private Model getSelectModel(QueryExecution qExec) {
Model model = ModelFactory.createDefaultModel();
Graph graph = model.getGraph();
ResultSet results = qExec.execSelect();
while (results.hasNext()) {
QuerySolution sol = results.next();
String subject;
String predicate;
RDFNode object;
try {
subject = sol.getResource("s").toString();
predicate = sol.getResource("p").toString();
object = sol.get("o");
} catch (NoSuchElementException e) {
logger.error("SELECT query does not return a (?s ?p ?o) Triple");
continue;
}
Node objNode;
if (object.isLiteral()) {
Literal obj = object.asLiteral();
objNode = NodeFactory.createLiteral(obj.getString(), obj.getDatatype());
} else {
objNode = NodeFactory.createLiteral(object.toString());
}
graph.add(
new Triple(NodeFactory.createURI(subject), NodeFactory.createURI(predicate), objNode));
}
return model;
}
@Override
public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception {
if (e.getCause() instanceof ClosedChannelException) {
// do nothing
} else if (e.getCause() instanceof UnresolvedAddressException) {
logger.error(
"Unable to resolve one of the server addresses: [{}]", e.getCause().getMessage());
} else if (e.getCause() instanceof ConnectException) {
logger.error("Unable to connect: [{}]", e.getCause().getMessage());
} else if (e.getCause().getMessage() != null
&& e.getCause().getMessage().contains("Connection reset by peer")) {
// still do nothing
} else {
super.exceptionCaught(ctx, e);
}
}
@Test
public void testRandomDocsNodeClient() throws Exception {
final NodeClient es =
new NodeClient()
.maxActionsPerBulkRequest(1000)
.flushInterval(TimeValue.timeValueSeconds(10))
.newClient(client("1"))
.newIndex("test");
try {
for (int i = 0; i < 12345; i++) {
es.index("test", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}");
}
es.flush();
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
es.shutdown();
assertEquals(13, es.getState().getTotalIngest().count());
if (es.hasThrowable()) {
logger.error("error", es.getThrowable());
}
assertFalse(es.hasThrowable());
}
}
@Before
public void startNodes() {
try {
setClusterName();
startNode("1");
startNode("2"); // we need 2 nodes for knapsack
findNodeAddress();
try {
ClusterHealthResponse healthResponse =
client("1")
.execute(
ClusterHealthAction.INSTANCE,
new ClusterHealthRequest()
.waitForYellowStatus()
.timeout(TimeValue.timeValueSeconds(30)))
.actionGet();
if (healthResponse != null && healthResponse.isTimedOut()) {
throw new IOException(
"cluster state is "
+ healthResponse.getStatus().name()
+ ", from here on, everything will fail!");
}
} catch (ElasticsearchTimeoutException e) {
throw new IOException(
"timeout, cluster does not respond to health request, cowardly refusing to continue with operations");
}
logger.info("ready");
} catch (Throwable t) {
logger.error("startNodes failed", t);
}
}
@Override
public void testFailure(Failure failure) throws Exception {
// Ignore assumptions.
if (failure.getException() instanceof AssumptionViolatedException) {
return;
}
final Description d = failure.getDescription();
final StringBuilder b = new StringBuilder();
b.append("FAILURE : ").append(d.getDisplayName()).append("\n");
b.append("REPRODUCE WITH : mvn test");
ReproduceErrorMessageBuilder builder =
reproduceErrorMessageBuilder(b).appendAllOpts(failure.getDescription());
if (mustAppendClusterSeed(failure)) {
appendClusterSeed(builder);
}
b.append("\n");
b.append("Throwable:\n");
if (failure.getException() != null) {
traces().formatThrowable(b, failure.getException());
}
logger.error(b.toString());
}
@Override
protected void doExecute(
final KnapsackPullRequest request, ActionListener<KnapsackPullResponse> listener) {
final KnapsackState state =
new KnapsackState().setMode("pull").setNodeName(nodeService.nodeName());
final KnapsackPullResponse response = new KnapsackPullResponse().setState(state);
try {
final BulkTransportClient transportClient =
ClientBuilder.builder()
.put(ClientBuilder.MAX_ACTIONS_PER_REQUEST, request.getMaxActionsPerBulkRequest())
.put(ClientBuilder.MAX_CONCURRENT_REQUESTS, request.getMaxBulkConcurrency())
.put(ClientBuilder.FLUSH_INTERVAL, TimeValue.timeValueSeconds(5))
.put(clientSettings(client, request))
.toBulkTransportClient();
final BulkNodeClient nodeClient =
ClientBuilder.builder()
.put(ClientBuilder.MAX_ACTIONS_PER_REQUEST, request.getMaxActionsPerBulkRequest())
.put(ClientBuilder.MAX_CONCURRENT_REQUESTS, request.getMaxBulkConcurrency())
.put(ClientBuilder.FLUSH_INTERVAL, TimeValue.timeValueSeconds(5))
.toBulkNodeClient(client);
state.setTimestamp(new DateTime());
response.setRunning(true);
knapsack.submit(
new Thread() {
public void run() {
performPull(request, state, transportClient, nodeClient);
}
});
listener.onResponse(response);
} catch (Throwable e) {
logger.error(e.getMessage(), e);
listener.onFailure(e);
}
}
private void handleException(final TransportResponseHandler handler, Throwable error) {
if (!(error instanceof RemoteTransportException)) {
error = new RemoteTransportException(error.getMessage(), error);
}
final RemoteTransportException rtx = (RemoteTransportException) error;
if (handler.executor() == ThreadPool.Names.SAME) {
try {
handler.handleException(rtx);
} catch (Throwable e) {
logger.error("failed to handle exception response [{}]", e, handler);
}
} else {
threadPool
.executor(handler.executor())
.execute(
new Runnable() {
@Override
public void run() {
try {
handler.handleException(rtx);
} catch (Throwable e) {
logger.error("failed to handle exception response [{}]", e, handler);
}
}
});
}
}
private boolean handleUnknownRevisionResponse(ChannelHandlerContext ctx) {
logger.error(
"Unknown revision response received on connection [{}]",
ctx.getChannel().getRemoteAddress(),
ctx.getChannel().getLocalAddress());
return false;
}
@Override
protected void doClose() throws ElasticsearchException {
int size = tasks.size();
if (size > 0) {
for (Future<?> f : tasks) {
if (!f.isDone()) {
logger.info("aborting knapsack task {}", f);
boolean b = f.cancel(true);
if (!b) {
logger.error("knapsack task {} could not be cancelled", f);
}
}
}
tasks.clear();
}
logger.info("knapsack shutdown...");
executor.shutdown();
try {
this.executor.awaitTermination(5, TimeUnit.SECONDS);
} catch (InterruptedException e) {
throw new ElasticsearchException(e.getMessage());
}
if (!executor.isShutdown()) {
logger.info("knapsack shutdown now");
executor.shutdownNow();
try {
this.executor.awaitTermination(5, TimeUnit.SECONDS);
} catch (InterruptedException e) {
throw new ElasticsearchException(e.getMessage());
}
}
logger.info("knapsack shutdown complete");
}
@Override
public BaseIngestTransportClient newIndex(String index) {
if (client == null) {
logger.warn("no client for create index");
return this;
}
if (index == null) {
logger.warn("no index name given to create index");
return this;
}
CreateIndexRequest request = new CreateIndexRequest(index);
if (getSettings() != null) {
request.settings(getSettings());
}
if (getMappings() != null) {
for (Map.Entry<String, String> me : getMappings().entrySet()) {
request.mapping(me.getKey(), me.getValue());
}
}
logger.info(
"creating index {} with settings = {}, mappings = {}",
index,
getSettings() != null ? getSettings().getAsMap() : null,
getMappings());
try {
client.admin().indices().create(request).actionGet();
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
return this;
}
private List<KnapsackState> get(String name) throws IOException {
ImmutableList.Builder<KnapsackState> builder = ImmutableList.builder();
try {
logger.debug("get knapsack states: {}", name);
final Client client = injector.getInstance(Client.class);
createIndexIfNotExist(client);
GetResponse getResponse =
client.prepareGet(INDEX_NAME, MAPPING_NAME, name).execute().actionGet();
if (!getResponse.isExists()) {
return builder.build();
}
XContentParser parser = xContent(JSON).createParser(getResponse.getSourceAsBytes());
while (parser.nextToken() != START_ARRAY) {
// forward
}
while (parser.nextToken() != END_ARRAY) {
KnapsackState state = new KnapsackState();
builder.add(state.fromXContent(parser));
}
return builder.build();
} catch (Throwable t) {
logger.error("get settings failed", t);
return null;
}
}
public static DigestBlob resumeTransfer(
BlobContainer blobContainer, String digest, UUID transferId, long currentPos) {
DigestBlob digestBlob = new DigestBlob(blobContainer, digest, transferId);
digestBlob.file = getTmpFilePath(blobContainer, digest, transferId);
try {
logger.trace("Resuming DigestBlob {}. CurrentPos {}", digest, currentPos);
digestBlob.headFileChannel = new FileOutputStream(digestBlob.file, false).getChannel();
digestBlob.headLength = currentPos;
digestBlob.headSize = new AtomicLong();
digestBlob.headCatchedUpLatch = new CountDownLatch(1);
RandomAccessFile raf = new RandomAccessFile(digestBlob.file, "rw");
raf.setLength(currentPos);
raf.close();
FileOutputStream outputStream = new FileOutputStream(digestBlob.file, true);
digestBlob.fileChannel = outputStream.getChannel();
} catch (IOException ex) {
logger.error("error resuming transfer of {}, id: {}", ex, digest, transferId);
return null;
}
return digestBlob;
}
private boolean handleRevision1Response(ChannelHandlerContext ctx, int payloadLength)
throws Exception {
int code = buffered.readInt();
int descriptionLength = buffered.readInt();
byte[] descBytes = new byte[descriptionLength];
buffered.readBytes(descBytes, 0, descBytes.length);
String description = new String(descBytes, StandardCharsets.UTF_8);
logger.debug(
"Decoded payload with length:[{}], code:[{}], descriptionLength:[{}], description:[{}] on connection [{}]",
payloadLength,
code,
descriptionLength,
description,
ctx.getChannel().getLocalAddress());
if (200 <= code && code <= 299) {
logger.info(
"Connected to Found Elasticsearch: [{}]: [{}] on connection [{}]",
code,
description,
ctx.getChannel().getLocalAddress());
return true;
} else {
logger.error(
"Unable to connect to Found Elasticsearch: [{}]: [{}] on connection [{}]",
code,
description,
ctx.getChannel().getLocalAddress());
return false;
}
}
@Override
public void onFailure(Throwable e) {
try {
restChannel.sendResponse(new XContentThrowableRestResponse(restRequest, e));
} catch (IOException e1) {
logger.error("Failed to send failure response", e1);
}
}
/** 加载用户配置的扩展词典到主词库表 */
private void loadExtDict() {
// 加载扩展词典配置
List<String> extDictFiles = configuration.getExtDictionarys();
if (extDictFiles != null) {
InputStream is = null;
for (String extDictName : extDictFiles) {
// 读取扩展词典文件
logger.info("[Dict Loading]" + extDictName);
File file = new File(configuration.getDictRoot(), extDictName);
try {
is = new FileInputStream(file);
} catch (FileNotFoundException e) {
logger.error("ik-analyzer", e);
}
// 如果找不到扩展的字典，则忽略
if (is == null) {
continue;
}
try {
BufferedReader br = new BufferedReader(new InputStreamReader(is, "UTF-8"), 512);
String theWord = null;
do {
theWord = br.readLine();
if (theWord != null && !"".equals(theWord.trim())) {
// 加载扩展词典数据到主内存词典中
_MainDict.fillSegment(theWord.trim().toCharArray());
}
} while (theWord != null);
} catch (IOException e) {
logger.error("ik-analyzer", e);
} finally {
try {
if (is != null) {
is.close();
is = null;
}
} catch (IOException e) {
logger.error("ik-analyzer", e);
}
}
}
}
}
/**
* 关闭数据库连接
*
* @param jedis redis.
*/
public static void closeConnection(Jedis jedis) {
if (null != jedis) {
try {
jedisPool.close();
} catch (Exception e) {
// e.printStackTrace();
logger.error(e.getMessage(), e);
}
}
}
/** Harvests all the triplets from each URI in the @rdfUris list */
private void harvestFromDumps() {
for (String uri : rdfUris) {
if (uri.isEmpty()) continue;
logger.info("Harvesting uri [{}]", uri);
Model model = ModelFactory.createDefaultModel();
try {
RDFDataMgr.read(model, uri.trim(), RDFLanguages.RDFXML);
BulkRequestBuilder bulkRequest = client.prepareBulk();
addModelToES(model, bulkRequest, true);
} catch (RiotException re) {
logger.error("Illegal xml character [{}]", re.getLocalizedMessage());
} catch (Exception e) {
logger.error(
"Exception when harvesting url: {}. Details: {}", uri, e.getLocalizedMessage());
}
}
}
public static void appendAMB(String content) {
try {
File file =
new File(AnsjElasticConfigurator.environment.configFile().toFile(), "ansj/ambiguity.dic");
appendFile(content, file);
} catch (IOException e) {
logger.error("read exception", e);
e.printStackTrace();
}
}
/**
* 获取数据库连接
*
* @return conn
*/
public static Jedis getConnection() {
Jedis jedis = null;
try {
jedis = jedisPool.getResource();
} catch (Exception e) {
e.printStackTrace();
logger.error(e.getMessage(), e);
}
return jedis;
}
private void remove(final String name) {
try {
logger.debug("remove: {}", name);
final Client client = injector.getInstance(Client.class);
createIndexIfNotExist(client);
client.prepareDelete(INDEX_NAME, MAPPING_NAME, name).setRefresh(true).execute().actionGet();
} catch (Throwable t) {
logger.error("remove failed", t);
}
}
@Test
public void testRenameAttribute() throws Throwable {
logger.debug("Start testRenameAttribute");
try {
logger.debug("Create river {}", getRiver());
String script = "ctx.document.score2 = ctx.document.score; delete ctx.document.score;";
super.createRiver(
"/test/elasticsearch/plugin/river/mongodb/script/test-mongodb-river-with-script.json",
getRiver(),
String.valueOf(getMongoPort1()),
String.valueOf(getMongoPort2()),
String.valueOf(getMongoPort3()),
getDatabase(),
getCollection(),
script,
getIndex(),
getDatabase());
String mongoDocument =
copyToStringFromClasspath(
"/test/elasticsearch/plugin/river/mongodb/script/test-simple-mongodb-document.json");
DBObject dbObject = (DBObject) JSON.parse(mongoDocument);
WriteResult result = mongoCollection.insert(dbObject);
Thread.sleep(wait);
String id = dbObject.get("_id").toString();
logger.info("WriteResult: {}", result.toString());
refreshIndex();
ActionFuture<IndicesExistsResponse> response =
getNode().client().admin().indices().exists(new IndicesExistsRequest(getIndex()));
assertThat(response.actionGet().isExists(), equalTo(true));
SearchResponse sr =
getNode()
.client()
.prepareSearch(getIndex())
.setQuery(fieldQuery("_id", id))
.execute()
.actionGet();
logger.debug("SearchResponse {}", sr.toString());
long totalHits = sr.getHits().getTotalHits();
logger.debug("TotalHits: {}", totalHits);
assertThat(totalHits, equalTo(1l));
assertThat(sr.getHits().getHits()[0].sourceAsMap().containsKey("score2"), equalTo(true));
mongoCollection.remove(dbObject);
} catch (Throwable t) {
logger.error("testRenameAttribute failed.", t);
t.printStackTrace();
throw t;
} finally {
super.deleteRiver();
super.deleteIndex();
}
}
/**
* Sets the {@link Harvester}'s {@link #rdfQueryType} parameter
*
* @param queryType - the type of any possible query
* @return the same {@link Harvester} with the {@link #rdfQueryType} parameter set
*/
public Harvester rdfQueryType(String queryType) {
try {
rdfQueryType = QueryType.valueOf(queryType.toUpperCase());
} catch (IllegalArgumentException e) {
logger.error("Bad query type: {}", queryType);
/* River process can't continue */
throw e;
}
return this;
}
public static Client getClient() {
if (client == null) {
try {
client = startClient();
} catch (IOException e) {
logger.error("can not start the client", e);
}
assertThat(client, notNullValue());
}
return client;
}
private String getLastUpdate() {
String res;
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
try {
GetResponse response =
client.prepareGet(indexName, "stats", "1").setFields("last_update").execute().actionGet();
if (!response.getFields().isEmpty()) {
res = (String) response.getField("last_update").getValue();
} else {
res = sdf.format(new Date(0));
}
} catch (ElasticsearchIllegalStateException ex) {
logger.error("Could not get last_update, use Date(0)", ex.getLocalizedMessage());
res = sdf.format(new Date(0));
} catch (NullPointerException ex) {
logger.error("Could not get last_update, use Date(0)", ex.getLocalizedMessage());
res = sdf.format(new Date(0));
}
return res;
}