protected boolean doBasicPathsMatch(final Path path1, final Path path2) {
if (path1.equals(path2)) {
return true;
}
// ignore count designators and attribute specifications whenc omparing paths
// ie, /a/b[3]/c/@foo for our purposes is equivalent to /a/b/c
String path1Relpaced =
StringUtils.chomp(path1.toString().replaceAll("\\[.*\\]", "").replaceAll("/@.*$", ""), "/");
String path2Replaced =
StringUtils.chomp(path2.toString().replaceAll("\\[.*\\]", "").replaceAll("/@.*$", ""), "/");
return path1Relpaced.equals(path2Replaced);
}
private String getRow(List<String> row) {
StringBuffer sb = new StringBuffer();
for (String data : row) {
sb.append(data).append(seperator);
}
return StringUtils.chomp(sb.toString(), seperator);
}
public String getServerLink() {
String url =
StringUtils.defaultIfEmpty(
StringUtils.trimToEmpty(getServerPublicUrl()), StringUtils.trimToEmpty(getServerUrl()));
url = StringUtils.defaultIfEmpty(url, MagicNames.DEFAULT_SONAR_URL);
return StringUtils.chomp(url, "/");
}
public ServerHttpClient(
String remoteServerUrl, Integer connectTimeoutMiliseconds, Integer readTimeoutMiliseconds) {
this.url = StringUtils.chomp(remoteServerUrl, "/");
if (connectTimeoutMiliseconds != null) {
this.connectTimeoutMiliseconds = connectTimeoutMiliseconds;
}
if (readTimeoutMiliseconds != null) {
this.readTimeoutMiliseconds = readTimeoutMiliseconds;
}
}
/**
* Executes <code>commandLine</code>. Sometimes (especially observed on MacOS) the commandLine
* isn't executed properly. In that cases another exec-method is to be used. To accomplish this
* please use the special delimiter '<code>@@</code>'. If <code>commandLine</code> contains this
* delimiter it is split into a String[] array and the special exec-method is used.
*
* <p>A possible {@link IOException} gets logged but no further processing is done.
*
* @param commandLine the command line to execute
* @param timeout timeout for execution in milliseconds
* @return response data from executed command line
*/
public static String executeCommandLineAndWaitResponse(String commandLine, int timeout) {
String retval = null;
CommandLine cmdLine = null;
if (commandLine.contains(CMD_LINE_DELIMITER)) {
String[] cmdArray = commandLine.split(CMD_LINE_DELIMITER);
cmdLine = new CommandLine(cmdArray[0]);
for (int i = 1; i < cmdArray.length; i++) {
cmdLine.addArgument(cmdArray[i], false);
}
} else {
cmdLine = CommandLine.parse(commandLine);
}
DefaultExecuteResultHandler resultHandler = new DefaultExecuteResultHandler();
ExecuteWatchdog watchdog = new ExecuteWatchdog(timeout);
Executor executor = new DefaultExecutor();
ByteArrayOutputStream stdout = new ByteArrayOutputStream();
PumpStreamHandler streamHandler = new PumpStreamHandler(stdout);
executor.setExitValue(1);
executor.setStreamHandler(streamHandler);
executor.setWatchdog(watchdog);
try {
executor.execute(cmdLine, resultHandler);
logger.debug("executed commandLine '{}'", commandLine);
} catch (ExecuteException e) {
logger.error("couldn't execute commandLine '" + commandLine + "'", e);
} catch (IOException e) {
logger.error("couldn't execute commandLine '" + commandLine + "'", e);
}
// some time later the result handler callback was invoked so we
// can safely request the exit code
try {
resultHandler.waitFor();
int exitCode = resultHandler.getExitValue();
retval = StringUtils.chomp(stdout.toString());
if (resultHandler.getException() != null) {
logger.warn(resultHandler.getException().getMessage());
} else {
logger.debug("exit code '{}', result '{}'", exitCode, retval);
}
} catch (InterruptedException e) {
logger.error("Timeout occured when executing commandLine '" + commandLine + "'", e);
}
return retval;
}
/**
* Executes the query against the specified index, and returns a bounded collection of sentences
* ordered by document id (so the sentence ordering is preserved in the collection).
*
* @param query the Boolean OR query computed from the top terms.
* @param weights the target location for the calculated weights.
* @return an array of sentences.
* @throws Exception if one is thrown.
*/
protected Double[] searchIndex(Query query, Double[] weights) throws Exception {
IndexReader reader = IndexReader.open(ramdir);
IndexSearcher searcher = new IndexSearcher(reader);
TopDocs topDocs = searcher.search(query, structure.getNumSentences());
for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
Document doc = searcher.doc(scoreDoc.doc);
String sentence = StringUtils.chomp(doc.get("text"));
weights[sentenceMap.get(sentence)] = (double) scoreDoc.score;
}
searcher.close();
return weights;
}
public static Object readObject(InputStream in, boolean allowJsonResolve)
throws ClassNotFoundException, IOException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
IOUtils.copy(in, baos);
in.close();
byte[] bytes = baos.toByteArray();
try {
char[] buffer = StringUtils.chomp(readString(bytes)).toCharArray();
return parseObject(buffer, 0, buffer.length, allowJsonResolve);
} catch (Exception e) {
}
ObjectInputStream oin = new ObjectInputStream(new ByteArrayInputStream(bytes));
Serializable value = (Serializable) oin.readObject();
oin.close();
return value;
}
/**
* Strip byte count from the bytebuffer. IRTrans devices include the number of bytes sent in each
* response it sends back to the connected host. This is a simple error checking mechanism - we do
* need that information, and so, we strip it
*
* @param byteBuffer the byte buffer
* @return the string
*/
protected String stripByteCount(ByteBuffer byteBuffer) {
/** {@link Pattern} which matches a binding configuration part */
Pattern RESPONSE_PATTERN = Pattern.compile("..(\\d{5}) (.*)");
String message = null;
String response = new String(byteBuffer.array(), 0, byteBuffer.limit());
response = StringUtils.chomp(response);
Matcher matcher = RESPONSE_PATTERN.matcher(response);
if (matcher.matches()) {
String byteCountAsString = matcher.group(1);
int byteCount = Integer.parseInt(byteCountAsString);
message = matcher.group(2);
}
return message;
}
public String forBrokerTenantCreate(String tenantName) {
return StringUtil.join(
"/", StringUtils.chomp(_baseUrl, "/"), "tenants", tenantName, "instances?type=broker");
}
public String forServerTenantCreate() {
return StringUtil.join("/", StringUtils.chomp(_baseUrl, "/"), "tenants?type=server");
}
// V2 API started
public String forTenantCreate() {
return StringUtil.join("/", StringUtils.chomp(_baseUrl, "/"), "tenants/");
}
public String forInstanceBulkCreate() {
return StringUtil.join("/", StringUtils.chomp(_baseUrl, "/"), "instances", "bulkAdd");
}
public String forTableCreate() {
return StringUtil.join("/", StringUtils.chomp(_baseUrl, "/"), "tables");
}
public String getResponseString() throws UnsupportedEncodingException {
return StringUtils.chomp(
new String(
getResponseBytes(), getResponseOffset(), getResponseLength(), getValidCharset()));
}
public String forBrokerTenantGet(String tenantName) {
return StringUtil.join(
"/", StringUtils.chomp(_baseUrl, "/"), "tenants", tenantName, "?type=broker");
}
public String forTableDelete(String tableName) {
return StringUtil.join("/", StringUtils.chomp(_baseUrl, "/"), "tables", tableName);
}
public String forResourceCreate() {
return StringUtil.join("/", StringUtils.chomp(_baseUrl, "/"), "dataresources");
}
public String forTableGet(String tableName) {
return StringUtil.join("/", StringUtils.chomp(_baseUrl, "/"), "tables", tableName, "instances");
}
public String forTableGetBrokerInstances(String tableName) {
return StringUtil.join(
"/", StringUtils.chomp(_baseUrl, "/"), "tables", tableName, "instances?type=broker");
}
public String forTableGetConfig(String tableName) {
return StringUtil.join("/", StringUtils.chomp(_baseUrl, "/"), "tables", tableName, "configs");
}
public String forTenantGet(String tenantName) {
return StringUtil.join("/", StringUtils.chomp(_baseUrl, "/"), "tenants", tenantName);
}
public String forDataFileUpload() {
return StringUtil.join("/", StringUtils.chomp(_baseUrl, "/"), "segments");
}
@Override
public String getValue() {
return StringUtils.chomp(Arrays.toString(values).substring(1), "]");
}
public String forInstanceCreate() {
return StringUtil.join("/", StringUtils.chomp(_baseUrl, "/"), "instances/");
}
public HelixBrokerStarter(
String helixClusterName, String zkServer, Configuration pinotHelixProperties)
throws Exception {
_liveInstancesListener = new LiveInstancesChangeListenerImpl(helixClusterName);
_pinotHelixProperties = DefaultHelixBrokerConfig.getDefaultBrokerConf(pinotHelixProperties);
final String brokerId =
_pinotHelixProperties.getString(
"instanceId",
CommonConstants.Helix.PREFIX_OF_BROKER_INSTANCE
+ NetUtil.getHostAddress()
+ "_"
+ _pinotHelixProperties.getInt(
CommonConstants.Helix.KEY_OF_BROKER_QUERY_PORT,
CommonConstants.Helix.DEFAULT_BROKER_QUERY_PORT));
_pinotHelixProperties.addProperty("pinot.broker.id", brokerId);
RoutingTableBuilder defaultOfflineRoutingTableBuilder =
getRoutingTableBuilder(
_pinotHelixProperties.subset(DEFAULT_OFFLINE_ROUTING_TABLE_BUILDER_KEY));
RoutingTableBuilder defaultRealtimeRoutingTableBuilder =
getRoutingTableBuilder(
_pinotHelixProperties.subset(DEFAULT_REALTIME_ROUTING_TABLE_BUILDER_KEY));
Map<String, RoutingTableBuilder> tableToRoutingTableBuilderMap =
getTableToRoutingTableBuilderMap(_pinotHelixProperties.subset(ROUTING_TABLE_BUILDER_KEY));
ZkClient zkClient =
new ZkClient(
StringUtil.join(
"/", StringUtils.chomp(zkServer, "/"), helixClusterName, "PROPERTYSTORE"),
ZkClient.DEFAULT_SESSION_TIMEOUT,
ZkClient.DEFAULT_CONNECTION_TIMEOUT,
new ZNRecordSerializer());
_propertyStore =
new ZkHelixPropertyStore<ZNRecord>(new ZkBaseDataAccessor<ZNRecord>(zkClient), "/", null);
_helixExternalViewBasedRouting =
new HelixExternalViewBasedRouting(
defaultOfflineRoutingTableBuilder,
defaultRealtimeRoutingTableBuilder,
tableToRoutingTableBuilderMap,
_propertyStore);
// _brokerServerBuilder = startBroker();
_brokerServerBuilder = startBroker(_pinotHelixProperties);
_helixManager =
HelixManagerFactory.getZKHelixManager(
helixClusterName, brokerId, InstanceType.PARTICIPANT, zkServer);
final StateMachineEngine stateMachineEngine = _helixManager.getStateMachineEngine();
final StateModelFactory<?> stateModelFactory =
new BrokerResourceOnlineOfflineStateModelFactory(
_helixManager, _helixExternalViewBasedRouting);
stateMachineEngine.registerStateModelFactory(
BrokerResourceOnlineOfflineStateModelFactory.getStateModelDef(), stateModelFactory);
_helixManager.connect();
_helixAdmin = _helixManager.getClusterManagmentTool();
_helixBrokerRoutingTable =
new HelixBrokerRoutingTable(_helixExternalViewBasedRouting, brokerId, _helixManager);
addInstanceTagIfNeeded(helixClusterName, brokerId);
_helixManager.addExternalViewChangeListener(_helixBrokerRoutingTable);
_helixManager.addInstanceConfigChangeListener(_helixBrokerRoutingTable);
_helixManager.addLiveInstanceChangeListener(_liveInstancesListener);
}
public String forInstanceEnable(String instanceName) {
return StringUtil.join(
"/", StringUtils.chomp(_baseUrl, "/"), "instances", instanceName, "?state=enable");
}
public String forResourceGet(String resourceName) {
return StringUtil.join("/", StringUtils.chomp(_baseUrl, "/"), "dataresources", resourceName);
}
public String forServerTenantDelete(String tenantName) {
return StringUtil.join(
"/", StringUtils.chomp(_baseUrl, "/"), "tenants", tenantName, "?type=server");
}