@Override
public boolean execute() throws Exception {
if (_brokerHost == null) {
_brokerHost = NetUtil.getHostAddress();
}
Configuration configuration = readConfigFromFile(_configFileName);
if (configuration == null) {
if (_configFileName != null) {
LOGGER.error("Error: Unable to find file {}.", _configFileName);
return false;
}
configuration = new PropertiesConfiguration();
configuration.addProperty(CommonConstants.Helix.KEY_OF_BROKER_QUERY_PORT, _brokerPort);
configuration.setProperty("pinot.broker.routing.table.builder.class", "random");
}
LOGGER.info("Executing command: " + toString());
final HelixBrokerStarter pinotHelixBrokerStarter =
new HelixBrokerStarter(_clusterName, _zkAddress, configuration);
savePID(System.getProperty("java.io.tmpdir") + File.separator + ".pinotAdminBroker.pid");
return true;
}
@Override
public boolean execute() throws Exception {
if (_controllerHost == null) {
_controllerHost = NetUtil.getHostAddress();
}
// Create a temp working directory.
File tmpDir = File.createTempFile(SEGMENT_UPLOADER, null, FileUtils.getTempDirectory());
FileUtils.deleteQuietly(tmpDir);
tmpDir.mkdir();
try {
LOGGER.info("Executing command: " + toString());
File dir = new File(_segmentDir);
File[] files = dir.listFiles();
for (File file : files) {
File tgzFile = file;
if (file.isDirectory()) {
LOGGER.info("Compressing segment {}", file.getName());
String srcDir = file.getAbsolutePath();
String tgzFileName =
TarGzCompressionUtils.createTarGzOfDirectory(
srcDir, tmpDir.getAbsolutePath() + File.separator + file.getName() + TAR_GZIP);
tgzFile = new File(tgzFileName);
}
LOGGER.info("Uploading segment {}", tgzFile.getName());
FileUploadUtils.sendSegmentFile(
_controllerHost,
_controllerPort,
tgzFile.getName(),
new FileInputStream(tgzFile),
tgzFile.length());
}
} catch (Exception e) {
LOGGER.error("Exception caught while uploading segment {}", _segmentDir, e);
} finally {
// Delete the temporary working directory.
FileUtils.deleteQuietly(tmpDir);
}
return true;
}
public HelixBrokerStarter(
String helixClusterName, String zkServer, Configuration pinotHelixProperties)
throws Exception {
_liveInstancesListener = new LiveInstancesChangeListenerImpl(helixClusterName);
_pinotHelixProperties = DefaultHelixBrokerConfig.getDefaultBrokerConf(pinotHelixProperties);
final String brokerId =
_pinotHelixProperties.getString(
"instanceId",
CommonConstants.Helix.PREFIX_OF_BROKER_INSTANCE
+ NetUtil.getHostAddress()
+ "_"
+ _pinotHelixProperties.getInt(
CommonConstants.Helix.KEY_OF_BROKER_QUERY_PORT,
CommonConstants.Helix.DEFAULT_BROKER_QUERY_PORT));
_pinotHelixProperties.addProperty("pinot.broker.id", brokerId);
RoutingTableBuilder defaultOfflineRoutingTableBuilder =
getRoutingTableBuilder(
_pinotHelixProperties.subset(DEFAULT_OFFLINE_ROUTING_TABLE_BUILDER_KEY));
RoutingTableBuilder defaultRealtimeRoutingTableBuilder =
getRoutingTableBuilder(
_pinotHelixProperties.subset(DEFAULT_REALTIME_ROUTING_TABLE_BUILDER_KEY));
Map<String, RoutingTableBuilder> tableToRoutingTableBuilderMap =
getTableToRoutingTableBuilderMap(_pinotHelixProperties.subset(ROUTING_TABLE_BUILDER_KEY));
ZkClient zkClient =
new ZkClient(
StringUtil.join(
"/", StringUtils.chomp(zkServer, "/"), helixClusterName, "PROPERTYSTORE"),
ZkClient.DEFAULT_SESSION_TIMEOUT,
ZkClient.DEFAULT_CONNECTION_TIMEOUT,
new ZNRecordSerializer());
_propertyStore =
new ZkHelixPropertyStore<ZNRecord>(new ZkBaseDataAccessor<ZNRecord>(zkClient), "/", null);
_helixExternalViewBasedRouting =
new HelixExternalViewBasedRouting(
defaultOfflineRoutingTableBuilder,
defaultRealtimeRoutingTableBuilder,
tableToRoutingTableBuilderMap,
_propertyStore);
// _brokerServerBuilder = startBroker();
_brokerServerBuilder = startBroker(_pinotHelixProperties);
_helixManager =
HelixManagerFactory.getZKHelixManager(
helixClusterName, brokerId, InstanceType.PARTICIPANT, zkServer);
final StateMachineEngine stateMachineEngine = _helixManager.getStateMachineEngine();
final StateModelFactory<?> stateModelFactory =
new BrokerResourceOnlineOfflineStateModelFactory(
_helixManager, _helixExternalViewBasedRouting);
stateMachineEngine.registerStateModelFactory(
BrokerResourceOnlineOfflineStateModelFactory.getStateModelDef(), stateModelFactory);
_helixManager.connect();
_helixAdmin = _helixManager.getClusterManagmentTool();
_helixBrokerRoutingTable =
new HelixBrokerRoutingTable(_helixExternalViewBasedRouting, brokerId, _helixManager);
addInstanceTagIfNeeded(helixClusterName, brokerId);
_helixManager.addExternalViewChangeListener(_helixBrokerRoutingTable);
_helixManager.addInstanceConfigChangeListener(_helixBrokerRoutingTable);
_helixManager.addLiveInstanceChangeListener(_liveInstancesListener);
}
