/**
* Returns an indexer instance which will automatically balance requests over the available Lily
* servers, and will retry operations according to what is specified in {@link RetryConf}.
*
* <p>To see some information when the client goes into retry mode, enable INFO logging for the
* category org.lilyproject.client.
*/
public Indexer getIndexer() {
return balancingAndRetryingLilyConnection.getIndexer();
}
/**
* Provides remote repository implementations.
*
* <p>Connects to zookeeper to find out available repository nodes.
*/
public class LilyClient implements Closeable {
private ZooKeeperItf zk;
private boolean managedZk;
private List<ServerNode> servers = Collections.synchronizedList(new ArrayList<ServerNode>());
private Set<String> serverAddresses = new HashSet<String>();
private RetryConf retryConf = new RetryConf();
private static final String nodesPath = "/lily/repositoryNodes";
private static final String hbaseConfigPath = "/lily/hbaseConfig";
private static final String blobDfsUriPath = "/lily/blobStoresConfig/dfsUri";
private static final String blobStoreAccessConfigPath = "/lily/blobStoresConfig/accessConfig";
private Log log = LogFactory.getLog(getClass());
private ZkWatcher watcher = new ZkWatcher();
private BalancingAndRetryingLilyConnection balancingAndRetryingLilyConnection =
BalancingAndRetryingLilyConnection.getInstance(this);
private RemoteSchemaCache schemaCache;
private HBaseConnections hbaseConnections = new HBaseConnections();
private boolean isClosed = true;
public LilyClient(ZooKeeperItf zk)
throws IOException, InterruptedException, KeeperException, ZkConnectException,
NoServersException, RepositoryException {
this.zk = zk;
schemaCache = new RemoteSchemaCache(zk, this);
init();
}
/**
* @throws NoServersException if the znode under which the repositories are published does not
* exist
*/
public LilyClient(String zookeeperConnectString, int sessionTimeout)
throws IOException, InterruptedException, KeeperException, ZkConnectException,
NoServersException, RepositoryException {
managedZk = true;
zk = ZkUtil.connect(zookeeperConnectString, sessionTimeout);
schemaCache = new RemoteSchemaCache(zk, this);
init();
}
private void init()
throws InterruptedException, KeeperException, NoServersException, RepositoryException {
zk.addDefaultWatcher(watcher);
refreshServers();
schemaCache.start();
this.isClosed = false;
}
@Override
public void close() throws IOException {
zk.removeDefaultWatcher(watcher);
schemaCache.close();
for (ServerNode node : servers) {
Closer.close(node.repository);
}
if (managedZk && zk != null) {
zk.close();
}
// Close pools related to the Configuration objects managed by this LilyClient instance
for (Configuration config : hbaseConnections.getConfigurations()) {
LocalHTable.closePool(config);
}
// Close HBase connections created by [only] this LilyClient instance.
// This will almost always contain only one connection, if not we would need a more
// advanced connection mgmt so that these connections don't stay open for the lifetime
// of LilyClient.
Closer.close(hbaseConnections);
this.isClosed = true;
}
/**
* Returns if the connection to the lily server has been closed.
*
* @return
*/
public boolean isClosed() {
return isClosed;
}
/**
* Returns a Repository that uses one of the available Lily servers (randomly selected). This
* repository instance will not automatically retry operations and to balance requests over
* multiple Lily servers, you need to recall this method regularly to retrieve other repository
* instances. Most of the time, you will rather use {@link #getRepository()}.
*/
public synchronized Repository getPlainRepository()
throws IOException, NoServersException, InterruptedException, KeeperException,
RepositoryException {
if (servers.size() == 0) {
throw new NoServersException("No servers available");
}
int pos = (int) Math.floor(Math.random() * servers.size());
ServerNode server = servers.get(pos);
if (server.repository == null) {
constructRepository(server);
}
return server.repository;
}
/**
* Returns a repository instance which will automatically balance requests over the available Lily
* servers, and will retry operations according to what is specified in {@link RetryConf}.
*
* <p>To see some information when the client goes into retry mode, enable INFO logging for the
* category org.lilyproject.client.
*/
public Repository getRepository() {
return balancingAndRetryingLilyConnection.getRepository();
}
/**
* Returns an Indexer that uses one of the available Lily servers (randomly selected). This
* indexer instance will not automatically retry operations and to balance requests over multiple
* Lily servers, you need to recall this method regularly to retrieve other indexer instances.
* Most of the time, you will rather use {@link #getIndexer()}.
*/
public synchronized Indexer getPlainIndexer()
throws IOException, NoServersException, InterruptedException, KeeperException,
RepositoryException {
if (servers.size() == 0) {
throw new NoServersException("No servers available");
}
int pos = (int) Math.floor(Math.random() * servers.size());
ServerNode server = servers.get(pos);
if (server.indexer == null) {
constructIndexer(server);
}
return server.indexer;
}
/**
* Returns an indexer instance which will automatically balance requests over the available Lily
* servers, and will retry operations according to what is specified in {@link RetryConf}.
*
* <p>To see some information when the client goes into retry mode, enable INFO logging for the
* category org.lilyproject.client.
*/
public Indexer getIndexer() {
return balancingAndRetryingLilyConnection.getIndexer();
}
public RetryConf getRetryConf() {
return retryConf;
}
public void setRetryConf(RetryConf retryConf) {
this.retryConf = retryConf;
}
private void constructRepository(ServerNode server)
throws IOException, InterruptedException, KeeperException, RepositoryException {
AvroConverter remoteConverter = new AvroConverter();
IdGeneratorImpl idGenerator = new IdGeneratorImpl();
RemoteTypeManager typeManager =
new RemoteTypeManager(
parseAddressAndPort(server.lilyAddressAndPort),
remoteConverter,
idGenerator,
zk,
schemaCache);
// TODO BlobManager can probably be shared across all repositories
BlobManager blobManager = getBlobManager(zk, hbaseConnections);
Configuration hbaseConf = getHBaseConfiguration(zk);
hbaseConf = hbaseConnections.getExisting(hbaseConf);
Repository repository =
new RemoteRepository(
parseAddressAndPort(server.lilyAddressAndPort),
remoteConverter,
typeManager,
idGenerator,
blobManager,
hbaseConf);
remoteConverter.setRepository(repository);
if ("true".equals(System.getProperty("lilyclient.trace"))) {
repository = TracingRepository.wrap(repository);
}
typeManager.start();
server.repository = repository;
}
public static BlobManager getBlobManager(ZooKeeperItf zk, HBaseConnections hbaseConns)
throws IOException {
Configuration configuration = getHBaseConfiguration(zk);
// Avoid HBase(Admin)/ZooKeeper connection leaks when using new Configuration objects each time.
configuration = hbaseConns.getExisting(configuration);
HBaseTableFactory hbaseTableFactory = new HBaseTableFactoryImpl(configuration);
URI dfsUri = getDfsUri(zk);
FileSystem fs = FileSystem.get(DfsUri.getBaseDfsUri(dfsUri), configuration);
Path blobRootPath = new Path(DfsUri.getDfsPath(dfsUri));
BlobStoreAccess dfsBlobStoreAccess = new DFSBlobStoreAccess(fs, blobRootPath);
BlobStoreAccess hbaseBlobStoreAccess = new HBaseBlobStoreAccess(configuration, true);
BlobStoreAccess inlineBlobStoreAccess = new InlineBlobStoreAccess();
List<BlobStoreAccess> blobStoreAccesses =
Arrays.asList(dfsBlobStoreAccess, hbaseBlobStoreAccess, inlineBlobStoreAccess);
SizeBasedBlobStoreAccessFactory blobStoreAccessFactory =
new SizeBasedBlobStoreAccessFactory(blobStoreAccesses, getBlobStoreAccessConfig(zk));
return new BlobManagerImpl(hbaseTableFactory, blobStoreAccessFactory, true);
}
private static BlobStoreAccessConfig getBlobStoreAccessConfig(ZooKeeperItf zk) {
try {
return new BlobStoreAccessConfig(zk.getData(blobStoreAccessConfigPath, false, new Stat()));
} catch (Exception e) {
throw new RuntimeException(
"Blob stores config lookup: failed to get blob store access config from ZooKeeper", e);
}
}
private static URI getDfsUri(ZooKeeperItf zk) {
try {
return new URI(new String(zk.getData(blobDfsUriPath, false, new Stat())));
} catch (Exception e) {
throw new RuntimeException(
"Blob stores config lookup: failed to get DFS URI from ZooKeeper", e);
}
}
public static Configuration getHBaseConfiguration(ZooKeeperItf zk) {
try {
Configuration configuration = HBaseConfiguration.create();
byte[] data = zk.getData(hbaseConfigPath, false, new Stat());
ObjectNode propertiesNode =
(ObjectNode) JsonFormat.deserializeSoft(data, "HBase configuration");
Iterator<Map.Entry<String, JsonNode>> it = propertiesNode.getFields();
while (it.hasNext()) {
Map.Entry<String, JsonNode> entry = it.next();
configuration.set(entry.getKey(), entry.getValue().getTextValue());
}
return configuration;
} catch (Exception e) {
throw new RuntimeException("Failed to get HBase configuration from ZooKeeper", e);
}
}
private void constructIndexer(ServerNode server)
throws IOException, InterruptedException, KeeperException, RepositoryException {
server.indexer =
new RemoteIndexer(parseAddressAndPort(server.lilyAddressAndPort), new AvroConverter());
}
private InetSocketAddress parseAddressAndPort(String addressAndPort) {
int colonPos = addressAndPort.indexOf(":");
if (colonPos == -1) {
// since these are produced by the server nodes, this should never occur
throw new RuntimeException("Unexpected situation: invalid addressAndPort: " + addressAndPort);
}
String address = addressAndPort.substring(0, colonPos);
int port = Integer.parseInt(addressAndPort.substring(colonPos + 1));
return new InetSocketAddress(address, port);
}
private class ServerNode {
private String lilyAddressAndPort;
private Repository repository;
private Indexer indexer;
public ServerNode(String lilyAddressAndPort) {
this.lilyAddressAndPort = lilyAddressAndPort;
}
}
private synchronized void refreshServers() throws InterruptedException, KeeperException {
Set<String> currentServers = new HashSet<String>();
boolean retry;
do {
retry = false;
try {
currentServers.addAll(zk.getChildren(nodesPath, true));
} catch (KeeperException.NoNodeException e) {
// The path does not exist: this can happen if the client is started before
// any Lily server has ever been started, or when using the LilyLauncher
// from the test framework and calling its resetLilyState JMX operation.
// In this case, put a watcher to be notified when the path is created.
Stat stat = zk.exists(nodesPath, true);
if (stat == null) {
if (log.isInfoEnabled()) {
log.info("The path with Lily servers does not exist in ZooKeeper: " + nodesPath);
}
clearServers();
return;
} else {
// The node was created in between the getChildren and exists calls: retry
retry = true;
}
}
} while (retry);
Set<String> removedServers = new HashSet<String>();
removedServers.addAll(serverAddresses);
removedServers.removeAll(currentServers);
Set<String> newServers = new HashSet<String>();
newServers.addAll(currentServers);
newServers.removeAll(serverAddresses);
if (log.isDebugEnabled()) {
log.debug(
"# current servers in ZK: "
+ currentServers.size()
+ ", # added servers: "
+ newServers.size()
+ ", # removed servers: "
+ removedServers.size());
}
// Remove removed servers
Iterator<ServerNode> serverIt = servers.iterator();
while (serverIt.hasNext()) {
ServerNode server = serverIt.next();
if (removedServers.contains(server.lilyAddressAndPort)) {
serverIt.remove();
Closer.close(server.repository);
}
}
serverAddresses.removeAll(removedServers);
// Add new servers
for (String server : newServers) {
servers.add(new ServerNode(server));
serverAddresses.add(server);
}
if (log.isInfoEnabled()) {
log.info("Current Lily servers = " + serverAddresses.toString());
}
}
private synchronized void clearServers() {
Iterator<ServerNode> serverIt = servers.iterator();
while (serverIt.hasNext()) {
ServerNode server = serverIt.next();
serverIt.remove();
Closer.close(server.repository);
}
serverAddresses.clear();
}
private class ZkWatcher implements Watcher {
@Override
public void process(WatchedEvent event) {
try {
if (event.getState() != Event.KeeperState.SyncConnected) {
if (log.isInfoEnabled()) {
log.info("Not connected to ZooKeeper, will clear list of servers.");
}
clearServers();
} else {
// We refresh the servers not only when /lily/repositoryNodes has changed, but also
// when we get a SyncConnected event, since our watcher might not be installed anymore
// (we do not have to check if it was still installed: ZK ignores double registration
// of the same watcher object)
refreshServers();
}
} catch (InterruptedException e) {
Thread.interrupted();
} catch (KeeperException.ConnectionLossException e) {
clearServers();
} catch (Throwable t) {
log.error("Error in ZooKeeper watcher.", t);
}
}
}
}
/**
* Returns a repository instance which will automatically balance requests over the available Lily
* servers, and will retry operations according to what is specified in {@link RetryConf}.
*
* <p>To see some information when the client goes into retry mode, enable INFO logging for the
* category org.lilyproject.client.
*/
public Repository getRepository() {
return balancingAndRetryingLilyConnection.getRepository();
}
