/** {@inheritDoc} */
@Override
protected GridConfiguration getConfiguration(String gridName) throws Exception {
GridConfiguration cfg = super.getConfiguration(gridName);
cfg.setCacheConfiguration(
cacheConfiguration(META_CACHE_NAME), cacheConfiguration(DATA_CACHE_NAME));
GridTcpDiscoverySpi discoSpi = new GridTcpDiscoverySpi();
discoSpi.setIpFinder(IP_FINDER);
cfg.setDiscoverySpi(discoSpi);
GridGgfsConfiguration ggfsCfg = new GridGgfsConfiguration();
ggfsCfg.setMetaCacheName(META_CACHE_NAME);
ggfsCfg.setDataCacheName(DATA_CACHE_NAME);
ggfsCfg.setName("ggfs");
ggfsCfg.setBlockSize(CFG_BLOCK_SIZE);
ggfsCfg.setFragmentizerEnabled(true);
cfg.setGgfsConfiguration(ggfsCfg);
return cfg;
}
/** @throws Exception If failed. */
public void testRemoteIfDataCacheNameEquals() throws Exception {
GridConfiguration g2Cfg = getConfiguration("g2");
GridGgfsConfiguration g2GgfsCfg1 = new GridGgfsConfiguration(g1GgfsCfg1);
GridGgfsConfiguration g2GgfsCfg2 = new GridGgfsConfiguration(g1GgfsCfg2);
g2GgfsCfg1.setName("g2GgfsCfg1");
g2GgfsCfg2.setName("g2GgfsCfg2");
g2GgfsCfg1.setMetaCacheName("g2MetaCache1");
g2GgfsCfg2.setMetaCacheName("g2MetaCache2");
g1Cfg.setCacheConfiguration(
concat(dataCaches(1024), metaCaches(), GridCacheConfiguration.class));
g2Cfg.setCacheConfiguration(
concat(
dataCaches(1024),
metaCaches("g2MetaCache1", "g2MetaCache2"),
GridCacheConfiguration.class));
g2Cfg.setGgfsConfiguration(g2GgfsCfg1, g2GgfsCfg2);
G.start(g1Cfg);
checkGridStartFails(
g2Cfg, "Data cache names should be different for different GGFS instances", false);
}
/** @throws Exception If failed. */
public void testLocalIfAffinityMapperIsWrongClass() throws Exception {
g1Cfg.setCacheConfiguration(
concat(dataCaches(1024), metaCaches(), GridCacheConfiguration.class));
for (GridCacheConfiguration cc : g1Cfg.getCacheConfiguration())
cc.setAffinityMapper(new GridCacheDefaultAffinityKeyMapper());
checkGridStartFails(
g1Cfg, "Invalid GGFS data cache configuration (key affinity mapper class should be", true);
}
/** @throws Exception If failed. */
public void testLocalIfBackupsEnabled() throws Exception {
g1Cfg.setCacheConfiguration(
concat(dataCaches(1024), metaCaches(), GridCacheConfiguration.class));
for (GridCacheConfiguration cc : g1Cfg.getCacheConfiguration()) {
cc.setCacheMode(PARTITIONED);
cc.setBackups(1);
}
checkGridStartFails(g1Cfg, "GGFS data cache cannot be used with backups", true);
}
/** @throws Exception If failed. */
public void testLocalIfOffheapIsEnabledAndMaxSpaceSizeIsGreater() throws Exception {
g1Cfg.setCacheConfiguration(
concat(dataCaches(1024), metaCaches(), GridCacheConfiguration.class));
for (GridCacheConfiguration cc : g1Cfg.getCacheConfiguration()) cc.setOffHeapMaxMemory(1000000);
g1GgfsCfg2.setMaxSpaceSize(999999999999999999L);
checkGridStartFails(
g1Cfg,
"Maximum GGFS space size cannot be greater than size of available heap memory and offheap storage",
true);
}
/** @throws Exception If failed. */
public void testRemoteIfAffinityMapperGroupSizeDiffers() throws Exception {
GridConfiguration g2Cfg = getConfiguration("g2");
g1Cfg.setCacheConfiguration(
concat(dataCaches(1024), metaCaches(), GridCacheConfiguration.class));
g2Cfg.setCacheConfiguration(
concat(dataCaches(4021), metaCaches(), GridCacheConfiguration.class));
G.start(g1Cfg);
checkGridStartFails(
g2Cfg,
"Affinity mapper group size should be the same on all nodes in grid for GGFS",
false);
}
/** @throws Exception If failed. */
public void testLocalIfOffheapIsDisabledAndMaxSpaceSizeIsGreater() throws Exception {
g1Cfg.setCacheConfiguration(
concat(dataCaches(1024), metaCaches(), GridCacheConfiguration.class));
g1GgfsCfg2.setMaxSpaceSize(999999999999999999L);
checkGridStartFails(
g1Cfg, "Maximum GGFS space size cannot be greater that size of available heap", true);
}
/** @throws Exception If failed. */
@SuppressWarnings("NullableProblems")
public void testLocalNullGgfsNameIsSupported() throws Exception {
g1Cfg.setCacheConfiguration(
concat(dataCaches(1024), metaCaches(), GridCacheConfiguration.class));
g1GgfsCfg1.setName(null);
assertFalse(G.start(g1Cfg).nodes().isEmpty());
}
/** @throws Exception If failed. */
public void testLocalIfQueryIndexingEnabledForDataCache() throws Exception {
GridCacheConfiguration[] dataCaches = dataCaches(1024);
dataCaches[0].setQueryIndexEnabled(true);
g1Cfg.setCacheConfiguration(concat(dataCaches, metaCaches(), GridCacheConfiguration.class));
checkGridStartFails(g1Cfg, "GGFS data cache cannot start with enabled query indexing", true);
}
/** @throws Exception If failed. */
public void testRemoteIfDataBlockSizeDiffers() throws Exception {
GridConfiguration g2Cfg = getConfiguration("g2");
g1Cfg.setCacheConfiguration(
concat(dataCaches(1024), metaCaches(), GridCacheConfiguration.class));
g2Cfg.setCacheConfiguration(
concat(dataCaches(1024), metaCaches(), GridCacheConfiguration.class));
GridGgfsConfiguration g2GgfsCfg1 = new GridGgfsConfiguration(g1GgfsCfg1);
g2GgfsCfg1.setBlockSize(g2GgfsCfg1.getBlockSize() + 100);
g2Cfg.setGgfsConfiguration(g2GgfsCfg1, g1GgfsCfg2);
G.start(g1Cfg);
checkGridStartFails(
g2Cfg, "Data block size should be same on all nodes in grid for GGFS", false);
}
/** @throws Exception If failed. */
public void testLocalIfNoMetadataCacheIsConfigured() throws Exception {
GridCacheConfiguration cc = defaultCacheConfiguration();
cc.setQueryIndexEnabled(false);
cc.setName(dataCache1Name);
g1Cfg.setCacheConfiguration(cc);
checkGridStartFails(g1Cfg, "Metadata cache is not configured locally for GGFS", true);
}
/** @throws Exception If failed. */
public void testLocalIfNonPrimaryModeAndHadoopFileSystemConfigPathIsNull() throws Exception {
g1Cfg.setCacheConfiguration(
concat(dataCaches(1024), metaCaches(), GridCacheConfiguration.class));
g1GgfsCfg2.setDefaultMode(PROXY);
g1GgfsCfg2.setSecondaryHadoopFileSystemUri("/");
checkGridStartFails(
g1Cfg, "secondaryHadoopFileSystemConfigPath cannot be null when mode is SECONDARY", true);
}
/** @throws Exception If failed. */
public void testLocalIfHadoopFileSystemUriIsNullAndConfigPathIsNot() throws Exception {
g1Cfg.setCacheConfiguration(
concat(dataCaches(1024), metaCaches(), GridCacheConfiguration.class));
g1GgfsCfg2.setSecondaryHadoopFileSystemConfigPath("/");
checkGridStartFails(
g1Cfg,
"secondaryHadoopFileSystemUri cannot be null when secondaryHadoopFileSystemConfigPath is set",
true);
}
/** @throws Exception If failed. */
public void testRemoteIfPathModeDiffers() throws Exception {
GridConfiguration g2Cfg = getConfiguration("g2");
GridGgfsConfiguration g2GgfsCfg1 = new GridGgfsConfiguration(g1GgfsCfg1);
GridGgfsConfiguration g2GgfsCfg2 = new GridGgfsConfiguration(g1GgfsCfg2);
g2GgfsCfg1.setPathModes(ImmutableMap.of("/somePath", DUAL_SYNC));
g2GgfsCfg2.setPathModes(ImmutableMap.of("/somePath", DUAL_SYNC));
g1Cfg.setCacheConfiguration(
concat(dataCaches(1024), metaCaches(), GridCacheConfiguration.class));
g2Cfg.setCacheConfiguration(
concat(dataCaches(1024), metaCaches(), GridCacheConfiguration.class));
g2Cfg.setGgfsConfiguration(g2GgfsCfg1, g2GgfsCfg2);
G.start(g1Cfg);
checkGridStartFails(
g2Cfg, "Path modes should be the same on all nodes in grid for GGFS", false);
}
/** @throws Exception If failed. */
public void testLocalIfGgfsConfigsHaveDifferentNames() throws Exception {
g1Cfg.setCacheConfiguration(
concat(dataCaches(1024), metaCaches(), GridCacheConfiguration.class));
String ggfsCfgName = "ggfs-cfg";
g1GgfsCfg1.setName(ggfsCfgName);
g1GgfsCfg2.setName(ggfsCfgName);
checkGridStartFails(
g1Cfg, "Duplicate GGFS name found (check configuration and assign unique name", true);
}
/** {@inheritDoc} */
@Override
protected GridConfiguration getConfiguration(String gridName) throws Exception {
GridConfiguration cfg = super.getConfiguration(gridName);
GridTcpDiscoverySpi discoSpi = new GridTcpDiscoverySpi();
discoSpi.setIpFinder(IP_FINDER);
cfg.setDiscoverySpi(discoSpi);
g1GgfsCfg1.setName("g1GgfsCfg1");
g1GgfsCfg1.setDataCacheName(dataCache1Name);
g1GgfsCfg1.setMetaCacheName(metaCache1Name);
g1GgfsCfg2.setName("g1GgfsCfg2");
g1GgfsCfg2.setDataCacheName(dataCache2Name);
g1GgfsCfg2.setMetaCacheName(metaCache2Name);
cfg.setGgfsConfiguration(g1GgfsCfg1, g1GgfsCfg2);
cfg.setLocalHost("127.0.0.1");
return cfg;
}
/** @throws Exception If failed. */
public void testRemoteIfDefaultModeDiffers() throws Exception {
GridConfiguration g2Cfg = getConfiguration("g2");
GridGgfsConfiguration g2GgfsCfg1 = new GridGgfsConfiguration(g1GgfsCfg1);
GridGgfsConfiguration g2GgfsCfg2 = new GridGgfsConfiguration(g1GgfsCfg2);
g1GgfsCfg1.setDefaultMode(DUAL_ASYNC);
g1GgfsCfg2.setDefaultMode(DUAL_ASYNC);
g2GgfsCfg1.setDefaultMode(DUAL_SYNC);
g2GgfsCfg2.setDefaultMode(DUAL_SYNC);
g1Cfg.setCacheConfiguration(
concat(dataCaches(1024), metaCaches(), GridCacheConfiguration.class));
g2Cfg.setCacheConfiguration(
concat(dataCaches(1024), metaCaches(), GridCacheConfiguration.class));
g2Cfg.setGgfsConfiguration(g2GgfsCfg1, g2GgfsCfg2);
G.start(g1Cfg);
checkGridStartFails(
g2Cfg, "Default mode should be the same on all nodes in grid for GGFS", false);
}
/** @throws Exception If failed. */
public void testRemoteIfMetaCacheNameDiffers() throws Exception {
GridConfiguration g2Cfg = getConfiguration("g2");
GridGgfsConfiguration g2GgfsCfg1 = new GridGgfsConfiguration(g1GgfsCfg1);
GridGgfsConfiguration g2GgfsCfg2 = new GridGgfsConfiguration(g1GgfsCfg2);
g2GgfsCfg1.setMetaCacheName("g2MetaCache1");
g2GgfsCfg2.setMetaCacheName("g2MetaCache2");
g1Cfg.setCacheConfiguration(
concat(dataCaches(1024), metaCaches(), GridCacheConfiguration.class));
g2Cfg.setCacheConfiguration(
concat(
dataCaches(1024),
metaCaches("g2MetaCache1", "g2MetaCache2"),
GridCacheConfiguration.class));
g2Cfg.setGgfsConfiguration(g2GgfsCfg1, g2GgfsCfg2);
G.start(g1Cfg);
checkGridStartFails(
g2Cfg, "Meta cache name should be the same on all nodes in grid for GGFS", false);
}
/** {@inheritDoc} */
@Override
protected GridConfiguration getConfiguration(String gridName) throws Exception {
GridConfiguration cfg = super.getConfiguration(gridName);
if (cache) {
GridCacheConfiguration cacheCfg = defaultCacheConfiguration();
cacheCfg.setCacheMode(PARTITIONED);
cacheCfg.setBackups(1);
cacheCfg.setDistributionMode(nearOnly ? NEAR_ONLY : NEAR_PARTITIONED);
cacheCfg.setPreloadMode(SYNC);
cacheCfg.setAtomicityMode(TRANSACTIONAL);
cfg.setCacheConfiguration(cacheCfg);
}
GridTcpDiscoverySpi discoSpi = new GridTcpDiscoverySpi();
discoSpi.setIpFinder(IP_FINDER);
cfg.setDiscoverySpi(discoSpi);
return cfg;
}
/** {@inheritDoc} */
@Override
protected GridConfiguration getConfiguration(String gridName) throws Exception {
GridConfiguration cfg = super.getConfiguration(gridName);
GridTcpDiscoverySpi discoSpi = new GridTcpDiscoverySpi();
discoSpi.setIpFinder(IP_FINDER);
cfg.setDiscoverySpi(discoSpi);
GridGgfsConfiguration ggfsCfg = new GridGgfsConfiguration();
ggfsCfg.setDataCacheName("partitioned");
ggfsCfg.setMetaCacheName("replicated");
ggfsCfg.setName("ggfs");
ggfsCfg.setManagementPort(GridGgfsConfiguration.DFLT_MGMT_PORT + cnt);
ggfsCfg.setIpcEndpointConfiguration(
new HashMap<String, String>() {
{
put("type", "shmem");
put("port", String.valueOf(GridIpcSharedMemoryServerEndpoint.DFLT_IPC_PORT + cnt));
}
});
ggfsCfg.setBlockSize(
512 * 1024); // Together with group blocks mapper will yield 64M per node groups.
cfg.setGgfsConfiguration(ggfsCfg);
cfg.setCacheConfiguration(cacheConfiguration());
cfg.setIncludeEventTypes(EVT_TASK_FAILED, EVT_TASK_FINISHED, EVT_JOB_MAPPED);
GridTcpCommunicationSpi commSpi = new GridTcpCommunicationSpi();
commSpi.setSharedMemoryPort(-1);
cfg.setCommunicationSpi(commSpi);
cnt++;
return cfg;
}
/** {@inheritDoc} */
@Override
protected GridConfiguration getConfiguration(String gridName) throws Exception {
GridConfiguration c = super.getConfiguration(gridName);
c.setLocalHost(HOST);
assert c.getClientConnectionConfiguration() == null;
GridClientConnectionConfiguration clientCfg = new GridClientConnectionConfiguration();
clientCfg.setRestTcpPort(REST_TCP_PORT_BASE);
GridSslContextFactory sslCtxFactory = sslContextFactory();
if (sslCtxFactory != null) {
clientCfg.setRestTcpSslEnabled(true);
clientCfg.setRestTcpSslContextFactory(sslCtxFactory);
}
c.setClientConnectionConfiguration(clientCfg);
GridTcpDiscoverySpi disco = new GridTcpDiscoverySpi();
disco.setIpFinder(IP_FINDER);
c.setDiscoverySpi(disco);
TestCommunicationSpi spi = new TestCommunicationSpi();
spi.setLocalPort(GridTestUtils.getNextCommPort(getClass()));
c.setCommunicationSpi(spi);
c.setCacheConfiguration(
cacheConfiguration(null),
cacheConfiguration(PARTITIONED_CACHE_NAME),
cacheConfiguration(REPLICATED_CACHE_NAME),
cacheConfiguration(REPLICATED_ASYNC_CACHE_NAME));
ThreadPoolExecutor exec =
new ThreadPoolExecutor(40, 40, 0, MILLISECONDS, new LinkedBlockingQueue<Runnable>());
exec.prestartAllCoreThreads();
c.setExecutorService(exec);
c.setExecutorServiceShutdown(true);
ThreadPoolExecutor sysExec =
new ThreadPoolExecutor(40, 40, 0, MILLISECONDS, new LinkedBlockingQueue<Runnable>());
sysExec.prestartAllCoreThreads();
c.setSystemExecutorService(sysExec);
c.setSystemExecutorServiceShutdown(true);
return c;
}