// FIXME: fails! needs MarcCombiningReader for mhld or at least a diff version of RawRecordReader
@Test
public void testMultMHLDsWithSameID() throws IOException {
// bib134, multMhlds1
String bibFilePath = testDataParentPath + File.separator + "mhldMergeBibs134.mrc";
String mhldFilePath = testDataParentPath + File.separator + "mhldMergeMhlds1Mult.mrc";
Map<String, Record> mergedRecs =
MergeSummaryHoldings.mergeMhldsIntoBibRecordsAsMap(bibFilePath, mhldFilePath);
Record mergedRec = mergedRecs.get("a1");
assertEquals("Expected three 852", 3, mergedRec.getVariableFields("852").size());
Set<String> expectedVals = new HashSet<String>();
expectedVals.add("Location1");
expectedVals.add("Location2");
RecordTestingUtils.assertSubfieldHasExpectedValues(mergedRec, "852", 'b', expectedVals);
expectedVals.clear();
expectedVals.add("(month)");
expectedVals.add("(season)");
RecordTestingUtils.assertSubfieldHasExpectedValues(mergedRec, "853", 'b', expectedVals);
assertEquals("Expected one 863", 2, mergedRec.getVariableFields("863").size());
assertEquals("Expected one 866", 1, mergedRec.getVariableFields("866").size());
// fail("Implement me");
System.out.println("Test testMultMHLDsWithSameID() successful");
}
@Test
public void testCloseable() throws IOException {
Set<HostAndPort> jedisClusterNode = new HashSet<HostAndPort>();
jedisClusterNode.add(new HostAndPort(nodeInfo1.getHost(), nodeInfo1.getPort()));
JedisCluster jc = null;
try {
jc = new JedisCluster(jedisClusterNode);
jc.set("51", "foo");
} finally {
if (jc != null) {
jc.close();
}
}
Iterator<JedisPool> poolIterator = jc.getClusterNodes().values().iterator();
while (poolIterator.hasNext()) {
JedisPool pool = poolIterator.next();
try {
pool.getResource();
fail("JedisCluster's internal pools should be already destroyed");
} catch (JedisConnectionException e) {
// ok to go...
}
}
}
@Test
public void testJedisClusterRunsWithMultithreaded()
throws InterruptedException, ExecutionException, IOException {
Set<HostAndPort> jedisClusterNode = new HashSet<HostAndPort>();
jedisClusterNode.add(new HostAndPort("127.0.0.1", 7379));
final JedisCluster jc = new JedisCluster(jedisClusterNode);
jc.set("foo", "bar");
ThreadPoolExecutor executor =
new ThreadPoolExecutor(10, 100, 0, TimeUnit.SECONDS, new ArrayBlockingQueue<Runnable>(10));
List<Future<String>> futures = new ArrayList<Future<String>>();
for (int i = 0; i < 50; i++) {
executor.submit(
new Callable<String>() {
@Override
public String call() throws Exception {
// FIXME : invalidate slot cache from JedisCluster to test
// random connection also does work
return jc.get("foo");
}
});
}
for (Future<String> future : futures) {
String value = future.get();
assertEquals("bar", value);
}
jc.close();
}
@Test
public void scanTest() {
Map<String, Map<String, ByteIterator>> keyMap = new HashMap<>();
for (int i = 0; i < 5; i++) {
String insertKey = KEY_PREFIX + i;
keyMap.put(insertKey, insertRow(insertKey));
}
Set<String> fieldSet = new HashSet<>();
fieldSet.add("FIELD0");
fieldSet.add("FIELD1");
int startIndex = 1;
int resultRows = 3;
Vector<HashMap<String, ByteIterator>> resultVector = new Vector<>();
orientDBClient.scan(CLASS, KEY_PREFIX + startIndex, resultRows, fieldSet, resultVector);
// Check the resultVector is the correct size
assertEquals(
"Assert the correct number of results rows were returned", resultRows, resultVector.size());
// Check each vector row to make sure we have the correct fields
int testIndex = startIndex;
for (HashMap<String, ByteIterator> result : resultVector) {
assertEquals(
"Assert that this row has the correct number of fields", fieldSet.size(), result.size());
for (String field : fieldSet) {
assertEquals(
"Assert this field is correct in this row",
keyMap.get(KEY_PREFIX + testIndex).get(field).toString(),
result.get(field).toString());
}
testIndex++;
}
}
private List<String> executeDDL(
String ddlFileName,
@Nullable String dataFileName,
boolean isLocalTable,
@Nullable String[] args)
throws Exception {
Path ddlFilePath = new Path(currentQueryPath, ddlFileName);
FileSystem fs = ddlFilePath.getFileSystem(conf);
assertTrue(ddlFilePath + " existence check", fs.exists(ddlFilePath));
String template = FileUtil.readTextFile(new File(ddlFilePath.toUri()));
String dataFilePath = null;
if (dataFileName != null) {
dataFilePath = getDataSetFile(dataFileName).toString();
}
String compiled = compileTemplate(template, dataFilePath, args);
List<ParsedResult> parsedResults = SimpleParser.parseScript(compiled);
List<String> createdTableNames = new ArrayList<String>();
for (ParsedResult parsedResult : parsedResults) {
// parse a statement
Expr expr = sqlParser.parse(parsedResult.getStatement());
assertNotNull(ddlFilePath + " cannot be parsed", expr);
if (expr.getType() == OpType.CreateTable) {
CreateTable createTable = (CreateTable) expr;
String tableName = createTable.getTableName();
assertTrue("Table creation is failed.", client.updateQuery(parsedResult.getStatement()));
TableDesc createdTable = client.getTableDesc(tableName);
String createdTableName = createdTable.getName();
assertTrue(
"table '" + createdTableName + "' creation check", client.existTable(createdTableName));
if (isLocalTable) {
createdTableGlobalSet.add(createdTableName);
createdTableNames.add(tableName);
}
} else if (expr.getType() == OpType.DropTable) {
DropTable dropTable = (DropTable) expr;
String tableName = dropTable.getTableName();
assertTrue(
"table '" + tableName + "' existence check",
client.existTable(CatalogUtil.buildFQName(currentDatabase, tableName)));
assertTrue("table drop is failed.", client.updateQuery(parsedResult.getStatement()));
assertFalse(
"table '" + tableName + "' dropped check",
client.existTable(CatalogUtil.buildFQName(currentDatabase, tableName)));
if (isLocalTable) {
createdTableGlobalSet.remove(tableName);
}
} else {
assertTrue(ddlFilePath + " is not a Create or Drop Table statement", false);
}
}
return createdTableNames;
}
/**
* these are the masses XTandem has
*
* @return
*/
public static Set<Integer> buildMassSet() {
Set<Integer> holder = new HashSet<Integer>();
for (int i = 0; i < peaks.length; i += 2) {
holder.add(asInt(peaks[i]));
}
return holder;
}
@Test
public void testHash() {
Set<Integer> set = new HashSet<Integer>();
for (WordNgram w : myNgrams) {
set.add(w.hashCode());
}
assertTrue("hash code test", set.size() > 8);
}
@Test(expected = JedisClusterMaxRedirectionsException.class)
public void testRedisClusterMaxRedirections() {
Set<HostAndPort> jedisClusterNode = new HashSet<HostAndPort>();
jedisClusterNode.add(new HostAndPort("127.0.0.1", 7379));
JedisCluster jc = new JedisCluster(jedisClusterNode);
int slot51 = JedisClusterCRC16.getSlot("51");
// This will cause an infinite redirection loop
node2.clusterSetSlotMigrating(slot51, JedisClusterTestUtil.getNodeId(node3.clusterNodes()));
jc.set("51", "foo");
}
@Test
public void testDiscoverNodesAutomatically() {
Set<HostAndPort> jedisClusterNode = new HashSet<HostAndPort>();
jedisClusterNode.add(new HostAndPort("127.0.0.1", 7379));
JedisCluster jc = new JedisCluster(jedisClusterNode);
assertEquals(3, jc.getClusterNodes().size());
JedisCluster jc2 = new JedisCluster(new HostAndPort("127.0.0.1", 7379));
assertEquals(3, jc2.getClusterNodes().size());
}
@Test(expected = JedisConnectionException.class)
public void testIfPoolConfigAppliesToClusterPools() {
GenericObjectPoolConfig config = new GenericObjectPoolConfig();
config.setMaxTotal(0);
config.setMaxWaitMillis(2000);
Set<HostAndPort> jedisClusterNode = new HashSet<HostAndPort>();
jedisClusterNode.add(new HostAndPort("127.0.0.1", 7379));
JedisCluster jc = new JedisCluster(jedisClusterNode, config);
jc.set("52", "poolTestValue");
}
@Test
public void testAskResponse() throws InterruptedException {
Set<HostAndPort> jedisClusterNode = new HashSet<HostAndPort>();
jedisClusterNode.add(new HostAndPort("127.0.0.1", 7379));
JedisCluster jc = new JedisCluster(jedisClusterNode);
int slot51 = JedisClusterCRC16.getSlot("51");
node3.clusterSetSlotImporting(slot51, JedisClusterTestUtil.getNodeId(node2.clusterNodes()));
node2.clusterSetSlotMigrating(slot51, JedisClusterTestUtil.getNodeId(node3.clusterNodes()));
jc.set("51", "foo");
assertEquals("foo", jc.get("51"));
}
@Test(expected = JedisClusterMaxRedirectionsException.class, timeout = 2000)
public void testReturnConnectionOnRedirection() {
Set<HostAndPort> jedisClusterNode = new HashSet<HostAndPort>();
jedisClusterNode.add(new HostAndPort("127.0.0.1", 7379));
JedisPoolConfig config = new JedisPoolConfig();
config.setMaxTotal(1);
JedisCluster jc = new JedisCluster(jedisClusterNode, 0, 2, config);
// This will cause an infinite redirection between node 2 and 3
node3.clusterSetSlotMigrating(15363, JedisClusterTestUtil.getNodeId(node2.clusterNodes()));
jc.get("e");
}
@Test
public void testMigrateToNewNode() throws InterruptedException {
log.info("test migrate slot to new node");
Set<HostAndPort> jedisClusterNode = new HashSet<HostAndPort>();
jedisClusterNode.add(nodeInfo1);
JedisCluster jc = new JedisCluster(jedisClusterNode);
node4.clusterMeet(localHost, nodeInfo1.getPort());
String node3Id = JedisClusterTestUtil.getNodeId(node3.clusterNodes());
String node4Id = JedisClusterTestUtil.getNodeId(node4.clusterNodes());
JedisClusterTestUtil.waitForClusterReady(node4);
node3.clusterSetSlotMigrating(15363, node4Id);
node4.clusterSetSlotImporting(15363, node3Id);
try {
node4.set("e", "e");
} catch (JedisMovedDataException jme) {
assertEquals(15363, jme.getSlot());
assertEquals(new HostAndPort(localHost, nodeInfo3.getPort()), jme.getTargetNode());
}
try {
node3.set("e", "e");
} catch (JedisAskDataException jae) {
assertEquals(15363, jae.getSlot());
assertEquals(new HostAndPort(localHost, nodeInfo4.getPort()), jae.getTargetNode());
}
jc.set("e", "e");
try {
node4.get("e");
} catch (JedisMovedDataException jme) {
assertEquals(15363, jme.getSlot());
assertEquals(new HostAndPort(localHost, nodeInfo3.getPort()), jme.getTargetNode());
}
try {
node3.get("e");
} catch (JedisAskDataException jae) {
assertEquals(15363, jae.getSlot());
assertEquals(new HostAndPort(localHost, nodeInfo4.getPort()), jae.getTargetNode());
}
assertEquals("e", jc.get("e"));
node4.clusterSetSlotNode(15363, node4Id);
node3.clusterSetSlotNode(15363, node4Id);
// assertEquals("e", jc.get("e"));
assertEquals("e", node4.get("e"));
// assertEquals("e", node3.get("e"));
}
@Test
public void testTopologicalSort() {
final Set<Object> nodes = new HashSet<Object>();
final Object centerNode = "centerNode";
nodes.add(centerNode);
final Object topNode = "topNode";
nodes.add(topNode);
final Object leftNode = "leftNode";
nodes.add(leftNode);
final Object leftiestNode = "leftiestNode";
nodes.add(leftiestNode);
final Object rightNode = "rightNode";
nodes.add(rightNode);
final Set<DirectedEdge<Object>> edges = new HashSet<DirectedEdge<Object>>();
final DirectedEdge<Object> centerTopEdge =
new ImmutableDirectedEdge<Object>(centerNode, topNode);
edges.add(centerTopEdge);
final DirectedEdge<Object> centerLeftEdge =
new ImmutableDirectedEdge<Object>(centerNode, leftNode);
edges.add(centerLeftEdge);
final DirectedEdge<Object> leftLeftiestEdge =
new ImmutableDirectedEdge<Object>(leftNode, leftiestNode);
edges.add(leftLeftiestEdge);
final DirectedEdge<Object> centerRightEdge =
new ImmutableDirectedEdge<Object>(centerNode, rightNode);
edges.add(centerRightEdge);
final BidirectedGraph<Object, DirectedEdge<Object>> graph =
new ImmutableDirectedAdjacencyGraph<Object, DirectedEdge<Object>>(nodes, edges);
final TopologicalSorter<Object> sorter = new SimpleTopologicalRanker<Object>();
final List<Object> sortedNodes = sorter.sort(graph);
Assert.assertTrue("center node is not the first node!", sortedNodes.get(0) == centerNode);
Assert.assertTrue(
"left node is not before leftiest node!",
sortedNodes.indexOf(leftNode) < sortedNodes.indexOf(leftiestNode));
}
public static Set<BigInteger> createPrimesSet(int max) {
Set<BigInteger> primes = new HashSet<BigInteger>();
BigInteger n = new BigInteger("2");
BigInteger maxValue = new BigInteger(String.valueOf(max));
while (n.compareTo(maxValue) < 0) {
primes.add(n);
n = n.nextProbablePrime();
}
System.out.println("Primes under " + max + " generated.");
return primes;
}
@Test
public void testRemoveService() {
ServiceHolder tested = new ServiceHolder();
Object service = new Object();
// Get the hash set.
Set<Object> servicesSet = getField(tested, Set.class);
servicesSet.add(service);
tested.removeService(service);
assertTrue("Set should be empty after removeal.", servicesSet.isEmpty());
}
@Test
public void testClusterCountKeysInSlot() {
Set<HostAndPort> jedisClusterNode = new HashSet<HostAndPort>();
jedisClusterNode.add(new HostAndPort(nodeInfo1.getHost(), nodeInfo1.getPort()));
JedisCluster jc = new JedisCluster(jedisClusterNode);
for (int index = 0; index < 5; index++) {
jc.set("foo{bar}" + index, "hello");
}
int slot = JedisClusterCRC16.getSlot("foo{bar}");
assertEquals(5, node1.clusterCountKeysInSlot(slot).intValue());
}
@Test
public void testJedisClusterTimeout() {
Set<HostAndPort> jedisClusterNode = new HashSet<HostAndPort>();
jedisClusterNode.add(new HostAndPort(nodeInfo1.getHost(), nodeInfo1.getPort()));
JedisCluster jc = new JedisCluster(jedisClusterNode, 4000);
for (JedisPool pool : jc.getClusterNodes().values()) {
Jedis jedis = pool.getResource();
assertEquals(jedis.getClient().getConnectionTimeout(), 4000);
assertEquals(jedis.getClient().getSoTimeout(), 4000);
jedis.close();
}
}
@Test
public void testRecalculateSlotsWhenMoved() throws InterruptedException {
Set<HostAndPort> jedisClusterNode = new HashSet<HostAndPort>();
jedisClusterNode.add(new HostAndPort("127.0.0.1", 7379));
JedisCluster jc = new JedisCluster(jedisClusterNode);
int slot51 = JedisClusterCRC16.getSlot("51");
node2.clusterDelSlots(slot51);
node3.clusterDelSlots(slot51);
node3.clusterAddSlots(slot51);
JedisClusterTestUtil.waitForClusterReady(node1, node2, node3);
jc.set("51", "foo");
assertEquals("foo", jc.get("51"));
}
/** slot->nodes 15363 node3 e */
@Test
public void testMigrate() {
log.info("test migrate slot");
Set<HostAndPort> jedisClusterNode = new HashSet<HostAndPort>();
jedisClusterNode.add(nodeInfo1);
JedisCluster jc = new JedisCluster(jedisClusterNode);
String node3Id = JedisClusterTestUtil.getNodeId(node3.clusterNodes());
String node2Id = JedisClusterTestUtil.getNodeId(node2.clusterNodes());
node3.clusterSetSlotMigrating(15363, node2Id);
node2.clusterSetSlotImporting(15363, node3Id);
try {
node2.set("e", "e");
} catch (JedisMovedDataException jme) {
assertEquals(15363, jme.getSlot());
assertEquals(new HostAndPort(localHost, nodeInfo3.getPort()), jme.getTargetNode());
}
try {
node3.set("e", "e");
} catch (JedisAskDataException jae) {
assertEquals(15363, jae.getSlot());
assertEquals(new HostAndPort(localHost, nodeInfo2.getPort()), jae.getTargetNode());
}
jc.set("e", "e");
try {
node2.get("e");
} catch (JedisMovedDataException jme) {
assertEquals(15363, jme.getSlot());
assertEquals(new HostAndPort(localHost, nodeInfo3.getPort()), jme.getTargetNode());
}
try {
node3.get("e");
} catch (JedisAskDataException jae) {
assertEquals(15363, jae.getSlot());
assertEquals(new HostAndPort(localHost, nodeInfo2.getPort()), jae.getTargetNode());
}
assertEquals("e", jc.get("e"));
node2.clusterSetSlotNode(15363, node2Id);
node3.clusterSetSlotNode(15363, node2Id);
// assertEquals("e", jc.get("e"));
assertEquals("e", node2.get("e"));
// assertEquals("e", node3.get("e"));
}
@Test(timeout = 2000)
public void testReturnConnectionOnJedisConnectionException() throws InterruptedException {
Set<HostAndPort> jedisClusterNode = new HashSet<HostAndPort>();
jedisClusterNode.add(new HostAndPort("127.0.0.1", 7379));
JedisPoolConfig config = new JedisPoolConfig();
config.setMaxTotal(1);
JedisCluster jc = new JedisCluster(jedisClusterNode, config);
Jedis j = jc.getClusterNodes().get("127.0.0.1:7380").getResource();
ClientKillerUtil.tagClient(j, "DEAD");
ClientKillerUtil.killClient(j, "DEAD");
j.close();
jc.get("test");
}
private void checkValue(
List<BusinessObject.Value> boValues, boolean strict, String name, Object... values) {
Set<Object> expected = CollectionUtils.newSetFromIterator(Arrays.asList(values).iterator());
Set<Object> actual = CollectionUtils.newSet();
for (BusinessObject.Value boValue : boValues) {
Map<?, ?> data = (Map<?, ?>) boValue.getValue();
actual.add(data.get(name));
}
if (strict) {
Assert.assertEquals("Values: ", expected, actual);
} else {
expected.removeAll(actual);
Assert.assertTrue("Missing values: " + expected, expected.isEmpty());
}
}
@Test
public void testCalculateConnectionPerSlot() {
Set<HostAndPort> jedisClusterNode = new HashSet<HostAndPort>();
jedisClusterNode.add(new HostAndPort("127.0.0.1", 7379));
JedisCluster jc = new JedisCluster(jedisClusterNode);
jc.set("foo", "bar");
jc.set("test", "test");
assertEquals("bar", node3.get("foo"));
assertEquals("test", node2.get("test"));
JedisCluster jc2 = new JedisCluster(new HostAndPort("127.0.0.1", 7379));
jc2.set("foo", "bar");
jc2.set("test", "test");
assertEquals("bar", node3.get("foo"));
assertEquals("test", node2.get("test"));
}
@Test
public void testStableSlotWhenMigratingNodeOrImportingNodeIsNotSpecified()
throws InterruptedException {
Set<HostAndPort> jedisClusterNode = new HashSet<HostAndPort>();
jedisClusterNode.add(new HostAndPort(nodeInfo1.getHost(), nodeInfo1.getPort()));
JedisCluster jc = new JedisCluster(jedisClusterNode);
int slot51 = JedisClusterCRC16.getSlot("51");
jc.set("51", "foo");
// node2 is responsible of taking care of slot51 (7186)
node3.clusterSetSlotImporting(slot51, JedisClusterTestUtil.getNodeId(node2.clusterNodes()));
assertEquals("foo", jc.get("51"));
node3.clusterSetSlotStable(slot51);
assertEquals("foo", jc.get("51"));
node2.clusterSetSlotMigrating(slot51, JedisClusterTestUtil.getNodeId(node3.clusterNodes()));
// assertEquals("foo", jc.get("51")); // it leads Max Redirections
node2.clusterSetSlotStable(slot51);
assertEquals("foo", jc.get("51"));
}
private <T> Collection<T> random(Store<T> s, int rows, int sampleSize) {
if (rows > 0) {
Set<T> ret = new HashSet<T>();
Random r = new Random();
for (int i = 0; i < sampleSize; i++) {
try {
int rand = r.nextInt(rows);
if (rand != 0) {
T t;
try {
Method lazy = s.getClass().getDeclaredMethod("lazyGet", Long.TYPE);
t = (T) lazy.invoke(s, new Long(rand));
} catch (NoSuchMethodException e) {
System.out.println("WARN:: Unable to lazily get object. Using full get.");
t = s.get(Integer.valueOf(rand).longValue());
} catch (InvocationTargetException e) {
System.out.println("WARN:: Unable to lazily get object. Using full get.");
t = s.get(Integer.valueOf(rand).longValue());
} catch (IllegalAccessException e) {
System.out.println("WARN:: Unable to lazily get object. Using full get.");
t = s.get(Integer.valueOf(rand).longValue());
}
if (t != null) {
ret.add(t);
}
}
} catch (IOException e) {
System.out.println("ERROR:: could not get random object from store");
}
}
return ret;
} else {
return Collections.emptySet();
}
}
public void setTokenTypeChannel(int ttype, int channel) {
hide.add(ttype);
}
