/** Basic tests */
public void testBasics() throws IOException {
Properties props = new Properties();
RecordManager recman =
RecordManagerFactory.createRecordManager(TestRecordFile.testFileName, props);
HashBucket bucket = new HashBucket(0);
// add
bucket.addElement("key", "value");
String s = (String) bucket.getValue("key");
assertEquals("value", s);
// replace
bucket.addElement("key", "value2");
s = (String) bucket.getValue("key");
assertEquals("value2", s);
// add
bucket.addElement("key2", "value3");
s = (String) bucket.getValue("key2");
assertEquals("value3", s);
// remove
bucket.removeElement("key2");
s = (String) bucket.getValue("key2");
assertEquals(null, s);
bucket.removeElement("key");
s = (String) bucket.getValue("key");
assertEquals(null, s);
recman.close();
}
public Graph(WeightedBVGraph graph, String[] names) {
org.apache.log4j.Logger logger =
org.apache.log4j.Logger.getLogger("it.unimi.dsi.webgraph.ImmutableGraph");
logger.setLevel(org.apache.log4j.Level.FATAL);
if (names.length != graph.numNodes())
throw new Error("Problem with the list of names for the nodes in the graph.");
try {
File auxFile = File.createTempFile("graph-maps-" + System.currentTimeMillis(), "aux");
auxFile.deleteOnExit();
RecordManager recMan = RecordManagerFactory.createRecordManager(auxFile.getAbsolutePath());
nodes = recMan.hashMap("nodes");
nodesReverse = recMan.hashMap("nodesReverse");
} catch (IOException ex) {
throw new Error(ex);
}
nodes.clear();
nodesReverse.clear();
Constructor[] cons = WeightedArc.class.getDeclaredConstructors();
for (int i = 0; i < cons.length; i++) cons[i].setAccessible(true);
this.graph = graph;
WeightedArcSet list = new WeightedArcSet();
ArcLabelledNodeIterator it = graph.nodeIterator();
while (it.hasNext()) {
if (commit++ % COMMIT_SIZE == 0) {
commit();
list.commit();
}
Integer aux1 = it.nextInt();
Integer aux2 = null;
ArcLabelledNodeIterator.LabelledArcIterator suc = it.successors();
while ((aux2 = suc.nextInt()) != null && aux2 >= 0 && (aux2 < graph.numNodes()))
try {
WeightedArc arc = (WeightedArc) cons[0].newInstance(aux2, aux1, suc.label().getFloat());
list.add(arc);
this.nodes.put(aux1, names[aux1]);
this.nodes.put(aux2, names[aux2]);
this.nodesReverse.put(names[aux1], aux1);
this.nodesReverse.put(names[aux2], aux2);
} catch (Exception ex) {
throw new Error(ex);
}
}
reverse = new WeightedBVGraph(list.toArray(new WeightedArc[0]));
numArcs = list.size();
iterator = nodeIterator();
try {
File auxFile = File.createTempFile("graph" + System.currentTimeMillis(), "aux");
auxFile.deleteOnExit();
String basename = auxFile.getAbsolutePath();
store(basename);
} catch (IOException ex) {
throw new Error(ex);
}
commit();
}
public DataStruc(RecordManager recman, String objectname) throws IOException {
// Constructor, get the hashtable from db, build a new one if it does not exist
long recid = recman.getNamedObject(objectname);
// System.out.println(objectname + recid);
if (recid != 0) {
hashtable = HTree.load(recman, recid);
} else {
hashtable = HTree.createInstance(recman);
recman.setNamedObject(objectname, hashtable.getRecid());
}
}
/** Stop the stage. */
public void shutdown() {
// Commit and close the database
if (recman != null) {
try {
recman.commit();
recman.close();
} catch (Exception ex) {
logger.warn("Unable to commit and close the database.");
}
}
// Set stop so the isDown method will return the correct value.
stop = true;
}
/**
* Update the indexes for this object. This stage tracks the following data:
*
* <ul>
* <li>Processing date (today)
* <li>PatientID
* <li>StudyInstanceUID
* <li>SeriesInstanceUID
* <li>SOPInstanceUID
* </ul>
*
* It creates table entries for the values in this object. IDs which are not unique may be
* overwritten by subsequent objects (e.g. duplicates). Thus, the tables contain only records of
* unique objects that have been processed.
*
* @param fileObject the object to process.
* @return the same FileObject if the result is true; otherwise null.
*/
public FileObject process(FileObject fileObject) {
lastFileIn = new File(fileObject.getFile().getAbsolutePath());
lastTimeIn = System.currentTimeMillis();
try {
if (fileObject instanceof DicomObject) {
DicomObject dob = (DicomObject) fileObject;
String date = StringUtil.getDate("");
String patientID = dob.getPatientID();
String studyInstanceUID = dob.getStudyInstanceUID();
String seriesInstanceUID = dob.getSeriesInstanceUID();
String sopInstanceUID = dob.getSOPInstanceUID();
index(dateIndex, date, patientID);
index(patientIndex, patientID, studyInstanceUID);
index(studyIndex, studyInstanceUID, seriesInstanceUID);
index(seriesIndex, seriesInstanceUID, sopInstanceUID);
recman.commit();
}
} catch (Exception skip) {
logger.debug("Unable to process " + fileObject.getFile());
}
lastFileOut = new File(fileObject.getFile().getAbsolutePath());
lastTimeOut = System.currentTimeMillis();
return fileObject;
}
/** Update the lookup table and the database. */
public synchronized void update(Document doc) {
LookupTable lut = LookupTable.getInstance(lutFile);
Properties props = lut.getProperties();
Element root = doc.getDocumentElement();
boolean changed = false;
Node child = root.getFirstChild();
while (child != null) {
if (child instanceof Element) {
Element term = (Element) child;
String key = term.getAttribute("key");
String value = term.getAttribute("value");
props.setProperty(key, value);
try {
index.remove(key);
} catch (Exception ignore) {
}
changed = true;
}
child = child.getNextSibling();
}
if (changed) {
try {
recman.commit();
} catch (Exception ignore) {
}
lut.save();
}
}
/**
* Check a DicomObject and record any failing lookups in the database.
*
* @param fileObject the object to process.
* @return the same FileObject if the result is true; otherwise null.
*/
public FileObject process(FileObject fileObject) {
String cmd;
lastFileIn = new File(fileObject.getFile().getAbsolutePath());
lastTimeIn = System.currentTimeMillis();
if (fileObject instanceof DicomObject) {
DicomObject dob = (DicomObject) fileObject;
if (dcmScriptFile != null) {
DAScript daScript = DAScript.getInstance(dcmScriptFile);
scriptTable = new ScriptTable(daScript);
lutProps = LookupTable.getProperties(lutFile);
synchronized (this) {
Dataset ds = dob.getDataset();
charset = ds.getSpecificCharacterSet();
boolean ok = checkDataset(ds);
if (!ok) {
try {
recman.commit();
} catch (Exception unable) {
}
;
if (quarantine != null) quarantine.insert(fileObject);
return null;
}
}
}
}
lastFileOut = new File(fileObject.getFile().getAbsolutePath());
lastTimeOut = System.currentTimeMillis();
return fileObject;
}
/**
* Create a persistent hashtable.
*
* @param recman Record manager used for persistence.
*/
public static <K, V> HTree<K, V> createInstance(
RecordManager recman, Serializer<K> keySerializer, Serializer<V> valueSerializer)
throws IOException {
HashDirectory<K, V> root;
long recid;
HTree<K, V> tree = new HTree<K, V>();
tree.keySerializer = keySerializer;
tree.valueSerializer = valueSerializer;
tree._root = new HashDirectory<K, V>(tree, (byte) 0);
recid = recman.insert(tree._root, tree.SERIALIZER);
tree._root.setPersistenceContext(recman, recid);
return tree;
}
/**
* Load a persistent hashtable
*
* @param recman RecordManager used to store the persistent hashtable
* @param root_recid Record id of the root directory of the HTree
*/
public static <K, V> HTree<K, V> load(
RecordManager recman,
long root_recid,
Serializer<K> keySerializer,
Serializer<V> valueSerializer)
throws IOException {
HashDirectory<K, V> root;
HTree<K, V> tree = new HTree<K, V>();
tree.keySerializer = keySerializer;
tree.valueSerializer = valueSerializer;
tree._root = (HashDirectory<K, V>) recman.fetch(root_recid, tree.SERIALIZER);
tree._root.setPersistenceContext(recman, root_recid);
return tree;
}
public Graph neighbourhoodGraph(int nnodes[], int hops) {
PrimaryHashMap<Integer, String> nodes;
PrimaryHashMap<String, Integer> nodesReverse;
try {
File auxFile = File.createTempFile("graph-maps-" + System.currentTimeMillis(), "aux");
auxFile.deleteOnExit();
RecordManager recMan = RecordManagerFactory.createRecordManager(auxFile.getAbsolutePath());
nodes = recMan.hashMap("nodes");
nodesReverse = recMan.hashMap("nodesReverse");
} catch (IOException ex) {
throw new Error(ex);
}
nodes.clear();
nodesReverse.clear();
WeightedArcSet list1 = new WeightedArcSet();
Int2IntAVLTreeMap map = new Int2IntAVLTreeMap();
IntSet set = new IntLinkedOpenHashSet();
int numIterators = 100;
Constructor[] cons = WeightedArc.class.getDeclaredConstructors();
for (int i = 0; i < cons.length; i++) cons[i].setAccessible(true);
for (int n : nnodes) map.put(n, 0);
NodeIterator its[] = new NodeIterator[numIterators];
int itNum[] = new int[numIterators];
for (int n = 0; n < its.length; n++) {
its[n] = nodeIterator();
itNum[n] = 0;
}
while (map.size() != 0) {
Integer node = 0;
for (int n = 0; n < its.length; n++) if (itNum[n] <= node) node = itNum[n];
node = map.tailMap(node).firstKey();
if (node == null) map.firstKey();
NodeIterator it = null;
Integer aux1 = 0;
int iit = 0;
for (int n = 0; n < its.length; n++) {
if (!its[n].hasNext()) {
its[n] = nodeIterator();
itNum[n] = 0;
}
if (itNum[n] == node) {
it = its[n];
aux1 = itNum[n];
iit = 0;
break;
}
if (itNum[n] < node && itNum[n] >= aux1) {
it = its[n];
aux1 = itNum[n];
iit = n;
}
}
if (it == null) {
its[0] = nodeIterator();
itNum[0] = 0;
it = its[0];
}
while (it != null && (aux1 = it.nextInt()) != null && aux1 >= 0 && aux1 < node) {}
itNum[iit] = aux1 + 1;
Integer aux2 = null;
ArcLabelledNodeIterator.LabelledArcIterator suc = it.successors();
while ((aux2 = suc.nextInt()) != null && aux2 >= 0 && (aux2 < graph.numNodes()))
try {
if (commit++ % COMMIT_SIZE == 0) {
try {
nodes.getRecordManager().commit();
} catch (IOException e) {
throw new Error(e);
}
try {
nodesReverse.getRecordManager().commit();
} catch (IOException e) {
throw new Error(e);
}
}
if (!nodesReverse.containsKey(this.nodes.get(aux1))) {
nodes.put(nodes.size(), this.nodes.get(aux1));
nodesReverse.put(this.nodes.get(aux1), nodesReverse.size());
}
if (!nodesReverse.containsKey(this.nodes.get(aux2))) {
nodes.put(nodes.size(), this.nodes.get(aux2));
nodesReverse.put(this.nodes.get(aux2), nodesReverse.size());
}
int aaux1 = nodesReverse.get(this.nodes.get(aux1));
int aaux2 = nodesReverse.get(this.nodes.get(aux2));
WeightedArc arc1 =
(WeightedArc) cons[0].newInstance(aaux1, aaux2, suc.label().getFloat());
list1.add(arc1);
if (map.get(node) < hops) {
if (!set.contains(aux1) && (map.get(aux1) == null || map.get(aux1) > map.get(node) + 1))
map.put(aux1.intValue(), map.get(node) + 1);
if (!set.contains(aux2) && (map.get(aux2) == null || map.get(aux2) > map.get(node) + 1))
map.put(aux2.intValue(), map.get(node) + 1);
}
} catch (Exception ex) {
ex.printStackTrace();
throw new Error(ex);
}
ArcLabelledNodeIterator.LabelledArcIterator anc = it.ancestors();
while ((aux2 = anc.nextInt()) != null && aux2 >= 0 && (aux2 < graph.numNodes()))
try {
if (commit++ % COMMIT_SIZE == 0) {
try {
nodes.getRecordManager().commit();
} catch (IOException e) {
throw new Error(e);
}
try {
nodesReverse.getRecordManager().commit();
} catch (IOException e) {
throw new Error(e);
}
}
if (!nodesReverse.containsKey(this.nodes.get(aux1))) {
nodes.put(nodes.size(), this.nodes.get(aux1));
nodesReverse.put(this.nodes.get(aux1), nodesReverse.size());
}
if (!nodesReverse.containsKey(this.nodes.get(aux2))) {
nodes.put(nodes.size(), this.nodes.get(aux2));
nodesReverse.put(this.nodes.get(aux2), nodesReverse.size());
}
int aaux1 = nodesReverse.get(this.nodes.get(aux1));
int aaux2 = nodesReverse.get(this.nodes.get(aux2));
WeightedArc arc1 =
(WeightedArc) cons[0].newInstance(aaux2, aaux1, anc.label().getFloat());
list1.add(arc1);
if (map.get(node) < hops) {
if (!set.contains(aux1) && (map.get(aux1) == null || map.get(aux1) > map.get(node) + 1))
map.put(aux1.intValue(), map.get(node) + 1);
if (!set.contains(aux2) && (map.get(aux2) == null || map.get(aux2) > map.get(node) + 1))
map.put(aux2.intValue(), map.get(node) + 1);
}
} catch (Exception ex) {
ex.printStackTrace();
throw new Error(ex);
}
map.remove(node);
set.add(node);
}
Graph newGraph = new Graph(list1.toArray(new WeightedArc[0]));
newGraph.nodes.clear();
newGraph.nodesReverse.clear();
newGraph.nodes = nodes;
newGraph.nodesReverse = nodesReverse;
return newGraph;
}
public Graph(String file) throws IOException {
org.apache.log4j.Logger logger =
org.apache.log4j.Logger.getLogger("it.unimi.dsi.webgraph.ImmutableGraph");
logger.setLevel(org.apache.log4j.Level.FATAL);
try {
File auxFile = File.createTempFile("graph-maps-" + System.currentTimeMillis(), "aux");
auxFile.deleteOnExit();
RecordManager recMan = RecordManagerFactory.createRecordManager(auxFile.getAbsolutePath());
nodes = recMan.hashMap("nodes");
nodesReverse = recMan.hashMap("nodesReverse");
} catch (IOException ex) {
throw new Error(ex);
}
nodes.clear();
nodesReverse.clear();
Constructor[] cons = WeightedArc.class.getDeclaredConstructors();
for (int i = 0; i < cons.length; i++) cons[i].setAccessible(true);
String aux = null;
Float weight = (float) 1.0;
WeightedArcSet list = new WeightedArcSet();
BufferedReader br;
try {
br =
new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(file))));
} catch (Exception ex) {
br = new BufferedReader(new FileReader(file));
}
while ((aux = br.readLine()) != null)
try {
if (commit++ % COMMIT_SIZE == 0) {
commit();
list.commit();
}
String parts[] = aux.split("\t");
String l1 = new String(parts[0]);
String l2 = new String(parts[1]);
if (!nodesReverse.containsKey(l1)) {
nodesReverse.put(l1, nodesReverse.size());
nodes.put(nodes.size(), l1);
}
if (!nodesReverse.containsKey(l2)) {
nodesReverse.put(l2, nodesReverse.size());
nodes.put(nodes.size(), l2);
}
if (parts.length == 3) weight = new Float(parts[2]);
list.add(
(WeightedArc) cons[0].newInstance(nodesReverse.get(l1), nodesReverse.get(l2), weight));
} catch (Exception ex) {
throw new Error(ex);
}
this.graph = new WeightedBVGraph(list.toArray(new WeightedArc[0]));
br.close();
list = new WeightedArcSet();
br = new BufferedReader(new FileReader(file));
while ((aux = br.readLine()) != null)
try {
if (commit++ % COMMIT_SIZE == 0) {
commit();
list.commit();
}
String parts[] = aux.split("\t");
String l1 = new String(parts[0]);
String l2 = new String(parts[1]);
if (parts.length == 3) weight = new Float(parts[2]);
list.add(
(WeightedArc) cons[0].newInstance(nodesReverse.get(l2), nodesReverse.get(l1), weight));
} catch (Exception ex) {
throw new Error(ex);
}
br.close();
this.reverse = new WeightedBVGraph(list.toArray(new WeightedArc[0]));
numArcs = list.size();
iterator = nodeIterator();
try {
File auxFile = File.createTempFile("graph" + System.currentTimeMillis(), "aux");
auxFile.deleteOnExit();
String basename = auxFile.getAbsolutePath();
store(basename);
} catch (IOException ex) {
throw new Error(ex);
}
commit();
}
public Graph(BVGraph graph) {
org.apache.log4j.Logger logger =
org.apache.log4j.Logger.getLogger("it.unimi.dsi.webgraph.ImmutableGraph");
logger.setLevel(org.apache.log4j.Level.FATAL);
try {
File auxFile = File.createTempFile("graph-maps-" + System.currentTimeMillis(), "aux");
auxFile.deleteOnExit();
RecordManager recMan = RecordManagerFactory.createRecordManager(auxFile.getAbsolutePath());
nodes = recMan.hashMap("nodes");
nodesReverse = recMan.hashMap("nodesReverse");
} catch (IOException ex) {
throw new Error(ex);
}
nodes.clear();
nodesReverse.clear();
Constructor[] cons = WeightedArc.class.getDeclaredConstructors();
for (int i = 0; i < cons.length; i++) cons[i].setAccessible(true);
Integer aux1 = null;
WeightedArcSet list = new WeightedArcSet();
it.unimi.dsi.webgraph.NodeIterator it = graph.nodeIterator();
while ((aux1 = it.nextInt()) != null) {
LazyIntIterator suc = it.successors();
Integer aux2 = null;
while ((aux2 = suc.nextInt()) != null && aux2 >= 0 && (aux2 < graph.numNodes()))
try {
if (commit++ % COMMIT_SIZE == 0) {
list.commit();
}
list.add((WeightedArc) cons[0].newInstance(aux1, aux2, (float) 1.0));
} catch (Exception ex) {
throw new Error(ex);
}
}
this.graph = new WeightedBVGraph(list.toArray(new WeightedArc[0]));
list = new WeightedArcSet();
it = graph.nodeIterator();
while ((aux1 = it.nextInt()) != null) {
LazyIntIterator suc = it.successors();
Integer aux2 = null;
while ((aux2 = suc.nextInt()) != null && aux2 >= 0 && (aux2 < graph.numNodes()))
try {
if (commit++ % COMMIT_SIZE == 0) {
commit();
list.commit();
}
list.add((WeightedArc) cons[0].newInstance(aux2, aux1, (float) 1.0));
this.nodes.put(aux1, "" + aux1);
this.nodes.put(aux2, "" + aux2);
this.nodesReverse.put("" + aux1, aux1);
this.nodesReverse.put("" + aux2, aux2);
} catch (Exception ex) {
throw new Error(ex);
}
}
this.reverse = new WeightedBVGraph(list.toArray(new WeightedArc[0]));
numArcs = list.size();
iterator = nodeIterator();
try {
File auxFile = File.createTempFile("graph" + System.currentTimeMillis(), "aux");
auxFile.deleteOnExit();
String basename = auxFile.getAbsolutePath();
store(basename);
} catch (IOException ex) {
throw new Error(ex);
}
commit();
}