protected Collection<EnforcedRelation> match( TIntHashSet subset, TIntObjectHashMap<Collection<EnforcedRelation>> index) { TIntIterator keyI = subset.iterator(); int firstKey = keyI.next(); if (index.get(firstKey) == null) { return Collections.emptyList(); } Collection<EnforcedRelation> result = new ArrayList<EnforcedRelation>(index.get(firstKey)); if (result.isEmpty()) { return Collections.emptyList(); } while (keyI.hasNext()) { int key = keyI.next(); if (result.isEmpty()) return result; Collection<EnforcedRelation> rest = index.get(key); if (rest == null) { return Collections.emptyList(); } result.retainAll(rest); } return result; }
public void testIterator() { TIntSet set = new TIntHashSet(); set.add(1); set.add(2); set.add(3); set.add(4); TIntIterator iter = set.iterator(); assertTrue("iterator should have a next item", iter.hasNext()); int last = -1; while (iter.hasNext()) { int next = iter.next(); assertTrue(Integer.valueOf(next).toString(), next >= 1 && next <= 4); assertTrue(Integer.valueOf(next).toString(), next != last); last = next; } assertFalse("iterator should not have a next item", iter.hasNext()); assertTrue("set should contain 1", set.contains(1)); assertTrue("set should contain 2", set.contains(2)); assertTrue("set should contain 3", set.contains(3)); assertTrue("set should contain 4", set.contains(4)); assertEquals(4, set.size()); }
/** {@inheritDoc} */ public boolean containsAll(TIntCollection collection) { TIntIterator iter = collection.iterator(); while (iter.hasNext()) { if (!TIntIntHashMap.this.containsValue(iter.next())) { return false; } } return true; }
// Kien added this mathRolesAnhType1AndType2 public Collection<EnforcedRelation> matchRolesAndType1AndType2( TIntHashSet candidateRoles, TIntHashSet candidateType1, TIntHashSet candidateType2) { TIntIterator roleKeyI = candidateRoles.iterator(); int firstRoleKey = roleKeyI.next(); if (rolesIndex.get(firstRoleKey) == null) { return Collections.emptyList(); } Collection<EnforcedRelation> result = new ArrayList<EnforcedRelation>(rolesIndex.get(firstRoleKey)); if (result.isEmpty()) { return Collections.emptyList(); } while (roleKeyI.hasNext()) { int key = roleKeyI.next(); if (result.isEmpty()) return result; Collection<EnforcedRelation> rest = rolesIndex.get(key); if (rest == null) { return Collections.emptyList(); } result.retainAll(rest); } TIntIterator type1KeyI = candidateType1.iterator(); while (type1KeyI.hasNext()) { int key = type1KeyI.next(); if (result.isEmpty()) return result; Collection<EnforcedRelation> rest = type1Index.get(key); if (rest == null) { return Collections.emptyList(); } result.retainAll(rest); } TIntIterator type2KeyI = candidateType2.iterator(); // Old code: while (roleKeyI.hasNext()) { // Kien changed roleKeyI to type2KeyI while (type2KeyI.hasNext()) { int key = type2KeyI.next(); if (result.isEmpty()) return result; Collection<EnforcedRelation> rest2 = type2Index.get(key); if (rest2 == null) { return Collections.emptyList(); } result.retainAll(rest2); } return result; }
@Override public Set<Integer> getFeatureIndicies() { Set<Integer> out = Sets.newHashSet(); TIntIterator it = w.keySet().iterator(); while (it.hasNext()) { int next = it.next(); out.add(next); } return out; }
@Override public boolean add(EnforcedRelation enf) { if (!this.subsubmedEnf(enf)) { enfs.add(enf); TIntHashSet type1 = enf.getType1(); TIntIterator type1Iterator = type1.iterator(); while (type1Iterator.hasNext()) { int key = type1Iterator.next(); ensureContainsKey(type1Index, key); type1Index.get(key).add(enf); } TIntHashSet type2 = enf.getType2(); TIntIterator type2Iterator = type2.iterator(); while (type2Iterator.hasNext()) { int key = type2Iterator.next(); ensureContainsKey(type2Index, key); type2Index.get(key).add(enf); } TIntHashSet roles = enf.getRoles(); TIntIterator rolesIterator = roles.iterator(); while (rolesIterator.hasNext()) { int key = rolesIterator.next(); ensureContainsKey(rolesIndex, key); rolesIndex.get(key).add(enf); } return true; } else { return false; } }
@Override public boolean remove(Object o) { EnforcedRelation enf = (EnforcedRelation) o; enfs.remove(enf); TIntHashSet type1 = enf.getType1(); TIntIterator type1Iterator = type1.iterator(); while (type1Iterator.hasNext()) { int key = type1Iterator.next(); ensureContainsKey(type1Index, key); type1Index.get(key).remove(enf); } TIntHashSet type2 = enf.getType2(); TIntIterator type2Iterator = type2.iterator(); while (type2Iterator.hasNext()) { int key = type2Iterator.next(); ensureContainsKey(type2Index, key); type2Index.get(key).remove(enf); } TIntHashSet roles = enf.getRoles(); TIntIterator rolesIterator = roles.iterator(); while (rolesIterator.hasNext()) { int key = rolesIterator.next(); ensureContainsKey(rolesIndex, key); rolesIndex.get(key).remove(enf); } return true; }
public boolean containsWord(int word, Mention mention) { if (!indexWithoutStopWords.containsKey(word)) return false; TIntLinkedList positions = indexIncludingStopWords.get(word); int mentionStart = mention.getStartToken(); int mentionEnd = mention.getEndToken(); for (TIntIterator itr = positions.iterator(); itr.hasNext(); ) { int position = itr.next(); if (position < mentionStart || position > mentionEnd) return true; } return false; }
/** {@inheritDoc} */ @SuppressWarnings({"SuspiciousMethodCalls"}) public boolean retainAll(Collection<?> collection) { boolean modified = false; TIntIterator iter = iterator(); while (iter.hasNext()) { if (!collection.contains(Integer.valueOf(iter.next()))) { iter.remove(); modified = true; } } return modified; }
/** * Write top lin and balanced inclusion (cover) scores to a file * * @param writer * @param entailedElementId * @param entailingElements * @param entailedElement2NormMap * @param entailingElement2NormMap */ private void writeEntailedElementScores( PrintWriter writer, Integer entailedElementId, TIntObjectMap<Pair<DoubleContainer, DoubleContainer>> entailingElements, TIntDoubleHashMap entailedElement2NormMap, TIntDoubleHashMap entailingElement2NormMap) { BoundedPriorityQueue<DistSimRule> linScores = new BoundedPriorityQueue<DistSimRule>(new DistSimRuleComparator(), m_maxRulesPerElement); BoundedPriorityQueue<DistSimRule> coverScores = new BoundedPriorityQueue<DistSimRule>(new DistSimRuleComparator(), m_maxRulesPerElement); double entailedElementNorm = entailedElement2NormMap.get(entailedElementId); TIntIterator iter = entailingElements.keySet().iterator(); while (iter.hasNext()) { Integer entailingElementId = iter.next(); // if there are no features for this element - then it is not similar to any other and we can // move on if (entailingElement2NormMap.get(entailingElementId) == 0.0) continue; double entailingElementNorm = entailingElement2NormMap.get(entailingElementId); Pair<DoubleContainer, DoubleContainer> scores = entailingElements.get(entailingElementId); Double linNominator = scores.key().value(); Double coverNominator = scores.value().value(); double linScore = linNominator / (entailedElementNorm + entailingElementNorm); double coverScore = coverNominator / entailingElementNorm; coverScore = Math.sqrt(linScore * coverScore); if (linScore > 0.0) { linScores.offer(new DistSimRule(entailedElementId, entailingElementId, linScore)); } if (coverScore > 0.0) { coverScores.offer(new DistSimRule(entailedElementId, entailingElementId, coverScore)); } } while (!linScores.isEmpty()) { DistSimRule linRule = linScores.poll(); writer.println("LIN\t" + linRule); } while (!coverScores.isEmpty()) { DistSimRule coverRule = coverScores.poll(); writer.println("COVER\t" + coverRule); } // we are done with the left element so we clear rightElements entailingElements.clear(); }
/** Creates a deep copy of a TIntObjectHashMap<TIntHashSet>. */ public static TIntObjectHashMap<TIntHashSet> deepClone(TIntObjectHashMap<TIntHashSet> map) { TIntObjectHashMap<TIntHashSet> clone = new TIntObjectHashMap<TIntHashSet>(); for (TIntIterator keyit = map.keySet().iterator(); keyit.hasNext(); ) { int key = keyit.next(); clone.put(key, new TIntHashSet(map.get(key).capacity())); clone.get(key).addAll(map.get(key)); } return clone; }
/** {@inheritDoc} */ public boolean retainAll(TIntCollection collection) { if (this == collection) { return false; } boolean modified = false; TIntIterator iter = iterator(); while (iter.hasNext()) { if (!collection.contains(iter.next())) { iter.remove(); modified = true; } } return modified; }
/** {@inheritDoc} */ public boolean removeAll(TIntCollection collection) { if (this == collection) { clear(); return true; } boolean changed = false; TIntIterator iter = collection.iterator(); while (iter.hasNext()) { int element = iter.next(); if (remove(element)) { changed = true; } } return changed; }
// bulk load public double loadTable(Table t, String dataFile, String delimiter) { switch (t.type.getId()) { case TABLE: if (t.partitions == null) return execute( "COPY " + t.name + " FROM '" + dataFile + "' DELIMITER '" + delimiter + "'"); else { this.t.reset(); this.t.start(); List<Table> partitionTables = TableUtils.perPartitionTables(t); TableRowIterator itr = new TableRowIterator(t, dataFile, delimiter); int count = 0; while (itr.hasNext()) { TableRow row = itr.next(); for (TIntIterator keyit = t.partitions.keySet().iterator(); keyit.hasNext(); ) { int key = keyit.next(); TableRow partialRow = TableRowUtils.partialTableRow( row, partitionTables.get(key), t.partitions.get(key).toArray()); partialRow.add(count); insertRow(partitionTables.get(key), partialRow); } count++; if (count % 10000 == 0) log("Inserted " + count + " rows"); } close(); this.t.stop(); return this.t.getElapsedTime(); } case CTABLE: this.t.reset(); this.t.start(); TableRowIterator itr = new TableRowIterator(t, dataFile, delimiter); int count = 0; while (itr.hasNext()) { insertRow(t, itr.next()); count++; if (count % 10000 == 0) log("Inserted " + count + " rows"); } this.t.stop(); return this.t.getElapsedTime(); default: throw new UnsupportedOperationException("Unknown table type !"); } }
public void testToArrayMatchesIteratorOrder() { TIntSet set = new TIntHashSet(); int[] ints = {42, 1138, 13, 86, 99}; set.addAll(ints); int[] toarray_ints = set.toArray(); int[] iter_ints = new int[5]; TIntIterator iter = set.iterator(); int index = 0; while (iter.hasNext()) { iter_ints[index++] = iter.next(); } assertTrue(Arrays.equals(iter_ints, toarray_ints)); }
// If there is enf(T1,R,T2) such that R contains Rho, T2 contain type of X // Then substitute all role wich contain X by T1(X) private Set<Atom> addTypeOfXToBody(Term x, TIntHashSet type, Set<Atom> body) { Set<Atom> addedAtoms = new HashSet<Atom>(); TIntIterator iterator = type.iterator(); while (iterator.hasNext()) { int i = iterator.next(); // for (int i = type.nextSetBit(0); i >= 0; i = type.nextSetBit(i + // 1)) { if (i != ClipperManager.getInstance().getThing()) { Predicate predicate = new DLPredicate(i, 1); List<Term> terms = new ArrayList<Term>(); terms.add(x); Atom newAtom = new Atom(predicate, terms); if (!body.contains(newAtom) && !addedAtoms.contains(newAtom)) { addedAtoms.add(newAtom); } } } return addedAtoms; }
private int calculateTotal(int base, TFloatList multipliers, TIntList modifiers) { // For now, add all modifiers and multiply by all multipliers. Negative modifiers cap to zero, // but negative // multipliers remain (so damage can be flipped to healing) float total = base; TIntIterator modifierIter = modifiers.iterator(); while (modifierIter.hasNext()) { total += modifierIter.next(); } total = Math.max(0, total); if (total == 0) { return 0; } TFloatIterator multiplierIter = multipliers.iterator(); while (multiplierIter.hasNext()) { total *= multiplierIter.next(); } return TeraMath.floorToInt(total); }
// returns number of docs public int readCorpusDat(String inFile, boolean updatePWord) throws Exception { BufferedReader brIn = new BufferedReader(new InputStreamReader(new FileInputStream(inFile), "UTF8")); String sLine; int i = 0; long toks = 0L; ArrayList<TIntArrayList> aldocs = new ArrayList<TIntArrayList>(); if (updatePWord) _pWord = new double[_VOCABSIZE]; int ct = 0; while ((sLine = brIn.readLine()) != null) { TIntArrayList ll = lineToList(sLine); aldocs.add(ll); if (updatePWord) { TIntIterator it = ll.iterator(); while (it.hasNext()) { _pWord[it.next()]++; } } toks += (long) ll.size(); ct++; } brIn.close(); System.out.println("Tokens: " + toks); System.out.println("Lines processed: " + ct + " Lines saved: " + aldocs.size()); _NUMTOKENS = toks; double dubToks = (double) toks; if (updatePWord) for (int j = 0; j < _pWord.length; j++) if (_pWord[j] > 0.0) _pWord[j] /= dubToks; _NUMDOCS = aldocs.size(); _docs = aldocs.toArray(new TIntArrayList[aldocs.size()]); _changeFactor = new TDoubleArrayList[_docs.length][_NUMLAYERS]; for (int lay = 0; lay < _NUMLAYERS; lay++) { for (int j = 0; j < _docs.length; j++) { _changeFactor[j][lay] = new TDoubleArrayList(_docs[j].size()); for (int k = 0; k < _docs[j].size(); k++) { _changeFactor[j][lay].add(1.0); // start with maximal change } } } return i; }
private void sendEntities(NetData.NetMessage.Builder message) { TIntIterator dirtyIterator = netDirty.iterator(); while (dirtyIterator.hasNext()) { int netId = dirtyIterator.next(); EntityRef entity = networkSystem.getEntity(netId); if (isOwned(entity)) { Set<Class<? extends Component>> emptyComponentClassSet = Collections.emptySet(); EntityData.PackedEntity entityData = entitySerializer.serialize( entity, emptyComponentClassSet, changedComponents.get(netId), emptyComponentClassSet, new ClientComponentFieldCheck()); if (entityData != null) { message.addUpdateEntity( NetData.UpdateEntityMessage.newBuilder().setEntity(entityData).setNetId(netId)); } } } netDirty.clear(); }
@Override public Vec gradient(final Vec mu) { final Vec grad = new ArrayVec(mu.dim()); for (int k = 0; k < grad.dim(); k++) { final TIntList idxs = classesIdxs.get(k); double val = 0.0; for (final TIntIterator listIter = idxs.iterator(); listIter.hasNext(); ) { final Vec x = ds.data().row(listIter.next()); final double trans = binClassifier.value(x); final double sigmoid = MathTools.sigmoid(trans); val -= (2 * sigmoid - 1) / (mu.get(k) * sigmoid + (1 - mu.get(k)) * (1 - sigmoid)); grad.set(k, val); } } final double norm = VecTools.norm(grad); VecTools.scale(grad, 1 / norm); for (int k = 0; k < grad.dim(); k++) { final double val = VecTools.multiply(laplacian.row(k), mu); grad.adjust(k, val); } return grad; }
public void mergeRules(File vectorsDir, int maxVectorLen) throws NumberFormatException, IOException { TIntObjectHashMap<BoundedPriorityQueue<Pair<Integer, Double>>> linScores = new TIntObjectHashMap<BoundedPriorityQueue<Pair<Integer, Double>>>(); TIntObjectHashMap<BoundedPriorityQueue<Pair<Integer, Double>>> balScores = new TIntObjectHashMap<BoundedPriorityQueue<Pair<Integer, Double>>>(); String line; BufferedReader reader = null; for (String fileName : vectorsDir.list()) { if (fileName.endsWith(".rules") && fileName.contains("Trunc" + maxVectorLen)) { System.out.println("Reading: " + fileName); reader = new BufferedReader(new FileReader(vectorsDir.getAbsolutePath() + "/" + fileName)); while ((line = reader.readLine()) != null) { String[] tokens = line.split("\t"); String scoreType = tokens[0]; Integer entailedElement = Integer.parseInt(tokens[1]); Integer entailingElement = Integer.parseInt(tokens[2]); Double score = Double.parseDouble(tokens[3]); if (scoreType.equals("LIN")) { if (linScores.contains(entailedElement)) linScores .get(entailedElement) .offer(new Pair<Integer, Double>(entailingElement, score)); else { linScores.put( entailedElement, new BoundedPriorityQueue<Pair<Integer, Double>>( new obj.PairComparator(), m_maxRulesPerElement)); linScores .get(entailedElement) .offer(new Pair<Integer, Double>(entailingElement, score)); } } else if (scoreType.equals("COVER")) if (balScores.contains(entailedElement)) balScores .get(entailedElement) .offer(new Pair<Integer, Double>(entailingElement, score)); else { balScores.put( entailedElement, new BoundedPriorityQueue<Pair<Integer, Double>>( new obj.PairComparator(), m_maxRulesPerElement)); balScores .get(entailedElement) .offer(new Pair<Integer, Double>(entailingElement, score)); } } reader.close(); } } System.out.println("Uploading elements"); TIntObjectMap<String> id2elementDesc = new TIntObjectHashMap<String>(); TIntObjectMap<String> id2targetElementDesc = new TIntObjectHashMap<String>(); reader = new BufferedReader(new FileReader(vectorsDir.getAbsolutePath() + "/elements.txt")); while ((line = reader.readLine()) != null) { id2elementDesc.put(Integer.parseInt(line.split("\t")[1]), line.split("\t")[0]); } reader.close(); reader = new BufferedReader(new FileReader(vectorsDir.getAbsolutePath() + "/targetElements.txt")); while ((line = reader.readLine()) != null) { id2targetElementDesc.put(Integer.parseInt(line.split("\t")[1]), line.split("\t")[0]); } reader.close(); PrintWriter writer = new PrintWriter(new FileOutputStream(vectorsDir.getAbsolutePath() + "/linRules.txt")); TIntIterator iter = linScores.keySet().iterator(); while (iter.hasNext()) { Integer entailedElementId = iter.next(); String entailedStr = id2targetElementDesc.get(entailedElementId); // if (entailedStr.contains("_")) { // String confDir = entailedStr.substring(entailedStr.indexOf("_")+1); // File resultsDir = new File(f.getAbsolutePath()+"/" + confDir); // if(!resultsDir.exists()) // resultsDir.mkdir(); // writer = new PrintWriter(new // FileOutputStream(resultsDir.getAbsolutePath()+"/"+entailedStr.substring(0,entailedStr.indexOf("_")))); // writer = new PrintWriter(new // FileOutputStream(resultsDir.getAbsolutePath()+"/"+entailedElementId)); // } // else // writer = new PrintWriter(new FileOutputStream(f.getAbsoluteFile()+"/"+entailedStr)); // writer = new PrintWriter(new // FileOutputStream(f.getAbsoluteFile()+"/"+entailedElementId)); while (!linScores.get(entailedElementId).isEmpty()) { Pair<Integer, Double> linRule = linScores.get(entailedElementId).poll(); writer.println( entailedStr + "\t" + id2elementDesc.get(linRule.key()) + "\t" + linRule.value()); } } if (writer != null) writer.close(); writer = new PrintWriter(new FileOutputStream(vectorsDir.getAbsolutePath() + "/balRules.txt")); iter = balScores.keySet().iterator(); while (iter.hasNext()) { Integer entailedElementId = iter.next(); String entailedStr = id2targetElementDesc.get(entailedElementId); // if (entailedStr.contains("_")) { // String confDir = entailedStr.substring(entailedStr.indexOf("_")+1); // File resultsDir = new File(f.getAbsolutePath()+"/" + confDir); // if(!resultsDir.exists()) // resultsDir.mkdir(); // writer = new PrintWriter(new // FileOutputStream(resultsDir.getAbsolutePath()+"/"+entailedStr.substring(0,entailedStr.indexOf("_")))); // writer = new PrintWriter(new // FileOutputStream(resultsDir.getAbsolutePath()+"/"+entailedElementId)); // } // else // writer = new PrintWriter(new FileOutputStream(f.getAbsoluteFile()+"/"+entailedStr)); // writer = new PrintWriter(new // FileOutputStream(f.getAbsoluteFile()+"/"+entailedElementId)); while (!balScores.get(entailedElementId).isEmpty()) { Pair<Integer, Double> balRule = balScores.get(entailedElementId).poll(); writer.println( entailedStr + "\t" + id2elementDesc.get(balRule.key()) + "\t" + balRule.value()); } } writer.close(); }
/** * Write the resulting clusters to an output stream. * * @param outStream output stream * @param data Generated data * @throws IOException thrown on write errors */ public void writeClusters(OutputStreamWriter outStream, MultipleObjectsBundle data) throws IOException { int modelcol = -1; { // Find model column for (int i = 0; i < data.metaLength(); i++) { if (TypeUtil.MODEL.isAssignableFromType(data.meta(i))) { modelcol = i; break; } } } if (modelcol < 0) { throw new AbortException("No model column found in bundle."); } ArrayList<Model> models = new ArrayList<>(); Map<Model, TIntList> modelMap = new HashMap<>(); { // Build a map from model to the actual objects for (int i = 0; i < data.dataLength(); i++) { Model model = (Model) data.data(i, modelcol); TIntList modelids = modelMap.get(model); if (modelids == null) { models.add(model); modelids = new TIntArrayList(); modelMap.put(model, modelids); } modelids.add(i); } } // compute global discard values int totalsize = 0, totaldisc = 0; for (Entry<Model, TIntList> ent : modelMap.entrySet()) { totalsize += ent.getValue().size(); if (ent.getKey() instanceof GeneratorSingleCluster) { totaldisc += ((GeneratorSingleCluster) ent.getKey()).getDiscarded(); } } double globdens = (double) (totalsize + totaldisc) / totalsize; outStream .append("########################################################") .append(LINE_SEPARATOR); outStream.append("## Number of clusters: " + models.size()).append(LINE_SEPARATOR); for (Model model : models) { TIntList ids = modelMap.get(model); outStream .append("########################################################") .append(LINE_SEPARATOR); outStream.append("## Size: " + ids.size()).append(LINE_SEPARATOR); if (model instanceof GeneratorSingleCluster) { GeneratorSingleCluster cursclus = (GeneratorSingleCluster) model; outStream .append("########################################################") .append(LINE_SEPARATOR); outStream.append("## Cluster: ").append(cursclus.getName()).append(LINE_SEPARATOR); Vector cmin = cursclus.getClipmin(); Vector cmax = cursclus.getClipmax(); if (cmin != null && cmax != null) { outStream .append("## Clipping: ") .append(cmin.toString()) // .append(" - ") .append(cmax.toString()) .append(LINE_SEPARATOR); } outStream .append("## Density correction factor: " + cursclus.getDensityCorrection()) .append(LINE_SEPARATOR); outStream.append("## Generators:").append(LINE_SEPARATOR); for (int i = 0; i < cursclus.getDim(); i++) { Distribution gen = cursclus.getDistribution(i); outStream.append("## ").append(gen.toString()).append(LINE_SEPARATOR); } if (cursclus.getTransformation() != null && cursclus.getTransformation().getTransformation() != null) { outStream.append("## Affine transformation matrix:").append(LINE_SEPARATOR); outStream .append(FormatUtil.format(cursclus.getTransformation().getTransformation(), "## ")) .append(LINE_SEPARATOR); } outStream .append( "## Discards: " + cursclus.getDiscarded() + " Retries left: " + cursclus.getRetries()) .append(LINE_SEPARATOR); double corf = /* cursclus.overweight */ (double) (cursclus.getSize() + cursclus.getDiscarded()) / cursclus.getSize() / globdens; outStream.append("## Density correction factor estimation: " + corf).append(LINE_SEPARATOR); } outStream .append("########################################################") .append(LINE_SEPARATOR); for (TIntIterator iter = ids.iterator(); iter.hasNext(); ) { int num = iter.next(); for (int c = 0; c < data.metaLength(); c++) { if (c != modelcol) { if (c > 0) { outStream.append(' '); } outStream.append(data.data(num, c).toString()); } } outStream.append(LINE_SEPARATOR); } } }