private static void setRowKeyOffset(Filter filter, int offset) { if (filter instanceof BooleanExpressionFilter) { BooleanExpressionFilter boolFilter = (BooleanExpressionFilter) filter; IndexUtil.setRowKeyExpressionOffset(boolFilter.getExpression(), offset); } else if (filter instanceof SkipScanFilter) { SkipScanFilter skipScanFilter = (SkipScanFilter) filter; skipScanFilter.setOffset(offset); } }
/** {@inheritDoc} */ public boolean handleEvent( final ResourceResolver resolver, final String resourceType, final String eventTopics, final String nodePath, final String[] resourceChangedAttributes, final String[] resourceAddedAttributes, final String[] resourceRemovedAttributes, final Enum<EventTopicsType> eventTopicsType) throws Exception { final boolean isDelete = (eventTopicsType == EventTopicsType.DELETED) ? true : false; if (isDelete) { // Check to see if the root is the index root node or not final String componentName = IndexUtil.getComponentName(resolver, nodePath); if (componentName != null && componentName.endsWith(IndexUtil.INDEX_FOLDER)) { return false; } } else if (resourceType == null) { return false; } else if (checkResourceType(resolver, nodePath, resourceType, addedResourceTypes) && eventTopics.equals(SlingConstants.TOPIC_RESOURCE_ADDED)) { // A target resource type has been added to the repository. } else if (checkResourceType(resolver, nodePath, resourceType, changedResourceTypes) && eventTopics.equals(SlingConstants.TOPIC_RESOURCE_CHANGED)) { // A target attribute may have been modified. if (!(checkAttributes(resourceChangedAttributes, targetModifiedAttributes) || checkAttributes(resourceAddedAttributes, targetAddedRemovedAttributes) || checkAttributes(resourceRemovedAttributes, targetAddedRemovedAttributes))) { // This attribute does not trigger an index update. return false; } } else { return false; } indexExec(resolver, nodePath, eventTopicsType); return true; }
/** * optimise * * @param structureName * @param index * @param statsCounter * @param numEntries * @throws IOException */ @SuppressWarnings("unchecked") public static void optimise( String structureName, Index index, LexiconBuilder.CollectionStatisticsCounter statsCounter, int numEntries) throws IOException { final String mapFileFilename = constructFilename(structureName, index.getPath(), index.getPrefix(), MAPFILE_EXT); final FixedSizeWriteableFactory<Text> keyFactory = (FixedSizeWriteableFactory<Text>) index.getIndexStructure(structureName + "-keyfactory"); final FixedSizeWriteableFactory<LexiconEntry> valueFactory = (FixedSizeWriteableFactory<LexiconEntry>) index.getIndexStructure(structureName + "-valuefactory"); // logger.info("Optimsing lexicon with "+ numEntries + " entries"); // term id lookups boolean termIdsAligned = true; int[] termid2index = new int[numEntries]; Arrays.fill(termid2index, -1); int counter = 0; int lastTermId = -1; // bsearch reduction int previousFirstChar = -1; int firstChar = 0; final TIntObjectHashMap<int[]> map = new TIntObjectHashMap<int[]>(); Iterator<Map.Entry<Text, LexiconEntry>> iterator = new FSOrderedMapFile.EntryIterator<Text, LexiconEntry>( mapFileFilename, keyFactory, valueFactory); while (iterator.hasNext()) { Map.Entry<Text, LexiconEntry> lee = iterator.next(); // System.err.println(lee.toString()); // term id int termId = lee.getValue().getTermId(); if (!(termId == lastTermId + 1)) termIdsAligned = false; if (termid2index[termId] != -1) { throw new WrappedIOException( new IllegalArgumentException( "Termid " + termId + " is not unique - used at entries " + termid2index[termId] + " and" + counter)); } termid2index[termId] = counter; lastTermId = termId; // bsearch reduction optimisaion firstChar = lee.getKey().charAt(0); if (firstChar != previousFirstChar) { int[] boundaries = new int[] {counter, 0}; map.put(firstChar, boundaries); previousFirstChar = firstChar; } // increments statsCounter.count(lee.getValue()); counter++; } if (counter != numEntries) termIdsAligned = false; IndexUtil.close(iterator); // deal with termids if (termIdsAligned) { index.setIndexProperty("index." + structureName + ".termids", "aligned"); // logger.info("All ids for structure "+structureName+ " are aligned, skipping " // +ID_EXT+ " file"); } else { DataOutputStream dos = new DataOutputStream( Files.writeFileStream( constructFilename(structureName, index.getPath(), index.getPrefix(), ID_EXT))); for (int indexof : termid2index) dos.writeInt(indexof); dos.close(); index.setIndexProperty( "index." + structureName + ".termids", (numEntries > 15000000) ? "file" : "fileinmem"); } int[] mapKeys = map.keys(); Arrays.sort(mapKeys); final int mapKeysSize = mapKeys.length; for (int i = 0; i < mapKeysSize - 1; i++) { int nextLowerBoundary = (map.get(mapKeys[i + 1]))[0]; int[] currentBoundaries = map.get(mapKeys[i]); currentBoundaries[1] = nextLowerBoundary; map.put(mapKeys[i], currentBoundaries); } // do something about the last entry int nextLowerBoundary = counter; int[] currentBoundaries = (int[]) map.get(mapKeys[mapKeysSize - 1]); currentBoundaries[1] = nextLowerBoundary; map.put(mapKeys[mapKeysSize - 1], currentBoundaries); final ObjectOutputStream oos = new ObjectOutputStream( Files.writeFileStream( constructFilename(structureName, index.getPath(), index.getPrefix(), HASH_EXT))); oos.writeObject(map); oos.close(); index.setIndexProperty("index." + structureName + ".bsearchshortcut", "charmap"); index.flush(); }