public static Query createQuery() { try { return (Query) prototypeObj.clone(); } catch (CloneNotSupportedException e) { e.printStackTrace(); return null; } }
/** * Returns the singleton application object matching the given {@code applicationClass} within * the given {@code database}. */ @SuppressWarnings("unchecked") public static <T extends Application> T getInstanceUsing( Class<T> applicationClass, Database database) { ObjectType type = database.getEnvironment().getTypeByClass(applicationClass); Query<T> query = Query.from(applicationClass).where("_type = ?", type.getId()).using(database); T app = query.first(); if (app == null) { DistributedLock lock = DistributedLock.Static.getInstance(database, applicationClass.getName()); lock.lock(); try { app = query.clone().noCache().first(); if (app == null) { app = (T) type.createObject(null); app.setName(type.getDisplayName()); app.saveImmediately(); return app; } } finally { lock.unlock(); } } String oldName = app.getName(); String newName = type.getDisplayName(); if (!ObjectUtils.equals(oldName, newName)) { app.setName(newName); app.save(); } return app; }
/** * Search for pages matching a query, eliminating excessive hits with matching values for a named * field. Hits after the first <code>maxHitsPerDup</code> are removed from results. The remaining * hits have {@link Hit#moreFromDupExcluded()} set. * * <p>If maxHitsPerDup is zero then all hits are returned. * * @param query query * @param numHits number of requested hits * @param maxHitsPerDup the maximum hits returned with matching values, or zero * @param dedupField field name to check for duplicates * @param sortField Field to sort on (or null if no sorting). * @param reverse True if we are to reverse sort by <code>sortField</code>. * @return Hits the matching hits * @throws IOException */ public Hits search( Query query, int numHits, int maxHitsPerDup, String dedupField, String sortField, boolean reverse) throws IOException { if (maxHitsPerDup <= 0) // disable dup checking return search(query, numHits, dedupField, sortField, reverse); final float rawHitsFactor = this.conf.getFloat("searcher.hostgrouping.rawhits.factor", 2.0f); int numHitsRaw = (int) (numHits * rawHitsFactor); if (LOG.isInfoEnabled()) { LOG.info("searching for " + numHitsRaw + " raw hits"); } Hits hits = searchBean.search(query, numHitsRaw, dedupField, sortField, reverse); final long total = hits.getTotal(); final Map<String, DupHits> dupToHits = new HashMap<String, DupHits>(); final List<Hit> resultList = new ArrayList<Hit>(); final Set<Hit> seen = new HashSet<Hit>(); final List<String> excludedValues = new ArrayList<String>(); boolean totalIsExact = true; for (int rawHitNum = 0; rawHitNum < hits.getTotal(); rawHitNum++) { // get the next raw hit if (rawHitNum >= hits.getLength()) { // optimize query by prohibiting more matches on some excluded values final Query optQuery = (Query) query.clone(); for (int i = 0; i < excludedValues.size(); i++) { if (i == MAX_PROHIBITED_TERMS) break; optQuery.addProhibitedTerm(excludedValues.get(i), dedupField); } numHitsRaw = (int) (numHitsRaw * rawHitsFactor); if (LOG.isInfoEnabled()) { LOG.info("re-searching for " + numHitsRaw + " raw hits, query: " + optQuery); } hits = searchBean.search(optQuery, numHitsRaw, dedupField, sortField, reverse); if (LOG.isInfoEnabled()) { LOG.info("found " + hits.getTotal() + " raw hits"); } rawHitNum = -1; continue; } final Hit hit = hits.getHit(rawHitNum); if (seen.contains(hit)) continue; seen.add(hit); // get dup hits for its value final String value = hit.getDedupValue(); DupHits dupHits = dupToHits.get(value); if (dupHits == null) dupToHits.put(value, dupHits = new DupHits()); // does this hit exceed maxHitsPerDup? if (dupHits.size() == maxHitsPerDup) { // yes -- ignore the hit if (!dupHits.maxSizeExceeded) { // mark prior hits with moreFromDupExcluded for (int i = 0; i < dupHits.size(); i++) { dupHits.get(i).setMoreFromDupExcluded(true); } dupHits.maxSizeExceeded = true; excludedValues.add(value); // exclude dup } totalIsExact = false; } else { // no -- collect the hit resultList.add(hit); dupHits.add(hit); // are we done? // we need to find one more than asked for, so that we can tell if // there are more hits to be shown if (resultList.size() > numHits) break; } } final Hits results = new Hits(total, resultList.toArray(new Hit[resultList.size()])); results.setTotalIsExact(totalIsExact); return results; }
public static void writeContents( Database database, BootstrapPackage pkg, Set<ObjectType> additionalTypes, Writer writer, String projectName) throws IOException { boolean first = true; boolean needsObjectTypeMap = false; Set<ObjectType> typeMaps = new HashSet<ObjectType>(); ObjectType objType = database.getEnvironment().getTypeByClass(ObjectType.class); Set<ObjectType> exportTypes = new HashSet<ObjectType>(); Query<?> query = Query.fromAll().using(database); Set<ObjectType> allTypeMappableTypes = new HashSet<ObjectType>(); Set<UUID> concreteTypeIds = new HashSet<UUID>(); for (ObjectType type : database.getEnvironment().getTypes()) { if (!type.as(TypeData.class).getTypeMappableGroups().isEmpty() && type.as(TypeData.class).getTypeMappableUniqueKey() != null) { allTypeMappableTypes.add(type); } } // Package: writer.write(PACKAGE_NAME_HEADER + ": "); writer.write(pkg.getName()); writer.write('\n'); // Project: writer.write(PROJECT_HEADER + ": "); writer.write(projectName); writer.write('\n'); // Date: writer.write(DATE_HEADER + ": "); writer.write(new DateTime().toString("yyyy-MM-dd HH:mm:ss z")); writer.write('\n'); // Types: writer.write(TYPES_HEADER + ": "); if (pkg.isInit()) { writer.write(ALL_TYPES_HEADER_VALUE); writer.write('\n'); } else { exportTypes.addAll(getAllTypes(database, pkg)); exportTypes.addAll(getAllTypes(database, additionalTypes)); for (ObjectType typeMappableType : allTypeMappableTypes) { GETTYPEMAPPABLETYPE: for (ObjectType type : exportTypes) { for (String group : typeMappableType.as(TypeData.class).getTypeMappableGroups()) { if (type.getGroups().contains(group)) { typeMaps.add(typeMappableType); break GETTYPEMAPPABLETYPE; } } } } GETOBJECTTYPE: for (ObjectType type : exportTypes) { for (ObjectField field : type.getFields()) { if (field.getTypes().contains(objType)) { needsObjectTypeMap = true; break GETOBJECTTYPE; } } } for (ObjectType exportType : exportTypes) { String clsName = exportType.getObjectClassName(); if (!ObjectUtils.isBlank(clsName)) { for (ObjectType concreteType : database.getEnvironment().getTypesByGroup(clsName)) { if (!concreteType.isAbstract()) { concreteTypeIds.add(concreteType.getId()); } } } } if (exportTypes.contains(objType)) { if (!first) { writer.write(','); } else { first = false; } writer.write(objType.getInternalName()); } for (ObjectType type : exportTypes) { if (type.equals(objType)) { continue; } if (!first) { writer.write(','); } else { first = false; } writer.write(type.getInternalName()); } writer.write('\n'); } // Determine if there are any fields that need references followed Map<UUID, Map<String, ObjectType>> followReferences = new HashMap<UUID, Map<String, ObjectType>>(); if (!pkg.isInit()) { for (ObjectType type : exportTypes) { for (String fieldName : type.as(TypeData.class).getFollowReferencesFields()) { ObjectField field = type.getField(fieldName); if (field != null) { for (ObjectType fieldType : field.getTypes()) { if (!exportTypes.contains(fieldType)) { if (!followReferences.containsKey(type.getId())) { followReferences.put(type.getId(), new HashMap<String, ObjectType>()); } followReferences.get(type.getId()).put(fieldName, fieldType); } } } } } } if (!typeMaps.isEmpty()) { for (ObjectType typeMapType : typeMaps) { String clsName = typeMapType.getObjectClassName(); if (!ObjectUtils.isBlank(clsName)) { for (ObjectType type : database.getEnvironment().getTypesByGroup(clsName)) { concreteTypeIds.remove(type.getId()); } } } } // Mapping Types: if (pkg.isInit() || needsObjectTypeMap || !typeMaps.isEmpty()) { first = true; writer.write(TYPE_MAP_HEADER + ": "); if (pkg.isInit() || needsObjectTypeMap) { writer.write(objType.getInternalName()); writer.write("/internalName"); first = false; } for (ObjectType typeMapType : typeMaps) { if (!first) { writer.write(','); } else { first = false; } writer.write(typeMapType.getInternalName()); writer.write('/'); writer.write(typeMapType.as(TypeData.class).getTypeMappableUniqueKey()); } writer.write('\n'); } // Row Count: Long count = 0L; try { if (concreteTypeIds.isEmpty()) { count = Query.fromAll().using(database).noCache().count(); } else { for (UUID concreteTypeId : concreteTypeIds) { long concreteCount = Query.fromAll() .using(database) .noCache() .where("_type = ?", concreteTypeId) .count(); count = count + concreteCount; } if (needsObjectTypeMap) { long objectCount = Query.fromAll().using(database).noCache().where("_type = ?", objType).count(); count = count + objectCount; } if (!typeMaps.isEmpty()) { for (ObjectType typeMapType : typeMaps) { long typeMapCount = Query.fromAll().using(database).noCache().where("_type = ?", typeMapType).count(); count = count + typeMapCount; } } } writer.write(ROW_COUNT_HEADER + ": "); writer.write(ObjectUtils.to(String.class, count)); writer.write('\n'); } catch (RuntimeException e) { // Count query timed out. Just don't write the row count header. count = null; } // blank line between headers and data writer.write('\n'); writer.flush(); // ObjectType records first if (exportTypes.isEmpty() || exportTypes.contains(objType) || needsObjectTypeMap) { for (Object r : Query.fromType(objType) .using(database) .noCache() .resolveToReferenceOnly() .iterable(100)) { writer.write(ObjectUtils.toJson(((Recordable) r).getState().getSimpleValues(true))); writer.write('\n'); } } // Then other mapping types for (ObjectType typeMapType : typeMaps) { for (Object r : Query.fromType(typeMapType) .using(database) .noCache() .resolveToReferenceOnly() .iterable(100)) { writer.write(ObjectUtils.toJson(((Recordable) r).getState().getSimpleValues(true))); writer.write('\n'); } } // Then everything else if (pkg.isInit()) { concreteTypeIds.clear(); // should already be empty concreteTypeIds.add(null); } UUID lastTypeId = null; Set<UUID> seenIds = new HashSet<UUID>(); query.getOptions().put(SqlDatabase.USE_JDBC_FETCH_SIZE_QUERY_OPTION, false); for (UUID typeId : concreteTypeIds) { Query<?> concreteQuery = query.clone(); if (typeId != null) { concreteQuery.where("_type = ?", typeId); } for (Object o : concreteQuery.noCache().resolveToReferenceOnly().iterable(100)) { if (o instanceof Recordable) { Recordable r = (Recordable) o; writer.write(ObjectUtils.toJson(r.getState().getSimpleValues(true))); writer.write('\n'); if (!pkg.isInit()) { if (lastTypeId == null || !lastTypeId.equals(r.getState().getTypeId())) { seenIds.clear(); } else if (seenIds.size() > MAX_SEEN_REFERENCE_IDS_SIZE) { seenIds.clear(); } lastTypeId = r.getState().getTypeId(); Map<String, ObjectType> followReferencesFieldMap; if ((followReferencesFieldMap = followReferences.get(r.getState().getTypeId())) != null) { for (Map.Entry<String, ObjectType> entry : followReferencesFieldMap.entrySet()) { Object reference = r.getState().getRawValue(entry.getKey()); Set<UUID> referenceIds = new HashSet<UUID>(); if (reference instanceof Collection) { for (Object referenceObj : ((Collection<?>) reference)) { if (referenceObj instanceof Recordable) { UUID referenceUUID = ObjectUtils.to( UUID.class, ((Recordable) referenceObj).getState().getId()); if (referenceUUID != null) { if (!seenIds.contains(referenceUUID)) { referenceIds.add(referenceUUID); } } } } } else if (reference instanceof Recordable) { UUID referenceUUID = ObjectUtils.to(UUID.class, ((Recordable) reference).getState().getId()); if (referenceUUID != null) { if (!seenIds.contains(referenceUUID)) { referenceIds.add(referenceUUID); } } } if (!referenceIds.isEmpty()) { for (Object ref : Query.fromType(entry.getValue()) .noCache() .using(database) .where("_id = ?", referenceIds) .selectAll()) { if (ref instanceof Recordable) { Recordable refr = (Recordable) ref; seenIds.add(refr.getState().getId()); writer.write(ObjectUtils.toJson(refr.getState().getSimpleValues(true))); writer.write('\n'); } } } } } } } } } writer.flush(); }