void grsImport(final GrsSource grsSource, final boolean rebuild) { final AuthoritativeResource authoritativeResource = grsSource.getAuthoritativeResource(); if (sourceContext.isVirtual(grsSource.getName())) { grsSource.getLogger().info("Not updating GRS data"); } else { acquireAndUpdateGrsData(grsSource, rebuild, authoritativeResource); } resourceTagger.tagObjects(grsSource); }
private void acquireAndUpdateGrsData( final GrsSource grsSource, final boolean rebuild, final AuthoritativeResource authoritativeData) { final Logger logger = grsSource.getLogger(); new Runnable() { private final RpslAttribute sourceAttribute = new RpslAttribute(AttributeType.SOURCE, grsSource.getName().toUpperCase()); private int nrCreated; private int nrUpdated; private int nrDeleted; private int nrIgnored; private Set<Integer> currentObjectIds; private Set<Integer> incompletelyIndexedObjectIds = Sets.newHashSet(); @Override public void run() { final Path dump = downloadDir.resolve(String.format("%s-DMP", grsSource.getName().toUpperCase())); try { grsSource.acquireDump(dump); } catch (IOException e) { throw new RuntimeException("Unable to acquire dump", e); } final Stopwatch stopwatch = Stopwatch.createStarted(); if (rebuild) { grsSource.getDao().cleanDatabase(); currentObjectIds = Collections.emptySet(); logger.info("Rebuilding database"); } else { currentObjectIds = Sets.newHashSet(grsSource.getDao().getCurrentObjectIds()); logger.info("Updating {} current objects in database", currentObjectIds.size()); } try { // TODO: [AH] continue from here to switch File to Path importObjects(dump.toFile()); deleteNotFoundInImport(); } catch (IOException e) { throw new RuntimeException(e); } finally { logger.info( "created {} / updated {} / deleted {} / ignored {} in {}", nrCreated, nrUpdated, nrDeleted, nrIgnored, stopwatch.stop()); } updateIndexes(); } private void importObjects(final File dumpFile) throws IOException { grsSource.handleObjects( dumpFile, new ObjectHandler() { @Override public void handle(final List<String> lines) { final String rpslObjectString = LINE_JOINER.join(lines); final RpslObject rpslObject; try { rpslObject = RpslObject.parse(rpslObjectString); } catch (RuntimeException e) { logger.info( "Unable to parse input as object: {}\n\n{}\n", e.getMessage(), rpslObjectString); return; } handle(FILTER_CHANGED_FUNCTION.apply(rpslObject)); } @Override public void handle(final RpslObject rpslObject) { if (rpslObject.getType() == null) { logger.debug("Unknown type: \n\n{}\n", rpslObject); nrIgnored++; } else { final ObjectMessages messages = new ObjectMessages(); final RpslObject filteredObject = filterObject(rpslObject); final RpslObject cleanObject = sanitizer.sanitize(filteredObject, messages); final RpslAttribute typeAttribute = cleanObject.getTypeAttribute(); typeAttribute.validateSyntax(cleanObject.getType(), messages); if (messages.hasErrors()) { logger.debug("Errors for object with key {}: {}", typeAttribute, messages); nrIgnored++; } else if (authoritativeData.isMaintainedInRirSpace(cleanObject)) { createOrUpdate(cleanObject); } } } private RpslObject filterObject(final RpslObject rpslObject) { final ObjectTemplate objectTemplate = ObjectTemplateProvider.getTemplate(rpslObject.getType()); final RpslObjectBuilder builder = new RpslObjectBuilder(rpslObject); for (int i = 0; i < builder.size(); i++) { final RpslAttribute rpslAttribute = builder.get(i); final AttributeType attributeType = rpslAttribute.getType(); if (attributeType == null || !objectTemplate.hasAttribute(attributeType)) { logger.debug( "Ignoring attribute in object {}: {}", rpslObject.getFormattedKey(), rpslAttribute); builder.remove(i--); } else if (attributeType.equals(AttributeType.SOURCE)) { builder.remove(i--); } } // best not to sort to avoid reordering remarks: attributes builder.append(sourceAttribute); return builder.get(); } @Transactional private void createOrUpdate(final RpslObject importedObject) { final String pkey = importedObject.getKey().toString(); final ObjectType type = importedObject.getType(); final GrsObjectInfo grsObjectInfo = grsSource.getDao().find(pkey, type); if (grsObjectInfo == null) { if (type == ObjectType.PERSON && grsSource.getDao().find(pkey, ObjectType.ROLE) != null) { return; } if (type == ObjectType.ROLE && grsSource.getDao().find(pkey, ObjectType.PERSON) != null) { return; } create(importedObject); } else { currentObjectIds.remove(grsObjectInfo.getObjectId()); if (!grsObjectInfo.getRpslObject().equals(importedObject)) { update(importedObject, grsObjectInfo); } } final int nrImported = nrCreated + nrUpdated; if ((nrImported % LOG_EVERY_NR_HANDLED == 0) && (nrImported > 0)) { logger.info("Imported {} objects", nrImported); } } private void create(final RpslObject importedObject) { final GrsDao.UpdateResult updateResult = grsSource.getDao().createObject(importedObject); if (updateResult.hasMissingReferences()) { incompletelyIndexedObjectIds.add(updateResult.getObjectId()); } nrCreated++; } private void update( final RpslObject importedObject, final GrsObjectInfo grsObjectInfo) { final GrsDao.UpdateResult updateResult = grsSource.getDao().updateObject(grsObjectInfo, importedObject); if (updateResult.hasMissingReferences()) { incompletelyIndexedObjectIds.add(updateResult.getObjectId()); } nrUpdated++; } }); } private void deleteNotFoundInImport() { if (nrCreated == 0 && nrUpdated == 0) { logger.warn("Skipping deletion since there were no other updates"); return; } logger.info("Cleaning up {} currently unreferenced objects", currentObjectIds.size()); for (final Integer objectId : currentObjectIds) { try { grsSource.getDao().deleteObject(objectId); nrDeleted++; } catch (RuntimeException e) { logger.error("Deleting object with id: {}", objectId, e); } } } private void updateIndexes() { logger.info( "Updating indexes for {} changed objects with missing references", incompletelyIndexedObjectIds.size()); int nrUpdated = 0; for (final Integer objectId : incompletelyIndexedObjectIds) { try { grsSource.getDao().updateIndexes(objectId); } catch (RuntimeException e) { logger.error("Updating index for object with id: {}", objectId, e); } nrUpdated++; if (nrUpdated % LOG_EVERY_NR_HANDLED == 0) { logger.info("Updated {} indexes", nrUpdated); } } } }.run(); }