Beispiel #1
0
public class BatchInserterImpl implements BatchInserter {
  private static final long MAX_NODE_ID = IdType.NODE.getMaxValue();

  private final Function<RelationshipRecord, Long> REL_RECORD_TO_ID =
      new Function<RelationshipRecord, Long>() {
        @Override
        public Long apply(RelationshipRecord relRecord) {
          return relRecord.getId();
        }
      };
  private final Function<RelationshipRecord, BatchRelationship> REL_RECORD_TO_BATCH_REL =
      new Function<RelationshipRecord, BatchRelationship>() {
        @Override
        public BatchRelationship apply(RelationshipRecord relRecord) {
          RelationshipType type =
              new RelationshipTypeImpl(relationshipTypeTokens.nameOf(relRecord.getType()));
          return new BatchRelationship(
              relRecord.getId(), relRecord.getFirstNode(), relRecord.getSecondNode(), type);
        }
      };

  private final LifeSupport life;
  private final NeoStore neoStore;
  private final IndexConfigStore indexStore;
  private final File storeDir;
  private final BatchTokenHolder propertyKeyTokens;
  private final BatchTokenHolder relationshipTypeTokens;
  private final BatchTokenHolder labelTokens;
  private final IdGeneratorFactory idGeneratorFactory;
  private final SchemaIndexProviderMap schemaIndexProviders;
  private final LabelScanStore labelScanStore;
  // TODO use Logging instead
  private final StringLogger msgLog;
  private final Logging logging;
  private final FileSystemAbstraction fileSystem;
  private final SchemaCache schemaCache;
  private final Config config;
  private final BatchInserterImpl.BatchSchemaActions actions;
  private final StoreLocker storeLocker;
  private boolean labelsTouched;

  private final FunctionFromPrimitiveLong<Label> labelIdToLabelFunction =
      new FunctionFromPrimitiveLong<Label>() {
        @Override
        public Label apply(long from) {
          return label(labelTokens.nameOf(safeCastLongToInt(from)));
        }
      };

  private boolean isShutdown = false;

  // Helper structure for setNodeProperty
  private final RelationshipCreator relationshipCreator;
  private final DirectRecordAccessSet recordAccess;
  private final PropertyTraverser propertyTraverser;
  private final PropertyCreator propertyCreator;
  private final PropertyDeleter propertyDeletor;

  BatchInserterImpl(String storeDir, Map<String, String> stringParams) {
    this(
        storeDir,
        new DefaultFileSystemAbstraction(),
        stringParams,
        Collections.<KernelExtensionFactory<?>>emptyList());
  }

  BatchInserterImpl(
      String storeDir,
      FileSystemAbstraction fileSystem,
      Map<String, String> stringParams,
      Iterable<KernelExtensionFactory<?>> kernelExtensions) {
    rejectAutoUpgrade(stringParams);
    Map<String, String> params = getDefaultParams();
    params.putAll(stringParams);
    config =
        StoreFactory.configForStoreDir(
            new Config(params, GraphDatabaseSettings.class), new File(storeDir));

    life = new LifeSupport();
    this.fileSystem = fileSystem;
    this.storeDir = new File(FileUtils.fixSeparatorsInPath(storeDir));
    Neo4jJobScheduler jobScheduler = life.add(new Neo4jJobScheduler());
    LifecycledPageCache pageCache =
        life.add(new LifecycledPageCache(fileSystem, jobScheduler, config));

    msgLog = StringLogger.loggerDirectory(fileSystem, this.storeDir);
    logging = new SingleLoggingService(msgLog);
    storeLocker = new StoreLocker(fileSystem);
    storeLocker.checkLock(this.storeDir);

    boolean dump = config.get(GraphDatabaseSettings.dump_configuration);
    this.idGeneratorFactory = new DefaultIdGeneratorFactory();

    Monitors monitors = new Monitors();
    StoreFactory sf =
        new StoreFactory(config, idGeneratorFactory, pageCache, fileSystem, msgLog, monitors);

    if (dump) {
      dumpConfiguration(params);
    }
    msgLog.logMessage(Thread.currentThread() + " Starting BatchInserter(" + this + ")");
    neoStore = sf.newNeoStore(true);
    if (!neoStore.isStoreOk()) {
      throw new IllegalStateException(storeDir + " store is not cleanly shutdown.");
    }
    neoStore.makeStoreOk();
    Token[] indexes = getPropertyKeyTokenStore().getTokens(10000);
    propertyKeyTokens = new BatchTokenHolder(indexes);
    labelTokens = new BatchTokenHolder(neoStore.getLabelTokenStore().getTokens(Integer.MAX_VALUE));
    Token[] types = getRelationshipTypeStore().getTokens(Integer.MAX_VALUE);
    relationshipTypeTokens = new BatchTokenHolder(types);
    indexStore = life.add(new IndexConfigStore(this.storeDir, fileSystem));
    schemaCache = new SchemaCache(neoStore.getSchemaStore());

    KernelExtensions extensions =
        life.add(
            new KernelExtensions(
                kernelExtensions,
                config,
                new DependencyResolverImpl(),
                UnsatisfiedDependencyStrategies.ignore()));

    life.start();

    SchemaIndexProvider provider =
        extensions.resolveDependency(
            SchemaIndexProvider.class, SchemaIndexProvider.HIGHEST_PRIORITIZED_OR_NONE);
    schemaIndexProviders = new DefaultSchemaIndexProviderMap(provider);
    labelScanStore =
        life.add(
            extensions
                .resolveDependency(
                    LabelScanStoreProvider.class, LabelScanStoreProvider.HIGHEST_PRIORITIZED)
                .getLabelScanStore());
    actions = new BatchSchemaActions();

    // Record access
    recordAccess = new DirectRecordAccessSet(neoStore);
    relationshipCreator =
        new RelationshipCreator(
            RelationshipLocker.NO_LOCKING,
            new RelationshipGroupGetter(neoStore.getRelationshipGroupStore()),
            neoStore.getDenseNodeThreshold());
    propertyTraverser = new PropertyTraverser();
    propertyCreator = new PropertyCreator(getPropertyStore(), propertyTraverser);
    propertyDeletor = new PropertyDeleter(getPropertyStore(), propertyTraverser);
  }

  private Map<String, String> getDefaultParams() {
    Map<String, String> params = new HashMap<>();
    params.put(GraphDatabaseSettings.mapped_memory_total_size.name(), "1%");
    return params;
  }

  @Override
  public boolean nodeHasProperty(long node, String propertyName) {
    return primitiveHasProperty(getNodeRecord(node).forChangingData(), propertyName);
  }

  @Override
  public boolean relationshipHasProperty(long relationship, String propertyName) {
    return primitiveHasProperty(
        recordAccess.getRelRecords().getOrLoad(relationship, null).forReadingData(), propertyName);
  }

  @Override
  public void setNodeProperty(long node, String propertyName, Object newValue) {
    propertyCreator.setPrimitiveProperty(
        getNodeRecord(node),
        getOrCreatePropertyKeyId(propertyName),
        newValue,
        recordAccess.getPropertyRecords());
    recordAccess.commit();
  }

  @Override
  public void setRelationshipProperty(
      long relationship, String propertyName, Object propertyValue) {
    propertyCreator.setPrimitiveProperty(
        getRelationshipRecord(relationship),
        getOrCreatePropertyKeyId(propertyName),
        propertyValue,
        recordAccess.getPropertyRecords());
    recordAccess.commit();
  }

  @Override
  public void removeNodeProperty(long node, String propertyName) {
    int propertyKey = getOrCreatePropertyKeyId(propertyName);
    propertyDeletor.removeProperty(
        getNodeRecord(node), propertyKey, recordAccess.getPropertyRecords());
    recordAccess.commit();
  }

  @Override
  public void removeRelationshipProperty(long relationship, String propertyName) {
    int propertyKey = getOrCreatePropertyKeyId(propertyName);
    propertyDeletor.removeProperty(
        getRelationshipRecord(relationship), propertyKey, recordAccess.getPropertyRecords());
    recordAccess.commit();
  }

  @Override
  public IndexCreator createDeferredSchemaIndex(Label label) {
    return new IndexCreatorImpl(actions, label);
  }

  private void createIndexRule(Label label, String propertyKey) {
    // TODO: Do not create duplicate index

    SchemaStore schemaStore = getSchemaStore();
    IndexRule schemaRule =
        IndexRule.indexRule(
            schemaStore.nextId(),
            getOrCreateLabelId(label.name()),
            getOrCreatePropertyKeyId(propertyKey),
            this.schemaIndexProviders.getDefaultProvider().getProviderDescriptor());
    for (DynamicRecord record : schemaStore.allocateFrom(schemaRule)) {
      schemaStore.updateRecord(record);
    }
    schemaCache.addSchemaRule(schemaRule);
    labelsTouched = true;
    recordAccess.commit();
  }

  private void repopulateAllIndexes() throws IOException {
    if (!labelsTouched) {
      return;
    }

    final IndexRule[] rules = getIndexesNeedingPopulation();
    final IndexPopulator[] populators = new IndexPopulator[rules.length];
    // the store is uncontended at this point, so creating a local LockService is safe.
    LockService locks = new ReentrantLockService();
    IndexStoreView storeView = new NeoStoreIndexStoreView(locks, neoStore);

    final int[] labelIds = new int[rules.length];
    final int[] propertyKeyIds = new int[rules.length];

    for (int i = 0; i < labelIds.length; i++) {
      IndexRule rule = rules[i];
      int labelId = rule.getLabel();
      int propertyKeyId = rule.getPropertyKey();
      labelIds[i] = labelId;
      propertyKeyIds[i] = propertyKeyId;

      IndexDescriptor descriptor = new IndexDescriptor(labelId, propertyKeyId);
      populators[i] =
          schemaIndexProviders
              .apply(rule.getProviderDescriptor())
              .getPopulator(
                  rule.getId(), descriptor, new IndexConfiguration(rule.isConstraintIndex()));
      populators[i].create();
    }

    Visitor<NodePropertyUpdate, IOException> propertyUpdateVisitor =
        new Visitor<NodePropertyUpdate, IOException>() {
          @Override
          public boolean visit(NodePropertyUpdate update) throws IOException {
            // Do a lookup from which property has changed to a list of indexes worried about that
            // property.
            int propertyKeyInQuestion = update.getPropertyKeyId();
            for (int i = 0; i < propertyKeyIds.length; i++) {
              if (propertyKeyIds[i] == propertyKeyInQuestion) {
                if (update.forLabel(labelIds[i])) {
                  try {
                    populators[i].add(update.getNodeId(), update.getValueAfter());
                  } catch (IndexEntryConflictException conflict) {
                    throw conflict.notAllowed(rules[i].getLabel(), rules[i].getPropertyKey());
                  }
                }
              }
            }
            return true;
          }
        };

    InitialNodeLabelCreationVisitor labelUpdateVisitor = new InitialNodeLabelCreationVisitor();
    StoreScan<IOException> storeScan =
        storeView.visitNodes(labelIds, propertyKeyIds, propertyUpdateVisitor, labelUpdateVisitor);
    storeScan.run();

    for (IndexPopulator populator : populators) {
      populator.close(true);
    }
    labelUpdateVisitor.close();
  }

  private class InitialNodeLabelCreationVisitor implements Visitor<NodeLabelUpdate, IOException> {
    LabelScanWriter writer = labelScanStore.newWriter();

    @Override
    public boolean visit(NodeLabelUpdate update) throws IOException {
      writer.write(update);
      return true;
    }

    public void close() throws IOException {
      writer.close();
    }
  }

  private IndexRule[] getIndexesNeedingPopulation() {
    List<IndexRule> indexesNeedingPopulation = new ArrayList<>();
    for (SchemaRule rule : schemaCache.schemaRules()) {
      if (rule.getKind().isIndex()) {
        IndexRule indexRule = (IndexRule) rule;
        SchemaIndexProvider provider =
            schemaIndexProviders.apply(indexRule.getProviderDescriptor());
        if (provider.getInitialState(indexRule.getId()) != InternalIndexState.FAILED) {
          indexesNeedingPopulation.add(indexRule);
        }
      }
    }
    return indexesNeedingPopulation.toArray(new IndexRule[indexesNeedingPopulation.size()]);
  }

  @Override
  public ConstraintCreator createDeferredConstraint(Label label) {
    return new BaseConstraintCreator(new BatchSchemaActions(), label);
  }

  private void createConstraintRule(UniquenessConstraint constraint) {
    // TODO: Do not create duplicate index

    SchemaStore schemaStore = getSchemaStore();

    long indexRuleId = schemaStore.nextId();
    long constraintRuleId = schemaStore.nextId();

    IndexRule indexRule =
        IndexRule.constraintIndexRule(
            indexRuleId,
            constraint.label(),
            constraint.propertyKeyId(),
            this.schemaIndexProviders.getDefaultProvider().getProviderDescriptor(),
            constraintRuleId);
    UniquenessConstraintRule constraintRule =
        UniquenessConstraintRule.uniquenessConstraintRule(
            constraintRuleId, constraint.label(), constraint.propertyKeyId(), indexRuleId);

    for (DynamicRecord record : schemaStore.allocateFrom(constraintRule)) {
      schemaStore.updateRecord(record);
    }
    schemaCache.addSchemaRule(constraintRule);
    for (DynamicRecord record : schemaStore.allocateFrom(indexRule)) {
      schemaStore.updateRecord(record);
    }
    schemaCache.addSchemaRule(indexRule);
    labelsTouched = true;
    recordAccess.commit();
  }

  private int getOrCreatePropertyKeyId(String name) {
    int propertyKeyId = getPropertyKeyId(name);
    if (propertyKeyId == -1) {
      propertyKeyId = createNewPropertyKeyId(name);
    }
    return propertyKeyId;
  }

  private int getOrCreateRelationshipTypeToken(RelationshipType type) {
    int typeId = relationshipTypeTokens.idOf(type.name());
    if (typeId == -1) {
      typeId = createNewRelationshipType(type.name());
    }
    return typeId;
  }

  private int getPropertyKeyId(String name) {
    return propertyKeyTokens.idOf(name);
  }

  private int getOrCreateLabelId(String name) {
    int labelId = getLabelId(name);
    if (labelId == -1) {
      labelId = createNewLabelId(name);
    }
    return labelId;
  }

  private int getLabelId(String name) {
    return labelTokens.idOf(name);
  }

  private boolean primitiveHasProperty(PrimitiveRecord record, String propertyName) {
    int propertyKeyId = propertyKeyTokens.idOf(propertyName);
    return propertyKeyId != -1
        && propertyTraverser.findPropertyRecordContaining(
                record, propertyKeyId, recordAccess.getPropertyRecords(), false)
            != Record.NO_NEXT_PROPERTY.intValue();
  }

  private void rejectAutoUpgrade(Map<String, String> params) {
    if (parseBoolean(params.get(GraphDatabaseSettings.allow_store_upgrade.name()))) {
      throw new IllegalArgumentException(
          "Batch inserter is not allowed to do upgrade of a store"
              + ", use "
              + EmbeddedGraphDatabase.class.getSimpleName()
              + " instead");
    }
  }

  @Override
  public long createNode(Map<String, Object> properties, Label... labels) {
    return internalCreateNode(getNodeStore().nextId(), properties, labels);
  }

  private long internalCreateNode(long nodeId, Map<String, Object> properties, Label... labels) {
    NodeRecord nodeRecord = recordAccess.getNodeRecords().create(nodeId, null).forChangingData();
    nodeRecord.setInUse(true);
    nodeRecord.setCreated();
    nodeRecord.setNextProp(
        propertyCreator.createPropertyChain(
            nodeRecord, propertiesIterator(properties), recordAccess.getPropertyRecords()));

    if (labels.length > 0) {
      setNodeLabels(nodeRecord, labels);
    }

    recordAccess.commit();
    return nodeId;
  }

  private Iterator<PropertyBlock> propertiesIterator(Map<String, Object> properties) {
    if (properties == null || properties.isEmpty()) {
      return IteratorUtil.emptyIterator();
    }
    return new IteratorWrapper<PropertyBlock, Map.Entry<String, Object>>(
        properties.entrySet().iterator()) {
      @Override
      protected PropertyBlock underlyingObjectToObject(Entry<String, Object> property) {
        return propertyCreator.encodePropertyValue(
            getOrCreatePropertyKeyId(property.getKey()), property.getValue());
      }
    };
  }

  private void setNodeLabels(NodeRecord nodeRecord, Label... labels) {
    NodeLabels nodeLabels = parseLabelsField(nodeRecord);
    getNodeStore()
        .updateDynamicLabelRecords(
            nodeLabels.put(
                getOrCreateLabelIds(labels),
                getNodeStore(),
                getNodeStore().getDynamicLabelStore()));
    labelsTouched = true;
  }

  private long[] getOrCreateLabelIds(Label[] labels) {
    long[] ids = new long[labels.length];
    for (int i = 0; i < ids.length; i++) {
      ids[i] = getOrCreateLabelId(labels[i].name());
    }
    return ids;
  }

  @Override
  public void createNode(long id, Map<String, Object> properties, Label... labels) {
    if (id < 0 || id > MAX_NODE_ID) {
      throw new IllegalArgumentException("id=" + id);
    }
    if (id == IdGeneratorImpl.INTEGER_MINUS_ONE) {
      throw new IllegalArgumentException("id " + id + " is reserved for internal use");
    }
    NodeStore nodeStore = neoStore.getNodeStore();
    if (neoStore.getNodeStore().loadLightNode(id) != null) {
      throw new IllegalArgumentException("id=" + id + " already in use");
    }
    long highId = nodeStore.getHighId();
    if (highId <= id) {
      nodeStore.setHighId(id + 1);
    }
    internalCreateNode(id, properties, labels);
  }

  @Override
  public void setNodeLabels(long node, Label... labels) {
    NodeRecord record = getNodeRecord(node).forChangingData();
    setNodeLabels(record, labels);
    recordAccess.commit();
  }

  @Override
  public Iterable<Label> getNodeLabels(final long node) {
    return new Iterable<Label>() {
      @Override
      public Iterator<Label> iterator() {
        NodeRecord record = getNodeRecord(node).forReadingData();
        long[] labels = parseLabelsField(record).get(getNodeStore());
        return map(labelIdToLabelFunction, PrimitiveLongCollections.iterator(labels));
      }
    };
  }

  @Override
  public boolean nodeHasLabel(long node, Label label) {
    int labelId = getLabelId(label.name());
    return labelId != -1 && nodeHasLabel(node, labelId);
  }

  private boolean nodeHasLabel(long node, int labelId) {
    NodeRecord record = getNodeRecord(node).forReadingData();
    for (long label : parseLabelsField(record).get(getNodeStore())) {
      if (label == labelId) {
        return true;
      }
    }
    return false;
  }

  @Override
  public long createRelationship(
      long node1, long node2, RelationshipType type, Map<String, Object> properties) {
    long id = neoStore.getRelationshipStore().nextId();
    int typeId = getOrCreateRelationshipTypeToken(type);
    relationshipCreator.relationshipCreate(id, typeId, node1, node2, recordAccess);
    if (properties != null && !properties.isEmpty()) {
      RelationshipRecord record =
          recordAccess.getRelRecords().getOrLoad(id, null).forChangingData();
      record.setNextProp(
          propertyCreator.createPropertyChain(
              record, propertiesIterator(properties), recordAccess.getPropertyRecords()));
    }
    recordAccess.commit();
    return id;
  }

  @Override
  public void setNodeProperties(long node, Map<String, Object> properties) {
    NodeRecord record = getNodeRecord(node).forChangingData();
    if (record.getNextProp() != Record.NO_NEXT_PROPERTY.intValue()) {
      propertyDeletor.getAndDeletePropertyChain(record, recordAccess.getPropertyRecords());
    }
    record.setNextProp(
        propertyCreator.createPropertyChain(
            record, propertiesIterator(properties), recordAccess.getPropertyRecords()));
    recordAccess.commit();
  }

  @Override
  public void setRelationshipProperties(long rel, Map<String, Object> properties) {
    RelationshipRecord record = recordAccess.getRelRecords().getOrLoad(rel, null).forChangingData();
    if (record.getNextProp() != Record.NO_NEXT_PROPERTY.intValue()) {
      propertyDeletor.getAndDeletePropertyChain(record, recordAccess.getPropertyRecords());
    }
    record.setNextProp(
        propertyCreator.createPropertyChain(
            record, propertiesIterator(properties), recordAccess.getPropertyRecords()));
    recordAccess.commit();
  }

  @Override
  public boolean nodeExists(long nodeId) {
    return neoStore.getNodeStore().loadLightNode(nodeId) != null;
  }

  @Override
  public Map<String, Object> getNodeProperties(long nodeId) {
    NodeRecord record = getNodeRecord(nodeId).forReadingData();
    if (record.getNextProp() != Record.NO_NEXT_PROPERTY.intValue()) {
      return getPropertyChain(record.getNextProp());
    }
    return Collections.emptyMap();
  }

  @Override
  public Iterable<Long> getRelationshipIds(long nodeId) {
    return map(REL_RECORD_TO_ID, new BatchRelationshipIterable(neoStore, nodeId));
  }

  @Override
  public Iterable<BatchRelationship> getRelationships(long nodeId) {
    return map(REL_RECORD_TO_BATCH_REL, new BatchRelationshipIterable(neoStore, nodeId));
  }

  @Override
  public BatchRelationship getRelationshipById(long relId) {
    RelationshipRecord record = getRelationshipRecord(relId).forReadingData();
    RelationshipType type =
        new RelationshipTypeImpl(relationshipTypeTokens.nameOf(record.getType()));
    return new BatchRelationship(
        record.getId(), record.getFirstNode(), record.getSecondNode(), type);
  }

  @Override
  public Map<String, Object> getRelationshipProperties(long relId) {
    RelationshipRecord record =
        recordAccess.getRelRecords().getOrLoad(relId, null).forChangingData();
    if (record.getNextProp() != Record.NO_NEXT_PROPERTY.intValue()) {
      return getPropertyChain(record.getNextProp());
    }
    return Collections.emptyMap();
  }

  @Override
  public void shutdown() {
    recordAccess.close();

    if (isShutdown) {
      throw new IllegalStateException("Batch inserter already has shutdown");
    }
    isShutdown = true;

    try {
      repopulateAllIndexes();
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
    neoStore.close();

    try {
      storeLocker.release();
    } catch (IOException e) {
      throw new UnderlyingStorageException("Could not release store lock", e);
    }

    msgLog.logMessage(
        Thread.currentThread() + " Clean shutdown on BatchInserter(" + this + ")", true);
    msgLog.close();
    life.shutdown();
  }

  @Override
  public String toString() {
    return "EmbeddedBatchInserter[" + storeDir + "]";
  }

  private static class RelationshipTypeImpl implements RelationshipType {
    private final String name;

    RelationshipTypeImpl(String name) {
      this.name = name;
    }

    @Override
    public String name() {
      return name;
    }
  }

  private Map<String, Object> getPropertyChain(long nextProp) {
    final Map<String, Object> map = new HashMap<>();
    propertyTraverser.getPropertyChain(
        nextProp,
        recordAccess.getPropertyRecords(),
        new Listener<PropertyBlock>() {
          @Override
          public void receive(PropertyBlock propBlock) {
            String key = propertyKeyTokens.nameOf(propBlock.getKeyIndexId());
            DefinedProperty propertyData = propBlock.newPropertyData(getPropertyStore());
            Object value =
                propertyData.value() != null
                    ? propertyData.value()
                    : propBlock.getType().getValue(propBlock, getPropertyStore());
            map.put(key, value);
          }
        });
    return map;
  }

  private int createNewPropertyKeyId(String stringKey) {
    PropertyKeyTokenStore idxStore = getPropertyKeyTokenStore();
    int keyId = (int) idxStore.nextId();
    PropertyKeyTokenRecord record = new PropertyKeyTokenRecord(keyId);
    record.setInUse(true);
    record.setCreated();
    Collection<DynamicRecord> keyRecords = idxStore.allocateNameRecords(encodeString(stringKey));
    record.setNameId((int) first(keyRecords).getId());
    record.addNameRecords(keyRecords);
    idxStore.updateRecord(record);
    propertyKeyTokens.addToken(stringKey, keyId);
    return keyId;
  }

  private int createNewLabelId(String stringKey) {
    LabelTokenStore labelTokenStore = neoStore.getLabelTokenStore();
    int keyId = (int) labelTokenStore.nextId();
    LabelTokenRecord record = new LabelTokenRecord(keyId);
    record.setInUse(true);
    record.setCreated();
    Collection<DynamicRecord> keyRecords =
        labelTokenStore.allocateNameRecords(encodeString(stringKey));
    record.setNameId((int) first(keyRecords).getId());
    record.addNameRecords(keyRecords);
    labelTokenStore.updateRecord(record);
    labelTokens.addToken(stringKey, keyId);
    return keyId;
  }

  private int createNewRelationshipType(String name) {
    RelationshipTypeTokenStore typeStore = getRelationshipTypeStore();
    int id = (int) typeStore.nextId();
    RelationshipTypeTokenRecord record = new RelationshipTypeTokenRecord(id);
    record.setInUse(true);
    record.setCreated();
    Collection<DynamicRecord> nameRecords = typeStore.allocateNameRecords(encodeString(name));
    record.setNameId((int) first(nameRecords).getId());
    record.addNameRecords(nameRecords);
    typeStore.updateRecord(record);
    relationshipTypeTokens.addToken(name, id);
    return id;
  }

  private NodeStore getNodeStore() {
    return neoStore.getNodeStore();
  }

  private RelationshipStore getRelationshipStore() {
    return neoStore.getRelationshipStore();
  }

  private PropertyStore getPropertyStore() {
    return neoStore.getPropertyStore();
  }

  private PropertyKeyTokenStore getPropertyKeyTokenStore() {
    return neoStore.getPropertyKeyTokenStore();
  }

  private RelationshipTypeTokenStore getRelationshipTypeStore() {
    return neoStore.getRelationshipTypeTokenStore();
  }

  private SchemaStore getSchemaStore() {
    return neoStore.getSchemaStore();
  }

  private RecordProxy<Long, NodeRecord, Void> getNodeRecord(long id) {
    if (id < 0 || id >= getNodeStore().getHighId()) {
      throw new NotFoundException("id=" + id);
    }
    return recordAccess.getNodeRecords().getOrLoad(id, null);
  }

  private RecordProxy<Long, RelationshipRecord, Void> getRelationshipRecord(long id) {
    if (id < 0 || id >= getRelationshipStore().getHighId()) {
      throw new NotFoundException("id=" + id);
    }
    return recordAccess.getRelRecords().getOrLoad(id, null);
  }

  @Override
  public String getStoreDir() {
    return storeDir.getPath();
  }

  // needed by lucene-index
  public IndexConfigStore getIndexStore() {
    return this.indexStore;
  }

  public IdGeneratorFactory getIdGeneratorFactory() {
    return idGeneratorFactory;
  }

  private void dumpConfiguration(Map<String, String> config) {
    for (String key : config.keySet()) {
      Object value = config.get(key);
      if (value != null) {
        // TODO no, No, NO NO NO!!! No. Pass in the PrintStream instead.
        System.out.println(key + "=" + value);
      }
    }
  }

  private class BatchSchemaActions implements InternalSchemaActions {
    @Override
    public IndexDefinition createIndexDefinition(Label label, String propertyKey) {
      createIndexRule(label, propertyKey);
      return new IndexDefinitionImpl(this, label, propertyKey, false);
    }

    @Override
    public void dropIndexDefinitions(Label label, String propertyKey) {
      throw unsupportedException();
    }

    @Override
    public ConstraintDefinition createPropertyUniquenessConstraint(
        Label label, String propertyKey) {
      int labelId = getOrCreateLabelId(label.name());
      int propertyKeyId = getOrCreatePropertyKeyId(propertyKey);
      createConstraintRule(new UniquenessConstraint(labelId, propertyKeyId));
      return new PropertyUniqueConstraintDefinition(this, label, propertyKey);
    }

    @Override
    public void dropPropertyUniquenessConstraint(Label label, String propertyKey) {
      throw unsupportedException();
    }

    @Override
    public String getUserMessage(KernelException e) {
      throw unsupportedException();
    }

    @Override
    public void assertInUnterminatedTransaction() {
      // BatchInserterImpl always is expected to be running in one big single "transaction"
    }

    private UnsupportedOperationException unsupportedException() {
      return new UnsupportedOperationException("Batch inserter doesn't support this");
    }
  }

  private class DependencyResolverImpl extends DependencyResolver.Adapter {
    @Override
    public <T> T resolveDependency(Class<T> type, SelectionStrategy selector)
        throws IllegalArgumentException {
      if (type.isInstance(fileSystem)) {
        return type.cast(fileSystem);
      }
      if (type.isInstance(config)) {
        return type.cast(config);
      }
      if (type.isInstance(logging)) {
        return type.cast(logging);
      }
      if (NeoStoreProvider.class.isAssignableFrom(type)) {
        return type.cast(
            new NeoStoreProvider() {
              @Override
              public NeoStore evaluate() {
                return neoStore;
              }
            });
      }
      throw new IllegalArgumentException("Unknown dependency " + type);
    }
  }
}
Beispiel #2
0
public class BatchInserterImpl implements BatchInserter {
  private static final long MAX_NODE_ID = IdType.NODE.getMaxValue();
  private static final long MAX_RELATIONSHIP_ID = IdType.RELATIONSHIP.getMaxValue();

  private final NeoStore neoStore;
  private final IndexStore indexStore;
  private final File storeDir;

  private final PropertyIndexHolder indexHolder;
  private final RelationshipTypeHolder typeHolder;

  private final IdGeneratorFactory idGeneratorFactory;

  private final StringLogger msgLog;
  private final FileSystemAbstraction fileSystem;

  BatchInserterImpl(String storeDir) {
    this(storeDir, new HashMap<String, String>());
  }

  BatchInserterImpl(String storeDir, Map<String, String> stringParams) {
    this(storeDir, new DefaultFileSystemAbstraction(), stringParams);
  }

  BatchInserterImpl(
      String storeDir, FileSystemAbstraction fileSystem, Map<String, String> stringParams) {
    this.fileSystem = fileSystem;
    this.storeDir = new File(FileUtils.fixSeparatorsInPath(storeDir));

    rejectAutoUpgrade(stringParams);
    msgLog = StringLogger.loggerDirectory(fileSystem, this.storeDir);
    Map<String, String> params = getDefaultParams();
    params.put(GraphDatabaseSettings.use_memory_mapped_buffers.name(), Settings.FALSE);
    params.put(InternalAbstractGraphDatabase.Configuration.store_dir.name(), storeDir);
    params.putAll(stringParams);

    Config config = new Config(params, GraphDatabaseSettings.class);
    boolean dump = config.get(GraphDatabaseSettings.dump_configuration);
    this.idGeneratorFactory = new DefaultIdGeneratorFactory();

    StoreFactory sf =
        new StoreFactory(
            config,
            idGeneratorFactory,
            new DefaultWindowPoolFactory(),
            fileSystem,
            StringLogger.DEV_NULL,
            null);

    File store = fixPath(this.storeDir, sf);

    if (dump) {
      dumpConfiguration(params);
    }
    msgLog.logMessage(Thread.currentThread() + " Starting BatchInserter(" + this + ")");
    neoStore = sf.newNeoStore(store);
    if (!neoStore.isStoreOk()) {
      throw new IllegalStateException(storeDir + " store is not cleanly shutdown.");
    }
    neoStore.makeStoreOk();
    NameData[] indexes = getPropertyIndexStore().getNames(10000);
    indexHolder = new PropertyIndexHolder(indexes);
    NameData[] types = getRelationshipTypeStore().getNames(Integer.MAX_VALUE);
    typeHolder = new RelationshipTypeHolder(types);
    indexStore = new IndexStore(this.storeDir, fileSystem);
    indexStore.start();
  }

  private Map<String, String> getDefaultParams() {
    Map<String, String> params = new HashMap<String, String>();
    params.put("neostore.nodestore.db.mapped_memory", "20M");
    params.put("neostore.propertystore.db.mapped_memory", "90M");
    params.put("neostore.propertystore.db.index.mapped_memory", "1M");
    params.put("neostore.propertystore.db.index.keys.mapped_memory", "1M");
    params.put("neostore.propertystore.db.strings.mapped_memory", "130M");
    params.put("neostore.propertystore.db.arrays.mapped_memory", "130M");
    params.put("neostore.relationshipstore.db.mapped_memory", "50M");
    return params;
  }

  @Override
  public boolean nodeHasProperty(long node, String propertyName) {
    return primitiveHasProperty(getNodeRecord(node), propertyName);
  }

  @Override
  public boolean relationshipHasProperty(long relationship, String propertyName) {
    return primitiveHasProperty(getRelationshipRecord(relationship), propertyName);
  }

  @Override
  public void setNodeProperty(long node, String propertyName, Object propertyValue) {
    NodeRecord nodeRec = getNodeRecord(node);
    if (setPrimitiveProperty(nodeRec, propertyName, propertyValue)) {
      getNodeStore().updateRecord(nodeRec);
    }
  }

  @Override
  public void setRelationshipProperty(
      long relationship, String propertyName, Object propertyValue) {
    RelationshipRecord relRec = getRelationshipRecord(relationship);
    if (setPrimitiveProperty(relRec, propertyName, propertyValue)) {
      getRelationshipStore().updateRecord(relRec);
    }
  }

  @Override
  public void removeNodeProperty(long node, String propertyName) {
    NodeRecord nodeRec = getNodeRecord(node);
    if (removePrimitiveProperty(nodeRec, propertyName)) {
      getNodeStore().updateRecord(nodeRec);
    }
  }

  @Override
  public void removeRelationshipProperty(long relationship, String propertyName) {
    RelationshipRecord relationshipRec = getRelationshipRecord(relationship);
    if (removePrimitiveProperty(relationshipRec, propertyName)) {
      getRelationshipStore().updateRecord(relationshipRec);
    }
  }

  private boolean removePrimitiveProperty(PrimitiveRecord primitive, String property) {
    PropertyRecord current = null;
    PropertyBlock target = null;
    long nextProp = primitive.getNextProp();
    int propIndex = indexHolder.getKeyId(property);
    if (nextProp == Record.NO_NEXT_PROPERTY.intValue() || propIndex == -1) {
      // No properties or no one has that property, nothing changed
      return false;
    }
    while (nextProp != Record.NO_NEXT_PROPERTY.intValue()) {
      current = getPropertyStore().getRecord(nextProp);
      if ((target = current.removePropertyBlock(propIndex)) != null) {
        if (target.isLight()) {
          getPropertyStore().makeHeavy(target);
        }
        for (DynamicRecord dynRec : target.getValueRecords()) {
          current.addDeletedRecord(dynRec);
        }
        break;
      }
      nextProp = current.getNextProp();
    }
    if (current.size() > 0) {
      getPropertyStore().updateRecord(current);
      return false;
    } else {
      return unlinkPropertyRecord(current, primitive);
    }
  }

  private boolean unlinkPropertyRecord(PropertyRecord propRecord, PrimitiveRecord primitive) {
    assert propRecord.size() == 0;
    boolean primitiveChanged = false;
    long prevProp = propRecord.getPrevProp();
    long nextProp = propRecord.getNextProp();
    if (primitive.getNextProp() == propRecord.getId()) {
      assert propRecord.getPrevProp() == Record.NO_PREVIOUS_PROPERTY.intValue()
          : propRecord + " for " + primitive;
      primitive.setNextProp(nextProp);
      primitiveChanged = true;
    }
    if (prevProp != Record.NO_PREVIOUS_PROPERTY.intValue()) {
      PropertyRecord prevPropRecord = getPropertyStore().getRecord(prevProp);
      assert prevPropRecord.inUse() : prevPropRecord + "->" + propRecord + " for " + primitive;
      prevPropRecord.setNextProp(nextProp);
      getPropertyStore().updateRecord(prevPropRecord);
    }
    if (nextProp != Record.NO_NEXT_PROPERTY.intValue()) {
      PropertyRecord nextPropRecord = getPropertyStore().getRecord(nextProp);
      assert nextPropRecord.inUse() : propRecord + "->" + nextPropRecord + " for " + primitive;
      nextPropRecord.setPrevProp(prevProp);
      getPropertyStore().updateRecord(nextPropRecord);
    }
    propRecord.setInUse(false);
    /*
     *  The following two are not needed - the above line does all the work (PropertyStore
     *  does not write out the prev/next for !inUse records). It is nice to set this
     *  however to check for consistency when assertPropertyChain().
     */
    propRecord.setPrevProp(Record.NO_PREVIOUS_PROPERTY.intValue());
    propRecord.setNextProp(Record.NO_NEXT_PROPERTY.intValue());
    getPropertyStore().updateRecord(propRecord);
    return primitiveChanged;
  }

  /** @return true if the passed primitive needs updating in the store. */
  private boolean setPrimitiveProperty(PrimitiveRecord primitive, String name, Object value) {
    boolean result = false;
    long nextProp = primitive.getNextProp();
    int index = indexHolder.getKeyId(name);

    if (index == -1) {
      index = createNewPropertyIndex(name);
    }
    PropertyBlock block = new PropertyBlock();
    getPropertyStore().encodeValue(block, index, value);
    int size = block.getSize();

    /*
     * current is the current record traversed
     * thatFits is the earliest record that can host the block
     * thatHas is the record that already has a block for this index
     */
    PropertyRecord current = null, thatFits = null, thatHas = null;
    /*
     * We keep going while there are records or until we both found the
     * property if it exists and the place to put it, if exists.
     */
    while (!(nextProp == Record.NO_NEXT_PROPERTY.intValue()
        || (thatHas != null && thatFits != null))) {
      current = getPropertyStore().getRecord(nextProp);
      /*
       * current.getPropertyBlock() is cheap but not free. If we already
       * have found thatHas, then we can skip this lookup.
       */
      if (thatHas == null && current.getPropertyBlock(index) != null) {
        thatHas = current;
        PropertyBlock removed = thatHas.removePropertyBlock(index);
        if (removed.isLight()) {
          getPropertyStore().makeHeavy(removed);
          for (DynamicRecord dynRec : removed.getValueRecords()) {
            thatHas.addDeletedRecord(dynRec);
          }
        }
        getPropertyStore().updateRecord(thatHas);
      }
      /*
       * We check the size after we remove - potentially we can put in the same record.
       *
       * current.size() is cheap but not free. If we already found somewhere
       * where it fits, no need to look again.
       */
      if (thatFits == null && (PropertyType.getPayloadSize() - current.size() >= size)) {
        thatFits = current;
      }
      nextProp = current.getNextProp();
    }
    /*
     * thatHas is of no importance here. We know that the block is definitely not there.
     * However, we can be sure that if the property existed, thatHas is not null and does
     * not contain the block.
     *
     * thatFits is interesting. If null, we need to create a new record and link, otherwise
     * just add the block there.
     */
    if (thatFits == null) {
      thatFits = new PropertyRecord(getPropertyStore().nextId());
      thatFits.setInUse(true);
      result = true;

      if (primitive.getNextProp() != Record.NO_NEXT_PROPERTY.intValue()) {
        PropertyRecord first = getPropertyStore().getRecord(primitive.getNextProp());
        thatFits.setNextProp(first.getId());
        first.setPrevProp(thatFits.getId());
        getPropertyStore().updateRecord(first);
      }
      primitive.setNextProp(thatFits.getId());
    }
    thatFits.addPropertyBlock(block);
    getPropertyStore().updateRecord(thatFits);
    return result;
  }

  private boolean primitiveHasProperty(PrimitiveRecord record, String propertyName) {
    long nextProp = record.getNextProp();
    int propertyIndex = indexHolder.getKeyId(propertyName);
    if (nextProp == Record.NO_NEXT_PROPERTY.intValue() || propertyIndex == -1) {
      return false;
    }

    PropertyRecord current = null;
    while (nextProp != Record.NO_NEXT_PROPERTY.intValue()) {
      current = getPropertyStore().getRecord(nextProp);
      if (current.getPropertyBlock(propertyIndex) != null) {
        return true;
      }
      nextProp = current.getNextProp();
    }
    return false;
  }

  private void rejectAutoUpgrade(Map<String, String> params) {
    if (parseBoolean(params.get(GraphDatabaseSettings.allow_store_upgrade.name()))) {
      throw new IllegalArgumentException(
          "Batch inserter is not allowed to do upgrade of a store"
              + ", use "
              + EmbeddedGraphDatabase.class.getSimpleName()
              + " instead");
    }
  }

  @Override
  public long createNode(Map<String, Object> properties) {
    return internalCreateNode(getNodeStore().nextId(), properties);
  }

  private long internalCreateNode(long nodeId, Map<String, Object> properties) {
    NodeRecord nodeRecord =
        new NodeRecord(
            nodeId, Record.NO_NEXT_RELATIONSHIP.intValue(), Record.NO_NEXT_PROPERTY.intValue());
    nodeRecord.setInUse(true);
    nodeRecord.setCreated();
    nodeRecord.setNextProp(createPropertyChain(properties));
    getNodeStore().updateRecord(nodeRecord);
    return nodeId;
  }

  @Override
  public void createNode(long id, Map<String, Object> properties) {
    if (id < 0 || id > MAX_NODE_ID) {
      throw new IllegalArgumentException("id=" + id);
    }
    if (id == IdGeneratorImpl.INTEGER_MINUS_ONE) {
      throw new IllegalArgumentException("id " + id + " is reserved for internal use");
    }
    long nodeId = id;
    NodeStore nodeStore = neoStore.getNodeStore();
    if (neoStore.getNodeStore().loadLightNode(nodeId) != null) {
      throw new IllegalArgumentException("id=" + id + " already in use");
    }
    long highId = nodeStore.getHighId();
    if (highId <= id) {
      nodeStore.setHighId(nodeId + 1);
    }
    internalCreateNode(nodeId, properties);
  }

  @Override
  public long createRelationship(
      long node1, long node2, RelationshipType type, Map<String, Object> properties) {
    NodeRecord firstNode = getNodeRecord(node1);
    NodeRecord secondNode = getNodeRecord(node2);
    int typeId = typeHolder.getTypeId(type.name());
    if (typeId == -1) {
      typeId = createNewRelationshipType(type.name());
    }
    long id = getRelationshipStore().nextId();
    RelationshipRecord record = new RelationshipRecord(id, node1, node2, typeId);
    record.setInUse(true);
    record.setCreated();
    connectRelationship(firstNode, secondNode, record);
    getNodeStore().updateRecord(firstNode);
    getNodeStore().updateRecord(secondNode);
    record.setNextProp(createPropertyChain(properties));
    getRelationshipStore().updateRecord(record);
    return id;
  }

  private void connectRelationship(
      NodeRecord firstNode, NodeRecord secondNode, RelationshipRecord rel) {
    assert firstNode.getNextRel() != rel.getId();
    assert secondNode.getNextRel() != rel.getId();
    rel.setFirstNextRel(firstNode.getNextRel());
    rel.setSecondNextRel(secondNode.getNextRel());
    connect(firstNode, rel);
    connect(secondNode, rel);
    firstNode.setNextRel(rel.getId());
    secondNode.setNextRel(rel.getId());
  }

  private void connect(NodeRecord node, RelationshipRecord rel) {
    if (node.getNextRel() != Record.NO_NEXT_RELATIONSHIP.intValue()) {
      RelationshipRecord nextRel = getRelationshipStore().getRecord(node.getNextRel());
      boolean changed = false;
      if (nextRel.getFirstNode() == node.getId()) {
        nextRel.setFirstPrevRel(rel.getId());
        changed = true;
      }
      if (nextRel.getSecondNode() == node.getId()) {
        nextRel.setSecondPrevRel(rel.getId());
        changed = true;
      }
      if (!changed) {
        throw new InvalidRecordException(node + " dont match " + nextRel);
      }
      getRelationshipStore().updateRecord(nextRel);
    }
  }

  @Override
  public void setNodeProperties(long node, Map<String, Object> properties) {
    NodeRecord record = getNodeRecord(node);
    if (record.getNextProp() != Record.NO_NEXT_PROPERTY.intValue()) {
      deletePropertyChain(record.getNextProp());
      /*
       * Batch inserter does not make any attempt to maintain the store's
       * integrity. It makes sense however to keep some things intact where
       * the cost is relatively low. So here, when we delete the property
       * chain we first make sure that the node record (or the relationship
       * record below) does not point anymore to the deleted properties. This
       * way, if during creation, something goes wrong, it will not have the properties
       * expected instead of throwing invalid record exceptions.
       */
      record.setNextProp(Record.NO_NEXT_PROPERTY.intValue());
      getNodeStore().updateRecord(record);
    }
    record.setNextProp(createPropertyChain(properties));
    getNodeStore().updateRecord(record);
  }

  @Override
  public void setRelationshipProperties(long rel, Map<String, Object> properties) {
    RelationshipRecord record = getRelationshipRecord(rel);
    if (record.getNextProp() != Record.NO_NEXT_PROPERTY.intValue()) {
      deletePropertyChain(record.getNextProp());
      /*
       * See setNodeProperties above for an explanation of what goes on
       * here
       */
      record.setNextProp(Record.NO_NEXT_PROPERTY.intValue());
      getRelationshipStore().updateRecord(record);
    }
    record.setNextProp(createPropertyChain(properties));
    getRelationshipStore().updateRecord(record);
  }

  @Override
  public boolean nodeExists(long nodeId) {
    return neoStore.getNodeStore().loadLightNode(nodeId) != null;
  }

  @Override
  public Map<String, Object> getNodeProperties(long nodeId) {
    NodeRecord record = getNodeRecord(nodeId);
    if (record.getNextProp() != Record.NO_NEXT_PROPERTY.intValue()) {
      return getPropertyChain(record.getNextProp());
    }
    return Collections.emptyMap();
  }

  @Override
  public Iterable<Long> getRelationshipIds(long nodeId) {
    NodeRecord nodeRecord = getNodeRecord(nodeId);
    long nextRel = nodeRecord.getNextRel();
    List<Long> ids = new ArrayList<Long>();
    while (nextRel != Record.NO_NEXT_RELATIONSHIP.intValue()) {
      RelationshipRecord relRecord = getRelationshipRecord(nextRel);
      ids.add(relRecord.getId());
      long firstNode = relRecord.getFirstNode();
      long secondNode = relRecord.getSecondNode();
      if (firstNode == nodeId) {
        nextRel = relRecord.getFirstNextRel();
      } else if (secondNode == nodeId) {
        nextRel = relRecord.getSecondNextRel();
      } else {
        throw new InvalidRecordException(
            "Node["
                + nodeId
                + "] not part of firstNode["
                + firstNode
                + "] or secondNode["
                + secondNode
                + "]");
      }
    }
    return ids;
  }

  @Override
  public Iterable<BatchRelationship> getRelationships(long nodeId) {
    NodeRecord nodeRecord = getNodeRecord(nodeId);
    long nextRel = nodeRecord.getNextRel();
    List<BatchRelationship> rels = new ArrayList<BatchRelationship>();
    while (nextRel != Record.NO_NEXT_RELATIONSHIP.intValue()) {
      RelationshipRecord relRecord = getRelationshipRecord(nextRel);
      RelationshipType type = new RelationshipTypeImpl(typeHolder.getName(relRecord.getType()));
      rels.add(
          new BatchRelationship(
              relRecord.getId(), relRecord.getFirstNode(), relRecord.getSecondNode(), type));
      long firstNode = relRecord.getFirstNode();
      long secondNode = relRecord.getSecondNode();
      if (firstNode == nodeId) {
        nextRel = relRecord.getFirstNextRel();
      } else if (secondNode == nodeId) {
        nextRel = relRecord.getSecondNextRel();
      } else {
        throw new InvalidRecordException(
            "Node["
                + nodeId
                + "] not part of firstNode["
                + firstNode
                + "] or secondNode["
                + secondNode
                + "]");
      }
    }
    return rels;
  }

  public Iterable<SimpleRelationship> getSimpleRelationships(long nodeId) {
    NodeRecord nodeRecord = getNodeRecord(nodeId);
    long nextRel = nodeRecord.getNextRel();
    List<SimpleRelationship> rels = new ArrayList<SimpleRelationship>();
    while (nextRel != Record.NO_NEXT_RELATIONSHIP.intValue()) {
      RelationshipRecord relRecord = getRelationshipRecord(nextRel);
      RelationshipType type = new RelationshipTypeImpl(typeHolder.getName(relRecord.getType()));
      rels.add(
          new SimpleRelationship(
              relRecord.getId(), relRecord.getFirstNode(), relRecord.getSecondNode(), type));
      long firstNode = relRecord.getFirstNode();
      long secondNode = relRecord.getSecondNode();
      if (firstNode == nodeId) {
        nextRel = relRecord.getFirstNextRel();
      } else if (secondNode == nodeId) {
        nextRel = relRecord.getSecondNextRel();
      } else {
        throw new InvalidRecordException(
            "Node["
                + nodeId
                + "] not part of firstNode["
                + firstNode
                + "] or secondNode["
                + secondNode
                + "]");
      }
    }
    return rels;
  }

  @Override
  public BatchRelationship getRelationshipById(long relId) {
    RelationshipRecord record = getRelationshipRecord(relId);
    RelationshipType type = new RelationshipTypeImpl(typeHolder.getName(record.getType()));
    return new BatchRelationship(
        record.getId(), record.getFirstNode(), record.getSecondNode(), type);
  }

  public SimpleRelationship getSimpleRelationshipById(long relId) {
    RelationshipRecord record = getRelationshipRecord(relId);
    RelationshipType type = new RelationshipTypeImpl(typeHolder.getName(record.getType()));
    return new SimpleRelationship(
        record.getId(), record.getFirstNode(), record.getSecondNode(), type);
  }

  @Override
  public Map<String, Object> getRelationshipProperties(long relId) {
    RelationshipRecord record = getRelationshipRecord(relId);
    if (record.getNextProp() != Record.NO_NEXT_PROPERTY.intValue()) {
      return getPropertyChain(record.getNextProp());
    }
    return Collections.emptyMap();
  }

  @Override
  public void shutdown() {
    neoStore.close();
    msgLog.logMessage(
        Thread.currentThread() + " Clean shutdown on BatchInserter(" + this + ")", true);
    msgLog.close();
  }

  @Override
  public String toString() {
    return "EmbeddedBatchInserter[" + storeDir + "]";
  }

  private static class RelationshipTypeImpl implements RelationshipType {
    private final String name;

    RelationshipTypeImpl(String name) {
      this.name = name;
    }

    @Override
    public String name() {
      return name;
    }
  }

  private long createPropertyChain(Map<String, Object> properties) {
    if (properties == null || properties.isEmpty()) {
      return Record.NO_NEXT_PROPERTY.intValue();
    }
    PropertyStore propStore = getPropertyStore();
    List<PropertyRecord> propRecords = new ArrayList<PropertyRecord>();
    PropertyRecord currentRecord = new PropertyRecord(propStore.nextId());
    currentRecord.setInUse(true);
    currentRecord.setCreated();
    propRecords.add(currentRecord);
    for (Entry<String, Object> entry : properties.entrySet()) {
      int keyId = indexHolder.getKeyId(entry.getKey());
      if (keyId == -1) {
        keyId = createNewPropertyIndex(entry.getKey());
      }

      PropertyBlock block = new PropertyBlock();
      propStore.encodeValue(block, keyId, entry.getValue());
      if (currentRecord.size() + block.getSize() > PropertyType.getPayloadSize()) {
        // Here it means the current block is done for
        PropertyRecord prevRecord = currentRecord;
        // Create new record
        long propertyId = propStore.nextId();
        currentRecord = new PropertyRecord(propertyId);
        currentRecord.setInUse(true);
        currentRecord.setCreated();
        // Set up links
        prevRecord.setNextProp(propertyId);
        currentRecord.setPrevProp(prevRecord.getId());
        propRecords.add(currentRecord);
        // Now current is ready to start picking up blocks
      }
      currentRecord.addPropertyBlock(block);
    }
    /*
     * Add the property records in reverse order, which means largest
     * id first. That is to make sure we expand the property store file
     * only once.
     */
    for (int i = propRecords.size() - 1; i >= 0; i--) {
      propStore.updateRecord(propRecords.get(i));
    }
    /*
     *  0 will always exist, if the map was empty we wouldn't be here
     *  and even one property will create at least one record.
     */
    return propRecords.get(0).getId();
  }

  private void deletePropertyChain(long nextProp) {
    PropertyStore propStore = getPropertyStore();
    while (nextProp != Record.NO_NEXT_PROPERTY.intValue()) {
      PropertyRecord propRecord = propStore.getRecord(nextProp);
      /*
       *  The only reason to loop over the blocks is to handle the dynamic
       *  records that possibly hang under them. Otherwise, we could just
       *  set the property record not in use and be done with it. The
       *  residue of the convenience is that we do not remove individual
       *  property blocks - we just mark the whole record as !inUse.
       */
      for (PropertyBlock propBlock : propRecord.getPropertyBlocks()) {
        if (propBlock.isLight()) {
          propStore.makeHeavy(propBlock);
        }
        for (DynamicRecord rec : propBlock.getValueRecords()) {
          rec.setInUse(false);
          propRecord.addDeletedRecord(rec);
        }
      }
      propRecord.setInUse(false);
      nextProp = propRecord.getNextProp();
      propStore.updateRecord(propRecord);
    }
  }

  private Map<String, Object> getPropertyChain(long nextProp) {
    PropertyStore propStore = getPropertyStore();
    Map<String, Object> properties = new HashMap<String, Object>();

    while (nextProp != Record.NO_NEXT_PROPERTY.intValue()) {
      PropertyRecord propRecord = propStore.getRecord(nextProp);
      for (PropertyBlock propBlock : propRecord.getPropertyBlocks()) {
        String key = indexHolder.getStringKey(propBlock.getKeyIndexId());
        PropertyData propertyData = propBlock.newPropertyData(propRecord);
        Object value =
            propertyData.getValue() != null
                ? propertyData.getValue()
                : propBlock.getType().getValue(propBlock, getPropertyStore());
        properties.put(key, value);
      }
      nextProp = propRecord.getNextProp();
    }
    return properties;
  }

  private int createNewPropertyIndex(String stringKey) {
    PropertyIndexStore idxStore = getPropertyIndexStore();
    int keyId = (int) idxStore.nextId();
    PropertyIndexRecord record = new PropertyIndexRecord(keyId);
    record.setInUse(true);
    record.setCreated();
    int nameId = idxStore.nextNameId();
    record.setNameId(nameId);
    Collection<DynamicRecord> keyRecords =
        idxStore.allocateNameRecords(nameId, encodeString(stringKey));
    for (DynamicRecord keyRecord : keyRecords) {
      record.addNameRecord(keyRecord);
    }
    idxStore.updateRecord(record);
    indexHolder.addPropertyIndex(stringKey, keyId);
    return keyId;
  }

  private int createNewRelationshipType(String name) {
    RelationshipTypeStore typeStore = getRelationshipTypeStore();
    int id = (int) typeStore.nextId();
    RelationshipTypeRecord record = new RelationshipTypeRecord(id);
    record.setInUse(true);
    record.setCreated();
    int nameId = typeStore.nextNameId();
    record.setNameId(nameId);
    Collection<DynamicRecord> nameRecords =
        typeStore.allocateNameRecords(nameId, encodeString(name));
    for (DynamicRecord typeRecord : nameRecords) {
      record.addNameRecord(typeRecord);
    }
    typeStore.updateRecord(record);
    typeHolder.addRelationshipType(name, id);
    return id;
  }

  private NodeStore getNodeStore() {
    return neoStore.getNodeStore();
  }

  private PropertyStore getPropertyStore() {
    return neoStore.getPropertyStore();
  }

  private PropertyIndexStore getPropertyIndexStore() {
    return getPropertyStore().getIndexStore();
  }

  private RelationshipStore getRelationshipStore() {
    return neoStore.getRelationshipStore();
  }

  private RelationshipTypeStore getRelationshipTypeStore() {
    return neoStore.getRelationshipTypeStore();
  }

  private NodeRecord getNodeRecord(long id) {
    if (id < 0 || id >= getNodeStore().getHighId()) {
      throw new NotFoundException("id=" + id);
    }
    return getNodeStore().getRecord(id);
  }

  private RelationshipRecord getRelationshipRecord(long id) {
    if (id < 0 || id >= getRelationshipStore().getHighId()) {
      throw new NotFoundException("id=" + id);
    }
    return getRelationshipStore().getRecord(id);
  }

  private File fixPath(File dir, StoreFactory sf) {
    try {
      fileSystem.mkdirs(dir);
    } catch (IOException e) {
      throw new UnderlyingStorageException(
          "Unable to create directory path[" + storeDir + "] for Neo4j kernel store.");
    }

    File store = new File(dir, NeoStore.DEFAULT_NAME);
    if (!fileSystem.fileExists(store)) {
      sf.createNeoStore(store).close();
    }
    return store;
  }

  @Override
  public String getStoreDir() {
    return storeDir.getPath();
  }

  @Override
  public long getReferenceNode() {
    if (nodeExists(0)) {
      return 0;
    }
    return -1;
  }

  /** @deprecated as of Neo4j 1.7 */
  @Deprecated
  public GraphDatabaseService getBatchGraphDbService() {
    return new BatchGraphDatabaseImpl(this);
  }

  public IndexStore getIndexStore() {
    return this.indexStore;
  }

  public IdGeneratorFactory getIdGeneratorFactory() {
    return idGeneratorFactory;
  }

  private void dumpConfiguration(Map<String, String> config) {
    for (Object key : config.keySet()) {
      Object value = config.get(key);
      if (value instanceof String) {
        System.out.println(key + "=" + value);
      }
    }
  }
}
Beispiel #3
0
public class BatchInserterImpl implements BatchInserter {
  private static final long MAX_NODE_ID = IdType.NODE.getMaxValue();

  private final LifeSupport life;
  private final NeoStore neoStore;
  private final IndexStore indexStore;
  private final File storeDir;
  private final BatchTokenHolder propertyKeyTokens;
  private final BatchTokenHolder relationshipTypeTokens;
  private final BatchTokenHolder labelTokens;
  private final IdGeneratorFactory idGeneratorFactory;
  private final SchemaIndexProviderMap schemaIndexProviders;
  private final LabelScanStore labelScanStore;
  // TODO use Logging instead
  private final StringLogger msgLog;
  private final Logging logging;
  private final FileSystemAbstraction fileSystem;
  private final SchemaCache schemaCache;
  private final Config config;
  private final BatchInserterImpl.BatchSchemaActions actions;
  private final StoreLocker storeLocker;

  private final FunctionFromPrimitiveLong<Label> labelIdToLabelFunction =
      new FunctionFromPrimitiveLong<Label>() {
        @Override
        public Label apply(long from) {
          return label(labelTokens.nameOf(safeCastLongToInt(from)));
        }
      };

  private boolean isShutdown = false;

  // Helper structure for setNodeProperty
  private Set<PropertyRecord> updatedRecords = new HashSet<PropertyRecord>();

  BatchInserterImpl(String storeDir, Map<String, String> stringParams) {
    this(
        storeDir,
        new DefaultFileSystemAbstraction(),
        stringParams,
        Collections.<KernelExtensionFactory<?>>emptyList());
  }

  BatchInserterImpl(
      String storeDir,
      FileSystemAbstraction fileSystem,
      Map<String, String> stringParams,
      Iterable<KernelExtensionFactory<?>> kernelExtensions) {
    life = new LifeSupport();
    this.fileSystem = fileSystem;
    this.storeDir = new File(FileUtils.fixSeparatorsInPath(storeDir));

    rejectAutoUpgrade(stringParams);
    msgLog = StringLogger.loggerDirectory(fileSystem, this.storeDir);
    logging = new SingleLoggingService(msgLog);
    Map<String, String> params = getDefaultParams();
    params.put(GraphDatabaseSettings.use_memory_mapped_buffers.name(), Settings.FALSE);
    params.put(InternalAbstractGraphDatabase.Configuration.store_dir.name(), storeDir);
    params.putAll(stringParams);

    storeLocker = new StoreLocker(fileSystem);
    storeLocker.checkLock(this.storeDir);

    config = new Config(params, GraphDatabaseSettings.class);
    boolean dump = config.get(GraphDatabaseSettings.dump_configuration);
    this.idGeneratorFactory = new DefaultIdGeneratorFactory();

    StoreFactory sf =
        new StoreFactory(
            config, idGeneratorFactory, new DefaultWindowPoolFactory(), fileSystem, msgLog, null);

    File store = fixPath(this.storeDir, sf);

    if (dump) {
      dumpConfiguration(params);
    }
    msgLog.logMessage(Thread.currentThread() + " Starting BatchInserter(" + this + ")");
    neoStore = sf.newNeoStore(store);
    if (!neoStore.isStoreOk()) {
      throw new IllegalStateException(storeDir + " store is not cleanly shutdown.");
    }
    neoStore.makeStoreOk();
    Token[] indexes = getPropertyKeyTokenStore().getTokens(10000);
    propertyKeyTokens = new BatchTokenHolder(indexes);
    labelTokens = new BatchTokenHolder(neoStore.getLabelTokenStore().getTokens(Integer.MAX_VALUE));
    Token[] types = getRelationshipTypeStore().getTokens(Integer.MAX_VALUE);
    relationshipTypeTokens = new BatchTokenHolder(types);
    indexStore = life.add(new IndexStore(this.storeDir, fileSystem));
    schemaCache = new SchemaCache(neoStore.getSchemaStore());

    KernelExtensions extensions =
        life.add(
            new KernelExtensions(
                kernelExtensions,
                config,
                new DependencyResolverImpl(),
                UnsatisfiedDependencyStrategies.ignore()));

    life.start();

    SchemaIndexProvider provider =
        extensions.resolveDependency(
            SchemaIndexProvider.class, SchemaIndexProvider.HIGHEST_PRIORITIZED_OR_NONE);
    schemaIndexProviders = new DefaultSchemaIndexProviderMap(provider);
    labelScanStore =
        life.add(
            extensions
                .resolveDependency(
                    LabelScanStoreProvider.class, LabelScanStoreProvider.HIGHEST_PRIORITIZED)
                .getLabelScanStore());
    actions = new BatchSchemaActions();
  }

  private Map<String, String> getDefaultParams() {
    Map<String, String> params = new HashMap<>();
    params.put("neostore.nodestore.db.mapped_memory", "20M");
    params.put("neostore.propertystore.db.mapped_memory", "90M");
    params.put("neostore.propertystore.db.index.mapped_memory", "1M");
    params.put("neostore.propertystore.db.index.keys.mapped_memory", "1M");
    params.put("neostore.propertystore.db.strings.mapped_memory", "130M");
    params.put("neostore.propertystore.db.arrays.mapped_memory", "130M");
    params.put("neostore.relationshipstore.db.mapped_memory", "50M");
    return params;
  }

  @Override
  public boolean nodeHasProperty(long node, String propertyName) {
    return primitiveHasProperty(getNodeRecord(node), propertyName);
  }

  @Override
  public boolean relationshipHasProperty(long relationship, String propertyName) {
    return primitiveHasProperty(getRelationshipRecord(relationship), propertyName);
  }

  @Override
  public void setNodeProperty(long node, String propertyName, Object newValue) {
    NodeRecord nodeRec = getNodeRecord(node);
    if (setPrimitiveProperty(nodeRec, propertyName, newValue)) {
      getNodeStore().updateRecord(nodeRec);
    }
  }

  @Override
  public void setRelationshipProperty(
      long relationship, String propertyName, Object propertyValue) {
    RelationshipRecord relRec = getRelationshipRecord(relationship);
    if (setPrimitiveProperty(relRec, propertyName, propertyValue)) {
      getRelationshipStore().updateRecord(relRec);
    }
  }

  @Override
  public void removeNodeProperty(long node, String propertyName) {
    NodeRecord nodeRec = getNodeRecord(node);
    if (removePrimitiveProperty(nodeRec, propertyName)) {
      getNodeStore().updateRecord(nodeRec);
    }
  }

  @Override
  public void removeRelationshipProperty(long relationship, String propertyName) {
    RelationshipRecord relationshipRec = getRelationshipRecord(relationship);
    if (removePrimitiveProperty(relationshipRec, propertyName)) {
      getRelationshipStore().updateRecord(relationshipRec);
    }
  }

  @Override
  public IndexCreator createDeferredSchemaIndex(Label label) {
    return new IndexCreatorImpl(actions, label);
  }

  private void createIndexRule(Label label, String propertyKey) {
    // TODO: Do not create duplicate index

    SchemaStore schemaStore = getSchemaStore();
    IndexRule schemaRule =
        IndexRule.indexRule(
            schemaStore.nextId(),
            getOrCreateLabelId(label.name()),
            getOrCreatePropertyKeyId(propertyKey),
            this.schemaIndexProviders.getDefaultProvider().getProviderDescriptor());
    for (DynamicRecord record : schemaStore.allocateFrom(schemaRule)) {
      schemaStore.updateRecord(record);
    }
    schemaCache.addSchemaRule(schemaRule);
  }

  private void repopulateAllIndexes() throws IOException {
    final IndexRule[] rules = getIndexesNeedingPopulation();
    final IndexPopulator[] populators = new IndexPopulator[rules.length];
    // the store is uncontended at this point, so creating a local LockService is safe.
    LockService locks = new ReentrantLockService();
    IndexStoreView storeView = new NeoStoreIndexStoreView(locks, neoStore);

    final int[] labelIds = new int[rules.length];
    final int[] propertyKeyIds = new int[rules.length];

    for (int i = 0; i < labelIds.length; i++) {
      IndexRule rule = rules[i];
      labelIds[i] = rule.getLabel();
      propertyKeyIds[i] = rule.getPropertyKey();

      populators[i] =
          schemaIndexProviders
              .apply(rule.getProviderDescriptor())
              .getPopulator(rule.getId(), new IndexConfiguration(rule.isConstraintIndex()));
      populators[i].create();
    }

    Visitor<NodePropertyUpdate, IOException> propertyUpdateVisitor =
        new Visitor<NodePropertyUpdate, IOException>() {
          @Override
          public boolean visit(NodePropertyUpdate update) throws IOException {
            // Do a lookup from which property has changed to a list of indexes worried about that
            // property.
            int propertyKeyInQuestion = update.getPropertyKeyId();
            for (int i = 0; i < propertyKeyIds.length; i++) {
              if (propertyKeyIds[i] == propertyKeyInQuestion) {
                if (update.forLabel(labelIds[i])) {
                  try {
                    populators[i].add(update.getNodeId(), update.getValueAfter());
                  } catch (IndexEntryConflictException conflict) {
                    throw conflict.notAllowed(rules[i].getLabel(), rules[i].getPropertyKey());
                  }
                }
              }
            }
            return true;
          }
        };

    NodeLabelUpdateVisitor labelUpdateVisitor = new NodeLabelUpdateVisitor();
    StoreScan<IOException> storeScan =
        storeView.visitNodes(labelIds, propertyKeyIds, propertyUpdateVisitor, labelUpdateVisitor);
    storeScan.run();

    for (IndexPopulator populator : populators) {
      populator.close(true);
    }
    labelUpdateVisitor.close();
  }

  private class NodeLabelUpdateVisitor implements Visitor<NodeLabelUpdate, IOException> {
    private final NodeLabelUpdate[] updateBatch = new NodeLabelUpdate[10000];
    private int cursor;

    @Override
    public boolean visit(NodeLabelUpdate update) throws IOException {
      if (cursor >= updateBatch.length) {
        writeAndResetBatch();
      }
      updateBatch[cursor++] = update;
      return true;
    }

    private void writeAndResetBatch() throws IOException {
      labelScanStore.updateAndCommit(iterator(cursor, updateBatch));
      cursor = 0;
    }

    void close() throws IOException {
      writeAndResetBatch();
    }
  }

  private IndexRule[] getIndexesNeedingPopulation() {
    List<IndexRule> indexesNeedingPopulation = new ArrayList<>();
    for (SchemaRule rule : schemaCache.schemaRules()) {
      if (rule.getKind().isIndex()) {
        IndexRule indexRule = (IndexRule) rule;
        SchemaIndexProvider provider =
            schemaIndexProviders.apply(indexRule.getProviderDescriptor());
        if (provider.getInitialState(indexRule.getId()) != InternalIndexState.FAILED) {
          indexesNeedingPopulation.add(indexRule);
        }
      }
    }
    return indexesNeedingPopulation.toArray(new IndexRule[indexesNeedingPopulation.size()]);
  }

  @Override
  public ConstraintCreator createDeferredConstraint(Label label) {
    return new BaseConstraintCreator(new BatchSchemaActions(), label);
  }

  private void createConstraintRule(UniquenessConstraint constraint) {
    // TODO: Do not create duplicate index

    SchemaStore schemaStore = getSchemaStore();

    long indexRuleId = schemaStore.nextId();
    long constraintRuleId = schemaStore.nextId();

    IndexRule indexRule =
        IndexRule.constraintIndexRule(
            indexRuleId,
            constraint.label(),
            constraint.propertyKeyId(),
            this.schemaIndexProviders.getDefaultProvider().getProviderDescriptor(),
            constraintRuleId);
    UniquenessConstraintRule constraintRule =
        UniquenessConstraintRule.uniquenessConstraintRule(
            schemaStore.nextId(), constraint.label(), constraint.propertyKeyId(), indexRuleId);

    for (DynamicRecord record : schemaStore.allocateFrom(constraintRule)) {
      schemaStore.updateRecord(record);
    }
    schemaCache.addSchemaRule(constraintRule);
    for (DynamicRecord record : schemaStore.allocateFrom(indexRule)) {
      schemaStore.updateRecord(record);
    }
    schemaCache.addSchemaRule(indexRule);
  }

  private boolean removePrimitiveProperty(PrimitiveRecord primitive, String property) {
    PropertyRecord current = null;
    PropertyBlock target;
    long nextProp = primitive.getNextProp();
    int propIndex = propertyKeyTokens.idOf(property);
    if (nextProp == Record.NO_NEXT_PROPERTY.intValue() || propIndex == -1) {
      // No properties or no one has that property, nothing changed
      return false;
    }
    while (nextProp != Record.NO_NEXT_PROPERTY.intValue()) {
      current = getPropertyStore().getRecord(nextProp);
      if ((target = current.removePropertyBlock(propIndex)) != null) {
        getPropertyStore().ensureHeavy(target);
        for (DynamicRecord dynRec : target.getValueRecords()) {
          dynRec.setInUse(false);
          current.addDeletedRecord(dynRec);
        }
        break;
      }
      nextProp = current.getNextProp();
    }
    assert current != null : "the if statement above prevents it";
    if (current.size() > 0) {
      getPropertyStore().updateRecord(current);
      return false;
    } else {
      current.setInUse(false);
      return unlinkPropertyRecord(current, primitive);
    }
  }

  private boolean unlinkPropertyRecord(PropertyRecord propRecord, PrimitiveRecord primitive) {
    assert propRecord.size() == 0;
    boolean primitiveChanged = false;
    long prevProp = propRecord.getPrevProp();
    long nextProp = propRecord.getNextProp();
    if (primitive.getNextProp() == propRecord.getId()) {
      assert propRecord.getPrevProp() == Record.NO_PREVIOUS_PROPERTY.intValue()
          : propRecord + " for " + primitive;
      primitive.setNextProp(nextProp);
      primitiveChanged = true;
    }
    if (prevProp != Record.NO_PREVIOUS_PROPERTY.intValue()) {
      PropertyRecord prevPropRecord = getPropertyStore().getRecord(prevProp);
      assert prevPropRecord.inUse() : prevPropRecord + "->" + propRecord + " for " + primitive;
      prevPropRecord.setNextProp(nextProp);
      getPropertyStore().updateRecord(prevPropRecord);
    }
    if (nextProp != Record.NO_NEXT_PROPERTY.intValue()) {
      PropertyRecord nextPropRecord = getPropertyStore().getRecord(nextProp);
      assert nextPropRecord.inUse() : propRecord + "->" + nextPropRecord + " for " + primitive;
      nextPropRecord.setPrevProp(prevProp);
      getPropertyStore().updateRecord(nextPropRecord);
    }
    propRecord.setInUse(false);
    /*
     *  The following two are not needed - the above line does all the work (PropertyStore
     *  does not write out the prev/next for !inUse records). It is nice to set this
     *  however to check for consistency when assertPropertyChain().
     */
    propRecord.setPrevProp(Record.NO_PREVIOUS_PROPERTY.intValue());
    propRecord.setNextProp(Record.NO_NEXT_PROPERTY.intValue());
    getPropertyStore().updateRecord(propRecord);
    return primitiveChanged;
  }

  /** @return true if the passed primitive needs updating in the store. */
  private boolean setPrimitiveProperty(PrimitiveRecord primitive, String name, Object value) {
    boolean result = false;
    long nextProp = primitive.getNextProp();
    int index = getOrCreatePropertyKeyId(name);
    PropertyBlock block = new PropertyBlock();
    getPropertyStore().encodeValue(block, index, value);
    int size = block.getSize();

    /*
     * current is the current record traversed
     * thatFits is the earliest record that can host the block
     * thatHas is the record that already has a block for this index
     */
    PropertyRecord current = null, thatFits = null, thatHas = null;
    updatedRecords.clear();

    /*
     * We keep going while there are records or until we both found the
     * property if it exists and the place to put it, if exists.
     */
    while (!(nextProp == Record.NO_NEXT_PROPERTY.intValue()
        || (thatHas != null && thatFits != null))) {
      current = getPropertyStore().getRecord(nextProp);
      /*
       * current.getPropertyBlock() is cheap but not free. If we already
       * have found thatHas, then we can skip this lookup.
       */
      if (thatHas == null && current.getPropertyBlock(index) != null) {
        thatHas = current;

        PropertyBlock removed = thatHas.removePropertyBlock(index);
        if (removed.isLight()) {
          getPropertyStore().makeHeavyIfLight(removed);
        }
        for (DynamicRecord dynRec : removed.getValueRecords()) {
          dynRec.setInUse(false);
          thatHas.addDeletedRecord(dynRec);
        }
        updatedRecords.add(thatHas);
      }
      /*
       * We check the size after we remove - potentially we can put in the same record.
       *
       * current.size() is cheap but not free. If we already found somewhere
       * where it fits, no need to look again.
       */
      if (thatFits == null && (PropertyType.getPayloadSize() - current.size() >= size)) {
        thatFits = current;
      }
      nextProp = current.getNextProp();
    }

    /*
     * If there was a record that contained a property for this key already, and if
     * we will not be able to re-use those records to store the new property,
     * release the old records.
     */
    if (thatHas != null && thatFits != thatHas) {
      PropertyBlock removed = thatHas.removePropertyBlock(index);
      getPropertyStore().ensureHeavy(removed);
      for (DynamicRecord dynRec : removed.getValueRecords()) {
        dynRec.setInUse(false);
        thatHas.addDeletedRecord(dynRec);
      }
      getPropertyStore().updateRecord(thatHas);
    }

    /*
     * thatHas is of no importance here. We know that the block is definitely not there.
     * However, we can be sure that if the property existed, thatHas is not null and does
     * not contain the block.
     *
     * thatFits is interesting. If null, we need to create a new record and link, otherwise
     * just add the block there.
     */
    if (thatFits == null) {
      thatFits = new PropertyRecord(getPropertyStore().nextId());
      thatFits.setInUse(true);
      result = true;

      if (primitive.getNextProp() != Record.NO_NEXT_PROPERTY.intValue()) {
        PropertyRecord first = getPropertyStore().getRecord(primitive.getNextProp());
        thatFits.setNextProp(first.getId());
        first.setPrevProp(thatFits.getId());
        getPropertyStore().updateRecord(first);
      }
      primitive.setNextProp(thatFits.getId());
    }

    thatFits.addPropertyBlock(block);
    updatedRecords.add(thatFits);

    // This ensures that a particular record is not updated twice in this method
    // It could lead to freeId being called multiple times for same id
    for (PropertyRecord updatedRecord : updatedRecords) {
      getPropertyStore().updateRecord(thatFits);
    }

    return result;
  }

  private int getOrCreatePropertyKeyId(String name) {
    int propertyKeyId = getPropertyKeyId(name);
    if (propertyKeyId == -1) {
      propertyKeyId = createNewPropertyKeyId(name);
    }
    return propertyKeyId;
  }

  public int getPropertyKeyId(String name) {
    return propertyKeyTokens.idOf(name);
  }

  public int getOrCreateLabelId(String name) {
    int labelId = getLabelId(name);
    if (labelId == -1) {
      labelId = createNewLabelId(name);
    }
    return labelId;
  }

  public int getLabelId(String name) {
    return labelTokens.idOf(name);
  }

  private boolean primitiveHasProperty(PrimitiveRecord record, String propertyName) {
    long nextProp = record.getNextProp();
    int propertyKeyId = propertyKeyTokens.idOf(propertyName);
    if (nextProp == Record.NO_NEXT_PROPERTY.intValue() || propertyKeyId == -1) {
      return false;
    }

    PropertyRecord current;
    while (nextProp != Record.NO_NEXT_PROPERTY.intValue()) {
      current = getPropertyStore().getRecord(nextProp);
      if (current.getPropertyBlock(propertyKeyId) != null) {
        return true;
      }
      nextProp = current.getNextProp();
    }
    return false;
  }

  private void rejectAutoUpgrade(Map<String, String> params) {
    if (parseBoolean(params.get(GraphDatabaseSettings.allow_store_upgrade.name()))) {
      throw new IllegalArgumentException(
          "Batch inserter is not allowed to do upgrade of a store"
              + ", use "
              + EmbeddedGraphDatabase.class.getSimpleName()
              + " instead");
    }
  }

  @Override
  public long createNode(Map<String, Object> properties, Label... labels) {
    return internalCreateNode(getNodeStore().nextId(), properties, labels);
  }

  private long internalCreateNode(long nodeId, Map<String, Object> properties, Label... labels) {
    NodeRecord nodeRecord =
        new NodeRecord(
            nodeId, Record.NO_NEXT_RELATIONSHIP.intValue(), Record.NO_NEXT_PROPERTY.intValue());
    nodeRecord.setInUse(true);
    nodeRecord.setCreated();
    nodeRecord.setNextProp(createPropertyChain(properties));

    setNodeLabels(nodeRecord, labels);

    getNodeStore().updateRecord(nodeRecord);
    return nodeId;
  }

  private void setNodeLabels(NodeRecord nodeRecord, Label... labels) {
    NodeLabels nodeLabels = parseLabelsField(nodeRecord);
    getNodeStore()
        .updateDynamicLabelRecords(nodeLabels.put(getOrCreateLabelIds(labels), getNodeStore()));
  }

  private long[] getOrCreateLabelIds(Label[] labels) {
    long[] ids = new long[labels.length];
    for (int i = 0; i < ids.length; i++) {
      ids[i] = getOrCreateLabelId(labels[i].name());
    }
    return ids;
  }

  @Override
  public void createNode(long id, Map<String, Object> properties, Label... labels) {
    if (id < 0 || id > MAX_NODE_ID) {
      throw new IllegalArgumentException("id=" + id);
    }
    if (id == IdGeneratorImpl.INTEGER_MINUS_ONE) {
      throw new IllegalArgumentException("id " + id + " is reserved for internal use");
    }
    NodeStore nodeStore = neoStore.getNodeStore();
    if (neoStore.getNodeStore().loadLightNode(id) != null) {
      throw new IllegalArgumentException("id=" + id + " already in use");
    }
    long highId = nodeStore.getHighId();
    if (highId <= id) {
      nodeStore.setHighId(id + 1);
    }
    internalCreateNode(id, properties, labels);
  }

  @Override
  public void setNodeLabels(long node, Label... labels) {
    NodeRecord record = getNodeRecord(node);
    setNodeLabels(record, labels);
    getNodeStore().updateRecord(record);
  }

  @Override
  public Iterable<Label> getNodeLabels(final long node) {
    return new Iterable<Label>() {
      @Override
      public Iterator<Label> iterator() {
        NodeStore nodeStore = neoStore.getNodeStore();
        long[] labels = parseLabelsField(nodeStore.getRecord(node)).get(getNodeStore());
        return map(labelIdToLabelFunction, asPrimitiveIterator(labels));
      }
    };
  }

  @Override
  public boolean nodeHasLabel(long node, Label label) {
    int labelId = getLabelId(label.name());
    return labelId != -1 && nodeHasLabel(node, labelId);
  }

  private boolean nodeHasLabel(long node, int labelId) {
    NodeStore nodeStore = neoStore.getNodeStore();
    for (long label : parseLabelsField(nodeStore.getRecord(node)).get(getNodeStore())) {
      if (label == labelId) {
        return true;
      }
    }
    return false;
  }

  @Override
  public long createRelationship(
      long node1, long node2, RelationshipType type, Map<String, Object> properties) {
    NodeRecord firstNode = getNodeRecord(node1);
    NodeRecord secondNode = getNodeRecord(node2);
    int typeId = relationshipTypeTokens.idOf(type.name());
    if (typeId == -1) {
      typeId = createNewRelationshipType(type.name());
    }
    long id = getRelationshipStore().nextId();
    RelationshipRecord record = new RelationshipRecord(id, node1, node2, typeId);
    record.setInUse(true);
    record.setCreated();
    connectRelationship(firstNode, secondNode, record);
    getNodeStore().updateRecord(firstNode);
    getNodeStore().updateRecord(secondNode);
    record.setNextProp(createPropertyChain(properties));
    getRelationshipStore().updateRecord(record);
    return id;
  }

  private void connectRelationship(
      NodeRecord firstNode, NodeRecord secondNode, RelationshipRecord rel) {
    assert firstNode.getNextRel() != rel.getId();
    assert secondNode.getNextRel() != rel.getId();
    rel.setFirstNextRel(firstNode.getNextRel());
    rel.setSecondNextRel(secondNode.getNextRel());
    connect(firstNode, rel);
    connect(secondNode, rel);
    firstNode.setNextRel(rel.getId());
    secondNode.setNextRel(rel.getId());
  }

  private void connect(NodeRecord node, RelationshipRecord rel) {
    if (node.getNextRel() != Record.NO_NEXT_RELATIONSHIP.intValue()) {
      RelationshipRecord nextRel = getRelationshipStore().getRecord(node.getNextRel());
      boolean changed = false;
      if (nextRel.getFirstNode() == node.getId()) {
        nextRel.setFirstPrevRel(rel.getId());
        changed = true;
      }
      if (nextRel.getSecondNode() == node.getId()) {
        nextRel.setSecondPrevRel(rel.getId());
        changed = true;
      }
      if (!changed) {
        throw new InvalidRecordException(node + " dont match " + nextRel);
      }
      getRelationshipStore().updateRecord(nextRel);
    }
  }

  @Override
  public void setNodeProperties(long node, Map<String, Object> properties) {
    NodeRecord record = getNodeRecord(node);
    if (record.getNextProp() != Record.NO_NEXT_PROPERTY.intValue()) {
      deletePropertyChain(record.getNextProp());
      /*
       * Batch inserter does not make any attempt to maintain the store's
       * integrity. It makes sense however to keep some things intact where
       * the cost is relatively low. So here, when we delete the property
       * chain we first make sure that the node record (or the relationship
       * record below) does not point anymore to the deleted properties. This
       * way, if during creation, something goes wrong, it will not have the properties
       * expected instead of throwing invalid record exceptions.
       */
      record.setNextProp(Record.NO_NEXT_PROPERTY.intValue());
      getNodeStore().updateRecord(record);
    }
    record.setNextProp(createPropertyChain(properties));
    getNodeStore().updateRecord(record);
  }

  @Override
  public void setRelationshipProperties(long rel, Map<String, Object> properties) {
    RelationshipRecord record = getRelationshipRecord(rel);
    if (record.getNextProp() != Record.NO_NEXT_PROPERTY.intValue()) {
      deletePropertyChain(record.getNextProp());
      /*
       * See setNodeProperties above for an explanation of what goes on
       * here
       */
      record.setNextProp(Record.NO_NEXT_PROPERTY.intValue());
      getRelationshipStore().updateRecord(record);
    }
    record.setNextProp(createPropertyChain(properties));
    getRelationshipStore().updateRecord(record);
  }

  @Override
  public boolean nodeExists(long nodeId) {
    return neoStore.getNodeStore().loadLightNode(nodeId) != null;
  }

  @Override
  public Map<String, Object> getNodeProperties(long nodeId) {
    NodeRecord record = getNodeRecord(nodeId);
    if (record.getNextProp() != Record.NO_NEXT_PROPERTY.intValue()) {
      return getPropertyChain(record.getNextProp());
    }
    return Collections.emptyMap();
  }

  @Override
  public Iterable<Long> getRelationshipIds(long nodeId) {
    NodeRecord nodeRecord = getNodeRecord(nodeId);
    long nextRel = nodeRecord.getNextRel();
    List<Long> ids = new ArrayList<>();
    while (nextRel != Record.NO_NEXT_RELATIONSHIP.intValue()) {
      RelationshipRecord relRecord = getRelationshipRecord(nextRel);
      ids.add(relRecord.getId());
      long firstNode = relRecord.getFirstNode();
      long secondNode = relRecord.getSecondNode();
      if (firstNode == nodeId) {
        nextRel = relRecord.getFirstNextRel();
      } else if (secondNode == nodeId) {
        nextRel = relRecord.getSecondNextRel();
      } else {
        throw new InvalidRecordException(
            "Node["
                + nodeId
                + "] not part of firstNode["
                + firstNode
                + "] or secondNode["
                + secondNode
                + "]");
      }
    }
    return ids;
  }

  @Override
  public Iterable<BatchRelationship> getRelationships(long nodeId) {
    NodeRecord nodeRecord = getNodeRecord(nodeId);
    long nextRel = nodeRecord.getNextRel();
    List<BatchRelationship> rels = new ArrayList<>();
    while (nextRel != Record.NO_NEXT_RELATIONSHIP.intValue()) {
      RelationshipRecord relRecord = getRelationshipRecord(nextRel);
      RelationshipType type =
          new RelationshipTypeImpl(relationshipTypeTokens.nameOf(relRecord.getType()));
      rels.add(
          new BatchRelationship(
              relRecord.getId(), relRecord.getFirstNode(), relRecord.getSecondNode(), type));
      long firstNode = relRecord.getFirstNode();
      long secondNode = relRecord.getSecondNode();
      if (firstNode == nodeId) {
        nextRel = relRecord.getFirstNextRel();
      } else if (secondNode == nodeId) {
        nextRel = relRecord.getSecondNextRel();
      } else {
        throw new InvalidRecordException(
            "Node["
                + nodeId
                + "] not part of firstNode["
                + firstNode
                + "] or secondNode["
                + secondNode
                + "]");
      }
    }
    return rels;
  }

  @Override
  public BatchRelationship getRelationshipById(long relId) {
    RelationshipRecord record = getRelationshipRecord(relId);
    RelationshipType type =
        new RelationshipTypeImpl(relationshipTypeTokens.nameOf(record.getType()));
    return new BatchRelationship(
        record.getId(), record.getFirstNode(), record.getSecondNode(), type);
  }

  @Override
  public Map<String, Object> getRelationshipProperties(long relId) {
    RelationshipRecord record = getRelationshipRecord(relId);
    if (record.getNextProp() != Record.NO_NEXT_PROPERTY.intValue()) {
      return getPropertyChain(record.getNextProp());
    }
    return Collections.emptyMap();
  }

  @Override
  public void shutdown() {
    if (isShutdown) {
      throw new IllegalStateException("Batch inserter already has shutdown");
    }
    isShutdown = true;

    try {
      repopulateAllIndexes();
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
    neoStore.close();

    try {
      storeLocker.release();
    } catch (IOException e) {
      throw new UnderlyingStorageException("Could not release store lock", e);
    }

    msgLog.logMessage(
        Thread.currentThread() + " Clean shutdown on BatchInserter(" + this + ")", true);
    msgLog.close();
    life.shutdown();
  }

  @Override
  public String toString() {
    return "EmbeddedBatchInserter[" + storeDir + "]";
  }

  private static class RelationshipTypeImpl implements RelationshipType {
    private final String name;

    RelationshipTypeImpl(String name) {
      this.name = name;
    }

    @Override
    public String name() {
      return name;
    }
  }

  private long createPropertyChain(Map<String, Object> properties) {
    if (properties == null || properties.isEmpty()) {
      return Record.NO_NEXT_PROPERTY.intValue();
    }
    PropertyStore propStore = getPropertyStore();
    List<PropertyRecord> propRecords = new ArrayList<>();
    PropertyRecord currentRecord = new PropertyRecord(propStore.nextId());
    currentRecord.setInUse(true);
    currentRecord.setCreated();
    propRecords.add(currentRecord);
    for (Entry<String, Object> entry : properties.entrySet()) {
      int keyId = propertyKeyTokens.idOf(entry.getKey());
      if (keyId == -1) {
        keyId = createNewPropertyKeyId(entry.getKey());
      }

      PropertyBlock block = new PropertyBlock();
      propStore.encodeValue(block, keyId, entry.getValue());
      if (currentRecord.size() + block.getSize() > PropertyType.getPayloadSize()) {
        // Here it means the current block is done for
        PropertyRecord prevRecord = currentRecord;
        // Create new record
        long propertyId = propStore.nextId();
        currentRecord = new PropertyRecord(propertyId);
        currentRecord.setInUse(true);
        currentRecord.setCreated();
        // Set up links
        prevRecord.setNextProp(propertyId);
        currentRecord.setPrevProp(prevRecord.getId());
        propRecords.add(currentRecord);
        // Now current is ready to start picking up blocks
      }
      currentRecord.addPropertyBlock(block);
    }
    /*
     * Add the property records in reverse order, which means largest
     * id first. That is to make sure we expand the property store file
     * only once.
     */
    for (int i = propRecords.size() - 1; i >= 0; i--) {
      propStore.updateRecord(propRecords.get(i));
    }
    /*
     *  0 will always exist, if the map was empty we wouldn't be here
     *  and even one property will create at least one record.
     */
    return propRecords.get(0).getId();
  }

  private void deletePropertyChain(long nextProp) {
    PropertyStore propStore = getPropertyStore();
    while (nextProp != Record.NO_NEXT_PROPERTY.intValue()) {
      PropertyRecord propRecord = propStore.getRecord(nextProp);
      /*
       *  The only reason to loop over the blocks is to handle the dynamic
       *  records that possibly hang under them. Otherwise, we could just
       *  set the property record not in use and be done with it. The
       *  residue of the convenience is that we do not remove individual
       *  property blocks - we just mark the whole record as !inUse.
       */
      for (PropertyBlock propBlock : propRecord.getPropertyBlocks()) {
        propStore.ensureHeavy(propBlock);
        for (DynamicRecord rec : propBlock.getValueRecords()) {
          rec.setInUse(false);
          propRecord.addDeletedRecord(rec);
        }
      }
      propRecord.setInUse(false);
      nextProp = propRecord.getNextProp();
      propStore.updateRecord(propRecord);
    }
  }

  private Map<String, Object> getPropertyChain(long nextProp) {
    PropertyStore propStore = getPropertyStore();
    Map<String, Object> properties = new HashMap<>();

    while (nextProp != Record.NO_NEXT_PROPERTY.intValue()) {
      PropertyRecord propRecord = propStore.getRecord(nextProp);
      for (PropertyBlock propBlock : propRecord.getPropertyBlocks()) {
        String key = propertyKeyTokens.nameOf(propBlock.getKeyIndexId());
        DefinedProperty propertyData = propBlock.newPropertyData(propStore);
        Object value =
            propertyData.value() != null
                ? propertyData.value()
                : propBlock.getType().getValue(propBlock, getPropertyStore());
        properties.put(key, value);
      }
      nextProp = propRecord.getNextProp();
    }
    return properties;
  }

  private int createNewPropertyKeyId(String stringKey) {
    PropertyKeyTokenStore idxStore = getPropertyKeyTokenStore();
    int keyId = (int) idxStore.nextId();
    PropertyKeyTokenRecord record = new PropertyKeyTokenRecord(keyId);
    record.setInUse(true);
    record.setCreated();
    Collection<DynamicRecord> keyRecords = idxStore.allocateNameRecords(encodeString(stringKey));
    record.setNameId((int) first(keyRecords).getId());
    record.addNameRecords(keyRecords);
    idxStore.updateRecord(record);
    propertyKeyTokens.addToken(stringKey, keyId);
    return keyId;
  }

  private int createNewLabelId(String stringKey) {
    LabelTokenStore labelTokenStore = neoStore.getLabelTokenStore();
    int keyId = (int) labelTokenStore.nextId();
    LabelTokenRecord record = new LabelTokenRecord(keyId);
    record.setInUse(true);
    record.setCreated();
    Collection<DynamicRecord> keyRecords =
        labelTokenStore.allocateNameRecords(encodeString(stringKey));
    record.setNameId((int) first(keyRecords).getId());
    record.addNameRecords(keyRecords);
    labelTokenStore.updateRecord(record);
    labelTokens.addToken(stringKey, keyId);
    return keyId;
  }

  private int createNewRelationshipType(String name) {
    RelationshipTypeTokenStore typeStore = getRelationshipTypeStore();
    int id = (int) typeStore.nextId();
    RelationshipTypeTokenRecord record = new RelationshipTypeTokenRecord(id);
    record.setInUse(true);
    record.setCreated();
    Collection<DynamicRecord> nameRecords = typeStore.allocateNameRecords(encodeString(name));
    record.setNameId((int) first(nameRecords).getId());
    record.addNameRecords(nameRecords);
    typeStore.updateRecord(record);
    relationshipTypeTokens.addToken(name, id);
    return id;
  }

  private NodeStore getNodeStore() {
    return neoStore.getNodeStore();
  }

  private PropertyStore getPropertyStore() {
    return neoStore.getPropertyStore();
  }

  private PropertyKeyTokenStore getPropertyKeyTokenStore() {
    return getPropertyStore().getPropertyKeyTokenStore();
  }

  private RelationshipStore getRelationshipStore() {
    return neoStore.getRelationshipStore();
  }

  private RelationshipTypeTokenStore getRelationshipTypeStore() {
    return neoStore.getRelationshipTypeStore();
  }

  private SchemaStore getSchemaStore() {
    return neoStore.getSchemaStore();
  }

  private NodeRecord getNodeRecord(long id) {
    if (id < 0 || id >= getNodeStore().getHighId()) {
      throw new NotFoundException("id=" + id);
    }
    return getNodeStore().getRecord(id);
  }

  private RelationshipRecord getRelationshipRecord(long id) {
    if (id < 0 || id >= getRelationshipStore().getHighId()) {
      throw new NotFoundException("id=" + id);
    }
    return getRelationshipStore().getRecord(id);
  }

  private File fixPath(File dir, StoreFactory sf) {
    try {
      fileSystem.mkdirs(dir);
    } catch (IOException e) {
      throw new UnderlyingStorageException(
          "Unable to create directory path[" + storeDir + "] for Neo4j kernel store.");
    }

    File store = new File(dir, NeoStore.DEFAULT_NAME);
    if (!fileSystem.fileExists(store)) {
      sf.createNeoStore(store).close();
    }
    return store;
  }

  @Override
  public String getStoreDir() {
    return storeDir.getPath();
  }

  // needed by lucene-index
  public IndexStore getIndexStore() {
    return this.indexStore;
  }

  public IdGeneratorFactory getIdGeneratorFactory() {
    return idGeneratorFactory;
  }

  public NeoStore getNeoStore() {
    return neoStore;
  }

  public int createAllPropertyIndexes(Collection<String> names) {
    int max = 0;
    for (String name : names) {
      if (getPropertyKeyId(name) != -1) continue;
      max = createNewPropertyKeyId(name);
    }
    return max;
  }

  public int createAllRelTypeIndexes(Collection<String> types) {
    int max = 0;
    for (String type : types) {
      if (getRelTypeId(type) != -1) continue;
      max = createNewRelationshipType(type);
    }
    return max;
  }

  public int getRelTypeId(String name) {
    return relationshipTypeTokens.idOf(name);
  }

  private void dumpConfiguration(Map<String, String> config) {
    for (String key : config.keySet()) {
      Object value = config.get(key);
      if (value != null) {
        System.out.println(key + "=" + value);
      }
    }
  }

  private class BatchSchemaActions implements InternalSchemaActions {
    @Override
    public IndexDefinition createIndexDefinition(Label label, String propertyKey) {
      createIndexRule(label, propertyKey);
      return new IndexDefinitionImpl(this, label, propertyKey, false);
    }

    @Override
    public void dropIndexDefinitions(Label label, String propertyKey) {
      throw unsupportedException();
    }

    @Override
    public ConstraintDefinition createPropertyUniquenessConstraint(
        Label label, String propertyKey) {
      int labelId = getOrCreateLabelId(label.name());
      int propertyKeyId = getOrCreatePropertyKeyId(propertyKey);
      createConstraintRule(new UniquenessConstraint(labelId, propertyKeyId));
      return new PropertyUniqueConstraintDefinition(this, label, propertyKey);
    }

    @Override
    public void dropPropertyUniquenessConstraint(Label label, String propertyKey) {
      throw unsupportedException();
    }

    @Override
    public String getUserMessage(KernelException e) {
      throw unsupportedException();
    }

    @Override
    public void assertInTransaction() {
      // BatchInserterImpl always is expected to be running in one big single "transaction"
    }

    private UnsupportedOperationException unsupportedException() {
      return new UnsupportedOperationException("Batch inserter doesn't support this");
    }
  }

  private class DependencyResolverImpl extends DependencyResolver.Adapter {
    @Override
    public <T> T resolveDependency(Class<T> type, SelectionStrategy selector)
        throws IllegalArgumentException {
      if (type.isInstance(fileSystem)) {
        return type.cast(fileSystem);
      }
      if (type.isInstance(config)) {
        return type.cast(config);
      }
      if (type.isInstance(logging)) {
        return type.cast(logging);
      }
      if (NeoStoreProvider.class.isAssignableFrom(type)) {
        return type.cast(
            new NeoStoreProvider() {
              @Override
              public NeoStore evaluate() {
                return neoStore;
              }
            });
      }
      throw new IllegalArgumentException("Unknown dependency " + type);
    }
  }
}