Ejemplo n.º 1
0
 /**
  * Reset the database
  *
  * @param connectionContext NodeCentric specific context, which contains connection to run queries
  *     against
  * @param hardReset if true, all tables are dropped and not just truncated
  * @param statements statements to use to initialize database
  * @return ValuePair containing servers new URI and ID
  * @throws AnzoException
  */
 protected URI resetDatabase(
     NodeCentricOperationContext connectionContext,
     boolean hardReset,
     MultiMap<URI, Statement> statements)
     throws AnzoException {
   datasource.resetDatasource();
   Pair<Long, URI> serverURI = null;
   boolean[] result = lockDB(connectionContext);
   if (result[1]) {
     try {
       serverURI = resetDatabaseTables(connectionContext, hardReset);
       if (statements != null && statements.size() > 0) {
         resetGraph(statements, connectionContext, serverURI.second);
       } else {
         if (rdfInitFiles != null && rdfInitFiles.size() > 0) {
           initializeProvidedRoles(connectionContext, serverURI.second);
         } else {
           initializeDefaultRoles(connectionContext, serverURI.second);
         }
       }
       unLockDB(connectionContext, true, serverURI.first);
       return serverURI.second;
     } catch (AnzoException ae) {
       if (result[0]) {
         BaseSQL.dropTable(
             connectionContext.getStatementProvider(),
             connectionContext.getConnection(),
             connectionContext.getConfiguration().getUsesUppercase()
                 ? NodeCentricDatasource.serverUpper
                 : NodeCentricDatasource.serverLower);
       }
       throw ae;
     }
   } else {
     if (result[0]) {
       BaseSQL.dropTable(
           connectionContext.getStatementProvider(),
           connectionContext.getConnection(),
           connectionContext.getConfiguration().getUsesUppercase()
               ? NodeCentricDatasource.serverUpper
               : NodeCentricDatasource.serverLower);
     }
     throw new AnzoException(ExceptionConstants.RDB.FAILED_INITIALZE_DB);
   }
 }
Ejemplo n.º 2
0
  private void dropTables(NodeCentricOperationContext connectionContext, boolean hardReset)
      throws AnzoException, SQLException {
    long start = System.currentTimeMillis();
    if (hardReset) {
      String[][] tableSets = {liveTables, nodeCentricTables, staticTables};
      datasource.begin(connectionContext.getConnection(), true, true);

      java.sql.Statement stmt = connectionContext.getConnection().createStatement();
      try {
        for (String view : views) {
          try {
            stmt.executeUpdate(
                dropView + datasource.getConfiguration().getDropExtras() + " " + view);
          } catch (SQLException sqle) {
            if (log.isTraceEnabled()) {
              log.trace(LogUtils.RDB_MARKER, "SQL Error droping view", sqle);
            }
          }
        }
        for (String[] sequenceVers : sequences) {
          if (sequenceVers != null) {
            for (String sequence : sequenceVers) {
              try {
                stmt.executeUpdate(
                    dropSequence + datasource.getConfiguration().getDropExtras() + " " + sequence);
              } catch (SQLException sqle) {
                if (log.isTraceEnabled()) {
                  log.trace(LogUtils.RDB_MARKER, "SQL Error droping sequence", sqle);
                }
              }
            }
          }
        }
        if (!connectionContext.getConfiguration().getSupportsIdentity()) {
          try {
            stmt.executeUpdate(
                dropSequence + datasource.getConfiguration().getDropExtras() + " LANG_SEQ");
          } catch (SQLException sqle) {
            if (log.isTraceEnabled()) {
              log.trace(LogUtils.RDB_MARKER, "SQL Error droping sequence", sqle);
            }
          }
          try {
            stmt.executeUpdate(
                dropSequence + datasource.getConfiguration().getDropExtras() + " DATATYPE_SEQ");
          } catch (SQLException sqle) {
            if (log.isTraceEnabled()) {
              log.trace(LogUtils.RDB_MARKER, "SQL Error droping sequence", sqle);
            }
          }
        }
        for (String[] tables : tableSets) {
          for (String table : tables) {
            try {
              stmt.executeUpdate(
                  dropTable + datasource.getConfiguration().getDropExtras() + " " + table);
            } catch (SQLException sqle) {
              if (log.isTraceEnabled()) {
                log.trace(LogUtils.RDB_MARKER, "SQL Error droping table", sqle);
              }
            }
          }
        }
        for (String table : tempTables) {
          try {
            stmt.executeUpdate(
                dropTable
                    + datasource.getConfiguration().getDropExtras()
                    + " "
                    + datasource.getConfiguration().getSessionPrefix()
                    + table);
          } catch (SQLException sqle) {
            if (log.isTraceEnabled()) {
              log.trace(LogUtils.RDB_MARKER, "SQL Error droping temptable", sqle);
            }
          }
        }
      } finally {
        try {
          stmt.close();
        } catch (SQLException sqle) {
          if (log.isTraceEnabled()) {
            log.trace(LogUtils.RDB_MARKER, "SQL Error closing statement", sqle);
          }
        }
        datasource.commit(connectionContext.getConnection(), true, true);
      }

      if (datasource.getConfiguration().getRequiresTempTablespace()
          && InsertStatementsRdbWrapper.getTempTablespaceDefined(
                  connectionContext.getStatementProvider(), connectionContext.getConnection())
              == 0) {
        datasource.begin(connectionContext.getConnection(), true, true);
        try {
          connectionContext
              .getStatementProvider()
              .runSQLGroup(initTablespace, new String[0], connectionContext.getConnection());
        } catch (SQLException sqle) {
          if (log.isErrorEnabled()) {
            log.error(LogUtils.RDB_MARKER, "SQL Error initializing tablespace", sqle);
          }
          throw sqle;
        } finally {
          datasource.commit(connectionContext.getConnection(), true, true);
        }
      }
      datasource.begin(connectionContext.getConnection(), true, true);
      try {
        connectionContext
            .getStatementProvider()
            .runSQLGroup(
                initDBtables, datasource.getInitParams(), connectionContext.getConnection());
        connectionContext
            .getStatementProvider()
            .runSQLGroup(
                initIndexes, datasource.getInitParams(), connectionContext.getConnection());
        connectionContext
            .getStatementProvider()
            .runSQLGroup(
                initSequences, datasource.getInitParams(), connectionContext.getConnection());
        datasource.commit(connectionContext.getConnection(), true, true);
      } catch (SQLException sqle) {
        log.error(LogUtils.RDB_MARKER, "Error initializing database tables", sqle);
        datasource.abort(connectionContext.getConnection(), true, true);
        throw new AnzoException(ExceptionConstants.RDB.FAILED_INITIALZE_DB, sqle);
      }
      try {
        datasource.begin(connectionContext.getConnection(), true, true);
        connectionContext
            .getStatementProvider()
            .runSQLGroup(
                "createTemporaryTables",
                datasource.getInitParams(),
                connectionContext.getConnection());
        datasource.commit(connectionContext.getConnection(), true, true);
      } catch (SQLException sqle) {
        log.error(LogUtils.RDB_MARKER, "Error initializing database temporary tables", sqle);
        datasource.abort(connectionContext.getConnection(), true, true);
        throw new AnzoException(ExceptionConstants.RDB.FAILED_INITIALZE_TEMPTABLES, sqle);
      }

    } else {
      String[][] tableSets = {liveTables, nodeCentricTables};
      datasource.begin(connectionContext.getConnection(), true, true);
      java.sql.Statement stmt = connectionContext.getConnection().createStatement();
      try {
        for (String[] tables : tableSets) {
          for (String table : tables) {
            try {
              BaseSQL.truncateTableMayCommit(
                  connectionContext.getStatementProvider(),
                  connectionContext.getConnection(),
                  table);
            } catch (RdbException sqle) {
              if (log.isErrorEnabled()) {
                log.error(LogUtils.RDB_MARKER, "SQL Error truncating table", sqle);
              }
            }
          }
        }
      } finally {
        try {
          stmt.close();
        } catch (SQLException sqle) {
          if (log.isTraceEnabled()) {
            log.trace(LogUtils.RDB_MARKER, "SQL Error closing statement", sqle);
          }
        }
        datasource.commit(connectionContext.getConnection(), true, true);
      }
    }
    if (log.isDebugEnabled()) {
      log.debug(LogUtils.RDB_MARKER, "[RESET DB] {}", (System.currentTimeMillis() - start));
    }
  }
Ejemplo n.º 3
0
  public void restoreData(String fileName) throws AnzoException {
    final NodeCentricOperationContext connectionContext =
        datasource.getWriteContext(
            new BaseOperationContext(
                "RESTORE",
                BaseOperationContext.generateOperationId(),
                new AnzoPrincipal("sysadmin", null, null, true, false)));
    if (getLockProvider() != null) {
      getLockProvider().writeLock().lock();
    }
    try {
      datasource.resetStarting();
      datasource.reset();
      if (eventAdmin != null)
        eventAdmin.sendEvent(
            new Event(OSGI.RESET_TOPIC, (Dictionary<Object, Object>) new Properties()));

      Pair<Long, URI> serverURI = null;

      datasource.resetDatasource();
      boolean[] result = lockDB(connectionContext);
      if (result[1]) {
        try {
          serverURI = resetDatabaseTables(connectionContext, hardReset);

          try {
            Reader fileReader = ReadWriteUtils.createSmartFileReader(fileName);
            XMLBackupReader reader = new XMLBackupReader(fileReader);
            try {
              reader.read(
                  new IBackupHandler() {
                    Long graphId;

                    Long metaId;

                    Long uuidId;

                    public void start() throws AnzoException {
                      datasource.begin(connectionContext.getConnection(), true, true);
                    }

                    public void handleStatement(
                        boolean metadata,
                        boolean revisioned,
                        Statement statement,
                        Long start,
                        Long end)
                        throws AnzoException {
                      Long s =
                          connectionContext
                              .getNodeLayout()
                              .store(statement.getSubject(), connectionContext.getConnection(), 1);
                      Long p =
                          connectionContext
                              .getNodeLayout()
                              .store(
                                  statement.getPredicate(), connectionContext.getConnection(), 1);
                      Long o =
                          connectionContext
                              .getNodeLayout()
                              .store(statement.getObject(), connectionContext.getConnection(), 1);
                      String stmtId =
                          MessageFormat.format(
                              ID_STRING,
                              (metadata) ? metaId.toString() : graphId.toString(),
                              s.toString(),
                              p.toString(),
                              o.toString());
                      if (revisioned) {
                        Backup.restoreStatement(
                            connectionContext.getStatementProvider(),
                            connectionContext.getConnection(),
                            stmtId,
                            metadata ? 1 : 0,
                            uuidId,
                            metadata ? metaId : graphId,
                            s,
                            p,
                            o,
                            start,
                            end);
                      } else {
                        Backup.restoreStatementNR(
                            connectionContext.getStatementProvider(),
                            connectionContext.getConnection(),
                            stmtId,
                            metadata ? 1 : 0,
                            metadata ? metaId : graphId,
                            s,
                            p,
                            o);
                      }
                    }

                    public boolean handleNamedGraph(
                        boolean revisioned,
                        URI namedGraphUri,
                        URI metadataURI,
                        URI uuid,
                        Collection<BackupRevision> revisions)
                        throws AnzoException {
                      try {
                        connectionContext.getConnection().commit();
                      } catch (SQLException sqle) {
                        throw new org.openanzo.jdbc.utils.RdbException(
                            org.openanzo.exceptions.ExceptionConstants.RDB
                                .FAILED_COMMIT_RDB_TRANSACTION,
                            sqle);
                      }
                      uuidId =
                          connectionContext
                              .getNodeLayout()
                              .store(uuid, connectionContext.getConnection(), 1);
                      graphId =
                          connectionContext
                              .getNodeLayout()
                              .store(namedGraphUri, connectionContext.getConnection(), 1);
                      metaId =
                          connectionContext
                              .getNodeLayout()
                              .store(metadataURI, connectionContext.getConnection(), 1);
                      for (BackupRevision backup : revisions) {
                        Long lastMod =
                            connectionContext
                                .getNodeLayout()
                                .store(backup.lastModifiedBy, connectionContext.getConnection(), 1);
                        if (revisioned) {
                          Backup.restoreNamedGraph(
                              connectionContext.getStatementProvider(),
                              connectionContext.getConnection(),
                              backup.start,
                              backup.end,
                              graphId,
                              metaId,
                              uuidId,
                              backup.revision,
                              lastMod);
                        } else {
                          Backup.restoreNamedGraphNR(
                              connectionContext.getStatementProvider(),
                              connectionContext.getConnection(),
                              backup.start,
                              graphId,
                              metaId,
                              uuidId,
                              backup.revision,
                              lastMod);
                        }
                      }
                      return true;
                    }

                    public void end() throws AnzoException {
                      datasource.commit(connectionContext.getConnection(), true, true);
                      connectionContext
                          .getNodeLayout()
                          .commitReferencedIds(connectionContext.getConnection(), 1);
                    }
                  });
            } catch (AnzoException ae) {
              datasource.abort(connectionContext.getConnection(), true, true);
            } finally {
              if (fileReader != null) {
                fileReader.close();
              }
            }
          } catch (Exception e) {
            log.error(LogUtils.RDB_MARKER, "Error resetting datasource", e);
          }
          datasource.getIndexHandler().rebuildIndex(true);
        } catch (AnzoException ae) {
          if (result[0]) {
            BaseSQL.dropTable(
                connectionContext.getStatementProvider(),
                connectionContext.getConnection(),
                connectionContext.getConfiguration().getUsesUppercase()
                    ? NodeCentricDatasource.serverUpper
                    : NodeCentricDatasource.serverLower);
          }
          throw ae;
        }
        unLockDB(connectionContext, true, serverURI.first);
        datasource.postReset();
        datasource.resetFinished();
      } else {
        if (result[0]) {
          BaseSQL.dropTable(
              connectionContext.getStatementProvider(),
              connectionContext.getConnection(),
              connectionContext.getConfiguration().getUsesUppercase()
                  ? NodeCentricDatasource.serverUpper
                  : NodeCentricDatasource.serverLower);
        }
        throw new AnzoException(ExceptionConstants.RDB.FAILED_INITIALZE_DB);
      }
    } finally {
      if (getLockProvider() != null) {
        getLockProvider().writeLock().unlock();
      }
      datasource.returnWriteContext(connectionContext);
    }
  }
  @Override
  protected void replicateInternal(
      IOperationContext context,
      Collection<NamedGraphRevision> newGraphs,
      Collection<NamedGraphRevision> diffGraphs,
      IReplicationHandler handler,
      ReplicationCache cache)
      throws AnzoException {
    NodeCentricOperationContext connectionContext = datasource.getQueryContext(context);
    long startAll = System.currentTimeMillis();
    int count = 0;
    try {
      Map<Long, NamedGraphRevision> idToGraphs = new HashMap<Long, NamedGraphRevision>();
      long revisionCount = diffGraphs.size();
      long norevisionCount = newGraphs.size();

      long start = System.currentTimeMillis();
      datasource.begin(connectionContext.getConnection(), false, false);
      try {
        StringBuilder sb = new StringBuilder("Replicating:");
        HashMap<URI, Boolean> newGraph = new HashMap<URI, Boolean>();
        for (NamedGraphRevision ngRevision : newGraphs) {
          newGraph.put(ngRevision.namedGraphUri, true);
          Long namedGraphId =
              connectionContext
                  .getNodeLayout()
                  .fetchId(ngRevision.namedGraphUri, connectionContext.getConnection());
          Long metadataId =
              connectionContext
                  .getNodeLayout()
                  .fetchId(ngRevision.metadataUri, connectionContext.getConnection());
          if (namedGraphId != null && metadataId != null) {
            idToGraphs.put(metadataId, ngRevision);
            idToGraphs.put(namedGraphId, ngRevision);

            long startInsertNamedGraphEntries = System.currentTimeMillis();
            long insertNamedGraphEntries =
                ReplicationRdbWrapper.insertNamedGraphNewRevisions(
                    connectionContext.getStatementProvider(),
                    connectionContext.getConnection(),
                    namedGraphId.toString(),
                    connectionContext.getConfiguration().getSessionPrefix(),
                    Long.toString(ngRevision.revision));
            if (log.isDebugEnabled()) {
              log.debug(
                  LogUtils.RDB_MARKER,
                  "[REPLICATE-INSERT_NAMEDGRAPH_ENTRIES] {}:{}",
                  insertNamedGraphEntries,
                  (System.currentTimeMillis() - startInsertNamedGraphEntries));
            }
            if (insertNamedGraphEntries == 0) {
              insertNamedGraphEntries =
                  ReplicationRdbWrapper.insertNamedGraphNRNewRevisions(
                      connectionContext.getStatementProvider(),
                      connectionContext.getConnection(),
                      namedGraphId.toString(),
                      connectionContext.getConfiguration().getSessionPrefix(),
                      Long.toString(ngRevision.revision));
              if (log.isDebugEnabled()) {
                log.debug(
                    LogUtils.RDB_MARKER,
                    "[REPLICATE-INSERT_NAMEDGRAPH_NR_ENTRIES] {}:{}",
                    insertNamedGraphEntries,
                    (System.currentTimeMillis() - startInsertNamedGraphEntries));
              }
            }
            sb.append(ngRevision.namedGraphUri + ",");
          }
        }
        for (NamedGraphRevision ngRevision : diffGraphs) {
          newGraph.put(ngRevision.namedGraphUri, false);
          Long namedGraphId =
              connectionContext
                  .getNodeLayout()
                  .fetchId(ngRevision.namedGraphUri, connectionContext.getConnection());
          Long metadataId =
              connectionContext
                  .getNodeLayout()
                  .fetchId(ngRevision.metadataUri, connectionContext.getConnection());
          if (namedGraphId != null && metadataId != null) {
            idToGraphs.put(metadataId, ngRevision);
            idToGraphs.put(namedGraphId, ngRevision);

            long startInsertNamedGraphEntries = System.currentTimeMillis();
            long insertNamedGraphEntries =
                ReplicationRdbWrapper.insertNamedGraphNewRevisions(
                    connectionContext.getStatementProvider(),
                    connectionContext.getConnection(),
                    namedGraphId.toString(),
                    connectionContext.getConfiguration().getSessionPrefix(),
                    Long.toString(ngRevision.revision));
            if (log.isDebugEnabled()) {
              log.debug(
                  LogUtils.RDB_MARKER,
                  "[REPLICATE-INSERT_NAMEDGRAPH_ENTRIES] {}:{}",
                  insertNamedGraphEntries,
                  (System.currentTimeMillis() - startInsertNamedGraphEntries));
            }
            if (insertNamedGraphEntries == 0) {
              insertNamedGraphEntries =
                  ReplicationRdbWrapper.insertNamedGraphNRNewRevisions(
                      connectionContext.getStatementProvider(),
                      connectionContext.getConnection(),
                      namedGraphId.toString(),
                      connectionContext.getConfiguration().getSessionPrefix(),
                      Long.toString(ngRevision.revision));
              if (log.isDebugEnabled()) {
                log.debug(
                    LogUtils.RDB_MARKER,
                    "[REPLICATE-INSERT_NAMEDGRAPH_NR_ENTRIES] {}:{}",
                    insertNamedGraphEntries,
                    (System.currentTimeMillis() - startInsertNamedGraphEntries));
              }
            }
            sb.append(ngRevision.namedGraphUri + ",");
          }
        }
        if (log.isDebugEnabled()) {
          log.debug(LogUtils.RDB_MARKER, "[REPLICATE-NAMEDGRAPHS] {}", sb.toString());
        }

        if (log.isDebugEnabled()) {
          log.debug(
              LogUtils.RDB_MARKER,
              "[REPLICATE-INSERT_REVISIONS] {}:{}",
              revisionCount + norevisionCount,
              (System.currentTimeMillis() - start));
        }

        long insertDeltaStatements = 0;
        long insertDeltaNRStatements = 0;
        if (revisionCount > 0) {
          long startInsertDeltaStatements = System.currentTimeMillis();
          insertDeltaStatements =
              ReplicationRdbWrapper.insertDeltaStatements(
                  connectionContext.getStatementProvider(),
                  connectionContext.getConnection(),
                  connectionContext.getConfiguration().getSessionPrefix());
          if (log.isDebugEnabled()) {
            log.debug(
                LogUtils.RDB_MARKER,
                "[REPLICATE-INSERT_DELTA_STATEMENTS] {}:{}",
                insertDeltaStatements,
                (System.currentTimeMillis() - startInsertDeltaStatements));
          }
          long startInsertDeltaNRStatements = System.currentTimeMillis();
          insertDeltaNRStatements =
              ReplicationRdbWrapper.insertDeltaNRStatements(
                  connectionContext.getStatementProvider(),
                  connectionContext.getConnection(),
                  connectionContext.getConfiguration().getSessionPrefix());
          if (log.isDebugEnabled()) {
            log.debug(
                LogUtils.RDB_MARKER,
                "[REPLICATE-INSERT_DELTA_NR_STATEMENTS] {}:{}",
                insertDeltaNRStatements,
                (System.currentTimeMillis() - startInsertDeltaNRStatements));
          }

          long startPurgeExtraStatements = System.currentTimeMillis();
          long purgeExtraStatements =
              ReplicationRdbWrapper.purgeExtraStatements(
                  connectionContext.getStatementProvider(),
                  connectionContext.getConnection(),
                  connectionContext.getConfiguration().getSessionPrefix());
          if (log.isDebugEnabled()) {
            log.debug(
                LogUtils.RDB_MARKER,
                "[REPLICATE-PURGE_EXTRA_STATEMENTS] {}:{}",
                purgeExtraStatements,
                (System.currentTimeMillis() - startPurgeExtraStatements));
          }
          insertDeltaStatements -= purgeExtraStatements;
        }
        if (insertDeltaStatements > 0 || insertDeltaNRStatements > 0 || norevisionCount > 0) {
          Deltas deltas = new Deltas(handler, connectionContext, newGraph, cache);
          count =
              processDeltaResults(
                  connectionContext,
                  deltas,
                  (insertDeltaStatements > 0 || insertDeltaNRStatements > 0),
                  (norevisionCount > 0),
                  idToGraphs);
          deltas.end();
        }
      } finally {
        // if (datasource.getConfiguration().getForceTempTablePurge()) {
        try {
          BaseSQL.truncateTableWithSessionMayCommit(
              datasource.getStatementProvider(),
              connectionContext.getConnection(),
              datasource.getConfiguration().getSessionPrefix(),
              "NGR_TMP");
          BaseSQL.truncateTableWithSessionMayCommit(
              datasource.getStatementProvider(),
              connectionContext.getConnection(),
              datasource.getConfiguration().getSessionPrefix(),
              "STMTS_REP_TMP");
        } catch (RdbException sqle) {
          log.error(LogUtils.RDB_MARKER, "Error clearing temporary tables", sqle);
        }
        // }
        datasource.commit(connectionContext.getConnection(), false, false);
      }

    } catch (SQLException sqle) {
      SQLException cause = sqle;
      while (cause != null) {
        log.error(LogUtils.RDB_MARKER, "SQL Error in replicate", cause);
        cause = cause.getNextException();
      }
      throw new AnzoException(
          ExceptionConstants.SERVER.REPLICATION_FAILED, sqle, sqle.getMessage());
    } finally {
      handler.end();
      if (log.isDebugEnabled()) {
        log.debug(
            LogUtils.RDB_MARKER,
            "[REPLICATE-TOTAL-TIME] {}:{}",
            count,
            (System.currentTimeMillis() - startAll));
      }
      if (connectionContext != null) {
        datasource.returnQueryContext(connectionContext);
      }
    }
  }