private void binMutations(
        MutationSet mutationsToProcess,
        Map<String, TabletServerMutations<Mutation>> binnedMutations) {
      try {
        Set<Entry<String, List<Mutation>>> es = mutationsToProcess.getMutations().entrySet();
        for (Entry<String, List<Mutation>> entry : es) {
          TabletLocator locator = getLocator(entry.getKey());

          String table = entry.getKey();
          List<Mutation> tableMutations = entry.getValue();

          if (tableMutations != null) {
            ArrayList<Mutation> tableFailures = new ArrayList<Mutation>();
            locator.binMutations(credentials, tableMutations, binnedMutations, tableFailures);

            if (tableFailures.size() > 0) {
              failedMutations.add(table, tableFailures);

              if (tableFailures.size() == tableMutations.size())
                if (!Tables.exists(instance, entry.getKey()))
                  throw new TableDeletedException(entry.getKey());
                else if (Tables.getTableState(instance, table) == TableState.OFFLINE)
                  throw new TableOfflineException(instance, entry.getKey());
            }
          }
        }
        return;
      } catch (AccumuloServerException ase) {
        updateServerErrors(ase.getServer(), ase);
      } catch (AccumuloException ae) {
        // assume an IOError communicating with !METADATA tablet
        failedMutations.add(mutationsToProcess);
      } catch (AccumuloSecurityException e) {
        updateAuthorizationFailures(
            Collections.singletonMap(
                new KeyExtent(new Text(MetadataTable.ID), null, null),
                SecurityErrorCode.valueOf(e.getSecurityErrorCode().name())));
      } catch (TableDeletedException e) {
        updateUnknownErrors(e.getMessage(), e);
      } catch (TableOfflineException e) {
        updateUnknownErrors(e.getMessage(), e);
      } catch (TableNotFoundException e) {
        updateUnknownErrors(e.getMessage(), e);
      }

      // an error ocurred
      binnedMutations.clear();
    }
Пример #2
0
 private boolean isTableDeleting() {
   try {
     return Tables.getTableState(tabletServer.getInstance(), extent.getTableId())
         == TableState.DELETING;
   } catch (Exception e) {
     log.warn("Failed to determine if table " + extent.getTableId() + " was deleting ", e);
     return false; // can not get positive confirmation that its deleting.
   }
 }
  private void updateAuthorizationFailures(
      Map<KeyExtent, SecurityErrorCode> authorizationFailures) {
    if (authorizationFailures.size() > 0) {

      // was a table deleted?
      HashSet<String> tableIds = new HashSet<String>();
      for (KeyExtent ke : authorizationFailures.keySet()) tableIds.add(ke.getTableId().toString());

      Tables.clearCache(instance);
      for (String tableId : tableIds)
        if (!Tables.exists(instance, tableId)) throw new TableDeletedException(tableId);

      synchronized (this) {
        somethingFailed = true;
        mergeAuthorizationFailures(this.authorizationFailures, authorizationFailures);
        this.notifyAll();
      }
    }
  }
Пример #4
0
 @Override
 public long isReady(long tid, Master environment) throws Exception {
   try {
     String namespaceId = Tables.getNamespaceId(environment.getInstance(), tableId);
     return Utils.reserveNamespace(namespaceId, tid, false, false, TableOperation.DELETE)
         + Utils.reserveTable(tableId, tid, true, true, TableOperation.DELETE);
   } catch (IllegalArgumentException ex) {
     if (ex.getCause() != null && ex.getCause() instanceof TableNotFoundException) {
       return 0;
     }
     throw ex;
   }
 }
Пример #5
0
 @Override
 public Repo<Master> call(long tid, Master environment) throws Exception {
   try {
     String namespaceId = Tables.getNamespaceId(environment.getInstance(), tableId);
     TableManager.getInstance().transitionTableState(tableId, TableState.DELETING);
     environment.getEventCoordinator().event("deleting table %s ", tableId);
     return new CleanUp(tableId, namespaceId);
   } catch (IllegalArgumentException ex) {
     if (ex.getCause() != null && ex.getCause() instanceof TableNotFoundException) {
       return null;
     }
     throw ex;
   }
 }
Пример #6
0
  @Override
  public Repo<Master> call(long id, Master master) throws Exception {

    Instance instance = master.getInstance();

    IZooReaderWriter zoo = ZooReaderWriter.getInstance();

    Utils.tableNameLock.lock();
    try {
      Utils.checkNamespaceDoesNotExist(instance, newName, namespaceId, TableOperation.RENAME);

      final String tap =
          ZooUtil.getRoot(instance)
              + Constants.ZNAMESPACES
              + "/"
              + namespaceId
              + Constants.ZNAMESPACE_NAME;

      zoo.mutate(
          tap,
          null,
          null,
          new Mutator() {
            @Override
            public byte[] mutate(byte[] current) throws Exception {
              final String currentName = new String(current);
              if (currentName.equals(newName))
                return null; // assume in this case the operation is running again, so we are done
              if (!currentName.equals(oldName)) {
                throw new AcceptableThriftTableOperationException(
                    null,
                    oldName,
                    TableOperation.RENAME,
                    TableOperationExceptionType.NAMESPACE_NOTFOUND,
                    "Name changed while processing");
              }
              return newName.getBytes();
            }
          });
      Tables.clearCache(instance);
    } finally {
      Utils.tableNameLock.unlock();
      Utils.unreserveNamespace(namespaceId, id, true);
    }

    LoggerFactory.getLogger(RenameNamespace.class)
        .debug("Renamed namespace " + namespaceId + " " + oldName + " " + newName);

    return null;
  }
Пример #7
0
 @Override
 public void undo(long tid, Master environment) throws Exception {
   String namespaceId = Tables.getNamespaceId(environment.getInstance(), tableId);
   Utils.unreserveNamespace(namespaceId, tid, false);
   Utils.unreserveTable(tableId, tid, true);
 }
  /**
   * Gets the splits of the tables that have been set on the job by reading the metadata table for
   * the specified ranges.
   *
   * @return the splits from the tables based on the ranges.
   * @throws java.io.IOException if a table set on the job doesn't exist or an error occurs
   *     initializing the tablet locator
   */
  @Override
  public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException {
    Level logLevel = getLogLevel(job);
    log.setLevel(logLevel);
    validateOptions(job);

    Random random = new Random();
    LinkedList<InputSplit> splits = new LinkedList<InputSplit>();
    Map<String, InputTableConfig> tableConfigs = getInputTableConfigs(job);
    for (Map.Entry<String, InputTableConfig> tableConfigEntry : tableConfigs.entrySet()) {
      String tableName = tableConfigEntry.getKey();
      InputTableConfig tableConfig = tableConfigEntry.getValue();

      Instance instance = getInstance(job);
      String tableId;
      // resolve table name to id once, and use id from this point forward
      if (DeprecationUtil.isMockInstance(instance)) {
        tableId = "";
      } else {
        try {
          tableId = Tables.getTableId(instance, tableName);
        } catch (TableNotFoundException e) {
          throw new IOException(e);
        }
      }

      Authorizations auths = getScanAuthorizations(job);
      String principal = getPrincipal(job);
      AuthenticationToken token = getAuthenticationToken(job);

      boolean batchScan = InputConfigurator.isBatchScan(CLASS, job);
      boolean supportBatchScan =
          !(tableConfig.isOfflineScan()
              || tableConfig.shouldUseIsolatedScanners()
              || tableConfig.shouldUseLocalIterators());
      if (batchScan && !supportBatchScan)
        throw new IllegalArgumentException(
            "BatchScanner optimization not available for offline scan, isolated, or local iterators");

      boolean autoAdjust = tableConfig.shouldAutoAdjustRanges();
      if (batchScan && !autoAdjust)
        throw new IllegalArgumentException(
            "AutoAdjustRanges must be enabled when using BatchScanner optimization");

      List<Range> ranges =
          autoAdjust ? Range.mergeOverlapping(tableConfig.getRanges()) : tableConfig.getRanges();
      if (ranges.isEmpty()) {
        ranges = new ArrayList<Range>(1);
        ranges.add(new Range());
      }

      // get the metadata information for these ranges
      Map<String, Map<KeyExtent, List<Range>>> binnedRanges =
          new HashMap<String, Map<KeyExtent, List<Range>>>();
      TabletLocator tl;
      try {
        if (tableConfig.isOfflineScan()) {
          binnedRanges = binOfflineTable(job, tableId, ranges);
          while (binnedRanges == null) {
            // Some tablets were still online, try again
            // sleep randomly between 100 and 200 ms
            sleepUninterruptibly(100 + random.nextInt(100), TimeUnit.MILLISECONDS);
            binnedRanges = binOfflineTable(job, tableId, ranges);
          }
        } else {
          tl = InputConfigurator.getTabletLocator(CLASS, job, tableId);
          // its possible that the cache could contain complete, but old information about a tables
          // tablets... so clear it
          tl.invalidateCache();

          ClientContext context =
              new ClientContext(
                  getInstance(job),
                  new Credentials(getPrincipal(job), getAuthenticationToken(job)),
                  getClientConfiguration(job));
          while (!tl.binRanges(context, ranges, binnedRanges).isEmpty()) {
            if (!DeprecationUtil.isMockInstance(instance)) {
              if (!Tables.exists(instance, tableId)) throw new TableDeletedException(tableId);
              if (Tables.getTableState(instance, tableId) == TableState.OFFLINE)
                throw new TableOfflineException(instance, tableId);
            }
            binnedRanges.clear();
            log.warn("Unable to locate bins for specified ranges. Retrying.");
            // sleep randomly between 100 and 200 ms
            sleepUninterruptibly(100 + random.nextInt(100), TimeUnit.MILLISECONDS);
            tl.invalidateCache();
          }
        }
      } catch (Exception e) {
        throw new IOException(e);
      }

      HashMap<Range, ArrayList<String>> splitsToAdd = null;

      if (!autoAdjust) splitsToAdd = new HashMap<Range, ArrayList<String>>();

      HashMap<String, String> hostNameCache = new HashMap<String, String>();
      for (Map.Entry<String, Map<KeyExtent, List<Range>>> tserverBin : binnedRanges.entrySet()) {
        String ip = tserverBin.getKey().split(":", 2)[0];
        String location = hostNameCache.get(ip);
        if (location == null) {
          InetAddress inetAddress = InetAddress.getByName(ip);
          location = inetAddress.getCanonicalHostName();
          hostNameCache.put(ip, location);
        }
        for (Map.Entry<KeyExtent, List<Range>> extentRanges : tserverBin.getValue().entrySet()) {
          Range ke = extentRanges.getKey().toDataRange();
          if (batchScan) {
            // group ranges by tablet to be read by a BatchScanner
            ArrayList<Range> clippedRanges = new ArrayList<Range>();
            for (Range r : extentRanges.getValue()) clippedRanges.add(ke.clip(r));

            BatchInputSplit split =
                new BatchInputSplit(tableName, tableId, clippedRanges, new String[] {location});
            SplitUtils.updateSplit(split, instance, tableConfig, principal, token, auths, logLevel);

            splits.add(split);
          } else {
            // not grouping by tablet
            for (Range r : extentRanges.getValue()) {
              if (autoAdjust) {
                // divide ranges into smaller ranges, based on the tablets
                RangeInputSplit split =
                    new RangeInputSplit(tableName, tableId, ke.clip(r), new String[] {location});
                SplitUtils.updateSplit(
                    split, instance, tableConfig, principal, token, auths, logLevel);
                split.setOffline(tableConfig.isOfflineScan());
                split.setIsolatedScan(tableConfig.shouldUseIsolatedScanners());
                split.setUsesLocalIterators(tableConfig.shouldUseLocalIterators());

                splits.add(split);
              } else {
                // don't divide ranges
                ArrayList<String> locations = splitsToAdd.get(r);
                if (locations == null) locations = new ArrayList<String>(1);
                locations.add(location);
                splitsToAdd.put(r, locations);
              }
            }
          }
        }
      }

      if (!autoAdjust)
        for (Map.Entry<Range, ArrayList<String>> entry : splitsToAdd.entrySet()) {
          RangeInputSplit split =
              new RangeInputSplit(
                  tableName, tableId, entry.getKey(), entry.getValue().toArray(new String[0]));
          SplitUtils.updateSplit(split, instance, tableConfig, principal, token, auths, logLevel);
          split.setOffline(tableConfig.isOfflineScan());
          split.setIsolatedScan(tableConfig.shouldUseIsolatedScanners());
          split.setUsesLocalIterators(tableConfig.shouldUseLocalIterators());

          splits.add(split);
        }
    }

    return splits.toArray(new InputSplit[splits.size()]);
  }
  @Override
  public long replicateLog(String tableId, WalEdits data, TCredentials tcreds)
      throws RemoteReplicationException, TException {
    log.debug("Got replication request to tableID {} with {} edits", tableId, data.getEditsSize());
    tabletServer.getSecurityOperation().authenticateUser(tabletServer.rpcCreds(), tcreds);

    String tableName;

    try {
      tableName = Tables.getTableName(tabletServer.getInstance(), tableId);
    } catch (TableNotFoundException e) {
      log.error("Could not find table with id {}", tableId);
      throw new RemoteReplicationException(
          RemoteReplicationErrorCode.TABLE_DOES_NOT_EXIST,
          "Table with id " + tableId + " does not exist");
    }

    AccumuloConfiguration conf = tabletServer.getConfiguration();

    Map<String, String> replicationHandlers =
        conf.getAllPropertiesWithPrefix(Property.TSERV_REPLICATION_REPLAYERS);
    String propertyForHandlerTable = Property.TSERV_REPLICATION_REPLAYERS.getKey() + tableId;

    String handlerClassForTable = replicationHandlers.get(propertyForHandlerTable);
    if (null == handlerClassForTable) {
      if (!replicationHandlers.isEmpty()) {
        log.debug("Could not find replication replayer for {}", tableId);
      }
      handlerClassForTable = conf.get(Property.TSERV_REPLICATION_DEFAULT_HANDLER);
    }

    log.debug("Using {} replication replayer for table {}", handlerClassForTable, tableId);

    // Get class for replayer
    Class<? extends AccumuloReplicationReplayer> clz;
    try {
      Class<?> untypedClz = Class.forName(handlerClassForTable);
      clz = untypedClz.asSubclass(AccumuloReplicationReplayer.class);
    } catch (ClassNotFoundException e) {
      log.error("Could not instantiate replayer class {}", handlerClassForTable, e);
      throw new RemoteReplicationException(
          RemoteReplicationErrorCode.CANNOT_INSTANTIATE_REPLAYER,
          "Could not instantiate replayer class " + handlerClassForTable);
    }

    // Create an instance
    AccumuloReplicationReplayer replayer;
    try {
      replayer = clz.newInstance();
    } catch (InstantiationException | IllegalAccessException e1) {
      log.error("Could not instantiate replayer class {}", clz.getName());
      throw new RemoteReplicationException(
          RemoteReplicationErrorCode.CANNOT_INSTANTIATE_REPLAYER,
          "Could not instantiate replayer class" + clz.getName());
    }

    long entriesReplicated;
    try {
      entriesReplicated = replayer.replicateLog(tabletServer, tableName, data);
    } catch (AccumuloException | AccumuloSecurityException e) {
      log.error("Could not get connection", e);
      throw new RemoteReplicationException(
          RemoteReplicationErrorCode.CANNOT_AUTHENTICATE,
          "Cannot get connector as " + tabletServer.getCredentials().getPrincipal());
    }

    log.debug("Replicated {} mutations to {}", entriesReplicated, tableName);

    return entriesReplicated;
  }
Пример #10
0
  public static List<KeyValue> scan(ClientContext context, ScanState scanState, int timeOut)
      throws ScanTimedOutException, AccumuloException, AccumuloSecurityException,
          TableNotFoundException {
    TabletLocation loc = null;
    Instance instance = context.getInstance();
    long startTime = System.currentTimeMillis();
    String lastError = null;
    String error = null;
    int tooManyFilesCount = 0;
    long sleepMillis = 100;

    List<KeyValue> results = null;

    Span span = Trace.start("scan");
    try {
      while (results == null && !scanState.finished) {
        if (Thread.currentThread().isInterrupted()) {
          throw new AccumuloException("Thread interrupted");
        }

        if ((System.currentTimeMillis() - startTime) / 1000.0 > timeOut)
          throw new ScanTimedOutException();

        while (loc == null) {
          long currentTime = System.currentTimeMillis();
          if ((currentTime - startTime) / 1000.0 > timeOut) throw new ScanTimedOutException();

          Span locateSpan = Trace.start("scan:locateTablet");
          try {
            loc =
                TabletLocator.getLocator(context, scanState.tableId)
                    .locateTablet(context, scanState.startRow, scanState.skipStartRow, false);

            if (loc == null) {
              if (!Tables.exists(instance, scanState.tableId))
                throw new TableDeletedException(scanState.tableId);
              else if (Tables.getTableState(instance, scanState.tableId) == TableState.OFFLINE)
                throw new TableOfflineException(instance, scanState.tableId);

              error =
                  "Failed to locate tablet for table : "
                      + scanState.tableId
                      + " row : "
                      + scanState.startRow;
              if (!error.equals(lastError)) log.debug("{}", error);
              else if (log.isTraceEnabled()) log.trace("{}", error);
              lastError = error;
              sleepMillis = pause(sleepMillis);
            } else {
              // when a tablet splits we do want to continue scanning the low child
              // of the split if we are already passed it
              Range dataRange = loc.tablet_extent.toDataRange();

              if (scanState.range.getStartKey() != null
                  && dataRange.afterEndKey(scanState.range.getStartKey())) {
                // go to the next tablet
                scanState.startRow = loc.tablet_extent.getEndRow();
                scanState.skipStartRow = true;
                loc = null;
              } else if (scanState.range.getEndKey() != null
                  && dataRange.beforeStartKey(scanState.range.getEndKey())) {
                // should not happen
                throw new RuntimeException(
                    "Unexpected tablet, extent : "
                        + loc.tablet_extent
                        + "  range : "
                        + scanState.range
                        + " startRow : "
                        + scanState.startRow);
              }
            }
          } catch (AccumuloServerException e) {
            log.debug("Scan failed, server side exception : {}", e.getMessage());
            throw e;
          } catch (AccumuloException e) {
            error = "exception from tablet loc " + e.getMessage();
            if (!error.equals(lastError)) log.debug("{}", error);
            else if (log.isTraceEnabled()) log.trace("{}", error);

            lastError = error;
            sleepMillis = pause(sleepMillis);
          } finally {
            locateSpan.stop();
          }
        }

        Span scanLocation = Trace.start("scan:location");
        scanLocation.data("tserver", loc.tablet_location);
        try {
          results = scan(loc, scanState, context);
        } catch (AccumuloSecurityException e) {
          Tables.clearCache(instance);
          if (!Tables.exists(instance, scanState.tableId))
            throw new TableDeletedException(scanState.tableId);
          e.setTableInfo(Tables.getPrintableTableInfoFromId(instance, scanState.tableId));
          throw e;
        } catch (TApplicationException tae) {
          throw new AccumuloServerException(loc.tablet_location, tae);
        } catch (TSampleNotPresentException tsnpe) {
          String message =
              "Table "
                  + Tables.getPrintableTableInfoFromId(instance, scanState.tableId)
                  + " does not have sampling configured or built";
          throw new SampleNotPresentException(message, tsnpe);
        } catch (NotServingTabletException e) {
          error = "Scan failed, not serving tablet " + loc;
          if (!error.equals(lastError)) log.debug("{}", error);
          else if (log.isTraceEnabled()) log.trace("{}", error);
          lastError = error;

          TabletLocator.getLocator(context, scanState.tableId).invalidateCache(loc.tablet_extent);
          loc = null;

          // no need to try the current scan id somewhere else
          scanState.scanID = null;

          if (scanState.isolated) throw new IsolationException();

          sleepMillis = pause(sleepMillis);
        } catch (NoSuchScanIDException e) {
          error = "Scan failed, no such scan id " + scanState.scanID + " " + loc;
          if (!error.equals(lastError)) log.debug("{}", error);
          else if (log.isTraceEnabled()) log.trace("{}", error);
          lastError = error;

          if (scanState.isolated) throw new IsolationException();

          scanState.scanID = null;
        } catch (TooManyFilesException e) {
          error = "Tablet has too many files " + loc + " retrying...";
          if (!error.equals(lastError)) {
            log.debug("{}", error);
            tooManyFilesCount = 0;
          } else {
            tooManyFilesCount++;
            if (tooManyFilesCount == 300) log.warn("{}", error);
            else if (log.isTraceEnabled()) log.trace("{}", error);
          }
          lastError = error;

          // not sure what state the scan session on the server side is
          // in after this occurs, so lets be cautious and start a new
          // scan session
          scanState.scanID = null;

          if (scanState.isolated) throw new IsolationException();

          sleepMillis = pause(sleepMillis);
        } catch (TException e) {
          TabletLocator.getLocator(context, scanState.tableId)
              .invalidateCache(context.getInstance(), loc.tablet_location);
          error =
              "Scan failed, thrift error "
                  + e.getClass().getName()
                  + "  "
                  + e.getMessage()
                  + " "
                  + loc;
          if (!error.equals(lastError)) log.debug("{}", error);
          else if (log.isTraceEnabled()) log.trace("{}", error);
          lastError = error;
          loc = null;

          // do not want to continue using the same scan id, if a timeout occurred could cause a
          // batch to be skipped
          // because a thread on the server side may still be processing the timed out continue scan
          scanState.scanID = null;

          if (scanState.isolated) throw new IsolationException();

          sleepMillis = pause(sleepMillis);
        } finally {
          scanLocation.stop();
        }
      }

      if (results != null && results.size() == 0 && scanState.finished) {
        results = null;
      }

      return results;
    } catch (InterruptedException ex) {
      throw new AccumuloException(ex);
    } finally {
      span.stop();
    }
  }
Пример #11
0
 public static AccumuloConfiguration getTableConfiguration(Connector conn, String tableId)
     throws TableNotFoundException, AccumuloException {
   String tableName = Tables.getTableName(conn.getInstance(), tableId);
   return new ConfigurationCopy(conn.tableOperations().getProperties(tableName));
 }
Пример #12
0
  @Override
  public void setUp(State state, Environment env) throws Exception {

    Connector conn = env.getConnector();
    Instance instance = env.getInstance();

    SortedSet<Text> splits = new TreeSet<Text>();
    for (int i = 1; i < 256; i++) {
      splits.add(new Text(String.format("%04x", i << 8)));
    }

    String hostname = InetAddress.getLocalHost().getHostName().replaceAll("[-.]", "_");
    String pid = env.getPid();

    imageTableName = String.format("img_%s_%s_%d", hostname, pid, System.currentTimeMillis());
    state.set("imageTableName", imageTableName);

    indexTableName = String.format("img_ndx_%s_%s_%d", hostname, pid, System.currentTimeMillis());
    state.set("indexTableName", indexTableName);

    try {
      conn.tableOperations().create(imageTableName);
      conn.tableOperations().addSplits(imageTableName, splits);
      log.debug(
          "Created table "
              + imageTableName
              + " (id:"
              + Tables.getNameToIdMap(instance).get(imageTableName)
              + ")");
    } catch (TableExistsException e) {
      log.error("Table " + imageTableName + " already exists.");
      throw e;
    }

    try {
      conn.tableOperations().create(indexTableName);
      log.debug(
          "Created table "
              + indexTableName
              + " (id:"
              + Tables.getNameToIdMap(instance).get(indexTableName)
              + ")");
    } catch (TableExistsException e) {
      log.error("Table " + imageTableName + " already exists.");
      throw e;
    }

    Random rand = new Random();
    if (rand.nextInt(10) < 5) {
      // setup locality groups
      Map<String, Set<Text>> groups = getLocalityGroups();

      conn.tableOperations().setLocalityGroups(imageTableName, groups);
      log.debug("Configured locality groups for " + imageTableName + " groups = " + groups);
    }

    state.set("numWrites", Long.valueOf(0));
    state.set("totalWrites", Long.valueOf(0));
    state.set("verified", Integer.valueOf(0));
    state.set("lastIndexRow", new Text(""));
  }
Пример #13
0
 public static String qualified(String tableName, String defaultNamespace) {
   Pair<String, String> qualifiedTableName = Tables.qualify(tableName, defaultNamespace);
   if (Constants.DEFAULT_NAMESPACE.equals(qualifiedTableName.getFirst()))
     return qualifiedTableName.getSecond();
   else return qualifiedTableName.toString("", ".", "");
 }