예제 #1
0
  public int logManyTablets(Map<CommitSession, List<Mutation>> mutations) throws IOException {

    final Map<CommitSession, List<Mutation>> loggables =
        new HashMap<CommitSession, List<Mutation>>(mutations);
    for (CommitSession t : mutations.keySet()) {
      if (!enabled(t)) loggables.remove(t);
    }
    if (loggables.size() == 0) return -1;

    int seq =
        write(
            loggables.keySet(),
            false,
            new Writer() {
              @Override
              public LoggerOperation write(DfsLogger logger, int ignored) throws Exception {
                List<TabletMutations> copy = new ArrayList<TabletMutations>(loggables.size());
                for (Entry<CommitSession, List<Mutation>> entry : loggables.entrySet()) {
                  CommitSession cs = entry.getKey();
                  copy.add(new TabletMutations(cs.getLogId(), cs.getWALogSeq(), entry.getValue()));
                }
                return logger.logManyTablets(copy);
              }
            });
    for (List<Mutation> entry : loggables.values()) {
      if (entry.size() < 1)
        throw new IllegalArgumentException("logManyTablets: logging empty mutation list");
      for (Mutation m : entry) {
        logSizeEstimate.addAndGet(m.numBytes());
      }
    }
    return seq;
  }
예제 #2
0
 public int log(final CommitSession commitSession, final int tabletSeq, final Mutation m)
     throws IOException {
   if (!enabled(commitSession)) return -1;
   int seq =
       write(
           commitSession,
           false,
           new Writer() {
             @Override
             public LoggerOperation write(DfsLogger logger, int ignored) throws Exception {
               return logger.log(tabletSeq, commitSession.getLogId(), m);
             }
           });
   logSizeEstimate.addAndGet(m.numBytes());
   return seq;
 }
예제 #3
0
  @Override
  public void client() {

    Connector conn = getConnector();
    String tableName = getTestProperty("TABLE");

    // get batch writer configuration
    long maxMemory = Long.parseLong(getTestProperty("MAX_MEMORY"));
    long maxLatency = Long.parseLong(getTestProperty("MAX_LATENCY"));
    int maxWriteThreads = Integer.parseInt(getTestProperty("NUM_THREADS"));

    // create batch writer
    BatchWriter bw = null;
    try {
      bw =
          conn.createBatchWriter(
              tableName,
              new BatchWriterConfig()
                  .setMaxMemory(maxMemory)
                  .setMaxLatency(maxLatency, TimeUnit.MILLISECONDS)
                  .setMaxWriteThreads(maxWriteThreads));
    } catch (TableNotFoundException e) {
      log.error("Table '" + tableName + "' not found.", e);
    }

    // configure writing
    Random r = new Random();
    String ingestInstanceId = UUID.randomUUID().toString();
    long numIngestEntries = Long.parseLong(getTestProperty("NUM_ENTRIES"));
    long minRow = 0L;
    long maxRow = 9223372036854775807L;
    int maxColF = 32767;
    int maxColQ = 32767;
    long count = 0;
    long totalBytes = 0;

    ColumnVisibility cv = new ColumnVisibility();

    // start timer
    startTimer();

    // write specified number of entries
    while (count < numIngestEntries) {
      count++;
      long rowId = ContinuousIngest.genLong(minRow, maxRow, r);
      Mutation m =
          ContinuousIngest.genMutation(
              rowId,
              r.nextInt(maxColF),
              r.nextInt(maxColQ),
              cv,
              ingestInstanceId.getBytes(StandardCharsets.UTF_8),
              count,
              null,
              r,
              false);
      totalBytes += m.numBytes();
      try {
        bw.addMutation(m);
      } catch (MutationsRejectedException e) {
        log.error("Mutations rejected.", e);
        System.exit(-1);
      }
    }

    // close writer
    try {
      bw.close();
    } catch (MutationsRejectedException e) {
      log.error("Could not close BatchWriter due to mutations being rejected.", e);
      System.exit(-1);
    }

    // stop timer
    stopTimer(count, totalBytes);
  }
    private MutationSet sendMutationsToTabletServer(
        String location, Map<KeyExtent, List<Mutation>> tabMuts, TimeoutTracker timeoutTracker)
        throws IOException, AccumuloSecurityException, AccumuloServerException {
      if (tabMuts.size() == 0) {
        return new MutationSet();
      }
      TInfo tinfo = Tracer.traceInfo();

      // TODO remove this
      TTransport transport = null;

      timeoutTracker.startingWrite();

      try {
        TabletClientService.Iface client;

        if (timeoutTracker.getTimeOut()
            < instance.getConfiguration().getTimeInMillis(Property.GENERAL_RPC_TIMEOUT))
          client = ThriftUtil.getTServerClient(location, timeoutTracker.getTimeOut());
        else client = ThriftUtil.getTServerClient(location, instance.getConfiguration());

        try {
          MutationSet allFailures = new MutationSet();

          if (tabMuts.size() == 1 && tabMuts.values().iterator().next().size() == 1) {
            Entry<KeyExtent, List<Mutation>> entry = tabMuts.entrySet().iterator().next();

            try {
              client.update(
                  tinfo,
                  credentials.toThrift(instance),
                  entry.getKey().toThrift(),
                  entry.getValue().get(0).toThrift());
            } catch (NotServingTabletException e) {
              allFailures.addAll(entry.getKey().getTableId().toString(), entry.getValue());
              TabletLocator.getLocator(instance, new Text(entry.getKey().getTableId()))
                  .invalidateCache(entry.getKey());
            } catch (ConstraintViolationException e) {
              updatedConstraintViolations(
                  Translator.translate(e.violationSummaries, Translator.TCVST));
            }
            timeoutTracker.madeProgress();
          } else {

            long usid = client.startUpdate(tinfo, credentials.toThrift(instance));

            List<TMutation> updates = new ArrayList<TMutation>();
            for (Entry<KeyExtent, List<Mutation>> entry : tabMuts.entrySet()) {
              long size = 0;
              Iterator<Mutation> iter = entry.getValue().iterator();
              while (iter.hasNext()) {
                while (size < MUTATION_BATCH_SIZE && iter.hasNext()) {
                  Mutation mutation = iter.next();
                  updates.add(mutation.toThrift());
                  size += mutation.numBytes();
                }

                client.applyUpdates(tinfo, usid, entry.getKey().toThrift(), updates);
                updates.clear();
                size = 0;
              }
            }

            UpdateErrors updateErrors = client.closeUpdate(tinfo, usid);

            Map<KeyExtent, Long> failures =
                Translator.translate(updateErrors.failedExtents, Translator.TKET);
            updatedConstraintViolations(
                Translator.translate(updateErrors.violationSummaries, Translator.TCVST));
            updateAuthorizationFailures(
                Translator.translate(updateErrors.authorizationFailures, Translator.TKET));

            long totalCommitted = 0;

            for (Entry<KeyExtent, Long> entry : failures.entrySet()) {
              KeyExtent failedExtent = entry.getKey();
              int numCommitted = (int) (long) entry.getValue();
              totalCommitted += numCommitted;

              String table = failedExtent.getTableId().toString();

              TabletLocator.getLocator(instance, new Text(table)).invalidateCache(failedExtent);

              ArrayList<Mutation> mutations = (ArrayList<Mutation>) tabMuts.get(failedExtent);
              allFailures.addAll(table, mutations.subList(numCommitted, mutations.size()));
            }

            if (failures.keySet().containsAll(tabMuts.keySet()) && totalCommitted == 0) {
              // nothing was successfully written
              timeoutTracker.wroteNothing();
            } else {
              // successfully wrote something to tablet server
              timeoutTracker.madeProgress();
            }
          }
          return allFailures;
        } finally {
          ThriftUtil.returnClient((TServiceClient) client);
        }
      } catch (TTransportException e) {
        timeoutTracker.errorOccured(e);
        throw new IOException(e);
      } catch (TApplicationException tae) {
        updateServerErrors(location, tae);
        throw new AccumuloServerException(location, tae);
      } catch (ThriftSecurityException e) {
        updateAuthorizationFailures(tabMuts.keySet(), e.code);
        throw new AccumuloSecurityException(e.user, e.code, e);
      } catch (NoSuchScanIDException e) {
        throw new IOException(e);
      } catch (TException e) {
        throw new IOException(e);
      } finally {
        ThriftTransportPool.getInstance().returnTransport(transport);
      }
    }