Example #1
0
  private static void buildCatalog() throws IOException {
    // build a real catalog
    File cat = File.createTempFile("temp-log-reinitiator", "catalog");
    cat.deleteOnExit();

    VoltProjectBuilder builder = new VoltProjectBuilder();
    String schema = "create table A (i integer not null, primary key (i));";
    builder.addLiteralSchema(schema);
    builder.addPartitionInfo("A", "i");
    builder.addStmtProcedure("hello", "select * from A where i = ?", "A.i: 0");

    if (!builder.compile(cat.getAbsolutePath())) {
      throw new IOException();
    }

    byte[] bytes = CatalogUtil.toBytes(cat);
    String serializedCat = CatalogUtil.loadCatalogFromJar(bytes, null);
    assertNotNull(serializedCat);
    Catalog catalog = new Catalog();
    catalog.execute(serializedCat);

    String deploymentPath = builder.getPathToDeployment();
    CatalogUtil.compileDeploymentAndGetCRC(catalog, deploymentPath, true);

    m_context = new CatalogContext(0, 0, catalog, bytes, 0, 0, 0);
    TheHashinator.initialize(LegacyHashinator.class, LegacyHashinator.getConfigureBytes(3));
  }
Example #2
0
    /*
     * Multi-partition/non-replicated fragment with collector and aggregator.
     */
    void addFragmentPair(
        int index,
        PlanFragment collectorFragment,
        PlanFragment aggregatorFragment,
        ByteBuffer params) {
      assert (index >= 0);
      assert (index < m_batchSize);
      assert (collectorFragment != null);
      assert (aggregatorFragment != null);
      assert (collectorFragment.getHasdependencies() == false);
      assert (aggregatorFragment.getHasdependencies() == true);

      // frags with no deps are usually collector frags that go to all partitions
      long distributedFragId = CatalogUtil.getUniqueIdForFragment(collectorFragment);
      long localFragId = CatalogUtil.getUniqueIdForFragment(aggregatorFragment);
      // if any frag is transactional, update this check
      if (aggregatorFragment.getNontransactional() == true) {
        m_localFragsAreNonTransactional = true;
      }
      int outputDepId = m_txnState.getNextDependencyId() | DtxnConstants.MULTIPARTITION_DEPENDENCY;
      m_depsForLocalTask[index] = outputDepId;
      // Add local and distributed fragments.
      m_localTask.addFragment(localFragId, m_depsToResume[index], params);
      m_distributedTask.addFragment(distributedFragId, outputDepId, params);
    }
Example #3
0
    @Override
    public void run() {
      CatalogAndIds catalogStuff = null;
      do {
        try {
          catalogStuff = CatalogUtil.getCatalogFromZK(m_rvdb.getHostMessenger().getZK());
        } catch (org.apache.zookeeper_voltpatches.KeeperException.NoNodeException e) {
        } catch (Exception e) {
          VoltDB.crashLocalVoltDB(
              "System was interrupted while waiting for a catalog.", false, null);
        }
      } while (catalogStuff == null || catalogStuff.catalogBytes.length == 0);

      String serializedCatalog = null;
      byte[] catalogJarBytes = catalogStuff.catalogBytes;
      try {
        Pair<InMemoryJarfile, String> loadResults =
            CatalogUtil.loadAndUpgradeCatalogFromJar(catalogStuff.catalogBytes);
        serializedCatalog = CatalogUtil.getSerializedCatalogStringFromJar(loadResults.getFirst());
        catalogJarBytes = loadResults.getFirst().getFullJarBytes();
      } catch (IOException e) {
        VoltDB.crashLocalVoltDB("Unable to load catalog", false, e);
      }

      if ((serializedCatalog == null) || (serializedCatalog.length() == 0))
        VoltDB.crashLocalVoltDB("Catalog loading failure", false, null);

      /* N.B. node recovery requires discovering the current catalog version. */
      Catalog catalog = new Catalog();
      catalog.execute(serializedCatalog);
      serializedCatalog = null;

      // note if this fails it will print an error first
      // This is where we compile real catalog and create runtime
      // catalog context. To validate deployment we compile and create
      // a starter context which uses a placeholder catalog.
      String result = CatalogUtil.compileDeployment(catalog, m_deployment, false);
      if (result != null) {
        hostLog.fatal(result);
        VoltDB.crashLocalVoltDB(result);
      }

      try {
        m_rvdb.m_catalogContext =
            new CatalogContext(
                catalogStuff.txnId,
                catalogStuff.uniqueId,
                catalog,
                catalogJarBytes,
                // Our starter catalog has set the deployment stuff, just yoink it out for now
                m_rvdb.m_catalogContext.getDeploymentBytes(),
                catalogStuff.version,
                -1);
      } catch (Exception e) {
        VoltDB.crashLocalVoltDB("Error agreeing on starting catalog version", true, e);
      }
    }
Example #4
0
    @Override
    public void run() {
      // if I'm the leader, send out the catalog
      if (m_rvdb.m_myHostId == m_rvdb.m_hostIdWithStartupCatalog) {

        try {
          // If no catalog was supplied provide an empty one.
          if (m_rvdb.m_pathToStartupCatalog == null) {
            try {
              File emptyJarFile = CatalogUtil.createTemporaryEmptyCatalogJarFile();
              if (emptyJarFile == null) {
                VoltDB.crashLocalVoltDB("Failed to generate empty catalog.");
              }
              m_rvdb.m_pathToStartupCatalog = emptyJarFile.getAbsolutePath();
            } catch (IOException e) {
              VoltDB.crashLocalVoltDB(
                  "I/O exception while creating empty catalog jar file.", false, e);
            }
          }

          // Get the catalog bytes and byte count.
          byte[] catalogBytes = readCatalog(m_rvdb.m_pathToStartupCatalog);

          // Export needs a cluster global unique id for the initial catalog version
          long catalogUniqueId =
              UniqueIdGenerator.makeIdFromComponents(
                  System.currentTimeMillis(), 0, MpInitiator.MP_INIT_PID);
          hostLog.debug(String.format("Sending %d catalog bytes", catalogBytes.length));

          long catalogTxnId;
          catalogTxnId = TxnEgo.makeZero(MpInitiator.MP_INIT_PID).getTxnId();

          // Need to get the deployment bytes from the starter catalog context
          byte[] deploymentBytes = m_rvdb.getCatalogContext().getDeploymentBytes();

          // publish the catalog bytes to ZK
          CatalogUtil.updateCatalogToZK(
              m_rvdb.getHostMessenger().getZK(),
              0,
              catalogTxnId,
              catalogUniqueId,
              catalogBytes,
              deploymentBytes);
        } catch (IOException e) {
          VoltDB.crashGlobalVoltDB("Unable to distribute catalog.", false, e);
        } catch (org.apache.zookeeper_voltpatches.KeeperException e) {
          VoltDB.crashGlobalVoltDB("Unable to publish catalog.", false, e);
        } catch (InterruptedException e) {
          VoltDB.crashGlobalVoltDB("Interrupted while publishing catalog.", false, e);
        }
      }
    }
Example #5
0
 @Override
 public String getColumnName(int columnIndex) {
   if (m_columns == null) {
     m_columns = CatalogUtil.getSortedCatalogItems(m_table.getColumns(), "index");
   }
   return m_columns.get(columnIndex).getTypeName();
 }
Example #6
0
    /*
     * Replicated fragment.
     */
    void addFragment(int index, PlanFragment frag, ByteBuffer params) {
      assert (index >= 0);
      assert (index < m_batchSize);
      assert (frag != null);
      assert (frag.getHasdependencies() == false);

      // if any frag is transactional, update this check
      if (frag.getNontransactional() == true) m_localFragsAreNonTransactional = true;

      long localFragId = CatalogUtil.getUniqueIdForFragment(frag);
      m_depsForLocalTask[index] = -1;
      // Add the local fragment data.
      m_localTask.addFragment(localFragId, m_depsToResume[index], params);
    }
Example #7
0
  protected VoltTable[] executeQueriesInABatch(List<QueuedSQL> batch, boolean isFinalSQL) {
    final int batchSize = batch.size();

    VoltTable[] results = null;

    if (batchSize == 0) {
      return new VoltTable[] {};
    }

    // IF THIS IS HSQL, RUN THE QUERIES DIRECTLY IN HSQL
    if (getHsqlBackendIfExists() != null) {
      results = new VoltTable[batchSize];
      int i = 0;
      for (QueuedSQL qs : batch) {
        List<StmtParameter> sparams;
        if (qs.stmt.catStmt != null) {
          CatalogMap<StmtParameter> sparamsMap = qs.stmt.catStmt.getParameters();
          sparams = CatalogUtil.getSortedCatalogItems(sparamsMap, "index");
        } else {
          assert (qs.stmt.plan != null);
          // TODO: For now extracted ad hoc SQL parameters are discarded
          qs.params = new ParameterSet();
          sparams = new ArrayList<StmtParameter>();
        }
        results[i++] =
            getHsqlBackendIfExists().runSQLWithSubstitutions(qs.stmt, qs.params, sparams);
      }
    } else if (m_catProc.getSinglepartition()) {
      results = fastPath(batch);
    } else {
      results = slowPath(batch, isFinalSQL);
    }

    // check expectations
    int i = 0;
    for (QueuedSQL qs : batch) {
      Expectation.check(m_procedureName, qs.stmt.sqlText, i, qs.expectation, results[i]);
      i++;
    }

    // clear the queued sql list for the next call
    batch.clear();

    return results;
  }
Example #8
0
  public static void initSQLStmt(SQLStmt stmt, Statement catStmt) {
    stmt.catStmt = catStmt;
    stmt.numFragGUIDs = catStmt.getFragments().size();
    PlanFragment fragments[] = new PlanFragment[stmt.numFragGUIDs];
    stmt.fragGUIDs = new long[stmt.numFragGUIDs];
    int i = 0;
    for (PlanFragment frag : stmt.catStmt.getFragments()) {
      fragments[i] = frag;
      stmt.fragGUIDs[i] = CatalogUtil.getUniqueIdForFragment(frag);
      i++;
    }

    stmt.numStatementParamJavaTypes = stmt.catStmt.getParameters().size();
    // StmtParameter parameters[] = new StmtParameter[stmt.numStatementParamJavaTypes];
    stmt.statementParamJavaTypes = new byte[stmt.numStatementParamJavaTypes];
    for (StmtParameter param : stmt.catStmt.getParameters()) {
      // parameters[i] = param;
      stmt.statementParamJavaTypes[param.getIndex()] = (byte) param.getJavatype();
      i++;
    }
  }
Example #9
0
  protected boolean createSetupInternal(
      String file_path,
      String file_nonce,
      long txnId,
      Map<Integer, Long> partitionTransactionIds,
      JSONObject jsData,
      SystemProcedureExecutionContext context,
      String hostname,
      final VoltTable result,
      Map<String, Map<Integer, Pair<Long, Long>>> exportSequenceNumbers,
      SiteTracker tracker,
      long timestamp)
      throws IOException {
    assert (SnapshotSiteProcessor.ExecutionSitesCurrentlySnapshotting.isEmpty());

    /*
     * List of partitions to include if this snapshot is
     * going to be deduped. Attempts to break up the work
     * by seeding and RNG selecting
     * a random replica to do the work. Will not work in failure
     * cases, but we don't use dedupe when we want durability.
     */
    List<Long> sitesToInclude = CSVSnapshotWritePlan.computeDedupedLocalSites(txnId, tracker);
    // If there's no work to do on this host, just claim success and get out:
    if (sitesToInclude.isEmpty() && !tracker.isFirstHost()) {
      return false;
    }

    NativeSnapshotWritePlan.createFileBasedCompletionTasks(
        file_path,
        file_nonce,
        txnId,
        partitionTransactionIds,
        context,
        exportSequenceNumbers,
        timestamp);

    final List<Table> tables = SnapshotUtil.getTablesToSave(context.getDatabase());
    final AtomicInteger numTables = new AtomicInteger(tables.size());
    final SnapshotRegistry.Snapshot snapshotRecord =
        SnapshotRegistry.startSnapshot(
            txnId,
            context.getHostId(),
            file_path,
            file_nonce,
            SnapshotFormat.CSV,
            tables.toArray(new Table[0]));

    SnapshotDataTarget sdt = null;
    boolean noTargetsCreated = true;

    final ArrayList<SnapshotTableTask> partitionedSnapshotTasks =
        new ArrayList<SnapshotTableTask>();
    final ArrayList<SnapshotTableTask> replicatedSnapshotTasks = new ArrayList<SnapshotTableTask>();
    for (final Table table : tables) {
      /*
       * For a deduped csv snapshot, only produce the replicated tables on the "leader"
       * host.
       */
      if (table.getIsreplicated() && !tracker.isFirstHost()) {
        snapshotRecord.removeTable(table.getTypeName());
        continue;
      }

      File saveFilePath = null;
      saveFilePath =
          SnapshotUtil.constructFileForTable(
              table, file_path, file_nonce, SnapshotFormat.CSV, context.getHostId());

      try {
        sdt = new SimpleFileSnapshotDataTarget(saveFilePath, !table.getIsreplicated());

        m_targets.add(sdt);
        final Runnable onClose =
            new TargetStatsClosure(sdt, table.getTypeName(), numTables, snapshotRecord);
        sdt.setOnCloseHandler(onClose);

        List<SnapshotDataFilter> filters = new ArrayList<SnapshotDataFilter>();
        filters.add(new CSVSnapshotFilter(CatalogUtil.getVoltTable(table), ',', null));

        final SnapshotTableTask task =
            new SnapshotTableTask(
                table.getRelativeIndex(),
                sdt,
                filters.toArray(new SnapshotDataFilter[filters.size()]),
                table.getIsreplicated(),
                table.getTypeName());

        if (table.getIsreplicated()) {
          replicatedSnapshotTasks.add(task);
        } else {
          partitionedSnapshotTasks.add(task);
        }

        noTargetsCreated = false;
        result.addRow(context.getHostId(), hostname, table.getTypeName(), "SUCCESS", "");
      } catch (IOException ex) {
        handleTargetCreationError(
            sdt, context, file_nonce, hostname, table.getTypeName(), ex, result);
      }
    }

    if (noTargetsCreated) {
      SnapshotRegistry.discardSnapshot(snapshotRecord);
    }

    // CSV snapshots do the partitioned work only on the specified sites for de-duping,
    // but since we've pre-filtered the replicated task list to only contain entries on
    // one node, we can go ahead and distribute them across all of the sites on that node.
    placePartitionedTasks(partitionedSnapshotTasks, sitesToInclude);
    placeReplicatedTasks(replicatedSnapshotTasks, tracker.getSitesForHost(context.getHostId()));
    return noTargetsCreated;
  }
Example #10
0
    @Override
    public void run() {
      if (!m_isRejoin && !m_config.m_isRejoinTest && !m_rvdb.m_joining) {
        String snapshotPath = null;
        if (m_rvdb
                .m_catalogContext
                .cluster
                .getDatabases()
                .get("database")
                .getSnapshotschedule()
                .get("default")
            != null) {
          snapshotPath =
              m_rvdb
                  .m_catalogContext
                  .cluster
                  .getDatabases()
                  .get("database")
                  .getSnapshotschedule()
                  .get("default")
                  .getPath();
        }

        int[] allPartitions = new int[m_rvdb.m_configuredNumberOfPartitions];
        for (int ii = 0; ii < allPartitions.length; ii++) {
          allPartitions[ii] = ii;
        }

        org.voltdb.catalog.CommandLog cl =
            m_rvdb.m_catalogContext.cluster.getLogconfig().get("log");

        try {
          m_rvdb.m_restoreAgent =
              new RestoreAgent(
                  m_rvdb.m_messenger,
                  m_rvdb.getSnapshotCompletionMonitor(),
                  m_rvdb,
                  m_config.m_startAction,
                  cl.getEnabled(),
                  cl.getLogpath(),
                  cl.getInternalsnapshotpath(),
                  snapshotPath,
                  allPartitions,
                  CatalogUtil.getVoltDbRoot(m_deployment.getPaths()).getAbsolutePath());
        } catch (IOException e) {
          VoltDB.crashLocalVoltDB("Unable to construct the RestoreAgent", true, e);
        }

        m_rvdb.m_globalServiceElector.registerService(m_rvdb.m_restoreAgent);
        m_rvdb.m_restoreAgent.setCatalogContext(m_rvdb.m_catalogContext);
        // Generate plans and get (hostID, catalogPath) pair
        Pair<Integer, String> catalog = m_rvdb.m_restoreAgent.findRestoreCatalog();

        // if the restore agent found a catalog, set the following info
        // so the right node can send it out to the others
        if (catalog != null) {
          // Make sure the catalog corresponds to the current server version.
          // Prevent automatic upgrades by rejecting mismatched versions.
          int hostId = catalog.getFirst().intValue();
          String catalogPath = catalog.getSecond();
          // Perform a version check when the catalog jar is available
          // on the current host.
          // Check that this host is the one providing the catalog.
          if (m_rvdb.m_myHostId == hostId) {
            try {
              byte[] catalogBytes = readCatalog(catalogPath);
              InMemoryJarfile inMemoryJar = CatalogUtil.loadInMemoryJarFile(catalogBytes);
              // This call pre-checks and returns the build info/version.
              String[] buildInfo = CatalogUtil.getBuildInfoFromJar(inMemoryJar);
              String catalogVersion = buildInfo[0];
              String serverVersion = m_rvdb.getVersionString();
              if (!catalogVersion.equals(serverVersion)) {
                VoltDB.crashLocalVoltDB(
                    String.format(
                        "Unable to load version %s catalog \"%s\" "
                            + "from snapshot into a version %s server.",
                        catalogVersion, catalogPath, serverVersion),
                    false,
                    null);
              }
            } catch (IOException e) {
              // Make it non-fatal with no check performed.
              hostLog.warn(
                  String.format(
                      "Unable to load catalog for version check due to exception: %s.",
                      e.getMessage()));
            }
          }
          hostLog.debug("Found catalog to load on host " + hostId + ": " + catalogPath);
          m_rvdb.m_hostIdWithStartupCatalog = hostId;
          assert (m_rvdb.m_hostIdWithStartupCatalog >= 0);
          m_rvdb.m_pathToStartupCatalog = catalogPath;
          assert (m_rvdb.m_pathToStartupCatalog != null);
        }
      }
    }
Example #11
0
  @SuppressWarnings("deprecation")
  @Override
  public void setUp() throws IOException, InterruptedException {
    VoltDB.instance().readBuildInfo("Test");

    // compile a catalog
    String testDir = BuildDirectoryUtils.getBuildDirectoryPath();
    String catalogJar = testDir + File.separator + JAR;

    TPCCProjectBuilder pb = new TPCCProjectBuilder();
    pb.addDefaultSchema();
    pb.addDefaultPartitioning();
    pb.addProcedures(MultiSiteSelect.class, InsertNewOrder.class);

    pb.compile(catalogJar, 2, 0);

    // load a catalog
    byte[] bytes = CatalogUtil.toBytes(new File(catalogJar));
    String serializedCatalog = CatalogUtil.loadCatalogFromJar(bytes, null);

    // create the catalog (that will be passed to the ClientInterface
    catalog = new Catalog();
    catalog.execute(serializedCatalog);

    // update the catalog with the data from the deployment file
    String pathToDeployment = pb.getPathToDeployment();
    assertTrue(CatalogUtil.compileDeploymentAndGetCRC(catalog, pathToDeployment, true) >= 0);

    cluster = catalog.getClusters().get("cluster");
    CatalogMap<Procedure> procedures = cluster.getDatabases().get("database").getProcedures();
    Procedure insertProc = procedures.get("InsertNewOrder");
    assert (insertProc != null);
    selectProc = procedures.get("MultiSiteSelect");
    assert (selectProc != null);

    // Each EE needs its own thread for correct initialization.
    final AtomicReference<ExecutionEngine> site1Reference = new AtomicReference<ExecutionEngine>();
    final byte configBytes[] = LegacyHashinator.getConfigureBytes(2);
    Thread site1Thread =
        new Thread() {
          @Override
          public void run() {
            site1Reference.set(
                new ExecutionEngineJNI(
                    cluster.getRelativeIndex(),
                    1,
                    0,
                    0,
                    "",
                    100,
                    HashinatorType.LEGACY,
                    configBytes));
          }
        };
    site1Thread.start();
    site1Thread.join();

    final AtomicReference<ExecutionEngine> site2Reference = new AtomicReference<ExecutionEngine>();
    Thread site2Thread =
        new Thread() {
          @Override
          public void run() {
            site2Reference.set(
                new ExecutionEngineJNI(
                    cluster.getRelativeIndex(),
                    2,
                    1,
                    0,
                    "",
                    100,
                    HashinatorType.LEGACY,
                    configBytes));
          }
        };
    site2Thread.start();
    site2Thread.join();

    // create two EEs
    site1 = new ExecutionSite(0); // site 0
    ee1 = site1Reference.get();
    ee1.loadCatalog(0, catalog.serialize());
    site2 = new ExecutionSite(1); // site 1
    ee2 = site2Reference.get();
    ee2.loadCatalog(0, catalog.serialize());

    // cache some plan fragments
    selectStmt = selectProc.getStatements().get("selectAll");
    assert (selectStmt != null);
    int i = 0;
    // this kinda assumes the right order
    for (PlanFragment f : selectStmt.getFragments()) {
      if (i == 0) selectTopFrag = f;
      else selectBottomFrag = f;
      i++;
    }
    assert (selectTopFrag != null);
    assert (selectBottomFrag != null);

    if (selectTopFrag.getHasdependencies() == false) {
      PlanFragment temp = selectTopFrag;
      selectTopFrag = selectBottomFrag;
      selectBottomFrag = temp;
    }

    // get the insert frag
    Statement insertStmt = insertProc.getStatements().get("insert");
    assert (insertStmt != null);

    for (PlanFragment f : insertStmt.getFragments()) insertFrag = f;

    // populate plan cache
    ActivePlanRepository.clear();
    ActivePlanRepository.addFragmentForTest(
        CatalogUtil.getUniqueIdForFragment(selectBottomFrag),
        Encoder.base64Decode(selectBottomFrag.getPlannodetree()));
    ActivePlanRepository.addFragmentForTest(
        CatalogUtil.getUniqueIdForFragment(selectTopFrag),
        Encoder.base64Decode(selectTopFrag.getPlannodetree()));
    ActivePlanRepository.addFragmentForTest(
        CatalogUtil.getUniqueIdForFragment(insertFrag),
        Encoder.base64Decode(insertFrag.getPlannodetree()));

    // insert some data
    ParameterSet params = ParameterSet.fromArrayNoCopy(1L, 1L, 1L);

    VoltTable[] results =
        ee2.executePlanFragments(
            1,
            new long[] {CatalogUtil.getUniqueIdForFragment(insertFrag)},
            null,
            new ParameterSet[] {params},
            1,
            0,
            42,
            Long.MAX_VALUE);
    assert (results.length == 1);
    assert (results[0].asScalarLong() == 1L);

    params = ParameterSet.fromArrayNoCopy(2L, 2L, 2L);

    results =
        ee1.executePlanFragments(
            1,
            new long[] {CatalogUtil.getUniqueIdForFragment(insertFrag)},
            null,
            new ParameterSet[] {params},
            2,
            1,
            42,
            Long.MAX_VALUE);
    assert (results.length == 1);
    assert (results[0].asScalarLong() == 1L);
  }
Example #12
0
  static String genrateStatementRow(Procedure procedure, Statement statement) {
    StringBuilder sb = new StringBuilder();
    sb.append("        <tr class='primaryrow2'>");

    // name column
    String anchor = (procedure.getTypeName() + "-" + statement.getTypeName()).toLowerCase();
    sb.append(
        "<td style='white-space: nowrap'><i id='p-"
            + anchor
            + "--icon' class='icon-chevron-right'></i> <a href='#' id='p-");
    sb.append(anchor).append("' class='togglex'>");
    sb.append(statement.getTypeName());
    sb.append("</a></td>");

    // sql column
    sb.append("<td><tt>");
    sb.append(statement.getSqltext());
    sb.append("</td></tt>");

    // params column
    sb.append("<td>");
    List<StmtParameter> params =
        CatalogUtil.getSortedCatalogItems(statement.getParameters(), "index");
    List<String> paramTypes = new ArrayList<String>();
    for (StmtParameter param : params) {
      paramTypes.add(VoltType.get((byte) param.getJavatype()).name());
    }
    if (paramTypes.size() == 0) {
      sb.append("<i>None</i>");
    }
    sb.append(StringUtils.join(paramTypes, ", "));
    sb.append("</td>");

    // r/w column
    sb.append("<td>");
    if (statement.getReadonly()) {
      tag(sb, "success", "Read");
    } else {
      tag(sb, "warning", "Write");
    }
    sb.append("</td>");

    // attributes
    sb.append("<td>");

    if (!statement.getIscontentdeterministic() || !statement.getIsorderdeterministic()) {
      tag(sb, "inverse", "Determinism");
    }

    if (statement.getSeqscancount() > 0) {
      tag(sb, "important", "Scans");
    }

    sb.append("</td>");

    sb.append("</tr>\n");

    // BUILD THE DROPDOWN FOR THE PLAN/DETAIL TABLE
    sb.append(
        "<tr class='dropdown2'><td colspan='5' id='p-"
            + procedure.getTypeName().toLowerCase()
            + "-"
            + statement.getTypeName().toLowerCase()
            + "--dropdown'>\n");

    sb.append("<div class='well well-small'><h4>Explain Plan:</h4>\n");
    StatementAnnotation annotation = (StatementAnnotation) statement.getAnnotation();
    if (annotation != null) {
      String plan = annotation.explainPlan;
      plan = plan.replace("\n", "<br/>");
      plan = plan.replace(" ", "&nbsp;");

      for (Table t : annotation.tablesRead) {
        String name = t.getTypeName().toUpperCase();
        String link = "\"<a href='#s-" + t.getTypeName() + "'>" + name + "</a>\"";
        plan = plan.replace("\"" + name + "\"", link);
      }
      for (Table t : annotation.tablesUpdated) {
        String name = t.getTypeName().toUpperCase();
        String link = "\"<a href='#s-" + t.getTypeName() + "'>" + name + "</a>\"";
        plan = plan.replace("\"" + name + "\"", link);
      }
      for (Index i : annotation.indexesUsed) {
        Table t = (Table) i.getParent();
        String name = i.getTypeName().toUpperCase();
        String link =
            "\"<a href='#s-" + t.getTypeName() + "-" + i.getTypeName() + "'>" + name + "</a>\"";
        plan = plan.replace("\"" + name + "\"", link);
      }

      sb.append("<tt>").append(plan).append("</tt>");
    } else {
      sb.append("<i>No SQL explain plan found.</i>\n");
    }
    sb.append("</div>\n");

    sb.append("</td></tr>\n");

    return sb.toString();
  }
Example #13
0
  public ClientResponseImpl call(Object... paramListIn) {
    // verify per-txn state has been reset
    assert (m_statusCode == ClientResponse.UNINITIALIZED_APP_STATUS_CODE);
    assert (m_statusString == null);
    assert (m_cachedRNG == null);

    // reset the hash of results
    m_inputCRC.reset();

    // use local var to avoid warnings about reassigning method argument
    Object[] paramList = paramListIn;

    ClientResponseImpl retval = null;
    // assert no sql is queued
    assert (m_batch.size() == 0);

    try {
      m_statsCollector.beginProcedure();

      byte status = ClientResponse.SUCCESS;
      VoltTable[] results = null;

      // inject sysproc execution context as the first parameter.
      if (isSystemProcedure()) {
        final Object[] combinedParams = new Object[paramList.length + 1];
        combinedParams[0] = m_systemProcedureContext;
        for (int i = 0; i < paramList.length; ++i) combinedParams[i + 1] = paramList[i];
        // swap the lists.
        paramList = combinedParams;
      }

      if (paramList.length != m_paramTypes.length) {
        m_statsCollector.endProcedure(false, true, null, null);
        String msg =
            "PROCEDURE "
                + m_procedureName
                + " EXPECTS "
                + String.valueOf(m_paramTypes.length)
                + " PARAMS, BUT RECEIVED "
                + String.valueOf(paramList.length);
        status = ClientResponse.GRACEFUL_FAILURE;
        return getErrorResponse(status, msg, null);
      }

      for (int i = 0; i < m_paramTypes.length; i++) {
        try {
          paramList[i] =
              ParameterConverter.tryToMakeCompatible(
                  m_paramTypeIsPrimitive[i],
                  m_paramTypeIsArray[i],
                  m_paramTypes[i],
                  m_paramTypeComponentType[i],
                  paramList[i]);
        } catch (Exception e) {
          m_statsCollector.endProcedure(false, true, null, null);
          String msg =
              "PROCEDURE "
                  + m_procedureName
                  + " TYPE ERROR FOR PARAMETER "
                  + i
                  + ": "
                  + e.toString();
          status = ClientResponse.GRACEFUL_FAILURE;
          return getErrorResponse(status, msg, null);
        }
      }

      boolean error = false;
      boolean abort = false;
      // run a regular java class
      if (m_catProc.getHasjava()) {
        try {
          if (log.isTraceEnabled()) {
            log.trace(
                "invoking... procMethod="
                    + m_procMethod.getName()
                    + ", class="
                    + getClass().getName());
          }
          try {
            Object rawResult = m_procMethod.invoke(m_procedure, paramList);
            results = getResultsFromRawResults(rawResult);
          } catch (IllegalAccessException e) {
            // If reflection fails, invoke the same error handling that other exceptions do
            throw new InvocationTargetException(e);
          }
          log.trace("invoked");
        } catch (InvocationTargetException itex) {
          // itex.printStackTrace();
          Throwable ex = itex.getCause();
          if (ex instanceof VoltAbortException && !(ex instanceof EEException)) {
            abort = true;
          } else {
            error = true;
          }
          if (ex instanceof Error) {
            m_statsCollector.endProcedure(false, true, null, null);
            throw (Error) ex;
          }

          retval = getErrorResponse(ex);
        }
      }
      // single statement only work
      // (this could be made faster, but with less code re-use)
      else {
        assert (m_catProc.getStatements().size() == 1);
        try {
          m_cachedSingleStmt.params = getCleanParams(m_cachedSingleStmt.stmt, paramList);
          if (getHsqlBackendIfExists() != null) {
            // HSQL handling
            CatalogMap<StmtParameter> sparamsMap = m_cachedSingleStmt.stmt.catStmt.getParameters();
            List<StmtParameter> sparams = CatalogUtil.getSortedCatalogItems(sparamsMap, "index");
            VoltTable table =
                getHsqlBackendIfExists()
                    .runSQLWithSubstitutions(
                        m_cachedSingleStmt.stmt, m_cachedSingleStmt.params, sparams);
            results = new VoltTable[] {table};
          } else {
            m_batch.add(m_cachedSingleStmt);
            results = voltExecuteSQL(true);
          }
        } catch (SerializableException ex) {
          retval = getErrorResponse(ex);
        }
      }

      // Record statistics for procedure call.
      StoredProcedureInvocation invoc = (m_txnState != null ? m_txnState.getInvocation() : null);
      ParameterSet paramSet = (invoc != null ? invoc.getParams() : null);
      m_statsCollector.endProcedure(abort, error, results, paramSet);

      // don't leave empty handed
      if (results == null) results = new VoltTable[0];

      if (retval == null)
        retval = new ClientResponseImpl(status, m_statusCode, m_statusString, results, null);

      int hash = (int) m_inputCRC.getValue();
      if ((retval.getStatus() == ClientResponse.SUCCESS) && (hash != 0)) {
        retval.setHash(hash);
      }
      if ((m_txnState != null)
          && // may be null for tests
          (m_txnState.getInvocation() != null)
          && (m_txnState.getInvocation().getType() == ProcedureInvocationType.REPLICATED)) {
        retval.convertResultsToHashForDeterminism();
      }
    } finally {
      // finally at the call(..) scope to ensure params can be
      // garbage collected and that the queue will be empty for
      // the next call
      m_batch.clear();

      // reset other per-txn state
      m_txnState = null;
      m_statusCode = ClientResponse.UNINITIALIZED_APP_STATUS_CODE;
      m_statusString = null;
      m_cachedRNG = null;
      m_cachedSingleStmt.params = null;
      m_cachedSingleStmt.expectation = null;
      m_seenFinalBatch = false;
    }

    return retval;
  }
Example #14
0
  static String genrateIndexRow(Table table, Index index) {
    StringBuilder sb = new StringBuilder();
    sb.append("        <tr class='primaryrow2'>");

    // name column
    String anchor = (table.getTypeName() + "-" + index.getTypeName()).toLowerCase();
    sb.append(
        "<td style='white-space: nowrap'><i id='s-"
            + anchor
            + "--icon' class='icon-chevron-right'></i> <a href='#' id='s-");
    sb.append(anchor).append("' class='togglex'>");
    sb.append(index.getTypeName());
    sb.append("</a></td>");

    // type column
    sb.append("<td>");
    sb.append(IndexType.get(index.getType()).toString());
    sb.append("</td>");

    // columns column
    sb.append("<td>");
    List<ColumnRef> cols = CatalogUtil.getSortedCatalogItems(index.getColumns(), "index");
    List<String> columnNames = new ArrayList<String>();
    for (ColumnRef colRef : cols) {
      columnNames.add(colRef.getColumn().getTypeName());
    }
    sb.append(StringUtils.join(columnNames, ", "));
    sb.append("</td>");

    // uniqueness column
    sb.append("<td>");
    if (index.getAssumeunique()) {
      tag(sb, "important", "AssumeUnique");
    } else if (index.getUnique()) {
      tag(sb, "important", "Unique");
    } else {
      tag(sb, "info", "Nonunique");
    }
    sb.append("</td>");

    sb.append("</tr>\n");

    // BUILD THE DROPDOWN FOR THE PLAN/DETAIL TABLE
    sb.append(
        "<tr class='dropdown2'><td colspan='5' id='s-"
            + table.getTypeName().toLowerCase()
            + "-"
            + index.getTypeName().toLowerCase()
            + "--dropdown'>\n");

    IndexAnnotation annotation = (IndexAnnotation) index.getAnnotation();
    if (annotation != null) {
      if (annotation.proceduresThatUseThis.size() > 0) {
        sb.append("<p>Used by procedures: ");
        List<String> procs = new ArrayList<String>();
        for (Procedure proc : annotation.proceduresThatUseThis) {
          procs.add("<a href='#p-" + proc.getTypeName() + "'>" + proc.getTypeName() + "</a>");
        }
        sb.append(StringUtils.join(procs, ", "));
        sb.append("</p>");
      }
    }

    sb.append("</td></tr>\n");

    return sb.toString();
  }
Example #15
0
  static String generateProcedureRow(Procedure procedure) {
    StringBuilder sb = new StringBuilder();
    sb.append("<tr class='primaryrow'>");

    // column 1: procedure name
    String anchor = procedure.getTypeName().toLowerCase();
    sb.append(
        "<td style='white-space: nowrap'><i id='p-"
            + anchor
            + "--icon' class='icon-chevron-right'></i> <a href='#p-");
    sb.append(anchor).append("' id='p-").append(anchor).append("' class='togglex'>");
    sb.append(procedure.getTypeName());
    sb.append("</a></td>");

    // column 2: parameter types
    sb.append("<td>");
    List<ProcParameter> params =
        CatalogUtil.getSortedCatalogItems(procedure.getParameters(), "index");
    List<String> paramTypes = new ArrayList<String>();
    for (ProcParameter param : params) {
      String paramType = VoltType.get((byte) param.getType()).name();
      if (param.getIsarray()) {
        paramType += "[]";
      }
      paramTypes.add(paramType);
    }
    if (paramTypes.size() == 0) {
      sb.append("<i>None</i>");
    }
    sb.append(StringUtils.join(paramTypes, ", "));
    sb.append("</td>");

    // column 3: partitioning
    sb.append("<td>");
    if (procedure.getSinglepartition()) {
      tag(sb, "success", "Single");
    } else {
      tag(sb, "warning", "Multi");
    }
    sb.append("</td>");

    // column 4: read/write
    sb.append("<td>");
    if (procedure.getReadonly()) {
      tag(sb, "success", "Read");
    } else {
      tag(sb, "warning", "Write");
    }
    sb.append("</td>");

    // column 5: access
    sb.append("<td>");
    List<String> groupNames = new ArrayList<String>();
    for (GroupRef groupRef : procedure.getAuthgroups()) {
      groupNames.add(groupRef.getGroup().getTypeName());
    }
    if (groupNames.size() == 0) {
      sb.append("<i>None</i>");
    }
    sb.append(StringUtils.join(groupNames, ", "));
    sb.append("</td>");

    // column 6: attributes
    sb.append("<td>");
    if (procedure.getHasjava()) {
      tag(sb, "info", "Java");
    } else {
      tag(sb, null, "Single-Stmt");
    }
    boolean isND = false;
    int scanCount = 0;
    for (Statement stmt : procedure.getStatements()) {
      scanCount += stmt.getSeqscancount();
      if (!stmt.getIscontentdeterministic() || !stmt.getIsorderdeterministic()) {
        isND = false;
      }
    }
    if (isND) {
      tag(sb, "inverse", "Determinism");
    }
    if (scanCount > 0) {
      tag(sb, "important", "Scans");
    }
    sb.append("</td>");

    sb.append("</tr>\n");

    // BUILD THE DROPDOWN FOR THE STATEMENT/DETAIL TABLE

    sb.append(
        "<tr class='tablesorter-childRow'><td class='invert' colspan='6' id='p-"
            + procedure.getTypeName().toLowerCase()
            + "--dropdown'>\n");

    // output partitioning parameter info
    if (procedure.getSinglepartition()) {
      String pTable = procedure.getPartitioncolumn().getParent().getTypeName();
      String pColumn = procedure.getPartitioncolumn().getTypeName();
      int pIndex = procedure.getPartitionparameter();

      sb.append(
          String.format(
              "<p>Partitioned on parameter %d which maps to column %s"
                  + " of table <a class='invert' href='#s-%s'>%s</a>.</p>",
              pIndex, pColumn, pTable, pTable));
    }

    // output what schema this interacts with
    ProcedureAnnotation annotation = (ProcedureAnnotation) procedure.getAnnotation();
    if (annotation != null) {
      // make sure tables appear in only one category
      annotation.tablesRead.removeAll(annotation.tablesUpdated);

      if (annotation.tablesRead.size() > 0) {
        sb.append("<p>Read-only access to tables: ");
        List<String> tables = new ArrayList<String>();
        for (Table table : annotation.tablesRead) {
          tables.add("<a href='#s-" + table.getTypeName() + "'>" + table.getTypeName() + "</a>");
        }
        sb.append(StringUtils.join(tables, ", "));
        sb.append("</p>");
      }
      if (annotation.tablesUpdated.size() > 0) {
        sb.append("<p>Read/Write access to tables: ");
        List<String> tables = new ArrayList<String>();
        for (Table table : annotation.tablesUpdated) {
          tables.add("<a href='#s-" + table.getTypeName() + "'>" + table.getTypeName() + "</a>");
        }
        sb.append(StringUtils.join(tables, ", "));
        sb.append("</p>");
      }
      if (annotation.indexesUsed.size() > 0) {
        sb.append("<p>Uses indexes: ");
        List<String> indexes = new ArrayList<String>();
        for (Index index : annotation.indexesUsed) {
          Table table = (Table) index.getParent();
          indexes.add(
              "<a href='#s-"
                  + table.getTypeName()
                  + "-"
                  + index.getTypeName()
                  + "'>"
                  + index.getTypeName()
                  + "</a>");
        }
        sb.append(StringUtils.join(indexes, ", "));
        sb.append("</p>");
      }
    }

    sb.append(generateStatementsTable(procedure));

    sb.append("</td></tr>\n");

    return sb.toString();
  }
Example #16
0
  static void compileSingleStmtProcedure(
      VoltCompiler compiler,
      HSQLInterface hsql,
      DatabaseEstimates estimates,
      Catalog catalog,
      Database db,
      ProcedureDescriptor procedureDescriptor)
      throws VoltCompiler.VoltCompilerException {
    final String className = procedureDescriptor.m_className;
    if (className.indexOf('@') != -1) {
      throw compiler.new VoltCompilerException("User procedure names can't contain \"@\".");
    }

    // get the short name of the class (no package if a user procedure)
    // use the Table.<builtin> name (allowing the period) if builtin.
    String shortName = className;
    if (procedureDescriptor.m_builtInStmt == false) {
      String[] parts = className.split("\\.");
      shortName = parts[parts.length - 1];
    }

    // add an entry to the catalog (using the full className)
    final Procedure procedure = db.getProcedures().add(shortName);
    for (String groupName : procedureDescriptor.m_authGroups) {
      final Group group = db.getGroups().get(groupName);
      if (group == null) {
        throw compiler
        .new VoltCompilerException(
            "Procedure "
                + className
                + " allows access by a role "
                + groupName
                + " that does not exist");
      }
      final GroupRef groupRef = procedure.getAuthgroups().add(groupName);
      groupRef.setGroup(group);
    }
    procedure.setClassname(className);
    // sysprocs don't use the procedure compiler
    procedure.setSystemproc(false);
    procedure.setDefaultproc(procedureDescriptor.m_builtInStmt);
    procedure.setHasjava(false);

    // get the annotation
    // first try to get one that has been passed from the compiler
    ProcInfoData info = compiler.getProcInfoOverride(shortName);
    // then check for the usual one in the class itself
    // and create a ProcInfo.Data instance for it
    if (info == null) {
      info = new ProcInfoData();
      if (procedureDescriptor.m_partitionString != null) {
        info.partitionInfo = procedureDescriptor.m_partitionString;
        info.singlePartition = true;
      }
    }
    assert (info != null);

    // ADD THE STATEMENT

    // add the statement to the catalog
    Statement catalogStmt = procedure.getStatements().add(VoltDB.ANON_STMT_NAME);

    // compile the statement
    StatementPartitioning partitioning =
        info.singlePartition ? StatementPartitioning.forceSP() : StatementPartitioning.forceMP();
    // default to FASTER detmode because stmt procs can't feed read output into writes
    StatementCompiler.compileFromSqlTextAndUpdateCatalog(
        compiler,
        hsql,
        catalog,
        db,
        estimates,
        catalogStmt,
        procedureDescriptor.m_singleStmt,
        procedureDescriptor.m_joinOrder,
        DeterminismMode.FASTER,
        partitioning);

    // if the single stmt is not read only, then the proc is not read only
    boolean procHasWriteStmts = (catalogStmt.getReadonly() == false);

    // set the read onlyness of a proc
    procedure.setReadonly(procHasWriteStmts == false);

    int seqs = catalogStmt.getSeqscancount();
    procedure.setHasseqscans(seqs > 0);

    // set procedure parameter types
    CatalogMap<ProcParameter> params = procedure.getParameters();
    CatalogMap<StmtParameter> stmtParams = catalogStmt.getParameters();

    // set the procedure parameter types from the statement parameter types
    int paramCount = 0;
    for (StmtParameter stmtParam : CatalogUtil.getSortedCatalogItems(stmtParams, "index")) {
      // name each parameter "param1", "param2", etc...
      ProcParameter procParam = params.add("param" + String.valueOf(paramCount));
      procParam.setIndex(stmtParam.getIndex());
      procParam.setIsarray(stmtParam.getIsarray());
      procParam.setType(stmtParam.getJavatype());
      paramCount++;
    }

    // parse the procinfo
    procedure.setSinglepartition(info.singlePartition);
    if (info.singlePartition) {
      parsePartitionInfo(compiler, db, procedure, info.partitionInfo);
      if (procedure.getPartitionparameter() >= params.size()) {
        String msg =
            "PartitionInfo parameter not a valid parameter for procedure: "
                + procedure.getClassname();
        throw compiler.new VoltCompilerException(msg);
      }
      // TODO: The planner does not currently validate that a single-statement plan declared as
      // single-partition correctly uses
      // the designated parameter as a partitioning filter, maybe some day.
      // In theory, the PartitioningForStatement would confirm the use of (only) a parameter as a
      // partition key --
      // or if the partition key was determined to be some other hard-coded constant (expression?)
      // it might display a warning
      // message that the passed parameter is assumed to be equal to that constant (expression).
    } else {
      if (partitioning.getCountOfIndependentlyPartitionedTables() == 1) {
        AbstractExpression statementPartitionExpression =
            partitioning.singlePartitioningExpressionForReport();
        if (statementPartitionExpression != null) {
          // The planner has uncovered an overlooked opportunity to run the statement SP.
          String msg = null;
          if (statementPartitionExpression instanceof ParameterValueExpression) {
            msg =
                "This procedure would benefit from setting the attribute 'partitioninfo="
                    + partitioning.getFullColumnName()
                    + ":"
                    + ((ParameterValueExpression) statementPartitionExpression).getParameterIndex()
                    + "'";
          } else {
            String valueDescription = null;
            Object partitionValue = partitioning.getInferredPartitioningValue();
            if (partitionValue == null) {
              // Statement partitioned on a runtime constant. This is likely to be cryptic, but
              // hopefully gets the idea across.
              valueDescription = "of " + statementPartitionExpression.explain("");
            } else {
              valueDescription =
                  partitionValue.toString(); // A simple constant value COULD have been a parameter.
            }
            msg =
                "This procedure would benefit from adding a parameter to be passed the value "
                    + valueDescription
                    + " and setting the attribute 'partitioninfo="
                    + partitioning.getFullColumnName()
                    + ":"
                    + paramCount
                    + "'";
          }
          compiler.addWarn(msg);
        }
      }
    }
  }
Example #17
0
  public void testMultiSiteSelectAll() {
    ParameterSet params = ParameterSet.emptyParameterSet();

    int outDepId = 1 | DtxnConstants.MULTIPARTITION_DEPENDENCY;
    VoltTable dependency1 =
        ee1.executePlanFragments(
                1,
                new long[] {CatalogUtil.getUniqueIdForFragment(selectBottomFrag)},
                null,
                new ParameterSet[] {params},
                3,
                2,
                42,
                Long.MAX_VALUE)[0];
    try {
      System.out.println(dependency1.toString());
    } catch (Exception e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }
    assertTrue(dependency1 != null);

    VoltTable dependency2 =
        ee2.executePlanFragments(
                1,
                new long[] {CatalogUtil.getUniqueIdForFragment(selectBottomFrag)},
                null,
                new ParameterSet[] {params},
                3,
                2,
                42,
                Long.MAX_VALUE)[0];
    try {
      System.out.println(dependency2.toString());
    } catch (Exception e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }
    assertTrue(dependency2 != null);

    ee1.stashDependency(outDepId, dependency1);
    ee1.stashDependency(outDepId, dependency2);

    dependency1 =
        ee1.executePlanFragments(
                1,
                new long[] {CatalogUtil.getUniqueIdForFragment(selectTopFrag)},
                new long[] {outDepId},
                new ParameterSet[] {params},
                3,
                2,
                42,
                Long.MAX_VALUE)[0];
    try {
      System.out.println("Final Result");
      System.out.println(dependency1.toString());
    } catch (Exception e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }
  }
Example #18
0
  /**
   * Create a new data source.
   *
   * @param db
   * @param tableName
   * @param isReplicated
   * @param partitionId
   * @param HSId
   * @param tableId
   * @param catalogMap
   */
  public ExportDataSource(
      Runnable onDrain,
      String db,
      String tableName,
      int partitionId,
      long HSId,
      String signature,
      long generation,
      CatalogMap<Column> catalogMap,
      String overflowPath)
      throws IOException {
    m_generation = generation;
    m_onDrain = onDrain;
    m_database = db;
    m_tableName = tableName;

    String nonce = signature + "_" + HSId + "_" + partitionId;

    m_committedBuffers = new StreamBlockQueue(overflowPath, nonce);

    /*
     * This is not the catalog relativeIndex(). This ID incorporates
     * a catalog version and a table id so that it is constant across
     * catalog updates that add or drop tables.
     */
    m_signature = signature;
    m_partitionId = partitionId;
    m_HSId = HSId;

    // Add the Export meta-data columns to the schema followed by the
    // catalog columns for this table.
    m_columnNames.add("VOLT_TRANSACTION_ID");
    m_columnTypes.add(((int) VoltType.BIGINT.getValue()));

    m_columnNames.add("VOLT_EXPORT_TIMESTAMP");
    m_columnTypes.add(((int) VoltType.BIGINT.getValue()));

    m_columnNames.add("VOLT_EXPORT_SEQUENCE_NUMBER");
    m_columnTypes.add(((int) VoltType.BIGINT.getValue()));

    m_columnNames.add("VOLT_PARTITION_ID");
    m_columnTypes.add(((int) VoltType.BIGINT.getValue()));

    m_columnNames.add("VOLT_SITE_ID");
    m_columnTypes.add(((int) VoltType.BIGINT.getValue()));

    m_columnNames.add("VOLT_EXPORT_OPERATION");
    m_columnTypes.add(((int) VoltType.TINYINT.getValue()));

    for (Column c : CatalogUtil.getSortedCatalogItems(catalogMap, "index")) {
      m_columnNames.add(c.getName());
      m_columnTypes.add(c.getType());
    }

    File adFile = new VoltFile(overflowPath, nonce + ".ad");
    exportLog.info("Creating ad for " + nonce);
    assert (!adFile.exists());
    FastSerializer fs = new FastSerializer();
    fs.writeLong(m_HSId);
    fs.writeString(m_database);
    writeAdvertisementTo(fs);
    FileOutputStream fos = new FileOutputStream(adFile);
    fos.write(fs.getBytes());
    fos.getFD().sync();
    fos.close();

    // compute the number of bytes necessary to hold one bit per
    // schema column
    m_nullArrayLength = ((m_columnTypes.size() + 7) & -8) >> 3;
  }