Beispiel #1
0
  /**
   * Get tablespace for the given URI. If uri is null, the default tablespace will be returned
   *
   * @param uri Table or Table Fragment URI.
   * @param <T> Tablespace class type
   * @return Tablespace. If uri is null, the default tablespace will be returned.
   */
  public static <T extends Tablespace> T get(@Nullable String uri) {

    if (uri == null || uri.isEmpty()) {
      return getDefault();
    }

    Tablespace lastOne = null;

    // Find the longest matched one. For example, assume that the caller tries to find /x/y/z, and
    // there are /x and /x/y. In this case, /x/y will be chosen because it is more specific.
    for (Map.Entry<URI, Tablespace> entry :
        TABLE_SPACES.headMap(URI.create(uri), true).entrySet()) {
      if (uri.startsWith(entry.getKey().toString())) {
        lastOne = entry.getValue();
      }
    }

    if (lastOne == null) {
      lastOne =
          initializeTableSpace(UUID.randomUUID().toString(), URI.create(uri), new JSONObject());
      try {
        lastOne.init(systemConf);
      } catch (IOException e) {
        throw new RuntimeException(e);
      }
    }

    return (T) lastOne;
  }
Beispiel #2
0
  public void executeDistributedQuery(
      QueryContext queryContext,
      Session session,
      LogicalPlan plan,
      String sql,
      String jsonExpr,
      SubmitQueryResponse.Builder responseBuilder)
      throws Exception {
    LogicalRootNode rootNode = plan.getRootBlock().getRoot();

    TableDesc tableDesc = PlannerUtil.getTableDesc(catalog, plan.getRootBlock().getRoot());
    if (tableDesc != null) {

      Tablespace space = TablespaceManager.get(tableDesc.getUri()).get();
      FormatProperty formatProperty = space.getFormatProperty(tableDesc.getMeta());

      if (!formatProperty.isInsertable()) {
        throw new UnsupportedException(
            String.format("INSERT operation on %s tablespace", tableDesc.getUri().toString()));
      }

      space.prepareTable(rootNode.getChild());
    }

    hookManager.doHooks(queryContext, plan);

    QueryManager queryManager = this.context.getQueryJobManager();
    QueryInfo queryInfo;

    queryInfo = queryManager.scheduleQuery(session, queryContext, sql, jsonExpr, rootNode);

    responseBuilder.setState(OK);
    responseBuilder.setQueryId(queryInfo.getQueryId().getProto());
    responseBuilder.setResultType(ResultType.FETCH);
    if (queryInfo.getQueryMasterHost() != null) {
      responseBuilder.setQueryMasterHost(queryInfo.getQueryMasterHost());
    }
    responseBuilder.setQueryMasterPort(queryInfo.getQueryMasterClientPort());
    LOG.info(
        "Query "
            + queryInfo.getQueryId().toString()
            + ","
            + queryInfo.getSql()
            + ","
            + " is forwarded to "
            + queryInfo.getQueryMasterHost()
            + ":"
            + queryInfo.getQueryMasterPort());
  }
Beispiel #3
0
  public MasterPlan compileMasterPlan(LogicalPlan plan, QueryContext context, GlobalPlanner planner)
      throws Exception {

    LogicalRootNode rootNode = plan.getRootBlock().getRoot();
    TableDesc tableDesc = PlannerUtil.getTableDesc(planner.getCatalog(), rootNode.getChild());

    if (tableDesc != null) {
      Tablespace space = TablespaceManager.get(tableDesc.getUri()).get();
      space.rewritePlan(context, plan);
    }

    MasterPlan masterPlan = new MasterPlan(QueryIdFactory.NULL_QUERY_ID, context, plan);
    planner.build(context, masterPlan);

    return masterPlan;
  }
Beispiel #4
0
  @VisibleForTesting
  public static Optional<Tablespace> addTableSpaceForTest(Tablespace space) {
    Tablespace existing;
    synchronized (SPACES_URIS_MAP) {
      String scheme = UriUtil.getScheme(space.getUri());
      if (!TABLE_SPACE_HANDLERS.containsKey(scheme)) {
        TABLE_SPACE_HANDLERS.put(scheme, space.getClass());
      }

      // Remove existing one
      SPACES_URIS_MAP.remove(space.getName());
      existing = TABLE_SPACES.remove(space.getUri());

      // Add anotherone for test
      registerTableSpace(space.name, space.uri, space.getConfig(), true, true);
    }
    // if there is an existing one, return it.
    return Optional.fromNullable(existing);
  }
Beispiel #5
0
  private static void registerTableSpace(
      String spaceName, URI uri, JSONObject spaceDesc, boolean visible, boolean override) {
    Tablespace tableSpace = initializeTableSpace(spaceName, uri, spaceDesc);
    tableSpace.setVisible(visible);

    try {
      tableSpace.init(systemConf);
    } catch (IOException e) {
      throw new RuntimeException(e);
    }

    putTablespace(tableSpace, override);

    // If the arbitrary path is allowed, root uri is also added as a tablespace
    if (tableSpace.getProperty().isArbitraryPathAllowed()) {
      URI rootUri = tableSpace.getRootUri();
      // if there already exists or the rootUri is 'file:/', it won't overwrite the tablespace.
      if (!TABLE_SPACES.containsKey(rootUri)
          && !rootUri.toString().startsWith(LOCAL_FS_URI.toString())) {
        String tmpName = UUID.randomUUID().toString();
        registerTableSpace(tmpName, rootUri, spaceDesc, false, override);
      }
    }
  }
Beispiel #6
0
  private static void putTablespace(Tablespace space, boolean override) {
    // It is a device to keep the relationship among name, URI, and tablespace 1:1:1.

    boolean nameExist = SPACES_URIS_MAP.containsKey(space.getName());
    boolean uriExist = TABLE_SPACES.containsKey(space.uri);

    boolean mismatch = nameExist && !SPACES_URIS_MAP.get(space.getName()).equals(space.getUri());
    mismatch = mismatch || uriExist && TABLE_SPACES.get(space.uri).equals(space);

    if (!override && mismatch) {
      throw new RuntimeException("Name or URI of Tablespace must be unique.");
    }

    SPACES_URIS_MAP.put(space.getName(), space.getUri());
    // We must guarantee that the same uri results in the same tablespace instance.
    TABLE_SPACES.put(space.getUri(), space);
  }
Beispiel #7
0
  /** Insert row values */
  private void insertRowValues(
      QueryContext queryContext,
      InsertNode insertNode,
      SubmitQueryResponse.Builder responseBuilder) {
    try {
      String nodeUniqName =
          insertNode.getTableName() == null
              ? new Path(insertNode.getUri()).getName()
              : insertNode.getTableName();
      String queryId = nodeUniqName + "_" + System.currentTimeMillis();

      URI finalOutputUri = insertNode.getUri();
      Tablespace space = TablespaceManager.get(finalOutputUri).get();
      TableMeta tableMeta = new TableMeta(insertNode.getStorageType(), insertNode.getOptions());
      tableMeta.putOption(StorageConstants.INSERT_DIRECTLY, Boolean.TRUE.toString());

      FormatProperty formatProperty = space.getFormatProperty(tableMeta);

      TaskAttemptContext taskAttemptContext;
      if (formatProperty
          .directInsertSupported()) { // if this format and storage supports direct insertion
        taskAttemptContext = new TaskAttemptContext(queryContext, null, null, null, null);
        taskAttemptContext.setOutputPath(new Path(finalOutputUri));

        EvalExprExec evalExprExec =
            new EvalExprExec(taskAttemptContext, (EvalExprNode) insertNode.getChild());
        InsertRowsExec exec = new InsertRowsExec(taskAttemptContext, insertNode, evalExprExec);

        try {
          exec.init();
          exec.next();
        } finally {
          exec.close();
        }
      } else {
        URI stagingSpaceUri =
            space.prepareStagingSpace(context.getConf(), queryId, queryContext, tableMeta);
        Path stagingDir = new Path(stagingSpaceUri);
        Path stagingResultDir = new Path(stagingDir, TajoConstants.RESULT_DIR_NAME);

        taskAttemptContext = new TaskAttemptContext(queryContext, null, null, null, stagingDir);
        taskAttemptContext.setOutputPath(new Path(stagingResultDir, "part-01-000000"));
        insertRowsThroughStaging(
            taskAttemptContext, insertNode, new Path(finalOutputUri), stagingDir, stagingResultDir);
      }

      // set insert stats (how many rows and bytes)
      TableStats stats = new TableStats();
      stats.setNumBytes(taskAttemptContext.getResultStats().getNumBytes());
      stats.setNumRows(taskAttemptContext.getResultStats().getNumRows());

      if (insertNode.hasTargetTable()) {
        CatalogProtos.UpdateTableStatsProto.Builder builder =
            CatalogProtos.UpdateTableStatsProto.newBuilder();
        builder.setTableName(insertNode.getTableName());
        builder.setStats(stats.getProto());

        catalog.updateTableStats(builder.build());

        TableDesc desc =
            new TableDesc(
                insertNode.getTableName(), insertNode.getTargetSchema(), tableMeta, finalOutputUri);
        responseBuilder.setTableDesc(desc.getProto());

      } else { // If INSERT INTO LOCATION

        // Empty TableDesc
        List<CatalogProtos.ColumnProto> columns = new ArrayList<CatalogProtos.ColumnProto>();
        CatalogProtos.TableDescProto tableDescProto =
            CatalogProtos.TableDescProto.newBuilder()
                .setTableName(nodeUniqName)
                .setMeta(
                    CatalogProtos.TableProto.newBuilder()
                        .setStoreType(BuiltinStorages.TEXT)
                        .build())
                .setSchema(CatalogProtos.SchemaProto.newBuilder().addAllFields(columns).build())
                .setStats(stats.getProto())
                .build();

        responseBuilder.setTableDesc(tableDescProto);
      }

      // If queryId == NULL_QUERY_ID and MaxRowNum == -1, TajoCli prints only number of inserted
      // rows.
      responseBuilder.setMaxRowNum(-1);
      responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
      responseBuilder.setResultType(ResultType.NO_RESULT);
      responseBuilder.setState(OK);
    } catch (Throwable t) {
      throw new RuntimeException(t);
    }
  }
Beispiel #8
0
 @Override
 public URI getTableURI(@Nullable String spaceName, String databaseName, String tableName) {
   Tablespace space = spaceName == null ? getDefault() : getByName(spaceName);
   return space.getTableUri(databaseName, tableName);
 }