/** Insert rows through staging phase */ private void insertRowsThroughStaging( TaskAttemptContext taskAttemptContext, InsertNode insertNode, Path finalOutputPath, Path stagingDir, Path stagingResultDir) throws IOException { EvalExprExec evalExprExec = new EvalExprExec(taskAttemptContext, (EvalExprNode) insertNode.getChild()); InsertRowsExec exec = new InsertRowsExec(taskAttemptContext, insertNode, evalExprExec); try { exec.init(); exec.next(); } finally { exec.close(); } FileSystem fs = TajoConf.getWarehouseDir(context.getConf()).getFileSystem(context.getConf()); if (insertNode.isOverwrite()) { // INSERT OVERWRITE INTO // it moves the original table into the temporary location. // Then it moves the new result table into the original table location. // Upon failed, it recovers the original table if possible. boolean movedToOldTable = false; boolean committed = false; Path oldTableDir = new Path(stagingDir, TajoConstants.INSERT_OVERWIRTE_OLD_TABLE_NAME); try { if (fs.exists(finalOutputPath)) { fs.rename(finalOutputPath, oldTableDir); movedToOldTable = fs.exists(oldTableDir); } else { // if the parent does not exist, make its parent directory. fs.mkdirs(finalOutputPath.getParent()); } fs.rename(stagingResultDir, finalOutputPath); committed = fs.exists(finalOutputPath); } catch (IOException ioe) { // recover the old table if (movedToOldTable && !committed) { fs.rename(oldTableDir, finalOutputPath); } } } else { FileStatus[] files = fs.listStatus(stagingResultDir); for (FileStatus eachFile : files) { Path targetFilePath = new Path(finalOutputPath, eachFile.getPath().getName()); if (fs.exists(targetFilePath)) { targetFilePath = new Path( finalOutputPath, eachFile.getPath().getName() + "_" + System.currentTimeMillis()); } fs.rename(eachFile.getPath(), targetFilePath); } } }
/** Insert row values */ private void insertRowValues( QueryContext queryContext, InsertNode insertNode, SubmitQueryResponse.Builder responseBuilder) { try { String nodeUniqName = insertNode.getTableName() == null ? new Path(insertNode.getUri()).getName() : insertNode.getTableName(); String queryId = nodeUniqName + "_" + System.currentTimeMillis(); URI finalOutputUri = insertNode.getUri(); Tablespace space = TablespaceManager.get(finalOutputUri).get(); TableMeta tableMeta = new TableMeta(insertNode.getStorageType(), insertNode.getOptions()); tableMeta.putOption(StorageConstants.INSERT_DIRECTLY, Boolean.TRUE.toString()); FormatProperty formatProperty = space.getFormatProperty(tableMeta); TaskAttemptContext taskAttemptContext; if (formatProperty .directInsertSupported()) { // if this format and storage supports direct insertion taskAttemptContext = new TaskAttemptContext(queryContext, null, null, null, null); taskAttemptContext.setOutputPath(new Path(finalOutputUri)); EvalExprExec evalExprExec = new EvalExprExec(taskAttemptContext, (EvalExprNode) insertNode.getChild()); InsertRowsExec exec = new InsertRowsExec(taskAttemptContext, insertNode, evalExprExec); try { exec.init(); exec.next(); } finally { exec.close(); } } else { URI stagingSpaceUri = space.prepareStagingSpace(context.getConf(), queryId, queryContext, tableMeta); Path stagingDir = new Path(stagingSpaceUri); Path stagingResultDir = new Path(stagingDir, TajoConstants.RESULT_DIR_NAME); taskAttemptContext = new TaskAttemptContext(queryContext, null, null, null, stagingDir); taskAttemptContext.setOutputPath(new Path(stagingResultDir, "part-01-000000")); insertRowsThroughStaging( taskAttemptContext, insertNode, new Path(finalOutputUri), stagingDir, stagingResultDir); } // set insert stats (how many rows and bytes) TableStats stats = new TableStats(); stats.setNumBytes(taskAttemptContext.getResultStats().getNumBytes()); stats.setNumRows(taskAttemptContext.getResultStats().getNumRows()); if (insertNode.hasTargetTable()) { CatalogProtos.UpdateTableStatsProto.Builder builder = CatalogProtos.UpdateTableStatsProto.newBuilder(); builder.setTableName(insertNode.getTableName()); builder.setStats(stats.getProto()); catalog.updateTableStats(builder.build()); TableDesc desc = new TableDesc( insertNode.getTableName(), insertNode.getTargetSchema(), tableMeta, finalOutputUri); responseBuilder.setTableDesc(desc.getProto()); } else { // If INSERT INTO LOCATION // Empty TableDesc List<CatalogProtos.ColumnProto> columns = new ArrayList<CatalogProtos.ColumnProto>(); CatalogProtos.TableDescProto tableDescProto = CatalogProtos.TableDescProto.newBuilder() .setTableName(nodeUniqName) .setMeta( CatalogProtos.TableProto.newBuilder() .setStoreType(BuiltinStorages.TEXT) .build()) .setSchema(CatalogProtos.SchemaProto.newBuilder().addAllFields(columns).build()) .setStats(stats.getProto()) .build(); responseBuilder.setTableDesc(tableDescProto); } // If queryId == NULL_QUERY_ID and MaxRowNum == -1, TajoCli prints only number of inserted // rows. responseBuilder.setMaxRowNum(-1); responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto()); responseBuilder.setResultType(ResultType.NO_RESULT); responseBuilder.setState(OK); } catch (Throwable t) { throw new RuntimeException(t); } }