private List<Tuple> getAllTableStats(Schema outSchema) {
    List<TableStatsProto> statList = masterContext.getCatalog().getAllTableStats();
    List<Tuple> tuples = new ArrayList<Tuple>(statList.size());
    List<Column> columns = outSchema.getRootColumns();
    Tuple aTuple;

    for (TableStatsProto stat : statList) {
      aTuple = new VTuple(outSchema.size());

      for (int fieldId = 0; fieldId < columns.size(); fieldId++) {
        Column column = columns.get(fieldId);

        if ("tid".equalsIgnoreCase(column.getSimpleName())) {
          aTuple.put(fieldId, DatumFactory.createInt4(stat.getTid()));
        } else if ("num_rows".equalsIgnoreCase(column.getSimpleName())) {
          aTuple.put(fieldId, DatumFactory.createInt8(stat.getNumRows()));
        } else if ("num_bytes".equalsIgnoreCase(column.getSimpleName())) {
          aTuple.put(fieldId, DatumFactory.createInt8(stat.getNumBytes()));
        }
      }

      tuples.add(aTuple);
    }

    return tuples;
  }
    @Override
    public Tuple next() throws IOException {
      Tuple aTuple;
      Tuple outTuple = new VTuple(outColumnNum);

      if (isClosed) {
        return null;
      }

      if (cachedData.size() == 0) {
        rescan();
      }

      if (!scanNode.hasQual()) {
        if (currentRow < cachedData.size()) {
          aTuple = cachedData.get(currentRow++);
          projector.eval(aTuple, outTuple);
          outTuple.setOffset(aTuple.getOffset());
          return outTuple;
        }
        return null;
      } else {
        while (currentRow < cachedData.size()) {
          aTuple = cachedData.get(currentRow++);
          if (qual.eval(aTuple).isTrue()) {
            projector.eval(aTuple, outTuple);
            return outTuple;
          }
        }
        return null;
      }
    }
  private List<Tuple> getDatabases(Schema outSchema) {
    List<DatabaseProto> databases = masterContext.getCatalog().getAllDatabases();
    List<Tuple> tuples = new ArrayList<Tuple>(databases.size());
    List<Column> columns = outSchema.getRootColumns();
    Tuple aTuple;

    for (DatabaseProto database : databases) {
      aTuple = new VTuple(outSchema.size());

      for (int fieldId = 0; fieldId < columns.size(); fieldId++) {
        Column column = columns.get(fieldId);
        if ("db_id".equalsIgnoreCase(column.getSimpleName())) {
          aTuple.put(fieldId, DatumFactory.createInt4(database.getId()));
        } else if ("db_name".equalsIgnoreCase(column.getSimpleName())) {
          aTuple.put(fieldId, DatumFactory.createText(database.getName()));
        } else if ("space_id".equalsIgnoreCase(column.getSimpleName())) {
          if (database.hasSpaceId()) {
            aTuple.put(fieldId, DatumFactory.createInt4(database.getSpaceId()));
          } else {
            aTuple.put(fieldId, DatumFactory.createNullDatum());
          }
        }
      }

      tuples.add(aTuple);
    }

    return tuples;
  }
  private List<Tuple> getAllTableOptions(Schema outSchema) {
    List<TableOptionProto> optionList = masterContext.getCatalog().getAllTableOptions();
    List<Tuple> tuples = new ArrayList<Tuple>(optionList.size());
    List<Column> columns = outSchema.getRootColumns();
    Tuple aTuple;

    for (TableOptionProto option : optionList) {
      aTuple = new VTuple(outSchema.size());

      for (int fieldId = 0; fieldId < columns.size(); fieldId++) {
        Column column = columns.get(fieldId);

        if ("tid".equalsIgnoreCase(column.getSimpleName())) {
          aTuple.put(fieldId, DatumFactory.createInt4(option.getTid()));
        } else if ("key_".equalsIgnoreCase(column.getSimpleName())) {
          aTuple.put(fieldId, DatumFactory.createText(option.getKeyval().getKey()));
        } else if ("value_".equalsIgnoreCase(column.getSimpleName())) {
          aTuple.put(fieldId, DatumFactory.createText(option.getKeyval().getValue()));
        }
      }

      tuples.add(aTuple);
    }

    return tuples;
  }
Beispiel #5
0
  @Override
  public Datum eval(Tuple params) {
    if (params.isBlankOrNull(0)) {
      return NullDatum.get();
    }

    return DatumFactory.createFloat8(Math.asin(params.getFloat8(0)));
  }
Beispiel #6
0
  @Override
  public Datum eval(Tuple params) {
    if (params.isBlankOrNull(0)) {
      return NullDatum.get();
    }

    String ipstr = params.getText(0);

    return new Int4Datum(IPconvertUtil.ipstr2int(ipstr));
  }
Beispiel #7
0
  @Override
  public Datum eval(Tuple params) {
    if (params.isBlankOrNull(0) || params.isBlankOrNull(1)) {
      return NullDatum.get();
    }

    String value = params.getText(0);
    String pattern = params.getText(1);

    TimeMeta tm = DateTimeFormat.parseDateTime(value, pattern);

    return new DateDatum(tm);
  }
Beispiel #8
0
  @Override
  public Datum eval(Tuple params) {
    if (params.isBlankOrNull(0)) {
      return NullDatum.get();
    }

    int value = params.getInt4(0);
    if (value <= 0 || value > 65525) {
      return NullDatum.get();
    } else {
      return DatumFactory.createText(String.valueOf((char) value));
    }
  }
  private List<Tuple> getTables(Schema outSchema) {
    List<TableDescriptorProto> tables = masterContext.getCatalog().getAllTables();
    List<Tuple> tuples = new ArrayList<Tuple>(tables.size());
    List<Column> columns = outSchema.getRootColumns();
    Tuple aTuple;

    for (TableDescriptorProto table : tables) {
      aTuple = new VTuple(outSchema.size());

      for (int fieldId = 0; fieldId < columns.size(); fieldId++) {
        Column column = columns.get(fieldId);
        if ("tid".equalsIgnoreCase(column.getSimpleName())) {
          aTuple.put(fieldId, DatumFactory.createInt4(table.getTid()));
        } else if ("db_id".equalsIgnoreCase(column.getSimpleName())) {
          aTuple.put(fieldId, DatumFactory.createInt4(table.getDbId()));
        } else if ("table_name".equalsIgnoreCase(column.getSimpleName())) {
          aTuple.put(fieldId, DatumFactory.createText(table.getName()));
        } else if ("table_type".equalsIgnoreCase(column.getSimpleName())) {
          if (table.hasTableType()) {
            aTuple.put(fieldId, DatumFactory.createText(table.getTableType()));
          } else {
            aTuple.put(fieldId, DatumFactory.createNullDatum());
          }
        } else if ("path".equalsIgnoreCase(column.getSimpleName())) {
          aTuple.put(fieldId, DatumFactory.createText(table.getPath()));
        } else if ("store_type".equalsIgnoreCase(column.getSimpleName())) {
          aTuple.put(fieldId, DatumFactory.createText(table.getStoreType()));
        }
      }

      tuples.add(aTuple);
    }

    return tuples;
  }
Beispiel #10
0
 public boolean equals(Tuple tuple) {
   for (int i = 0; i < keys.length; i++) {
     final int field = keyIndex[i];
     final boolean n1 = keys[i] == null;
     final boolean n2 = tuple.isBlankOrNull(field);
     if (n1 && n2) {
       continue;
     }
     if (n1 ^ n2) {
       return false;
     }
     switch (keyTypes[i]) {
       case BOOLEAN:
         if ((Boolean) keys[i] != tuple.getBool(field)) return false;
         continue;
       case BIT:
         if ((Byte) keys[i] != tuple.getByte(field)) return false;
         continue;
       case INT1:
       case INT2:
         if ((Short) keys[i] != tuple.getInt2(field)) return false;
         continue;
       case INT4:
       case DATE:
         if ((Integer) keys[i] != tuple.getInt4(field)) return false;
         continue;
       case INT8:
       case TIME:
       case TIMESTAMP:
         if ((Long) keys[i] != tuple.getInt8(field)) return false;
         continue;
       case FLOAT4:
         if ((Float) keys[i] != tuple.getFloat4(field)) return false;
         continue;
       case FLOAT8:
         if ((Double) keys[i] != tuple.getFloat8(field)) return false;
         continue;
       case TEXT:
       case CHAR:
       case BLOB:
         if (!Arrays.equals((byte[]) keys[i], tuple.getBytes(field))) return false;
         continue;
       case DATUM:
         if (!keys[i].equals(tuple.asDatum(field))) return false;
         continue;
     }
   }
   return true;
 }
  @Test
  public final void testComparatorsFromJoinQual() {
    Schema outerSchema = new Schema();
    outerSchema.addColumn("employee.id1", CatalogUtil.newSimpleDataType(Type.INT4));
    outerSchema.addColumn("employee.id2", CatalogUtil.newSimpleDataType(Type.INT4));
    Schema innerSchema = new Schema();
    innerSchema.addColumn("people.fid1", CatalogUtil.newSimpleDataType(Type.INT4));
    innerSchema.addColumn("people.fid2", CatalogUtil.newSimpleDataType(Type.INT4));

    FieldEval f1 = new FieldEval("employee.id1", CatalogUtil.newSimpleDataType(Type.INT4));
    FieldEval f2 = new FieldEval("people.fid1", CatalogUtil.newSimpleDataType(Type.INT4));
    FieldEval f3 = new FieldEval("employee.id2", CatalogUtil.newSimpleDataType(Type.INT4));
    FieldEval f4 = new FieldEval("people.fid2", CatalogUtil.newSimpleDataType(Type.INT4));

    EvalNode joinQual = new BinaryEval(EvalType.EQUAL, f1, f2);
    TupleComparator[] comparators =
        PlannerUtil.getComparatorsFromJoinQual(joinQual, outerSchema, innerSchema);

    Tuple t1 = new VTuple(2);
    t1.put(0, DatumFactory.createInt4(1));
    t1.put(1, DatumFactory.createInt4(2));

    Tuple t2 = new VTuple(2);
    t2.put(0, DatumFactory.createInt4(2));
    t2.put(1, DatumFactory.createInt4(3));

    TupleComparator outerComparator = comparators[0];
    assertTrue(outerComparator.compare(t1, t2) < 0);
    assertTrue(outerComparator.compare(t2, t1) > 0);

    TupleComparator innerComparator = comparators[1];
    assertTrue(innerComparator.compare(t1, t2) < 0);
    assertTrue(innerComparator.compare(t2, t1) > 0);

    // tests for composited join key
    EvalNode joinQual2 = new BinaryEval(EvalType.EQUAL, f3, f4);
    EvalNode compositedJoinQual = new BinaryEval(EvalType.AND, joinQual, joinQual2);
    comparators =
        PlannerUtil.getComparatorsFromJoinQual(compositedJoinQual, outerSchema, innerSchema);

    outerComparator = comparators[0];
    assertTrue(outerComparator.compare(t1, t2) < 0);
    assertTrue(outerComparator.compare(t2, t1) > 0);

    innerComparator = comparators[1];
    assertTrue(innerComparator.compare(t1, t2) < 0);
    assertTrue(innerComparator.compare(t2, t1) > 0);
  }
Beispiel #12
0
  @Override
  public Datum eval(Tuple params) {
    Datum datum = params.get(0);
    if (datum instanceof NullDatum) return NullDatum.get();

    return DatumFactory.createInt4(datum.asByteArray().length * 8);
  }
Beispiel #13
0
 public void set(Tuple tuple) {
   for (int i = 0; i < keyTypes.length; i++) {
     final int field = keyIndex[i];
     if (tuple.isBlankOrNull(field)) {
       keys[i] = null;
       continue;
     }
     switch (keyTypes[i]) {
       case BOOLEAN:
         keys[i] = tuple.getBool(field);
         break;
       case BIT:
         keys[i] = tuple.getByte(field);
         break;
       case INT1:
       case INT2:
         keys[i] = tuple.getInt2(field);
         break;
       case INT4:
       case DATE:
         keys[i] = tuple.getInt4(field);
         break;
       case INT8:
       case TIME:
       case TIMESTAMP:
         keys[i] = tuple.getInt8(field);
         break;
       case FLOAT4:
         keys[i] = tuple.getFloat4(field);
         break;
       case FLOAT8:
         keys[i] = tuple.getFloat8(field);
         break;
       case TEXT:
       case CHAR:
       case BLOB:
         keys[i] = tuple.getBytes(field);
         break;
       case DATUM:
         keys[i] = tuple.asDatum(field);
         break;
       default:
         throw new IllegalArgumentException();
     }
   }
 }
Beispiel #14
0
  @Override
  public Datum eval(Tuple tuple) {
    Datum resultDatum;

    for (int i = 0; i < tuple.size(); i++) {
      params[i] = WritableTypeConverter.convertDatum2Writable(tuple.asDatum(i));
    }

    try {
      Writable result = (Writable) evalMethod.invoke(instance, params);
      resultDatum = WritableTypeConverter.convertWritable2Datum(result);
    } catch (Exception e) {
      throw new TajoInternalError(e);
    }

    return resultDatum;
  }
  private List<Tuple> getTablespaces(Schema outSchema) {
    List<TablespaceProto> tablespaces = masterContext.getCatalog().getAllTablespaces();
    List<Tuple> tuples = new ArrayList<Tuple>(tablespaces.size());
    List<Column> columns = outSchema.getRootColumns();
    Tuple aTuple;

    for (TablespaceProto tablespace : tablespaces) {
      aTuple = new VTuple(outSchema.size());

      for (int fieldId = 0; fieldId < columns.size(); fieldId++) {
        Column column = columns.get(fieldId);
        if ("space_id".equalsIgnoreCase(column.getSimpleName())) {
          if (tablespace.hasId()) {
            aTuple.put(fieldId, DatumFactory.createInt4(tablespace.getId()));
          } else {
            aTuple.put(fieldId, DatumFactory.createNullDatum());
          }
        } else if ("space_name".equalsIgnoreCase(column.getSimpleName())) {
          aTuple.put(fieldId, DatumFactory.createText(tablespace.getSpaceName()));
        } else if ("space_handler".equalsIgnoreCase(column.getSimpleName())) {
          if (tablespace.hasHandler()) {
            aTuple.put(fieldId, DatumFactory.createText(tablespace.getHandler()));
          } else {
            aTuple.put(fieldId, DatumFactory.createNullDatum());
          }
        } else if ("space_uri".equalsIgnoreCase(column.getSimpleName())) {
          aTuple.put(fieldId, DatumFactory.createText(tablespace.getUri()));
        }
      }
      tuples.add(aTuple);
    }

    return tuples;
  }
  @Before
  public void setUp() throws Exception {
    this.conf = new TajoConf();
    util = new TajoTestingCluster();
    catalog = util.startCatalogCluster().getCatalog();
    testDir = CommonTestingUtil.getTestDir(TEST_PATH);
    conf.setVar(TajoConf.ConfVars.WORKER_TEMPORAL_DIR, testDir.toString());
    sm = StorageManagerFactory.getStorageManager(conf, testDir);

    Schema schema = new Schema();
    schema.addColumn("managerId", Type.INT4);
    schema.addColumn("empId", Type.INT4);
    schema.addColumn("deptName", Type.TEXT);

    TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV);
    Path employeePath = new Path(testDir, "employee.csv");
    Appender appender =
        StorageManagerFactory.getStorageManager(conf)
            .getAppender(employeeMeta, schema, employeePath);
    appender.enableStats();
    appender.init();
    Tuple tuple = new VTuple(schema.getColumnNum());
    for (int i = 0; i < numTuple; i++) {
      tuple.put(
          new Datum[] {
            DatumFactory.createInt4(rnd.nextInt(50)),
            DatumFactory.createInt4(rnd.nextInt(100)),
            DatumFactory.createText("dept_" + i),
          });
      appender.addTuple(tuple);
    }
    appender.flush();
    appender.close();

    System.out.println(
        appender.getStats().getNumRows()
            + " rows ("
            + (appender.getStats().getNumBytes() / 1048576)
            + " MB)");

    employee = new TableDesc("employee", schema, employeeMeta, employeePath);
    catalog.addTable(employee);
    analyzer = new SQLAnalyzer();
    planner = new LogicalPlanner(catalog);
  }
  private List<Tuple> getSessionInfo(Schema outSchema) {
    List<Tuple> outputs = Lists.newArrayList();
    Tuple eachVariable;

    try {
      for (Map.Entry<String, String> var :
          masterContext.getSessionManager().getAllVariables(sessionId).entrySet()) {
        eachVariable = new VTuple(outSchema.size());
        eachVariable.put(0, DatumFactory.createText(var.getKey()));
        eachVariable.put(1, DatumFactory.createText(var.getValue()));

        outputs.add(eachVariable);
      }
    } catch (InvalidSessionException e) {
      LOG.error(e);
    }

    return outputs;
  }
  /**
   * It computes the value cardinality of a tuple range.
   *
   * @return
   */
  public static BigDecimal computeCardinalityForAllColumns(
      Schema schema, TupleRange range, boolean inclusive) {
    Tuple start = range.getStart();
    Tuple end = range.getEnd();
    Column col;

    BigDecimal cardinality = new BigDecimal(1);
    BigDecimal columnCard;
    for (int i = 0; i < schema.getColumnNum(); i++) {
      col = schema.getColumn(i);
      columnCard = computeCardinality(col.getDataType(), start.get(i), end.get(i), inclusive);

      if (new BigDecimal(0).compareTo(columnCard) < 0) {
        cardinality = cardinality.multiply(columnCard);
      }
    }

    return cardinality;
  }
 /** Called after all fields have been processed. */
 @Override
 public void end() {
   for (int i = 0; i < projectionMap.length; ++i) {
     final int projectionIndex = projectionMap[i];
     Column column = tajoReadSchema.getColumn(projectionIndex);
     if (column.getDataType().getType() == TajoDataTypes.Type.NULL_TYPE
         || currentTuple.isBlankOrNull(i)) {
       set(projectionIndex, NullDatum.get());
     }
   }
 }
Beispiel #20
0
  private void writeValue(Column column, Tuple tuple, int index) {
    switch (column.getDataType().getType()) {
      case BOOLEAN:
        recordConsumer.addBoolean(tuple.getBool(index));
        break;
      case BIT:
      case INT2:
      case INT4:
        recordConsumer.addInteger(tuple.getInt4(index));
        break;
      case INT8:
        recordConsumer.addLong(tuple.getInt8(index));
        break;
      case FLOAT4:
        recordConsumer.addFloat(tuple.getFloat4(index));
        break;
      case FLOAT8:
        recordConsumer.addDouble(tuple.getFloat8(index));
        break;
      case CHAR:
        if (tuple.size(index) > column.getDataType().getLength()) {
          throw new ValueTooLongForTypeCharactersException(column.getDataType().getLength());
        }

        recordConsumer.addBinary(Binary.fromByteArray(tuple.getTextBytes(index)));
        break;
      case TEXT:
        recordConsumer.addBinary(Binary.fromByteArray(tuple.getTextBytes(index)));
        break;
      case DATE:
        // Parquet DATE type is based on Unix Epoch(Jan 1, 1970).
        recordConsumer.addInteger(tuple.getInt4(index) - DateTimeConstants.UNIX_EPOCH_JDATE);
        break;
      case PROTOBUF:
      case BLOB:
        recordConsumer.addBinary(Binary.fromByteArray(tuple.getBytes(index)));
        break;
      default:
        break;
    }
  }
  private List<Tuple> getAllPartitions(Schema outSchema) {
    List<TablePartitionProto> partitionList = masterContext.getCatalog().getAllPartitions();
    List<Tuple> tuples = new ArrayList<Tuple>(partitionList.size());
    List<Column> columns = outSchema.getRootColumns();
    Tuple aTuple;

    for (TablePartitionProto partition : partitionList) {
      aTuple = new VTuple(outSchema.size());

      for (int fieldId = 0; fieldId < columns.size(); fieldId++) {
        Column column = columns.get(fieldId);

        if ("partition_id".equalsIgnoreCase(column.getSimpleName())) {
          aTuple.put(fieldId, DatumFactory.createInt4(partition.getPartitionId()));
        } else if ("tid".equalsIgnoreCase(column.getSimpleName())) {
          aTuple.put(fieldId, DatumFactory.createInt4(partition.getTid()));
        } else if ("partition_name".equalsIgnoreCase(column.getSimpleName())) {
          if (partition.hasPartitionName()) {
            aTuple.put(fieldId, DatumFactory.createText(partition.getPartitionName()));
          } else {
            aTuple.put(fieldId, DatumFactory.createNullDatum());
          }
        } else if ("path".equalsIgnoreCase(column.getSimpleName())) {
          aTuple.put(fieldId, DatumFactory.createText(partition.getPath()));
        }
      }

      tuples.add(aTuple);
    }

    return tuples;
  }
Beispiel #22
0
  @Override
  public Datum eval(Tuple params) {

    if (params.isBlankOrNull(0) || params.isBlankOrNull(1)) {
      return NullDatum.get();
    }

    String fillText;
    if (hasFillCharacters) {
      fillText = params.getText(2);
    } else {
      fillText = " ";
    }

    String input = params.getText(0);
    int expected = params.getInt4(1);

    int templen = expected - params.size(0);

    if (templen <= 0) {
      return DatumFactory.createText(input.substring(0, expected));
    } else {
      return DatumFactory.createText(StringUtils.leftPad(input, expected, fillText));
    }
  }
  private List<Tuple> getIndexes(Schema outSchema) {
    List<IndexProto> indexList = masterContext.getCatalog().getAllIndexes();
    List<Tuple> tuples = new ArrayList<Tuple>(indexList.size());
    List<Column> columns = outSchema.getRootColumns();
    Tuple aTuple;

    for (IndexProto index : indexList) {
      aTuple = new VTuple(outSchema.size());

      for (int fieldId = 0; fieldId < columns.size(); fieldId++) {
        Column column = columns.get(fieldId);

        if ("db_id".equalsIgnoreCase(column.getSimpleName())) {
          aTuple.put(fieldId, DatumFactory.createInt4(index.getDbId()));
        } else if ("tid".equalsIgnoreCase(column.getSimpleName())) {
          aTuple.put(fieldId, DatumFactory.createInt4(index.getTId()));
        } else if ("index_name".equalsIgnoreCase(column.getSimpleName())) {
          aTuple.put(fieldId, DatumFactory.createText(index.getIndexName()));
        } else if ("column_name".equalsIgnoreCase(column.getSimpleName())) {
          aTuple.put(fieldId, DatumFactory.createText(index.getColumnName()));
        } else if ("data_type".equalsIgnoreCase(column.getSimpleName())) {
          aTuple.put(fieldId, DatumFactory.createText(index.getDataType()));
        } else if ("index_type".equalsIgnoreCase(column.getSimpleName())) {
          aTuple.put(fieldId, DatumFactory.createText(index.getIndexType()));
        } else if ("is_unique".equalsIgnoreCase(column.getSimpleName())) {
          aTuple.put(fieldId, DatumFactory.createBool(index.getIsUnique()));
        } else if ("is_clustered".equalsIgnoreCase(column.getSimpleName())) {
          aTuple.put(fieldId, DatumFactory.createBool(index.getIsClustered()));
        } else if ("is_ascending".equalsIgnoreCase(column.getSimpleName())) {
          aTuple.put(fieldId, DatumFactory.createBool(index.getIsAscending()));
        }
      }

      tuples.add(aTuple);
    }

    return tuples;
  }
Beispiel #24
0
 protected final void set(int index, Tuple tuple, int field) {
   if (tuple.isBlankOrNull(field)) {
     nulls.set(index);
     return;
   }
   nulls.clear(index);
   switch (type) {
     case 0:
       booleans[index] = tuple.getBool(field);
       break;
     case 1:
       bits[index] = tuple.getByte(field);
       break;
     case 2:
       shorts[index] = tuple.getInt2(field);
       break;
     case 3:
       ints[index] = tuple.getInt4(field);
       break;
     case 4:
       longs[index] = tuple.getInt8(field);
       break;
     case 5:
       floats[index] = tuple.getFloat4(field);
       break;
     case 6:
       doubles[index] = tuple.getFloat8(field);
       break;
     case 7:
       bytes[index] = tuple.getBytes(field);
       break;
     case 8:
       ints[index] = tuple.getInt4(field);
       break;
     default:
       throw new IllegalArgumentException();
   }
 }
  private List<Tuple> getColumns(Schema outSchema) {
    List<ColumnProto> columnsList = masterContext.getCatalog().getAllColumns();
    List<Tuple> tuples = new ArrayList<Tuple>(columnsList.size());
    List<Column> columns = outSchema.getRootColumns();
    Tuple aTuple;
    int columnId = 1, prevtid = -1, tid = 0;

    for (ColumnProto column : columnsList) {
      aTuple = new VTuple(outSchema.size());

      tid = column.getTid();
      if (prevtid != tid) {
        columnId = 1;
        prevtid = tid;
      }

      for (int fieldId = 0; fieldId < columns.size(); fieldId++) {
        Column colObj = columns.get(fieldId);

        if ("tid".equalsIgnoreCase(colObj.getSimpleName())) {
          if (column.hasTid()) {
            aTuple.put(fieldId, DatumFactory.createInt4(tid));
          } else {
            aTuple.put(fieldId, DatumFactory.createNullDatum());
          }
        } else if ("column_name".equalsIgnoreCase(colObj.getSimpleName())) {
          aTuple.put(fieldId, DatumFactory.createText(column.getName()));
        } else if ("ordinal_position".equalsIgnoreCase(colObj.getSimpleName())) {
          aTuple.put(fieldId, DatumFactory.createInt4(columnId));
        } else if ("data_type".equalsIgnoreCase(colObj.getSimpleName())) {
          aTuple.put(fieldId, DatumFactory.createText(column.getDataType().getType().toString()));
        } else if ("type_length".equalsIgnoreCase(colObj.getSimpleName())) {
          DataType dataType = column.getDataType();
          if (dataType.hasLength()) {
            aTuple.put(fieldId, DatumFactory.createInt4(dataType.getLength()));
          } else {
            aTuple.put(fieldId, DatumFactory.createNullDatum());
          }
        }
      }

      columnId++;
      tuples.add(aTuple);
    }

    return tuples;
  }
Beispiel #26
0
 private void writeRecordFields(GroupType schema, Schema tajoSchema, Tuple tuple) {
   List<Type> fields = schema.getFields();
   // Parquet ignores Tajo NULL_TYPE columns, so the index may differ.
   int index = 0;
   for (int tajoIndex = 0; tajoIndex < tajoSchema.size(); ++tajoIndex) {
     Column column = tajoSchema.getColumn(tajoIndex);
     if (column.getDataType().getType() == TajoDataTypes.Type.NULL_TYPE) {
       continue;
     }
     Type fieldType = fields.get(index);
     if (!tuple.isBlankOrNull(tajoIndex)) {
       recordConsumer.startField(fieldType.getName(), index);
       writeValue(column, tuple, tajoIndex);
       recordConsumer.endField(fieldType.getName(), index);
     } else if (fieldType.isRepetition(Type.Repetition.REQUIRED)) {
       throw new RuntimeException("Null-value for required field: " + column.getSimpleName());
     }
     ++index;
   }
 }
Beispiel #27
0
 @Override
 public void merge(FunctionContext ctx, Tuple part) {
   ((CountRowContext) ctx).count += part.get(0).asInt8();
 }
  public Tuple next() throws IOException {
    Tuple previous;

    for (; ; ) {
      if (!outerIterator.hasNext() && !innerIterator.hasNext()) {
        if (end) {
          return null;
        }

        if (outerTuple == null) {
          outerTuple = leftChild.next();
        }
        if (innerTuple == null) {
          innerTuple = rightChild.next();
        }

        outerTupleSlots.clear();
        innerTupleSlots.clear();

        int cmp;
        while ((cmp = joincomparator.compare(outerTuple, innerTuple)) != 0) {
          if (cmp > 0) {
            innerTuple = rightChild.next();
          } else if (cmp < 0) {
            outerTuple = leftChild.next();
          }
          if (innerTuple == null || outerTuple == null) {
            return null;
          }
        }

        try {
          previous = outerTuple.clone();
          do {
            outerTupleSlots.add(outerTuple.clone());
            outerTuple = leftChild.next();
            if (outerTuple == null) {
              end = true;
              break;
            }
          } while (tupleComparator[0].compare(previous, outerTuple) == 0);
          outerIterator = outerTupleSlots.iterator();
          outerNext = outerIterator.next();

          previous = innerTuple.clone();
          do {
            innerTupleSlots.add(innerTuple.clone());
            innerTuple = rightChild.next();
            if (innerTuple == null) {
              end = true;
              break;
            }
          } while (tupleComparator[1].compare(previous, innerTuple) == 0);
          innerIterator = innerTupleSlots.iterator();
        } catch (CloneNotSupportedException e) {

        }
      }

      if (!innerIterator.hasNext()) {
        outerNext = outerIterator.next();
        innerIterator = innerTupleSlots.iterator();
      }

      frameTuple.set(outerNext, innerIterator.next());
      joinQual.eval(qualCtx, inSchema, frameTuple);
      if (joinQual.terminate(qualCtx).asBool()) {
        projector.eval(evalContexts, frameTuple);
        projector.terminate(evalContexts, outTuple);
        return outTuple;
      }
    }
  }
 private void set(int index, Object value) {
   currentTuple.put(index, (Datum) value);
 }
  @Before
  public void setup() throws Exception {
    this.randomValues = new HashMap<Integer, Integer>();
    this.conf = new TajoConf();
    util = new TajoTestingCluster();
    util.startCatalogCluster();
    catalog = util.getMiniCatalogCluster().getCatalog();

    Path workDir = CommonTestingUtil.getTestDir();
    catalog.createTablespace(DEFAULT_TABLESPACE_NAME, workDir.toUri().toString());
    catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME);
    sm = StorageManagerFactory.getStorageManager(conf, workDir);

    idxPath = new Path(workDir, "test.idx");

    Schema schema = new Schema();
    schema.addColumn("managerid", Type.INT4);
    schema.addColumn("empid", Type.INT4);
    schema.addColumn("deptname", Type.TEXT);

    this.idxSchema = new Schema();
    idxSchema.addColumn("managerid", Type.INT4);
    SortSpec[] sortKeys = new SortSpec[1];
    sortKeys[0] = new SortSpec(idxSchema.getColumn("managerid"), true, false);
    this.comp = new TupleComparator(idxSchema, sortKeys);

    this.writer =
        new BSTIndex(conf)
            .getIndexWriter(idxPath, BSTIndex.TWO_LEVEL_INDEX, this.idxSchema, this.comp);
    writer.setLoadNum(100);
    writer.open();
    long offset;

    meta = CatalogUtil.newTableMeta(StoreType.CSV);
    tablePath = StorageUtil.concatPath(workDir, "employee", "table.csv");
    fs = tablePath.getFileSystem(conf);
    fs.mkdirs(tablePath.getParent());

    FileAppender appender =
        (FileAppender)
            StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath);
    appender.init();
    Tuple tuple = new VTuple(schema.size());
    for (int i = 0; i < 10000; i++) {

      Tuple key = new VTuple(this.idxSchema.size());
      int rndKey = rnd.nextInt(250);
      if (this.randomValues.containsKey(rndKey)) {
        int t = this.randomValues.remove(rndKey) + 1;
        this.randomValues.put(rndKey, t);
      } else {
        this.randomValues.put(rndKey, 1);
      }

      key.put(new Datum[] {DatumFactory.createInt4(rndKey)});
      tuple.put(
          new Datum[] {
            DatumFactory.createInt4(rndKey),
            DatumFactory.createInt4(rnd.nextInt(10)),
            DatumFactory.createText("dept_" + rnd.nextInt(10))
          });
      offset = appender.getOffset();
      appender.addTuple(tuple);
      writer.write(key, offset);
    }
    appender.flush();
    appender.close();
    writer.close();

    TableDesc desc =
        new TableDesc(
            CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"),
            schema,
            meta,
            sm.getTablePath("employee"));
    catalog.createTable(desc);

    analyzer = new SQLAnalyzer();
    planner = new LogicalPlanner(catalog);
    optimizer = new LogicalOptimizer(conf);
  }