private Object[] getFromCache(RowMetaInterface keyMeta, Object[] keyData)
      throws KettleValueException {
    if (meta.isMemoryPreservationActive()) {
      if (meta.isUsingSortedList()) {
        KeyValue keyValue = new KeyValue(keyData, null);
        int idx = Collections.binarySearch(data.list, keyValue, data.comparator);
        if (idx < 0) return null; // nothing found

        keyValue = (KeyValue) data.list.get(idx);
        return keyValue.getValue();
      } else {
        if (meta.isUsingIntegerPair()) {
          Long value = data.longIndex.get(keyMeta.getInteger(keyData, 0));
          if (value == null) return null;
          return new Object[] {
            value,
          };
        } else {
          try {
            byte[] value = data.hashIndex.get(RowMeta.extractData(keyMeta, keyData));
            if (value == null) return null;
            return RowMeta.getRow(data.valueMeta, value);
          } catch (Exception e) {
            logError("Oops", e);
            throw new RuntimeException(e);
          }
        }
      }
    } else {
      return (Object[]) data.look.get(new RowMetaAndData(keyMeta, keyData));
    }
  }
Ejemplo n.º 2
0
  @Test
  public void testQuoteReservedWords() {
    DatabaseMeta databaseMeta = mock(DatabaseMeta.class);
    doCallRealMethod().when(databaseMeta).quoteReservedWords(any(RowMetaInterface.class));
    doCallRealMethod().when(databaseMeta).quoteField(anyString());
    doCallRealMethod().when(databaseMeta).setDatabaseInterface(any(DatabaseInterface.class));
    doReturn("\"").when(databaseMeta).getStartQuote();
    doReturn("\"").when(databaseMeta).getEndQuote();
    final DatabaseInterface databaseInterface = mock(DatabaseInterface.class);
    doReturn(true).when(databaseInterface).isQuoteAllFields();
    databaseMeta.setDatabaseInterface(databaseInterface);

    final RowMeta fields = new RowMeta();
    for (int i = 0; i < 10; i++) {
      final ValueMeta valueMeta = new ValueMeta("test_" + i);
      fields.addValueMeta(valueMeta);
    }

    for (int i = 0; i < 10; i++) {
      databaseMeta.quoteReservedWords(fields);
    }

    for (int i = 0; i < 10; i++) {
      databaseMeta.quoteReservedWords(fields);
      final String name = fields.getValueMeta(i).getName();
      // check valueMeta index in list
      assertTrue(name.contains("test_" + i));
      // check valueMeta is found by quoted name
      assertNotNull(fields.searchValueMeta(name));
    }
  }
  @Test
  public void testHTTPResultDefaultRows() throws IOException {
    File localFileForUpload = getInputFile("existingFile1", ".tmp");
    File tempFileForDownload = File.createTempFile("downloadedFile1", ".tmp");
    localFileForUpload.deleteOnExit();
    tempFileForDownload.deleteOnExit();

    Object[] r =
        new Object[] {
          HTTP_SERVER_BASEURL + "/uploadFile",
          localFileForUpload.getCanonicalPath(),
          tempFileForDownload.getCanonicalPath()
        };
    RowMeta rowMetaDefault = new RowMeta();
    rowMetaDefault.addValueMeta(new ValueMetaString("URL"));
    rowMetaDefault.addValueMeta(new ValueMetaString("UPLOAD"));
    rowMetaDefault.addValueMeta(new ValueMetaString("DESTINATION"));
    List<RowMetaAndData> rows = new ArrayList<RowMetaAndData>();
    rows.add(new RowMetaAndData(rowMetaDefault, r));
    Result previousResult = new Result();
    previousResult.setRows(rows);

    JobEntryHTTP http = new JobEntryHTTP();
    http.setParentJob(new Job());
    http.setRunForEveryRow(true);
    http.setAddFilenameToResult(false);
    http.execute(previousResult, 0);
    assertTrue(FileUtils.contentEquals(localFileForUpload, tempFileForDownload));
  }
  @Test
  public void testConditionResolution() throws Exception {
    RowMeta rowMeta = new RowMeta();
    rowMeta.addValueMeta(new ValueMetaString("aString"));
    rowMeta.addValueMeta(new ValueMetaInteger("anInt"));
    rowMeta.addValueMeta(new ValueMetaDate("aDate"));

    String query =
        "SELECT * FROM "
            + DATA_SERVICE_NAME
            + " WHERE anInt = 2 AND aDate IN ('2014-12-05','2008-01-01')";

    when(transMeta.getStepFields(DATA_SERVICE_STEP)).thenReturn(rowMeta);

    DataServiceExecutor executor =
        new DataServiceExecutor.Builder(new SQL(query), dataService, context)
            .serviceTrans(transMeta)
            .prepareExecution(false)
            .build();

    Condition condition = executor.getSql().getWhereCondition().getCondition();

    Calendar calendar = Calendar.getInstance();
    calendar.clear();
    calendar.set(2014, Calendar.DECEMBER, 5);

    assertThat(
        condition.evaluate(rowMeta, new Object[] {"value", 2L, calendar.getTime()}), is(true));
    assertThat(condition.evaluate(rowMeta, new Object[] {"value", 2L, new Date()}), is(false));
  }
  @Test
  public void testReturnDigitsOnly() throws KettleException {
    RowMeta inputRowMeta = new RowMeta();
    ValueMetaString nameMeta = new ValueMetaString("Name");
    inputRowMeta.addValueMeta(nameMeta);
    ValueMetaString valueMeta = new ValueMetaString("Value");
    inputRowMeta.addValueMeta(valueMeta);

    RowSet inputRowSet =
        smh.getMockInputRowSet(new Object[][] {{"name1", "qwe123asd456zxc"}, {"name2", null}});
    inputRowSet.setRowMeta(inputRowMeta);

    Calculator calculator =
        new Calculator(smh.stepMeta, smh.stepDataInterface, 0, smh.transMeta, smh.trans);
    calculator.getInputRowSets().add(inputRowSet);
    calculator.setInputRowMeta(inputRowMeta);
    calculator.init(smh.initStepMetaInterface, smh.initStepDataInterface);

    CalculatorMeta meta = new CalculatorMeta();
    meta.setCalculation(
        new CalculatorMetaFunction[] {
          new CalculatorMetaFunction(
              "digits",
              CalculatorMetaFunction.CALC_GET_ONLY_DIGITS,
              "Value",
              null,
              null,
              ValueMetaInterface.TYPE_STRING,
              0,
              0,
              false,
              "",
              "",
              "",
              "")
        });

    // Verify output
    try {
      calculator.addRowListener(
          new RowAdapter() {
            @Override
            public void rowWrittenEvent(RowMetaInterface rowMeta, Object[] row)
                throws KettleStepException {
              assertEquals("123456", row[2]);
            }
          });
      calculator.processRow(meta, new CalculatorData());
    } catch (KettleException ke) {
      ke.printStackTrace();
      fail();
    }
  }
  private void addToCache(
      RowMetaInterface keyMeta, Object[] keyData, RowMetaInterface valueMeta, Object[] valueData)
      throws KettleValueException {
    if (meta.isMemoryPreservationActive()) {
      if (meta.isUsingSortedList()) {
        KeyValue keyValue = new KeyValue(keyData, valueData);
        int idx = Collections.binarySearch(data.list, keyValue, data.comparator);
        if (idx < 0) {
          int index = -idx - 1; // this is the insertion point
          data.list.add(index, keyValue); // insert to keep sorted.
        } else {
          data.list.set(idx, keyValue); // Overwrite to simulate Hashtable behaviour
        }
      } else {
        if (meta.isUsingIntegerPair()) {
          if (!data.metadataVerifiedIntegerPair) {
            data.metadataVerifiedIntegerPair = true;
            if (keyMeta.size() != 1
                || valueMeta.size() != 1
                || !keyMeta.getValueMeta(0).isInteger()
                || !valueMeta.getValueMeta(0).isInteger()) {

              throw new KettleValueException(
                  BaseMessages.getString(
                      PKG, "StreamLookup.Exception.CanNotUseIntegerPairAlgorithm"));
            }
          }

          Long key = keyMeta.getInteger(keyData, 0);
          Long value = valueMeta.getInteger(valueData, 0);
          data.longIndex.put(key, value);
        } else {
          if (data.hashIndex == null) {
            data.hashIndex = new ByteArrayHashIndex(keyMeta);
          }
          data.hashIndex.put(
              RowMeta.extractData(keyMeta, keyData), RowMeta.extractData(valueMeta, valueData));
        }
      }
    } else {
      // We can't just put Object[] in the map
      // The compare function is not in it.
      // We need to wrap in and use that.
      // Let's use RowMetaAndData for this one.
      //
      data.look.put(new RowMetaAndData(keyMeta, keyData), valueData);
    }
  }
  @Test
  public void calculatorShouldClearDataInstance() throws Exception {
    RowMeta inputRowMeta = new RowMeta();
    ValueMetaInteger valueMeta = new ValueMetaInteger("Value");
    inputRowMeta.addValueMeta(valueMeta);

    RowSet inputRowSet = smh.getMockInputRowSet(new Object[] {-1L});
    inputRowSet.setRowMeta(inputRowMeta);

    Calculator calculator =
        new Calculator(smh.stepMeta, smh.stepDataInterface, 0, smh.transMeta, smh.trans);
    calculator.getInputRowSets().add(inputRowSet);
    calculator.setInputRowMeta(inputRowMeta);
    calculator.init(smh.initStepMetaInterface, smh.initStepDataInterface);

    CalculatorMeta meta = new CalculatorMeta();
    meta.setCalculation(
        new CalculatorMetaFunction[] {
          new CalculatorMetaFunction(
              "test",
              CalculatorMetaFunction.CALC_ABS,
              "Value",
              null,
              null,
              ValueMetaInterface.TYPE_STRING,
              0,
              0,
              false,
              "",
              "",
              "",
              "")
        });

    CalculatorData data = new CalculatorData();
    data = spy(data);

    calculator.processRow(meta, data);
    verify(data).getValueMetaFor(eq(valueMeta.getType()), anyString());

    calculator.processRow(meta, data);
    verify(data).clearValuesMetaMapping();
  }
  @Test
  public void testWithLazyConversion() throws Exception {
    RowMeta rowMeta = new RowMeta();
    ValueMetaInterface vm = new ValueMetaString("aBinaryStoredString");
    vm.setStorageType(ValueMetaInterface.STORAGE_TYPE_BINARY_STRING);
    vm.setStorageMetadata(new ValueMetaString());
    rowMeta.addValueMeta(vm);

    String query = "SELECT * FROM " + DATA_SERVICE_NAME + " WHERE aBinaryStoredString = 'value'";

    when(transMeta.getStepFields(DATA_SERVICE_STEP)).thenReturn(rowMeta);

    DataServiceExecutor executor =
        new DataServiceExecutor.Builder(new SQL(query), dataService, context)
            .serviceTrans(new Trans(transMeta))
            .prepareExecution(false)
            .build();

    executor
        .getSql()
        .getWhereCondition()
        .getCondition()
        .evaluate(rowMeta, new Object[] {"value".getBytes()});
  }
Ejemplo n.º 9
0
  private void loadAllTableDataIntoTheCache() throws KettleException {
    DatabaseMeta dbMeta = meta.getDatabaseMeta();

    try {
      // We only want to get the used table fields...
      //
      String sql = "SELECT ";

      for (int i = 0; i < meta.getStreamKeyField1().length; i++) {
        if (i > 0) {
          sql += ", ";
        }
        sql += dbMeta.quoteField(meta.getTableKeyField()[i]);
      }

      // Also grab the return field...
      //
      for (int i = 0; i < meta.getReturnValueField().length; i++) {
        sql += ", " + dbMeta.quoteField(meta.getReturnValueField()[i]);
      }
      // The schema/table
      //
      sql +=
          " FROM "
              + dbMeta.getQuotedSchemaTableCombination(
                  environmentSubstitute(meta.getSchemaName()),
                  environmentSubstitute(meta.getTablename()));

      // order by?
      if (meta.getOrderByClause() != null && meta.getOrderByClause().length() != 0) {
        sql += " ORDER BY " + meta.getOrderByClause();
      }

      // Now that we have the SQL constructed, let's store the rows...
      //
      List<Object[]> rows = data.db.getRows(sql, 0);
      if (rows != null && rows.size() > 0) {
        RowMetaInterface returnRowMeta = data.db.getReturnRowMeta();
        // Copy the data into 2 parts: key and value...
        //
        for (Object[] row : rows) {
          int index = 0;
          RowMeta keyMeta = new RowMeta();
          Object[] keyData = new Object[meta.getStreamKeyField1().length];
          for (int i = 0; i < meta.getStreamKeyField1().length; i++) {
            keyData[i] = row[index];
            keyMeta.addValueMeta(returnRowMeta.getValueMeta(index++));
          }
          // RowMeta valueMeta = new RowMeta();
          Object[] valueData = new Object[data.returnMeta.size()];
          for (int i = 0; i < data.returnMeta.size(); i++) {
            valueData[i] = row[index++];
            // valueMeta.addValueMeta(returnRowMeta.getValueMeta(index++));
          }
          // Store the data...
          //
          storeRowInCache(keyMeta, keyData, valueData);
          incrementLinesInput();
        }
      }
    } catch (Exception e) {
      throw new KettleException(e);
    }
  }
Ejemplo n.º 10
0
  public ResultSet getColumns(
      String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern)
      throws SQLException {

    log.debug(
        "catalog:"
            + catalog
            + " , schemaPattern:"
            + schemaPattern
            + " , tableNamePattern:"
            + tableNamePattern
            + " ,columnNamePattern:"
            + columnNamePattern);

    List<RowMetaAndData> rowAndDatas = new ArrayList<RowMetaAndData>();
    // if USE_TRANSNAME_AS_SCHEMA is true, then we use the filename or
    // transformation name as the schema
    if (!isDir) {

      log.debug(helper.getRowMeta(tableNamePattern));
      RowMeta rm = helper.getRowMeta(tableNamePattern);
      ColInfo[] colInfo = KettleHelper.convert(rm);
      String[] columns = rm.getFieldNames();
      for (int i = 0; columns != null && i < columns.length; i++) {

        RowMetaAndData rd = new RowMetaAndData();
        rd.addValue("TABLE_CAT", ValueMetaInterface.TYPE_STRING, catalog);
        rd.addValue("TABLE_SCHEM", ValueMetaInterface.TYPE_STRING, schemaPattern);
        rd.addValue("TABLE_NAME", ValueMetaInterface.TYPE_STRING, tableNamePattern);
        rd.addValue("COLUMN_NAME", ValueMetaInterface.TYPE_STRING, columns[i]);
        rd.addValue("DATA_TYPE", ValueMetaInterface.TYPE_INTEGER, colInfo[i].jdbcType);
        rd.addValue("TYPE_NAME", ValueMetaInterface.TYPE_STRING, "");
        rd.addValue("COLUMN_SIZE", ValueMetaInterface.TYPE_INTEGER, columns.length);
        rd.addValue("BUFFER_LENGTH", ValueMetaInterface.TYPE_INTEGER, "20");

        rd.addValue("DECIMAL_DIGITS", ValueMetaInterface.TYPE_INTEGER, "20");
        rd.addValue("NUM_PREC_RADIX", ValueMetaInterface.TYPE_INTEGER, "20");
        rd.addValue("NULLABLE", ValueMetaInterface.TYPE_INTEGER, "20");
        rd.addValue("REMARKS", ValueMetaInterface.TYPE_STRING, "");
        rd.addValue("COLUMN_DEF", ValueMetaInterface.TYPE_STRING, "");
        rd.addValue("SQL_DATA_TYPE", ValueMetaInterface.TYPE_INTEGER, "20");
        rd.addValue("SQL_DATETIME_SUB", ValueMetaInterface.TYPE_INTEGER, "20");
        rd.addValue("CHAR_OCTET_LENGTH", ValueMetaInterface.TYPE_INTEGER, "1");
        rd.addValue("ORDINAL_POSITION", ValueMetaInterface.TYPE_INTEGER, "20");
        rd.addValue("IS_NULLABLE", ValueMetaInterface.TYPE_STRING, "0");
        rd.addValue("SCOPE_CATALOG", ValueMetaInterface.TYPE_STRING, "0");
        rd.addValue("SCOPE_SCHEMA", ValueMetaInterface.TYPE_STRING, "0");
        rd.addValue("SCOPE_TABLE", ValueMetaInterface.TYPE_STRING, "0");
        rd.addValue("SOURCE_DATA_TYPE", ValueMetaInterface.TYPE_INTEGER, "1");
        rowAndDatas.add(rd);
      }
      KettleJDBCResultSet rs = new KettleJDBCResultSet(null, rowAndDatas, "*");
      return rs;
    }

    //		log.debug("getRowMeta:" + helper.getRowMeta(tableNamePattern));
    RowMeta rm = helper.getRowMeta(tableNamePattern);
    ColInfo[] colInfo = KettleHelper.convert(rm);
    String[] columns = rm.getFieldNames();
    for (int i = 0; columns != null && i < columns.length; i++) {
      String name = columns[i];
      RowMetaAndData rd = new RowMetaAndData();
      rd.addValue("TABLE_CAT", ValueMetaInterface.TYPE_STRING, catalog);
      rd.addValue("TABLE_SCHEM", ValueMetaInterface.TYPE_STRING, schemaPattern);
      rd.addValue("TABLE_NAME", ValueMetaInterface.TYPE_STRING, tableNamePattern);
      rd.addValue("COLUMN_NAME", ValueMetaInterface.TYPE_STRING, name);
      rd.addValue("DATA_TYPE", ValueMetaInterface.TYPE_INTEGER, colInfo[i].getJdbcType());
      rd.addValue("TYPE_NAME", ValueMetaInterface.TYPE_STRING, "");
      rd.addValue("COLUMN_SIZE", ValueMetaInterface.TYPE_INTEGER, columns.length);
      rd.addValue("BUFFER_LENGTH", ValueMetaInterface.TYPE_INTEGER, name);
      rd.addValue("DECIMAL_DIGITS", ValueMetaInterface.TYPE_INTEGER, "20");
      rd.addValue("NUM_PREC_RADIX", ValueMetaInterface.TYPE_INTEGER, "20");
      rd.addValue("NULLABLE", ValueMetaInterface.TYPE_INTEGER, "20");
      rd.addValue("REMARKS", ValueMetaInterface.TYPE_STRING, name);
      rd.addValue("COLUMN_DEF", ValueMetaInterface.TYPE_STRING, name);
      rd.addValue("SQL_DATA_TYPE", ValueMetaInterface.TYPE_INTEGER, "20");
      rd.addValue("SQL_DATETIME_SUB", ValueMetaInterface.TYPE_INTEGER, "20");
      rd.addValue("CHAR_OCTET_LENGTH", ValueMetaInterface.TYPE_INTEGER, "1");
      rd.addValue("ORDINAL_POSITION", ValueMetaInterface.TYPE_INTEGER, "20");
      rd.addValue("IS_NULLABLE", ValueMetaInterface.TYPE_STRING, "0");
      rd.addValue("SCOPE_CATALOG", ValueMetaInterface.TYPE_STRING, "0");
      rd.addValue("SCOPE_SCHEMA", ValueMetaInterface.TYPE_STRING, "0");
      rd.addValue("SCOPE_TABLE", ValueMetaInterface.TYPE_STRING, "0");
      rd.addValue("SOURCE_DATA_TYPE", ValueMetaInterface.TYPE_INTEGER, "1");
      rowAndDatas.add(rd);
    }

    KettleJDBCResultSet rs = new KettleJDBCResultSet(null, rowAndDatas, "*");
    return rs;
  }
  public void assertRoundGeneral(
      final Object expectedResult,
      final int calcFunction,
      final Number value,
      final Long precision,
      final Long roundingMode,
      final int valueDataType,
      final int functionDataType)
      throws KettleException {

    final String msg =
        getKettleTypeName(valueDataType) + "->" + getKettleTypeName(functionDataType) + " ";

    final RowMeta inputRowMeta = new RowMeta();
    final List<Object> inputValues = new ArrayList<Object>(3);

    final String fieldValue = "Value";
    final ValueMetaInterface valueMeta;
    switch (valueDataType) {
      case ValueMetaInterface.TYPE_BIGNUMBER:
        valueMeta = new ValueMetaBigNumber(fieldValue);
        break;
      case ValueMetaInterface.TYPE_NUMBER:
        valueMeta = new ValueMetaNumber(fieldValue);
        break;
      case ValueMetaInterface.TYPE_INTEGER:
        valueMeta = new ValueMetaInteger(fieldValue);
        break;
      default:
        throw new IllegalArgumentException(
            msg
                + "Unexpected value dataType: "
                + value.getClass().getName()
                + ". Long, Double or BigDecimal expected.");
    }
    inputRowMeta.addValueMeta(valueMeta);
    inputValues.add(value);

    final String fieldPrecision;
    final ValueMetaInteger precisionMeta;
    if (precision == null) {
      fieldPrecision = null;
      precisionMeta = null;
    } else {
      fieldPrecision = "Precision";
      precisionMeta = new ValueMetaInteger(fieldPrecision);
      inputRowMeta.addValueMeta(precisionMeta);
      inputValues.add(precision);
    }

    final String fieldRoundingMode;
    final ValueMetaInteger roundingModeMeta;
    if (roundingMode == null) {
      fieldRoundingMode = null;
      roundingModeMeta = null;
    } else {
      fieldRoundingMode = "RoundingMode";
      roundingModeMeta = new ValueMetaInteger(fieldRoundingMode);
      inputRowMeta.addValueMeta(roundingModeMeta);
      inputValues.add(roundingMode);
    }

    RowSet inputRowSet = smh.getMockInputRowSet(inputValues.toArray());
    inputRowSet.setRowMeta(inputRowMeta);
    final String fieldA =
        inputRowMeta.size() > 0 ? inputRowMeta.getValueMetaList().get(0).getName() : null;
    final String fieldB =
        inputRowMeta.size() > 1 ? inputRowMeta.getValueMetaList().get(1).getName() : null;
    final String fieldC =
        inputRowMeta.size() > 2 ? inputRowMeta.getValueMetaList().get(2).getName() : null;

    final int resultDataType = functionDataType;

    final String fieldResult = "test";
    final int expectedResultRowSize = inputRowMeta.size() + 1;

    Calculator calculator =
        new Calculator(smh.stepMeta, smh.stepDataInterface, 0, smh.transMeta, smh.trans);
    calculator.getInputRowSets().add(inputRowSet);
    calculator.setInputRowMeta(inputRowMeta);
    calculator.init(smh.initStepMetaInterface, smh.initStepDataInterface);

    CalculatorMeta meta = new CalculatorMeta();
    meta.setCalculation(
        new CalculatorMetaFunction[] {
          new CalculatorMetaFunction(
              fieldResult,
              calcFunction,
              fieldA,
              fieldB,
              fieldC,
              resultDataType,
              2,
              0,
              false,
              "",
              "",
              "",
              "")
        });

    // Verify output
    try {
      calculator.addRowListener(
          new RowAdapter() {
            @Override
            public void rowWrittenEvent(RowMetaInterface rowMeta, Object[] row)
                throws KettleStepException {
              assertEquals(msg + " resultRowSize", expectedResultRowSize, rowMeta.size());
              final int fieldResultIndex = rowMeta.size() - 1;
              assertEquals(
                  msg + " fieldResult",
                  fieldResult,
                  rowMeta.getValueMeta(fieldResultIndex).getName());
              assertEquals(msg, expectedResult, row[fieldResultIndex]);
            }
          });
      calculator.processRow(meta, new CalculatorData());
    } catch (KettleException ke) {
      ke.printStackTrace();
      fail(msg + ke.getMessage());
    }
  }
  private ArrayList<Object[]> writeToTable(RowMetaInterface rowMeta, ArrayList<Object[]> rows)
      throws KettleException {

    if (rows.isEmpty()) // Stop: last line or error encountered
    {
      if (log.isDetailed()) logDetailed("Last line inserted: stop");
      return null;
    }

    PreparedStatement insertStatement = null;

    ArrayList<Object[]> insertRowsData = new ArrayList<Object[]>();
    ArrayList<Object[]> outputRowsData = rows;

    String tableName = null;

    boolean sendToErrorRow = false;
    String errorMessage = null;
    boolean rowIsSafe = false;
    int[] updateCounts = null;
    List<Exception> exceptionsList = null;
    boolean batchProblem = false;

    for (Object[] row : rows) {
      if (meta.isTableNameInField()) {
        // Cache the position of the table name field
        if (data.indexOfTableNameField < 0) {
          String realTablename = environmentSubstitute(meta.getTableNameField());
          data.indexOfTableNameField = rowMeta.indexOfValue(realTablename);
          if (data.indexOfTableNameField < 0) {
            String message = "Unable to find table name field [" + realTablename + "] in input row";
            logError(message);
            throw new KettleStepException(message);
          }
          if (!meta.isTableNameInTable()) {
            data.insertRowMeta.removeValueMeta(data.indexOfTableNameField);
          }
        }
        tableName = rowMeta.getString(rows.get(0), data.indexOfTableNameField);
        if (!meta.isTableNameInTable()) {
          // If the name of the table should not be inserted itself,
          // remove the table name
          // from the input row data as well. This forcibly creates a
          // copy of r
          insertRowsData.add(
              RowDataUtil.removeItem(rowMeta.cloneRow(row), data.indexOfTableNameField));
        } else {
          insertRowsData.add(row);
        }
      } else if (meta.isPartitioningEnabled()
          && (meta.isPartitioningDaily() || meta.isPartitioningMonthly())
          && (meta.getPartitioningField() != null && meta.getPartitioningField().length() > 0)) {
        // Initialize some stuff!
        if (data.indexOfPartitioningField < 0) {
          data.indexOfPartitioningField =
              rowMeta.indexOfValue(environmentSubstitute(meta.getPartitioningField()));
          if (data.indexOfPartitioningField < 0) {
            throw new KettleStepException(
                "Unable to find field [" + meta.getPartitioningField() + "] in the input row!");
          }

          if (meta.isPartitioningDaily()) {
            data.dateFormater = new SimpleDateFormat("yyyyMMdd");
          } else {
            data.dateFormater = new SimpleDateFormat("yyyyMM");
          }
        }

        ValueMetaInterface partitioningValue = rowMeta.getValueMeta(data.indexOfPartitioningField);
        if (!partitioningValue.isDate() || row[data.indexOfPartitioningField] == null) {
          throw new KettleStepException(
              "Sorry, the partitioning field needs to contain a data value and can't be empty!");
        }

        Object partitioningValueData = rowMeta.getDate(row, data.indexOfPartitioningField);
        tableName =
            environmentSubstitute(meta.getTablename())
                + "_"
                + data.dateFormater.format((Date) partitioningValueData);
        insertRowsData.add(row);
      } else {
        tableName = data.tableName;
        insertRowsData.add(row);
      }

      if (Const.isEmpty(tableName)) {
        throw new KettleStepException("The tablename is not defined (empty)");
      }
    }

    if (!data.preparedStatements.containsKey(tableName)) {
      data.preparedStatements.put(tableName, new Hashtable<Integer, PreparedStatement>());
    }

    insertStatement = (PreparedStatement) data.preparedStatements.get(tableName).get(rows.size());
    if (insertStatement == null) {
      String sql =
          getInsertStatement(
              environmentSubstitute(meta.getSchemaName()),
              tableName,
              data.insertRowMeta,
              rows.size());
      if (log.isDetailed()) logDetailed("Prepared statement : " + sql);
      insertStatement = data.db.prepareSQL(sql);

      if (!data.preparedStatements.containsKey(tableName)) {
        data.preparedStatements.put(tableName, new Hashtable<Integer, PreparedStatement>());
      }

      data.preparedStatements.get(tableName).put(rows.size(), insertStatement);
    }

    try {
      // For PG & GP, we add a savepoint before the row.
      // Then revert to the savepoint afterwards... (not a transaction, so
      // hopefully still fast)
      //
      if (data.specialErrorHandling) {
        data.savepoint = data.db.setSavepoint();
      }

      RowMeta insertRowMeta = new RowMeta();
      for (int i = 0; i < rows.size(); i++) {
        for (int j = 0; j < data.valuenrs.length; j++) {
          insertRowMeta.addValueMeta(data.insertRowMeta.getValueMeta(j));
        }
      }
      data.db.setValues(insertRowMeta, toArray(insertRowsData), insertStatement);
      data.db.insertRow(insertStatement, data.batchMode, false); // false:
      // no
      // commit,
      // it is
      // handled
      // in
      // this
      // step
      // different

      // Get a commit counter per prepared statement to keep track of
      // separate tables, etc.
      //
      Integer commitCounter = data.commitCounterMap.get(tableName);
      if (commitCounter == null) {
        commitCounter = Integer.valueOf(1);
      } else {
        commitCounter++;
      }
      data.commitCounterMap.put(tableName, Integer.valueOf(commitCounter.intValue()));

      // Release the savepoint if needed
      //
      if (data.specialErrorHandling) {
        if (data.releaseSavepoint) {
          data.db.releaseSavepoint(data.savepoint);
        }
      }

      // Perform a commit if needed
      //

      if ((data.commitSize > 0) && ((commitCounter % data.commitSize) == 0)) {
        if (data.batchMode) {
          try {
            insertStatement.executeBatch();
            data.db.commit();
            insertStatement.clearBatch();
          } catch (BatchUpdateException ex) {
            KettleDatabaseBatchException kdbe =
                new KettleDatabaseBatchException("Error updating batch", ex);
            kdbe.setUpdateCounts(ex.getUpdateCounts());
            List<Exception> exceptions = new ArrayList<Exception>();

            // 'seed' the loop with the root exception
            SQLException nextException = ex;
            do {
              exceptions.add(nextException);
              // while current exception has next exception, add
              // to list
            } while ((nextException = nextException.getNextException()) != null);
            kdbe.setExceptionsList(exceptions);
            throw kdbe;
          } catch (SQLException ex) {
            throw new KettleDatabaseException("Error inserting row", ex);
          } catch (Exception ex) {
            throw new KettleDatabaseException("Unexpected error inserting row", ex);
          }
        } else {
          // insertRow normal commit
          data.db.commit();
        }
        // Clear the batch/commit counter...
        //
        data.commitCounterMap.put(tableName, Integer.valueOf(0));
        rowIsSafe = true;
      } else {
        rowIsSafe = false;
      }
    } catch (KettleDatabaseBatchException be) {
      errorMessage = be.toString();
      batchProblem = true;
      sendToErrorRow = true;
      updateCounts = be.getUpdateCounts();
      exceptionsList = be.getExceptionsList();

      if (getStepMeta().isDoingErrorHandling()) {
        data.db.clearBatch(insertStatement);
        data.db.commit(true);
      } else {
        data.db.clearBatch(insertStatement);
        data.db.rollback();
        StringBuffer msg =
            new StringBuffer("Error batch inserting rows into table [" + tableName + "].");
        msg.append(Const.CR);
        msg.append("Errors encountered (first 10):").append(Const.CR);
        for (int x = 0; x < be.getExceptionsList().size() && x < 10; x++) {
          Exception exception = be.getExceptionsList().get(x);
          if (exception.getMessage() != null) msg.append(exception.getMessage()).append(Const.CR);
        }
        throw new KettleException(msg.toString(), be);
      }
    } catch (KettleDatabaseException dbe) {
      if (getStepMeta().isDoingErrorHandling()) {
        if (data.specialErrorHandling) {
          data.db.rollback(data.savepoint);
          if (data.releaseSavepoint) {
            data.db.releaseSavepoint(data.savepoint);
          }
          // data.db.commit(true); // force a commit on the connection
          // too.
        }

        sendToErrorRow = true;
        errorMessage = dbe.toString();
      } else {
        if (meta.ignoreErrors()) {
          if (data.warnings < 20) {
            if (log.isBasic())
              logBasic("WARNING: Couldn't insert row into table." + Const.CR + dbe.getMessage());
          } else if (data.warnings == 20) {
            if (log.isBasic())
              logBasic(
                  "FINAL WARNING (no more then 20 displayed): Couldn't insert row into table: "
                      + Const.CR
                      + dbe.getMessage());
          }
          data.warnings++;
        } else {
          setErrors(getErrors() + 1);
          data.db.rollback();
          throw new KettleException("Error inserting row into table [" + tableName + "]", dbe);
        }
      }
    }

    if (data.batchMode) {
      if (sendToErrorRow) {
        if (batchProblem) {
          for (Object[] row : outputRowsData) {
            data.batchBuffer.add(row);
          }
          outputRowsData = null;
          processBatchException(errorMessage, updateCounts, exceptionsList);
        } else {
          // Simply add this row to the error row
          for (Object[] row : outputRowsData) {
            putError(rowMeta, row, 1L, errorMessage, null, "TOP001");
          }
          outputRowsData = null;
        }
      } else {
        for (Object[] row : outputRowsData) {
          data.batchBuffer.add(row);
        }
        outputRowsData = null;

        if (rowIsSafe) // A commit was done and the rows are all safe
        // (no error)
        {
          for (int i = 0; i < data.batchBuffer.size(); i++) {
            Object[] row = (Object[]) data.batchBuffer.get(i);
            putRow(data.outputRowMeta, row);
            incrementLinesOutput();
          }
          // Clear the buffer
          data.batchBuffer.clear();
        }
      }
    } else {
      if (sendToErrorRow) {
        for (Object[] row : outputRowsData) {
          putError(rowMeta, row, 1L, errorMessage, null, "TOP001");
        }
        outputRowsData = null;
      }
    }

    return outputRowsData;
  }
Ejemplo n.º 13
0
 protected static RowMetaInterface createRowMeta(ValueMetaInterface... valueMetas) {
   RowMeta rowMeta = new RowMeta();
   rowMeta.setValueMetaList(Arrays.asList(valueMetas));
   return rowMeta;
 }