public ClusterSchema loadClusterSchema(ObjectId id_cluster_schema, List<SlaveServer> slaveServers)
      throws KettleException {
    ClusterSchema clusterSchema = new ClusterSchema();
    RowMetaAndData row = getClusterSchema(id_cluster_schema);

    clusterSchema.setObjectId(id_cluster_schema);
    clusterSchema.setName(row.getString(KettleDatabaseRepository.FIELD_CLUSTER_NAME, null));
    clusterSchema.setBasePort(
        row.getString(KettleDatabaseRepository.FIELD_CLUSTER_BASE_PORT, null));
    clusterSchema.setSocketsBufferSize(
        row.getString(KettleDatabaseRepository.FIELD_CLUSTER_SOCKETS_BUFFER_SIZE, null));
    clusterSchema.setSocketsFlushInterval(
        row.getString(KettleDatabaseRepository.FIELD_CLUSTER_SOCKETS_FLUSH_INTERVAL, null));
    clusterSchema.setSocketsCompressed(
        row.getBoolean(KettleDatabaseRepository.FIELD_CLUSTER_SOCKETS_COMPRESSED, true));
    clusterSchema.setDynamic(row.getBoolean(KettleDatabaseRepository.FIELD_CLUSTER_DYNAMIC, true));

    ObjectId[] pids = repository.getClusterSlaveIDs(id_cluster_schema);
    for (int i = 0; i < pids.length; i++) {
      SlaveServer slaveServer = repository.loadSlaveServer(pids[i], null); // Load last version
      SlaveServer reference = SlaveServer.findSlaveServer(slaveServers, slaveServer.getName());
      if (reference != null) {
        clusterSchema.getSlaveServers().add(reference);
      } else {
        clusterSchema.getSlaveServers().add(slaveServer);
      }
    }

    return clusterSchema;
  }
Esempio n. 2
0
  public void encodeClusterSchema(Element e, TransMeta transMeta) {
    JSONArray jsonArray = new JSONArray();
    for (int i = 0; i < transMeta.getClusterSchemas().size(); i++) {
      ClusterSchema clusterSchema = transMeta.getClusterSchemas().get(i);

      JSONObject jsonObject = new JSONObject();
      jsonObject.put("name", clusterSchema.getName());
      jsonObject.put("base_port", clusterSchema.getBasePort());
      jsonObject.put("sockets_buffer_size", clusterSchema.getSocketsBufferSize());

      jsonObject.put("sockets_flush_interval", clusterSchema.getSocketsFlushInterval());
      jsonObject.put("sockets_compressed", clusterSchema.isSocketsCompressed() ? "Y" : "N");
      jsonObject.put("dynamic", clusterSchema.isDynamic() ? "Y" : "N");

      JSONArray slaveservers = new JSONArray();
      for (int j = 0; j < clusterSchema.getSlaveServers().size(); j++) {
        SlaveServer slaveServer = clusterSchema.getSlaveServers().get(j);
        slaveservers.add(SlaveServerCodec.encode(slaveServer));
      }
      jsonObject.put("slaveservers", slaveservers);

      jsonArray.add(jsonObject);
    }
    e.setAttribute("clusterSchemas", jsonArray.toString());
  }
Esempio n. 3
0
 /**
  * Add clusters in the repository to this transformation if they are not yet present.
  *
  * @param TransMeta The transformation to load into.
  * @param overWriteShared if an object with the same name exists, overwrite
  */
 protected void readClusters(
     TransMeta transMeta, boolean overWriteShared, List<ClusterSchema> clusterSchemas) {
   for (ClusterSchema clusterSchema : clusterSchemas) {
     if (overWriteShared || transMeta.findClusterSchema(clusterSchema.getName()) == null) {
       if (!Const.isEmpty(clusterSchema.getName())) {
         clusterSchema.shareVariablesWith(transMeta);
         transMeta.addOrReplaceClusterSchema(clusterSchema);
         if (!overWriteShared) {
           clusterSchema.setChanged(false);
         }
       }
     }
   }
 }
Esempio n. 4
0
  public ClusterSchema decodeClusterSchema(
      JSONObject jsonObject, List<SlaveServer> referenceSlaveServers) {
    ClusterSchema clusterSchema = new ClusterSchema();
    clusterSchema.setName(jsonObject.optString("name"));
    clusterSchema.setBasePort(jsonObject.optString("base_port"));
    clusterSchema.setSocketsBufferSize(jsonObject.optString("sockets_buffer_size"));
    clusterSchema.setSocketsFlushInterval(jsonObject.optString("sockets_flush_interval"));
    clusterSchema.setSocketsCompressed(
        "Y".equalsIgnoreCase(jsonObject.optString("sockets_compressed")));
    clusterSchema.setDynamic("Y".equalsIgnoreCase(jsonObject.optString("dynamic")));

    ArrayList<SlaveServer> slaveServers = new ArrayList<SlaveServer>();
    JSONArray slavesNode = jsonObject.optJSONArray("slaveservers");
    if (slavesNode != null) {
      for (int i = 0; i < slavesNode.size(); i++) {
        JSONObject slaveServerJson = slavesNode.getJSONObject(i);
        SlaveServer slaveServer =
            SlaveServer.findSlaveServer(referenceSlaveServers, slaveServerJson.optString("name"));
        if (slaveServer != null) {
          slaveServers.add(slaveServer);
        }
      }
      clusterSchema.setSlaveServers(slaveServers);
    }

    return clusterSchema;
  }
Esempio n. 5
0
  public void saveSharedObjects(
      final RepositoryElementInterface element, final String versionComment)
      throws KettleException {
    TransMeta transMeta = (TransMeta) element;
    // First store the databases and other depending objects in the transformation.
    //

    // Only store if the database has actually changed or doesn't have an object ID (imported)
    //
    for (DatabaseMeta databaseMeta : transMeta.getDatabases()) {
      if (databaseMeta.hasChanged() || databaseMeta.getObjectId() == null) {
        if (databaseMeta.getObjectId() == null
            || unifiedRepositoryConnectionAclService.hasAccess(
                databaseMeta.getObjectId(), RepositoryFilePermission.WRITE)) {
          repo.save(databaseMeta, versionComment, null);
        } else {
          log.logError(
              BaseMessages.getString(
                  PKG,
                  "PurRepository.ERROR_0004_DATABASE_UPDATE_ACCESS_DENIED",
                  databaseMeta.getName()));
        }
      }
    }

    // Store the slave servers...
    //
    for (SlaveServer slaveServer : transMeta.getSlaveServers()) {
      if (slaveServer.hasChanged() || slaveServer.getObjectId() == null) {
        repo.save(slaveServer, versionComment, null);
      }
    }

    // Store the cluster schemas
    //
    for (ClusterSchema clusterSchema : transMeta.getClusterSchemas()) {
      if (clusterSchema.hasChanged() || clusterSchema.getObjectId() == null) {
        repo.save(clusterSchema, versionComment, null);
      }
    }

    // Save the partition schemas
    //
    for (PartitionSchema partitionSchema : transMeta.getPartitionSchemas()) {
      if (partitionSchema.hasChanged() || partitionSchema.getObjectId() == null) {
        repo.save(partitionSchema, versionComment, null);
      }
    }
  }
  protected void replaceSharedObjects(TransMeta transMeta) throws KettleException {
    replaceSharedObjects((AbstractMeta) transMeta);
    for (ClusterSchema clusterSchema : getSharedObjects(ClusterSchema.class)) {
      int index = transMeta.getClusterSchemas().indexOf(clusterSchema);
      if (index < 0) {
        transMeta.getClusterSchemas().add(clusterSchema);
      } else {
        ClusterSchema imported = transMeta.getClusterSchemas().get(index);
        // Preserve the object id so we can update without having to look up the id
        imported.setObjectId(clusterSchema.getObjectId());
        if (equals(clusterSchema, imported)
            || !getPromptResult(
                BaseMessages.getString(
                    PKG,
                    "RepositoryImporter.Dialog.ClusterSchemaExistsOverWrite.Message",
                    imported.getName()),
                BaseMessages.getString(
                    PKG,
                    "RepositoryImporter.Dialog.ConnectionExistsOverWrite.DontShowAnyMoreMessage"),
                IMPORT_ASK_ABOUT_REPLACE_CS)) {
          imported.replaceMeta(clusterSchema);
          // We didn't actually change anything
          imported.clearChanged();
        } else {
          imported.setChanged();
        }
      }
    }

    for (PartitionSchema partitionSchema : getSharedObjects(PartitionSchema.class)) {
      int index = transMeta.getPartitionSchemas().indexOf(partitionSchema);
      if (index < 0) {
        transMeta.getPartitionSchemas().add(partitionSchema);
      } else {
        PartitionSchema imported = transMeta.getPartitionSchemas().get(index);
        // Preserve the object id so we can update without having to look up the id
        imported.setObjectId(partitionSchema.getObjectId());
        if (equals(partitionSchema, imported)
            || !getPromptResult(
                BaseMessages.getString(
                    PKG,
                    "RepositoryImporter.Dialog.PartitionSchemaExistsOverWrite.Message",
                    imported.getName()),
                BaseMessages.getString(
                    PKG,
                    "RepositoryImporter.Dialog.ConnectionExistsOverWrite.DontShowAnyMoreMessage"),
                IMPORT_ASK_ABOUT_REPLACE_PS)) {
          imported.replaceMeta(partitionSchema);
          // We didn't actually change anything
          imported.clearChanged();
        } else {
          imported.setChanged();
        }
      }
    }
  }
  private synchronized ObjectId insertCluster(ClusterSchema clusterSchema) throws KettleException {
    if (getClusterID(clusterSchema.getName()) != null) {
      // This cluster schema name is already in use. Throw an exception.
      throw new KettleObjectExistsException(
          "Failed to create object in repository. Object ["
              + clusterSchema.getName()
              + "] already exists.");
    }

    ObjectId id = repository.connectionDelegate.getNextClusterID();

    RowMetaAndData table = new RowMetaAndData();

    table.addValue(
        new ValueMeta(
            KettleDatabaseRepository.FIELD_CLUSTER_ID_CLUSTER, ValueMetaInterface.TYPE_INTEGER),
        id);
    table.addValue(
        new ValueMeta(KettleDatabaseRepository.FIELD_CLUSTER_NAME, ValueMetaInterface.TYPE_STRING),
        clusterSchema.getName());
    table.addValue(
        new ValueMeta(
            KettleDatabaseRepository.FIELD_CLUSTER_BASE_PORT, ValueMetaInterface.TYPE_STRING),
        clusterSchema.getBasePort());
    table.addValue(
        new ValueMeta(
            KettleDatabaseRepository.FIELD_CLUSTER_SOCKETS_BUFFER_SIZE,
            ValueMetaInterface.TYPE_STRING),
        clusterSchema.getSocketsBufferSize());
    table.addValue(
        new ValueMeta(
            KettleDatabaseRepository.FIELD_CLUSTER_SOCKETS_FLUSH_INTERVAL,
            ValueMetaInterface.TYPE_STRING),
        clusterSchema.getSocketsFlushInterval());
    table.addValue(
        new ValueMeta(
            KettleDatabaseRepository.FIELD_CLUSTER_SOCKETS_COMPRESSED,
            ValueMetaInterface.TYPE_BOOLEAN),
        Boolean.valueOf(clusterSchema.isSocketsCompressed()));
    table.addValue(
        new ValueMeta(
            KettleDatabaseRepository.FIELD_CLUSTER_DYNAMIC, ValueMetaInterface.TYPE_BOOLEAN),
        Boolean.valueOf(clusterSchema.isDynamic()));

    repository
        .connectionDelegate
        .getDatabase()
        .prepareInsert(table.getRowMeta(), KettleDatabaseRepository.TABLE_R_CLUSTER);
    repository.connectionDelegate.getDatabase().setValuesInsert(table);
    repository.connectionDelegate.getDatabase().insertRow();
    repository.connectionDelegate.getDatabase().closeInsert();

    return id;
  }
  public void newClusteringSchema(TransMeta transMeta) {
    ClusterSchema clusterSchema = new ClusterSchema();

    ClusterSchemaDialog dialog =
        new ClusterSchemaDialog(
            spoon.getShell(),
            clusterSchema,
            transMeta.getClusterSchemas(),
            transMeta.getSlaveServers());

    if (dialog.open()) {
      List<ClusterSchema> clusterSchemas = transMeta.getClusterSchemas();
      if (isDuplicate(clusterSchemas, clusterSchema)) {
        new ErrorDialog(
            spoon.getShell(),
            getMessage("Spoon.Dialog.ErrorSavingCluster.Title"),
            getMessage("Spoon.Dialog.ErrorSavingCluster.Message", clusterSchema.getName()),
            new KettleException(getMessage("Spoon.Dialog.ErrorSavingCluster.NotUnique")));
        return;
      }

      clusterSchemas.add(clusterSchema);

      if (spoon.rep != null) {
        try {
          if (!spoon.rep.getSecurityProvider().isReadOnly()) {
            spoon.rep.save(clusterSchema, Const.VERSION_COMMENT_INITIAL_VERSION, null);
          } else {
            throw new KettleException(
                BaseMessages.getString(PKG, "Spoon.Dialog.Exception.ReadOnlyRepositoryUser"));
          }
        } catch (KettleException e) {
          new ErrorDialog(
              spoon.getShell(),
              getMessage("Spoon.Dialog.ErrorSavingCluster.Title"),
              getMessage("Spoon.Dialog.ErrorSavingCluster.Message", clusterSchema.getName()),
              e);
        }
      }

      spoon.refreshTree();
    }
  }
  public synchronized void updateCluster(ClusterSchema clusterSchema) throws KettleException {
    RowMetaAndData table = new RowMetaAndData();

    table.addValue(
        new ValueMeta(
            KettleDatabaseRepository.FIELD_CLUSTER_ID_CLUSTER, ValueMetaInterface.TYPE_INTEGER),
        clusterSchema.getObjectId());
    table.addValue(
        new ValueMeta(KettleDatabaseRepository.FIELD_CLUSTER_NAME, ValueMetaInterface.TYPE_STRING),
        clusterSchema.getName());
    table.addValue(
        new ValueMeta(
            KettleDatabaseRepository.FIELD_CLUSTER_BASE_PORT, ValueMetaInterface.TYPE_STRING),
        clusterSchema.getBasePort());
    table.addValue(
        new ValueMeta(
            KettleDatabaseRepository.FIELD_CLUSTER_SOCKETS_BUFFER_SIZE,
            ValueMetaInterface.TYPE_STRING),
        clusterSchema.getSocketsBufferSize());
    table.addValue(
        new ValueMeta(
            KettleDatabaseRepository.FIELD_CLUSTER_SOCKETS_FLUSH_INTERVAL,
            ValueMetaInterface.TYPE_STRING),
        clusterSchema.getSocketsFlushInterval());
    table.addValue(
        new ValueMeta(
            KettleDatabaseRepository.FIELD_CLUSTER_SOCKETS_COMPRESSED,
            ValueMetaInterface.TYPE_BOOLEAN),
        Boolean.valueOf(clusterSchema.isSocketsCompressed()));
    table.addValue(
        new ValueMeta(
            KettleDatabaseRepository.FIELD_CLUSTER_DYNAMIC, ValueMetaInterface.TYPE_BOOLEAN),
        Boolean.valueOf(clusterSchema.isDynamic()));

    repository.connectionDelegate.updateTableRow(
        KettleDatabaseRepository.TABLE_R_CLUSTER,
        KettleDatabaseRepository.FIELD_CLUSTER_ID_CLUSTER,
        table,
        clusterSchema.getObjectId());
  }
Esempio n. 10
0
  /**
   * @return The object that is selected in the tree or null if we couldn't figure it out. (titles
   *     etc. == null)
   */
  public TreeSelection[] getTreeObjects(final Tree tree, Tree selectionTree, Tree coreObjectsTree) {
    List<TreeSelection> objects = new ArrayList<TreeSelection>();

    if (selectionTree != null && !selectionTree.isDisposed() && tree.equals(selectionTree)) {
      TreeItem[] selection = selectionTree.getSelection();
      for (int s = 0; s < selection.length; s++) {
        TreeItem treeItem = selection[s];
        String[] path = ConstUI.getTreeStrings(treeItem);

        TreeSelection object = null;

        switch (path.length) {
          case 0:
            break;
          case 1: // ------complete-----
            if (path[0].equals(
                Spoon.STRING_TRANSFORMATIONS)) { // the top level Transformations entry

              object = new TreeSelection(path[0], TransMeta.class);
            }
            if (path[0].equals(Spoon.STRING_JOBS)) { // the top level Jobs entry

              object = new TreeSelection(path[0], JobMeta.class);
            }
            break;

          case 2: // ------complete-----
            if (path[0].equals(
                Spoon.STRING_BUILDING_BLOCKS)) { // the top level Transformations entry

              if (path[1].equals(Spoon.STRING_TRANS_BASE)) {
                object = new TreeSelection(path[1], PluginInterface.class);
              }
            }
            if (path[0].equals(Spoon.STRING_TRANSFORMATIONS)) { // Transformation title

              object = new TreeSelection(path[1], spoon.delegates.trans.getTransformation(path[1]));
            }
            if (path[0].equals(Spoon.STRING_JOBS)) { // Jobs title

              object = new TreeSelection(path[1], spoon.delegates.jobs.getJob(path[1]));
            }
            break;

          case 3: // ------complete-----
            if (path[0].equals(Spoon.STRING_TRANSFORMATIONS)) { // Transformations title

              TransMeta transMeta = spoon.delegates.trans.getTransformation(path[1]);
              if (path[2].equals(Spoon.STRING_CONNECTIONS)) {
                object = new TreeSelection(path[2], DatabaseMeta.class, transMeta);
              }
              if (path[2].equals(Spoon.STRING_STEPS)) {
                object = new TreeSelection(path[2], StepMeta.class, transMeta);
              }
              if (path[2].equals(Spoon.STRING_HOPS)) {
                object = new TreeSelection(path[2], TransHopMeta.class, transMeta);
              }
              if (path[2].equals(Spoon.STRING_PARTITIONS)) {
                object = new TreeSelection(path[2], PartitionSchema.class, transMeta);
              }
              if (path[2].equals(Spoon.STRING_SLAVES)) {
                object = new TreeSelection(path[2], SlaveServer.class, transMeta);
              }
              if (path[2].equals(Spoon.STRING_CLUSTERS)) {
                object = new TreeSelection(path[2], ClusterSchema.class, transMeta);
              }
            }
            if (path[0].equals(Spoon.STRING_JOBS)) { // Jobs title

              JobMeta jobMeta = spoon.delegates.jobs.getJob(path[1]);
              if (path[2].equals(Spoon.STRING_CONNECTIONS)) {
                object = new TreeSelection(path[2], DatabaseMeta.class, jobMeta);
              }
              if (path[2].equals(Spoon.STRING_JOB_ENTRIES)) {
                object = new TreeSelection(path[2], JobEntryCopy.class, jobMeta);
              }
              if (path[2].equals(Spoon.STRING_SLAVES)) {
                object = new TreeSelection(path[2], SlaveServer.class, jobMeta);
              }
            }
            break;

          case 4: // ------complete-----
            if (path[0].equals(Spoon.STRING_TRANSFORMATIONS)) { // The name of a transformation
              final TransMeta transMeta = spoon.delegates.trans.getTransformation(path[1]);

              if (transMeta != null) {
                if (path[2].equals(Spoon.STRING_CONNECTIONS)) {
                  String dbName = path[3];
                  DatabaseMeta databaseMeta = transMeta.findDatabase(dbName);
                  if (databaseMeta != null) {
                    dbName = databaseMeta.getName();
                  }

                  object = new TreeSelection(dbName, databaseMeta, transMeta);
                }
                if (path[2].equals(Spoon.STRING_STEPS)) {
                  object = new TreeSelection(path[3], transMeta.findStep(path[3]), transMeta);
                }
                if (path[2].equals(Spoon.STRING_HOPS)) {
                  object = new TreeSelection(path[3], transMeta.findTransHop(path[3]), transMeta);
                }
                if (path[2].equals(Spoon.STRING_PARTITIONS)) {
                  object =
                      new TreeSelection(path[3], transMeta.findPartitionSchema(path[3]), transMeta);
                }
                if (path[2].equals(Spoon.STRING_SLAVES)) {
                  object =
                      new TreeSelection(path[3], transMeta.findSlaveServer(path[3]), transMeta);
                }
                if (path[2].equals(Spoon.STRING_CLUSTERS)) {
                  object =
                      new TreeSelection(path[3], transMeta.findClusterSchema(path[3]), transMeta);
                }
              }
            }
            if (path[0].equals(Spoon.STRING_JOBS)) { // The name of a job
              JobMeta jobMeta = spoon.delegates.jobs.getJob(path[1]);
              if (jobMeta != null && path[2].equals(Spoon.STRING_CONNECTIONS)) {
                String dbName = path[3];
                DatabaseMeta databaseMeta = jobMeta.findDatabase(dbName);
                if (databaseMeta != null) {
                  dbName = databaseMeta.getName();
                }

                object = new TreeSelection(dbName, databaseMeta, jobMeta);
              }
              if (jobMeta != null && path[2].equals(Spoon.STRING_JOB_ENTRIES)) {
                object = new TreeSelection(path[3], jobMeta.findJobEntry(path[3]), jobMeta);
              }
              if (jobMeta != null && path[2].equals(Spoon.STRING_SLAVES)) {
                object = new TreeSelection(path[3], jobMeta.findSlaveServer(path[3]), jobMeta);
              }
            }
            break;

          case 5:
            if (path[0].equals(Spoon.STRING_TRANSFORMATIONS)) { // The name of a transformation

              TransMeta transMeta = spoon.delegates.trans.getTransformation(path[1]);
              if (transMeta != null && path[2].equals(Spoon.STRING_CLUSTERS)) {
                ClusterSchema clusterSchema = transMeta.findClusterSchema(path[3]);
                object =
                    new TreeSelection(
                        path[4], clusterSchema.findSlaveServer(path[4]), clusterSchema, transMeta);
              }
            }
            break;
          default:
            break;
        }

        if (object != null) {
          objects.add(object);
        }
      }
    }
    if (tree != null && coreObjectsTree != null && tree.equals(coreObjectsTree)) {
      TreeItem[] selection = coreObjectsTree.getSelection();
      for (int s = 0; s < selection.length; s++) {
        TreeItem treeItem = selection[s];
        String[] path = ConstUI.getTreeStrings(treeItem);

        TreeSelection object = null;

        switch (path.length) {
          case 0:
            break;
          case 2: // Job entries
            if (spoon.showJob) {
              PluginRegistry registry = PluginRegistry.getInstance();
              Class<? extends PluginTypeInterface> pluginType = JobEntryPluginType.class;
              PluginInterface plugin = registry.findPluginWithName(pluginType, path[1]);

              // Retry for Start
              //
              if (plugin == null) {
                if (path[1].equals(JobMeta.STRING_SPECIAL_START)) {
                  plugin = registry.findPluginWithId(pluginType, JobMeta.STRING_SPECIAL);
                }
              }
              // Retry for Dummy
              //
              if (plugin == null) {
                if (path[1].equals(JobMeta.STRING_SPECIAL_DUMMY)) {
                  plugin = registry.findPluginWithId(pluginType, JobMeta.STRING_SPECIAL);
                }
              }

              if (plugin != null) {
                object = new TreeSelection(path[1], plugin);
              }
            }

            if (spoon.showTrans) {
              // Steps
              object =
                  new TreeSelection(
                      path[1],
                      PluginRegistry.getInstance()
                          .findPluginWithName(StepPluginType.class, path[1]));
            }
            break;
          default:
            break;
        }

        if (object != null) {
          objects.add(object);
        }
      }
    }

    return objects.toArray(new TreeSelection[objects.size()]);
  }
  public void saveClusterSchema(
      ClusterSchema clusterSchema,
      String versionComment,
      ObjectId id_transformation,
      boolean isUsedByTransformation,
      boolean overwrite)
      throws KettleException {
    ObjectId existingClusterSchemaId = getClusterID(clusterSchema.getName());
    if (existingClusterSchemaId != null) {
      clusterSchema.setObjectId(existingClusterSchemaId);
    }

    if (clusterSchema.getObjectId() == null) {
      // New Slave Server
      clusterSchema.setObjectId(insertCluster(clusterSchema));
    } else {

      // If we received a clusterSchemaId and it is different from the cluster schema we are working
      // with...
      if (existingClusterSchemaId != null
          && !clusterSchema.getObjectId().equals(existingClusterSchemaId)) {
        // A cluster with this name already exists
        if (overwrite) {
          // Proceed with save, removing the original version from the repository first
          repository.deleteClusterSchema(existingClusterSchemaId);
          updateCluster(clusterSchema);
        } else {
          throw new KettleObjectExistsException(
              "Failed to save object to repository. Object ["
                  + clusterSchema.getName()
                  + "] already exists.");
        }
      } else {
        // There are no naming collisions (either it is the same object or the name is unique)
        updateCluster(clusterSchema);
      }
    }

    repository.delClusterSlaves(clusterSchema.getObjectId());

    // Also save the used slave server references.
    for (int i = 0; i < clusterSchema.getSlaveServers().size(); i++) {
      SlaveServer slaveServer = clusterSchema.getSlaveServers().get(i);
      if (slaveServer.getObjectId() == null) // oops, not yet saved!
      {
        repository.save(
            slaveServer,
            versionComment,
            null,
            id_transformation,
            isUsedByTransformation,
            overwrite);
      }
      repository.insertClusterSlave(clusterSchema, slaveServer);
    }

    // Save a link to the transformation to keep track of the use of this cluster schema
    // Only save it if it's really used by the transformation
    if (isUsedByTransformation) {
      repository.insertTransformationCluster(id_transformation, clusterSchema.getObjectId());
    }
  }
Esempio n. 12
0
  public void dataNodeToElement(final DataNode rootNode, final RepositoryElementInterface element)
      throws KettleException {
    TransMeta transMeta = (TransMeta) element;

    List<String> privateTransformationDatabases = null;
    // read the private databases
    DataNode privateDatabases = rootNode.getNode(NODE_TRANS_PRIVATE_DATABASES);
    // if we have node than we use new format we could remove unexpected node
    if (privateDatabases != null) {
      privateTransformationDatabases = new ArrayList<String>();
      for (DataNode privateDatabase : privateDatabases.getNodes()) {
        privateTransformationDatabases.add(privateDatabase.getName());
      }
    }
    transMeta.setPrivateTransformationDatabases(privateTransformationDatabases);

    // read the steps...
    //
    DataNode stepsNode = rootNode.getNode(NODE_STEPS);
    for (DataNode stepNode : stepsNode.getNodes()) {

      StepMeta stepMeta = new StepMeta(new StringObjectId(stepNode.getId().toString()));
      stepMeta.setParentTransMeta(transMeta); // for tracing, retain hierarchy

      // Read the basics
      //
      stepMeta.setName(getString(stepNode, PROP_NAME));
      if (stepNode.hasProperty(PROP_DESCRIPTION)) {
        stepMeta.setDescription(getString(stepNode, PROP_DESCRIPTION));
      }
      stepMeta.setDistributes(stepNode.getProperty(PROP_STEP_DISTRIBUTE).getBoolean());
      DataProperty rowDistributionProperty = stepNode.getProperty(PROP_STEP_ROW_DISTRIBUTION);
      String rowDistributionCode =
          rowDistributionProperty == null ? null : rowDistributionProperty.getString();
      RowDistributionInterface rowDistribution =
          PluginRegistry.getInstance()
              .loadClass(
                  RowDistributionPluginType.class,
                  rowDistributionCode,
                  RowDistributionInterface.class);
      stepMeta.setRowDistribution(rowDistribution);
      stepMeta.setDraw(stepNode.getProperty(PROP_STEP_GUI_DRAW).getBoolean());
      int copies = (int) stepNode.getProperty(PROP_STEP_COPIES).getLong();
      String copiesString =
          stepNode.getProperty(PROP_STEP_COPIES_STRING) != null
              ? stepNode.getProperty(PROP_STEP_COPIES_STRING).getString()
              : StringUtils.EMPTY;
      if (!Const.isEmpty(copiesString)) {
        stepMeta.setCopiesString(copiesString);
      } else {
        stepMeta.setCopies(copies); // for backward compatibility
      }

      int x = (int) stepNode.getProperty(PROP_STEP_GUI_LOCATION_X).getLong();
      int y = (int) stepNode.getProperty(PROP_STEP_GUI_LOCATION_Y).getLong();
      stepMeta.setLocation(x, y);

      // Load the group attributes map
      //
      AttributesMapUtil.loadAttributesMap(stepNode, stepMeta);

      String stepType = getString(stepNode, PROP_STEP_TYPE);

      // Create a new StepMetaInterface object...
      //
      PluginRegistry registry = PluginRegistry.getInstance();
      PluginInterface stepPlugin = registry.findPluginWithId(StepPluginType.class, stepType);

      StepMetaInterface stepMetaInterface = null;
      if (stepPlugin != null) {
        stepMetaInterface = (StepMetaInterface) registry.loadClass(stepPlugin);
        stepType =
            stepPlugin.getIds()[0]; // revert to the default in case we loaded an alternate version
      } else {
        stepMeta.setStepMetaInterface(
            (StepMetaInterface) new MissingTrans(stepMeta.getName(), stepType));
        transMeta.addMissingTrans((MissingTrans) stepMeta.getStepMetaInterface());
      }

      stepMeta.setStepID(stepType);

      // Read the metadata from the repository too...
      //
      RepositoryProxy proxy = new RepositoryProxy(stepNode.getNode(NODE_STEP_CUSTOM));
      if (!stepMeta.isMissing()) {
        readRepCompatibleStepMeta(stepMetaInterface, proxy, null, transMeta.getDatabases());
        stepMetaInterface.readRep(proxy, transMeta.getMetaStore(), null, transMeta.getDatabases());
        stepMeta.setStepMetaInterface(stepMetaInterface);
      }

      // Get the partitioning as well...
      StepPartitioningMeta stepPartitioningMeta = new StepPartitioningMeta();
      if (stepNode.hasProperty(PROP_PARTITIONING_SCHEMA)) {
        String partSchemaId =
            stepNode.getProperty(PROP_PARTITIONING_SCHEMA).getRef().getId().toString();
        String schemaName =
            repo.loadPartitionSchema(new StringObjectId(partSchemaId), null).getName();

        stepPartitioningMeta.setPartitionSchemaName(schemaName);
        String methodCode = getString(stepNode, PROP_PARTITIONING_METHOD);
        stepPartitioningMeta.setMethod(StepPartitioningMeta.getMethod(methodCode));
        if (stepPartitioningMeta.getPartitioner() != null) {
          proxy = new RepositoryProxy(stepNode.getNode(NODE_PARTITIONER_CUSTOM));
          stepPartitioningMeta.getPartitioner().loadRep(proxy, null);
        }
        stepPartitioningMeta.hasChanged(true);
      }
      stepMeta.setStepPartitioningMeta(stepPartitioningMeta);

      stepMeta
          .getStepPartitioningMeta()
          .setPartitionSchemaAfterLoading(transMeta.getPartitionSchemas());
      // Get the cluster schema name
      String clusterSchemaName = getString(stepNode, PROP_CLUSTER_SCHEMA);
      stepMeta.setClusterSchemaName(clusterSchemaName);
      if (clusterSchemaName != null && transMeta.getClusterSchemas() != null) {
        // Get the cluster schema from the given name
        for (ClusterSchema clusterSchema : transMeta.getClusterSchemas()) {
          if (clusterSchema.getName().equals(clusterSchemaName)) {
            stepMeta.setClusterSchema(clusterSchema);
            break;
          }
        }
      }

      transMeta.addStep(stepMeta);
    }

    for (DataNode stepNode : stepsNode.getNodes()) {

      ObjectId stepObjectId = new StringObjectId(stepNode.getId().toString());
      StepMeta stepMeta = StepMeta.findStep(transMeta.getSteps(), stepObjectId);

      // Also load the step error handling metadata
      //
      if (stepNode.hasProperty(PROP_STEP_ERROR_HANDLING_SOURCE_STEP)) {
        StepErrorMeta meta = new StepErrorMeta(transMeta, stepMeta);
        meta.setTargetStep(
            StepMeta.findStep(
                transMeta.getSteps(),
                stepNode.getProperty(PROP_STEP_ERROR_HANDLING_TARGET_STEP).getString()));
        meta.setEnabled(stepNode.getProperty(PROP_STEP_ERROR_HANDLING_IS_ENABLED).getBoolean());
        meta.setNrErrorsValuename(getString(stepNode, PROP_STEP_ERROR_HANDLING_NR_VALUENAME));
        meta.setErrorDescriptionsValuename(
            getString(stepNode, PROP_STEP_ERROR_HANDLING_DESCRIPTIONS_VALUENAME));
        meta.setErrorFieldsValuename(
            getString(stepNode, PROP_STEP_ERROR_HANDLING_FIELDS_VALUENAME));
        meta.setErrorCodesValuename(getString(stepNode, PROP_STEP_ERROR_HANDLING_CODES_VALUENAME));
        meta.setMaxErrors(getString(stepNode, PROP_STEP_ERROR_HANDLING_MAX_ERRORS));
        meta.setMaxPercentErrors(getString(stepNode, PROP_STEP_ERROR_HANDLING_MAX_PCT_ERRORS));
        meta.setMinPercentRows(getString(stepNode, PROP_STEP_ERROR_HANDLING_MIN_PCT_ROWS));
        meta.getSourceStep().setStepErrorMeta(meta); // a bit of a trick, I know.
      }
    }

    // Have all StreamValueLookups, etc. reference the correct source steps...
    //
    for (int i = 0; i < transMeta.nrSteps(); i++) {
      StepMeta stepMeta = transMeta.getStep(i);
      StepMetaInterface sii = stepMeta.getStepMetaInterface();
      if (sii != null) {
        sii.searchInfoAndTargetSteps(transMeta.getSteps());
      }
    }

    // Read the notes...
    //
    DataNode notesNode = rootNode.getNode(NODE_NOTES);
    int nrNotes = (int) notesNode.getProperty(PROP_NR_NOTES).getLong();
    for (DataNode noteNode : notesNode.getNodes()) {
      String xml = getString(noteNode, PROP_XML);
      transMeta.addNote(
          new NotePadMeta(
              XMLHandler.getSubNode(XMLHandler.loadXMLString(xml), NotePadMeta.XML_TAG)));
    }
    if (transMeta.nrNotes() != nrNotes) {
      throw new KettleException(
          "The number of notes read ["
              + transMeta.nrNotes()
              + "] was not the number we expected ["
              + nrNotes
              + "]");
    }

    // Read the hops...
    //
    DataNode hopsNode = rootNode.getNode(NODE_HOPS);
    int nrHops = (int) hopsNode.getProperty(PROP_NR_HOPS).getLong();
    for (DataNode hopNode : hopsNode.getNodes()) {
      String stepFromName = getString(hopNode, TRANS_HOP_FROM);
      String stepToName = getString(hopNode, TRANS_HOP_TO);
      boolean enabled = true;
      if (hopNode.hasProperty(TRANS_HOP_ENABLED)) {
        enabled = hopNode.getProperty(TRANS_HOP_ENABLED).getBoolean();
      }

      StepMeta stepFrom = StepMeta.findStep(transMeta.getSteps(), stepFromName);
      StepMeta stepTo = StepMeta.findStep(transMeta.getSteps(), stepToName);

      // Make sure to only accept valid hops PDI-5519
      //
      if (stepFrom != null && stepTo != null) {
        transMeta.addTransHop(new TransHopMeta(stepFrom, stepTo, enabled));
      }
    }
    if (transMeta.nrTransHops() != nrHops) {
      throw new KettleException(
          "The number of hops read ["
              + transMeta.nrTransHops()
              + "] was not the number we expected ["
              + nrHops
              + "]");
    }

    // Load the details at the end, to make sure we reference the databases correctly, etc.
    //
    loadTransformationDetails(rootNode, transMeta);

    transMeta.eraseParameters();

    DataNode paramsNode = rootNode.getNode(NODE_PARAMETERS);

    int count = (int) paramsNode.getProperty(PROP_NR_PARAMETERS).getLong();
    for (int idx = 0; idx < count; idx++) {
      DataNode paramNode = paramsNode.getNode(TRANS_PARAM_PREFIX + idx);
      String key = getString(paramNode, PARAM_KEY);
      String def = getString(paramNode, PARAM_DEFAULT);
      String desc = getString(paramNode, PARAM_DESC);
      transMeta.addParameterDefinition(key, def, desc);
    }

    transMeta.activateParameters();
  }
Esempio n. 13
0
  @Override
  public AbstractMeta decode(String graphXml) throws Exception {
    mxGraph graph = new mxGraph();
    mxCodec codec = new mxCodec();
    Document doc = mxUtils.parseXml(graphXml);
    codec.decode(doc.getDocumentElement(), graph.getModel());
    mxCell root = (mxCell) graph.getDefaultParent();

    TransMeta transMeta = new TransMeta();
    decodeCommRootAttr(root, transMeta);
    transMeta.setTransstatus(Const.toInt(root.getAttribute("trans_status"), -1));
    transMeta.setTransversion(root.getAttribute("trans_version"));

    if (transMeta.getRepository() != null)
      transMeta.setSharedObjects(transMeta.getRepository().readTransSharedObjects(transMeta));
    else transMeta.setSharedObjects(transMeta.readSharedObjects());

    transMeta.importFromMetaStore();

    decodeDatabases(root, transMeta);
    decodeNote(graph, transMeta);

    int count = graph.getModel().getChildCount(root);
    for (int i = 0; i < count; i++) {
      mxCell cell = (mxCell) graph.getModel().getChildAt(root, i);
      if (cell.isVertex()) {
        Element e = (Element) cell.getValue();
        if (PropsUI.TRANS_STEP_NAME.equals(e.getTagName())) {
          StepDecoder stepDecoder = (StepDecoder) PluginFactory.getBean(cell.getAttribute("ctype"));
          StepMeta stepMeta =
              stepDecoder.decodeStep(cell, transMeta.getDatabases(), transMeta.getMetaStore());
          stepMeta.setParentTransMeta(transMeta);
          if (stepMeta.isMissing()) {
            transMeta.addMissingTrans((MissingTrans) stepMeta.getStepMetaInterface());
          }

          StepMeta check = transMeta.findStep(stepMeta.getName());
          if (check != null) {
            if (!check.isShared()) {
              // Don't overwrite shared objects
              transMeta.addOrReplaceStep(stepMeta);
            } else {
              check.setDraw(stepMeta.isDrawn()); // Just keep the  drawn flag  and location
              check.setLocation(stepMeta.getLocation());
            }
          } else {
            transMeta.addStep(stepMeta); // simply add it.
          }
        }
      }
    }

    // Have all StreamValueLookups, etc. reference the correct source steps...
    //
    for (int i = 0; i < transMeta.nrSteps(); i++) {
      StepMeta stepMeta = transMeta.getStep(i);
      StepMetaInterface sii = stepMeta.getStepMetaInterface();
      if (sii != null) {
        sii.searchInfoAndTargetSteps(transMeta.getSteps());
      }
    }

    count = graph.getModel().getChildCount(root);
    for (int i = 0; i < count; i++) {
      mxCell cell = (mxCell) graph.getModel().getChildAt(root, i);
      if (cell.isEdge()) {
        mxCell source = (mxCell) cell.getSource();
        mxCell target = (mxCell) cell.getTarget();

        TransHopMeta hopinf = new TransHopMeta(null, null, true);
        String[] stepNames = transMeta.getStepNames();
        for (int j = 0; j < stepNames.length; j++) {
          if (stepNames[j].equalsIgnoreCase(source.getAttribute("label")))
            hopinf.setFromStep(transMeta.getStep(j));
          if (stepNames[j].equalsIgnoreCase(target.getAttribute("label")))
            hopinf.setToStep(transMeta.getStep(j));
        }
        transMeta.addTransHop(hopinf);
      }
    }

    JSONObject jsonObject = JSONObject.fromObject(root.getAttribute("transLogTable"));
    TransLogTable transLogTable = transMeta.getTransLogTable();
    transLogTable.setConnectionName(jsonObject.optString("connection"));
    transLogTable.setSchemaName(jsonObject.optString("schema"));
    transLogTable.setTableName(jsonObject.optString("table"));
    transLogTable.setLogSizeLimit(jsonObject.optString("size_limit_lines"));
    transLogTable.setLogInterval(jsonObject.optString("interval"));
    transLogTable.setTimeoutInDays(jsonObject.optString("timeout_days"));
    JSONArray jsonArray = jsonObject.optJSONArray("fields");
    if (jsonArray != null) {
      for (int i = 0; i < jsonArray.size(); i++) {
        JSONObject fieldJson = jsonArray.getJSONObject(i);
        String id = fieldJson.optString("id");
        LogTableField field = transLogTable.findField(id);
        if (field == null) {
          field = transLogTable.getFields().get(i);
        }
        if (field != null) {
          field.setFieldName(fieldJson.optString("name"));
          field.setEnabled(fieldJson.optBoolean("enabled"));
          field.setSubject(StepMeta.findStep(transMeta.getSteps(), fieldJson.optString("subject")));
        }
      }
    }

    jsonObject = JSONObject.fromObject(root.getAttribute("stepLogTable"));
    StepLogTable stepLogTable = transMeta.getStepLogTable();
    stepLogTable.setConnectionName(jsonObject.optString("connection"));
    stepLogTable.setSchemaName(jsonObject.optString("schema"));
    stepLogTable.setTableName(jsonObject.optString("table"));
    stepLogTable.setTimeoutInDays(jsonObject.optString("timeout_days"));
    jsonArray = jsonObject.optJSONArray("fields");
    if (jsonArray != null) {
      for (int i = 0; i < jsonArray.size(); i++) {
        JSONObject fieldJson = jsonArray.getJSONObject(i);
        String id = fieldJson.optString("id");
        LogTableField field = stepLogTable.findField(id);
        if (field == null && i < stepLogTable.getFields().size()) {
          field = stepLogTable.getFields().get(i);
        }
        if (field != null) {
          field.setFieldName(fieldJson.optString("name"));
          field.setEnabled(fieldJson.optBoolean("enabled"));
        }
      }
    }

    jsonObject = JSONObject.fromObject(root.getAttribute("performanceLogTable"));
    PerformanceLogTable performanceLogTable = transMeta.getPerformanceLogTable();
    performanceLogTable.setConnectionName(jsonObject.optString("connection"));
    performanceLogTable.setSchemaName(jsonObject.optString("schema"));
    performanceLogTable.setTableName(jsonObject.optString("table"));
    performanceLogTable.setLogInterval(jsonObject.optString("interval"));
    performanceLogTable.setTimeoutInDays(jsonObject.optString("timeout_days"));
    jsonArray = jsonObject.optJSONArray("fields");
    if (jsonArray != null) {
      for (int i = 0; i < jsonArray.size(); i++) {
        JSONObject fieldJson = jsonArray.getJSONObject(i);
        String id = fieldJson.optString("id");
        LogTableField field = performanceLogTable.findField(id);
        if (field == null && i < performanceLogTable.getFields().size()) {
          field = performanceLogTable.getFields().get(i);
        }
        if (field != null) {
          field.setFieldName(fieldJson.optString("name"));
          field.setEnabled(fieldJson.optBoolean("enabled"));
        }
      }
    }

    jsonObject = JSONObject.fromObject(root.getAttribute("metricsLogTable"));
    MetricsLogTable metricsLogTable = transMeta.getMetricsLogTable();
    metricsLogTable.setConnectionName(jsonObject.optString("connection"));
    metricsLogTable.setSchemaName(jsonObject.optString("schema"));
    metricsLogTable.setTableName(jsonObject.optString("table"));
    metricsLogTable.setTimeoutInDays(jsonObject.optString("timeout_days"));
    jsonArray = jsonObject.optJSONArray("fields");
    if (jsonArray != null) {
      for (int i = 0; i < jsonArray.size(); i++) {
        JSONObject fieldJson = jsonArray.getJSONObject(i);
        String id = fieldJson.optString("id");
        LogTableField field = metricsLogTable.findField(id);
        if (field == null && i < metricsLogTable.getFields().size()) {
          field = metricsLogTable.getFields().get(i);
        }
        if (field != null) {
          field.setFieldName(fieldJson.optString("name"));
          field.setEnabled(fieldJson.optBoolean("enabled"));
        }
      }
    }

    jsonArray = JSONArray.fromObject(root.getAttribute("partitionschemas"));
    for (int i = 0; i < jsonArray.size(); i++) {
      jsonObject = jsonArray.getJSONObject(i);
      PartitionSchema partitionSchema = decodePartitionSchema(jsonObject);
      PartitionSchema check = transMeta.findPartitionSchema(partitionSchema.getName());
      if (check != null) {
        if (!check.isShared()) {
          transMeta.addOrReplacePartitionSchema(partitionSchema);
        }
      } else {
        transMeta.getPartitionSchemas().add(partitionSchema);
      }
    }

    decodeSlaveServers(root, transMeta);

    jsonArray = JSONArray.fromObject(root.getAttribute("clusterSchemas"));
    for (int i = 0; i < jsonArray.size(); i++) {
      jsonObject = jsonArray.getJSONObject(i);
      ClusterSchema clusterSchema = decodeClusterSchema(jsonObject, transMeta.getSlaveServers());
      clusterSchema.shareVariablesWith(transMeta);

      ClusterSchema check = transMeta.findClusterSchema(clusterSchema.getName());
      if (check != null) {
        if (!check.isShared()) {
          transMeta.addOrReplaceClusterSchema(clusterSchema);
        }
      } else {
        transMeta.getClusterSchemas().add(clusterSchema);
      }
    }

    for (int i = 0; i < transMeta.nrSteps(); i++) {
      transMeta.getStep(i).setClusterSchemaAfterLoading(transMeta.getClusterSchemas());
    }

    transMeta.setSizeRowset(Const.toInt(root.getAttribute("size_rowset"), Const.ROWS_IN_ROWSET));
    transMeta.setSleepTimeEmpty(
        Const.toInt(root.getAttribute("sleep_time_empty"), Const.TIMEOUT_GET_MILLIS));
    transMeta.setSleepTimeFull(
        Const.toInt(root.getAttribute("sleep_time_full"), Const.TIMEOUT_PUT_MILLIS));
    transMeta.setUsingUniqueConnections(
        "Y".equalsIgnoreCase(root.getAttribute("unique_connections")));

    transMeta.setFeedbackShown(!"N".equalsIgnoreCase(root.getAttribute("feedback_shown")));
    transMeta.setFeedbackSize(Const.toInt(root.getAttribute("feedback_size"), Const.ROWS_UPDATE));
    transMeta.setUsingThreadPriorityManagment(
        !"N".equalsIgnoreCase(root.getAttribute("using_thread_priorities")));

    transMeta.setCapturingStepPerformanceSnapShots(
        "Y".equalsIgnoreCase(root.getAttribute("capture_step_performance")));
    transMeta.setStepPerformanceCapturingDelay(
        Const.toLong(root.getAttribute("step_performance_capturing_delay"), 1000));
    transMeta.setStepPerformanceCapturingSizeLimit(
        root.getAttribute("step_performance_capturing_size_limit"));

    transMeta.setKey(XMLHandler.stringToBinary(root.getAttribute("key_for_session_key")));
    transMeta.setPrivateKey("Y".equals(root.getAttribute("is_key_private")));

    return transMeta;
  }
 protected boolean equals(ClusterSchema clusterSchema, ClusterSchema clusterSchema2) {
   if (!equals(clusterSchema.getName(), clusterSchema2.getName())) {
     return false;
   } else if (!equals(clusterSchema.getBasePort(), clusterSchema2.getBasePort())) {
     return false;
   } else if (!equals(
       clusterSchema.getSocketsBufferSize(), clusterSchema2.getSocketsBufferSize())) {
     return false;
   } else if (!equals(
       clusterSchema.getSocketsFlushInterval(), clusterSchema2.getSocketsFlushInterval())) {
     return false;
   } else if (!equals(clusterSchema.isSocketsCompressed(), clusterSchema2.isSocketsCompressed())) {
     return false;
   } else if (!equals(clusterSchema.isDynamic(), clusterSchema2.isDynamic())) {
     return false;
   } else if (!equals(clusterSchema.getSlaveServers(), clusterSchema2.getSlaveServers())) {
     return false;
   }
   return true;
 }