private void generateRDFFromWorksheet(
      Worksheet wk,
      Workspace workspace,
      KR2RMLMapping mapping,
      List<KR2RMLRDFWriter> writers,
      String baseURI)
      throws IOException, JSONException, KarmaException {
    // Generate RDF for the remaining rows
    // Gets all the errors generated during the RDF generation
    ErrorReport errorReport = new ErrorReport();

    this.applyHistoryToWorksheet(workspace, wk, mapping);
    SuperSelection selection = SuperSelectionManager.DEFAULT_SELECTION;
    if (selectionName != null && !selectionName.trim().isEmpty())
      selection = wk.getSuperSelectionManager().getSuperSelection(selectionName);
    if (selection == null) return;
    // RDF generation object initialization
    KR2RMLWorksheetRDFGenerator rdfGen =
        new KR2RMLWorksheetRDFGenerator(
            wk,
            workspace.getFactory(),
            workspace.getOntologyManager(),
            writers,
            false,
            mapping,
            errorReport,
            selection);

    // Generate the rdf
    rdfGen.generateRDF(false);
  }
示例#2
0
 @Override
 public UpdateContainer doIt(Workspace workspace) throws CommandException {
   UpdateContainer undoEffects =
       workspace.getCommandHistory().undoOrRedoCommandsUntil(workspace, commandIdArg);
   UpdateContainer result = new UpdateContainer(new HistoryUpdate(workspace.getCommandHistory()));
   result.append(undoEffects);
   return result;
 }
示例#3
0
 @Override
 public UpdateContainer undoIt(Workspace workspace) {
   Node node = workspace.getFactory().getNode(nodeIdArg);
   SuperSelection sel = getSuperSelection(workspace);
   node.setValue(previousValue, previousStatus, workspace.getFactory());
   UpdateContainer uc =
       WorksheetUpdateFactory.createWorksheetHierarchicalAndCleaningResultsUpdates(
           worksheetId, sel);
   uc.add(new NodeChangedUpdate(worksheetId, nodeIdArg, previousValue, previousStatus));
   return uc;
 }
 @Override
 public UpdateContainer doIt(Workspace workspace) throws CommandException {
   UpdateContainer uc = new UpdateContainer();
   SuperSelection sel = this.getSuperSelection(workspace);
   for (int i = 0; i < updates.length(); i++) {
     String update = updates.getString(i);
     switch (update) {
       case "headers":
         uc.add(new WorksheetHeadersUpdate(worksheetId));
         break;
       case "list":
         uc.add(new WorksheetListUpdate());
         break;
       case "data":
         uc.add(new WorksheetDataUpdate(worksheetId, sel));
         break;
       case "alignment":
         {
           Alignment alignment =
               AlignmentManager.Instance()
                   .getAlignmentOrCreateIt(
                       workspace.getId(), worksheetId, workspace.getOntologyManager());
           uc.add(new AlignmentSVGVisualizationUpdate(worksheetId, alignment));
           break;
         }
       case "semanticTypes":
         {
           Alignment alignment =
               AlignmentManager.Instance()
                   .getAlignmentOrCreateIt(
                       workspace.getId(), worksheetId, workspace.getOntologyManager());
           uc.add(
               new SemanticTypesUpdate(
                   workspace.getWorksheet(worksheetId), worksheetId, alignment));
           break;
         }
       case "regenerate":
         uc.add(new RegenerateWorksheetUpdate(worksheetId));
         break;
       case "all":
         uc = WorksheetUpdateFactory.createRegenerateWorksheetUpdates(worksheetId, sel);
         break;
       case "cleaning":
         uc.add(new WorksheetCleaningUpdate(worksheetId, false, sel));
         break;
     }
   }
   return uc;
 }
  @Override
  public UpdateContainer doIt(VWorkspace vWorkspace) throws CommandException {
    Worksheet worksheet = vWorkspace.getViewFactory().getVWorksheet(vWorksheetId).getWorksheet();

    Workspace ws = vWorkspace.getWorkspace();
    if (worksheet.getSemanticTypes().getListOfTypes().size() == 0) {
      SemanticTypeUtil.populateSemanticTypesUsingCRF(
          worksheet,
          ws.getTagsContainer().getTag(TagName.Outlier),
          ws.getCrfModelHandler(),
          ws.getOntologyManager());
    }

    WorksheetGeospatialContent geo = new WorksheetGeospatialContent(worksheet);
    // Send an error update if no geospatial data found!
    if (geo.hasNoGeospatialData()) {
      return new UpdateContainer(new ErrorUpdate("No geospatial data found in the worksheet!"));
    }

    try {
      final File file = geo.publishKML();
      // Transfer the file to a public server
      final boolean transfer = transferFileToPublicServer(file);
      if (!transfer) {
        logger.error(
            "Published KML file could not be moved to a public server to display on Google Maps!");
      }

      return new UpdateContainer(
          new AbstractUpdate() {
            @Override
            public void generateJson(String prefix, PrintWriter pw, VWorkspace vWorkspace) {
              JSONObject outputObject = new JSONObject();
              try {
                outputObject.put(JsonKeys.updateType.name(), "PublishKMLUpdate");
                outputObject.put(JsonKeys.fileName.name(), publicKMLAddress + file.getName());
                outputObject.put(JsonKeys.transferSuccessful.name(), transfer);
                outputObject.put(JsonKeys.localFileName.name(), "KML/" + file.getName());
                pw.println(outputObject.toString(4));
              } catch (JSONException e) {
                logger.error("Error occured while generating JSON!");
              }
            }
          });
    } catch (FileNotFoundException e) {
      logger.error("KML File not found!", e);
      return new UpdateContainer(new ErrorUpdate("Error occurred while publishing KML layer!"));
    }
  }
 public static UpdateContainer createSemanticTypesAndSVGAlignmentUpdates(
     String worksheetId, Workspace workspace, Alignment alignment) {
   Worksheet worksheet = workspace.getWorksheet(worksheetId);
   UpdateContainer c = new UpdateContainer();
   c.add(new SemanticTypesUpdate(worksheet, worksheetId, alignment));
   c.add(new AlignmentSVGVisualizationUpdate(worksheetId, alignment));
   return c;
 }
  @Override
  public UpdateContainer doIt(Workspace workspace) throws CommandException {
    Worksheet worksheet = workspace.getWorksheet(worksheetId);
    RepFactory factory = workspace.getFactory();
    SuperSelection superSel = getSuperSelection(worksheet);
    HTable hTable = factory.getHTable(factory.getHNode(hNodeId).getHTableId());
    Selection currentSel = superSel.getSelection(hTable.getId());
    if (currentSel != null) {
      currentSel.updateSelection();
    }
    CommandHistory history = workspace.getCommandHistory();
    List<Command> tmp =
        gatherAllOperateSelectionCommands(
            history.getCommandsFromWorksheetId(worksheetId), workspace);
    if (tmp.size() > 0) {
      JSONArray inputJSON = new JSONArray();
      inputJSON.put(
          CommandInputJSONUtil.createJsonObject(
              "worksheetId", worksheetId, ParameterType.worksheetId));
      inputJSON.put(
          CommandInputJSONUtil.createJsonObject("hNodeId", hNodeId, ParameterType.hNodeId));
      inputJSON.put(
          CommandInputJSONUtil.createJsonObject(
              "operation", Operation.Intersect.name(), ParameterType.other));
      inputJSON.put(
          CommandInputJSONUtil.createJsonObject(
              "pythonCode", SelectionManager.defaultCode, ParameterType.other));
      inputJSON.put(CommandInputJSONUtil.createJsonObject("onError", "false", ParameterType.other));
      inputJSON.put(
          CommandInputJSONUtil.createJsonObject(
              "selectionName", superSel.getName(), ParameterType.other));
      Command t = null;
      try {
        t = new OperateSelectionCommandFactory().createCommand(inputJSON, workspace);
      } catch (Exception e) {

      }
      if (t != null) history._getHistory().add(t);
      history._getHistory().addAll(tmp);
    }
    UpdateContainer uc =
        WorksheetUpdateFactory.createWorksheetHierarchicalAndCleaningResultsUpdates(
            worksheetId, superSel);
    uc.add(new HistoryUpdate(history));
    return uc;
  }
 public static void detectSelectionStatusChange(
     String worksheetId, Workspace workspace, Command command) {
   Worksheet worksheet = workspace.getWorksheet(worksheetId);
   for (Selection sel : worksheet.getSelectionManager().getAllDefinedSelection()) {
     Set<String> inputColumns = new HashSet<String>(sel.getInputColumns());
     inputColumns.retainAll(command.getOutputColumns());
     if (inputColumns.size() > 0) sel.invalidateSelection();
   }
 }
示例#9
0
 @Override
 public UpdateContainer doIt(Workspace workspace) throws CommandException {
   Node node = workspace.getFactory().getNode(nodeIdArg);
   SuperSelection sel = getSuperSelection(workspace);
   inputColumns.clear();
   outputColumns.clear();
   inputColumns.add(node.getHNodeId());
   outputColumns.add(node.getHNodeId());
   previousValue = node.getValue();
   previousStatus = node.getStatus();
   if (node.hasNestedTable()) {
     throw new CommandException(
         this, "Cell " + nodeIdArg + " has a nested table. It cannot be edited.");
   }
   node.setValue(newValueArg, Node.NodeStatus.edited, workspace.getFactory());
   WorksheetUpdateFactory.detectSelectionStatusChange(worksheetId, workspace, this);
   UpdateContainer uc =
       WorksheetUpdateFactory.createWorksheetHierarchicalAndCleaningResultsUpdates(
           worksheetId, sel);
   uc.add(new NodeChangedUpdate(worksheetId, nodeIdArg, newValueArg, Node.NodeStatus.edited));
   return uc;
 }
 private List<Command> gatherAllOperateSelectionCommands(
     List<Command> commands, Workspace workspace) {
   List<Command> operationCommands = new ArrayList<Command>();
   for (Command c : commands) {
     if (c instanceof OperateSelectionCommand) {
       OperateSelectionCommand t = (OperateSelectionCommand) c;
       if (isSamehTableId(t.getHNodeId(), hNodeId, workspace)) {
         JSONObject obj = workspace.getCommandHistory().getCommandJSON(workspace, t);
         Command tmp = generateCommandFromJSON(workspace, obj);
         if (tmp != null) operationCommands.add(tmp);
       }
     }
   }
   return operationCommands;
 }
  private JSONArray extractHistoryFromModel(Workspace workspace, UpdateContainer uc)
      throws RepositoryException, RDFParseException, IOException, JSONException, KarmaException {

    Worksheet ws = workspace.getFactory().getWorksheet(worksheetId);
    R2RMLMappingIdentifier id =
        new R2RMLMappingIdentifier(ws.getTitle(), r2rmlModelFile.toURI().toURL());
    WorksheetR2RMLJenaModelParser parser = new WorksheetR2RMLJenaModelParser(id);
    KR2RMLMapping mapping = parser.parse();
    KR2RMLVersion version = mapping.getVersion();
    if (version.compareTo(KR2RMLVersion.current) < 0) {
      uc.add(
          new InfoUpdate(
              "Model version is "
                  + version.toString()
                  + ".  Current version is "
                  + KR2RMLVersion.current.toString()
                  + ".  Please publish it again."));
    }
    return mapping.getWorksheetHistory();
  }
 @Override
 public UpdateContainer doIt(Workspace workspace) throws CommandException {
   Worksheet wk = workspace.getWorksheet(worksheetId);
   SuperSelection selection = getSuperSelection(wk);
   String Msg =
       String.format("begin, Time,%d, Worksheet,%s", System.currentTimeMillis(), worksheetId);
   logger.info(Msg);
   // Get the HNode
   HashMap<String, HashMap<String, String>> rows = new HashMap<String, HashMap<String, String>>();
   HNodePath selectedPath = null;
   List<HNodePath> columnPaths = wk.getHeaders().getAllPaths();
   for (HNodePath path : columnPaths) {
     if (path.getLeaf().getId().equals(hNodeId)) {
       selectedPath = path;
     }
   }
   // random nodes
   Collection<Node> nodes = new ArrayList<Node>();
   wk.getDataTable().collectNodes(selectedPath, nodes, selection);
   HashSet<Integer> indSet = this.obtainIndexs(nodes.size());
   int index = 0;
   for (Iterator<Node> iterator = nodes.iterator(); iterator.hasNext(); ) {
     Node node = iterator.next();
     if (indSet.contains(index)) {
       String id = node.getId();
       String originalVal = node.getValue().asString();
       HashMap<String, String> x = new HashMap<String, String>();
       x.put("Org", originalVal);
       x.put("Tar", originalVal);
       x.put("Orgdis", originalVal);
       x.put("Tardis", originalVal);
       rows.put(id, x);
     }
     index++;
   }
   Msg = String.format("end, Time,%d, Worksheet,%s", System.currentTimeMillis(), worksheetId);
   logger.info(Msg);
   return new UpdateContainer(new FetchResultUpdate(hNodeId, rows));
 }
  @Override
  public UpdateContainer doIt(Workspace workspace) throws CommandException {
    Worksheet worksheet = workspace.getWorksheet(worksheetId);

    CSVFileExport csvFileExport = new CSVFileExport(worksheet);

    try {

      final String fileName = csvFileExport.publishCSV();
      if (fileName == null)
        return new UpdateContainer(
            new ErrorUpdate("No data to export! Have you aligned the worksheet?"));
      return new UpdateContainer(
          new AbstractUpdate() {
            @Override
            public void generateJson(String prefix, PrintWriter pw, VWorkspace vWorkspace) {
              JSONObject outputObject = new JSONObject();
              try {
                outputObject.put(JsonKeys.updateType.name(), "PublishCSVUpdate");
                outputObject.put(JsonKeys.fileUrl.name(), fileName);
                outputObject.put(JsonKeys.worksheetId.name(), worksheetId);
                pw.println(outputObject.toString(4));

              } catch (JSONException e) {
                logger.error("Error occured while generating JSON!");
              }
            }
          });
    } catch (FileNotFoundException e) {
      logger.error("CSV folder not found!", e);
      return new UpdateContainer(new ErrorUpdate("Error occurred while exporting CSV file!"));
    } catch (Exception e) {
      logger.error("CSV Export Error", e);
      return new UpdateContainer(new ErrorUpdate("Error occurred while exporting CSV file!"));
    }
  }
  private void generateRDF(
      String wkname,
      String query,
      List<KR2RMLRDFWriter> writers,
      R2RMLMappingIdentifier id,
      String baseURI)
      throws IOException, JSONException, KarmaException, SQLException, ClassNotFoundException {
    logger.debug("Generating RDF...");

    WorksheetR2RMLJenaModelParser parserTest = new WorksheetR2RMLJenaModelParser(id);
    KR2RMLMapping mapping = parserTest.parse();

    AbstractJDBCUtil dbUtil = JDBCUtilFactory.getInstance(dbType);
    Connection conn = dbUtil.getConnection(hostname, portnumber, username, password, dBorSIDName);
    conn.setAutoCommit(false);

    java.sql.Statement stmt =
        conn.createStatement(
            java.sql.ResultSet.TYPE_FORWARD_ONLY, java.sql.ResultSet.CONCUR_READ_ONLY);
    stmt.setFetchSize(DATABASE_TABLE_FETCH_SIZE);

    ResultSet r = stmt.executeQuery(query);
    ResultSetMetaData meta = r.getMetaData();
    ;

    // Get the column names
    List<String> columnNames = new ArrayList<>();
    for (int i = 1; i <= meta.getColumnCount(); i++) {
      columnNames.add(meta.getColumnName(i));
    }

    // Prepare required Karma objects
    Workspace workspace = initializeWorkspace();

    RepFactory factory = workspace.getFactory();
    Worksheet wk = factory.createWorksheet(wkname, workspace, encoding);
    List<String> headersList = addHeaders(wk, columnNames, factory);

    int counter = 0;

    ArrayList<String> rowValues = null;
    while ((rowValues = dbUtil.parseResultSetRow(r)) != null) {
      // Generate RDF and create a new worksheet for every DATABASE_TABLE_FETCH_SIZE rows
      if (counter % DATABASE_TABLE_FETCH_SIZE == 0 && counter != 0) {
        generateRDFFromWorksheet(wk, workspace, mapping, writers, baseURI);
        logger.debug("Done for " + counter + " rows ...");
        removeWorkspace(workspace);

        parserTest = new WorksheetR2RMLJenaModelParser(id);
        mapping = parserTest.parse();
        workspace = initializeWorkspace();
        factory = workspace.getFactory();
        wk = factory.createWorksheet(wkname, workspace, encoding);
        headersList = addHeaders(wk, columnNames, factory);
      }

      /** Add the data * */
      Table dataTable = wk.getDataTable();
      Row row = dataTable.addRow(factory);
      for (int i = 0; i < rowValues.size(); i++) {
        row.setValue(headersList.get(i), rowValues.get(i), factory);
      }

      counter++;
    }

    generateRDFFromWorksheet(wk, workspace, mapping, writers, baseURI);

    // Releasing all the resources
    r.close();
    conn.close();
    stmt.close();
    logger.debug("done");
  }
  @Override
  public UpdateContainer doIt(Workspace workspace) throws CommandException {
    ModelingConfiguration modelingConfiguration =
        ModelingConfigurationRegistry.getInstance()
            .getModelingConfiguration(
                WorkspaceKarmaHomeRegistry.getInstance().getKarmaHome(workspace.getId()));
    TripleStoreUtil utilObj = new TripleStoreUtil();
    boolean showModelsWithoutMatching = modelingConfiguration.isShowModelsWithoutMatching();
    try {
      HashMap<String, List<String>> metadata =
          utilObj.getMappingsWithMetadata(TripleStoreUrl, context);
      RepFactory factory = workspace.getFactory();
      List<String> model_Names = metadata.get("model_names");
      List<String> model_Urls = metadata.get("model_urls");
      List<String> model_Times = metadata.get("model_publishtimes");
      List<String> model_Contexts = metadata.get("model_contexts");
      List<String> model_inputColumns = metadata.get("model_inputcolumns");
      final List<JSONObject> list = new ArrayList<>();
      Set<String> worksheetcolumns = new HashSet<>();
      if (worksheetId != null && !worksheetId.trim().isEmpty()) {
        HTable htable = factory.getWorksheet(worksheetId).getHeaders();
        getHNodesForWorksheet(htable, worksheetcolumns, factory);
      }
      Iterator<String> nameitr = model_Names.iterator();
      Iterator<String> urlitr = model_Urls.iterator();
      Iterator<String> timeitr = model_Times.iterator();
      Iterator<String> contextitr = model_Contexts.iterator();
      Iterator<String> inputitr = model_inputColumns.iterator();
      while (nameitr.hasNext()
          && urlitr.hasNext()
          && timeitr.hasNext()
          && contextitr.hasNext()
          && inputitr.hasNext()) {
        JSONObject obj = new JSONObject();
        Set<String> inputs = new HashSet<>();
        obj.put("name", nameitr.next());
        obj.put("url", urlitr.next());
        obj.put("publishTime", timeitr.next());
        obj.put("context", contextitr.next());
        String columns = inputitr.next();
        if (columns != null && !columns.isEmpty()) {
          JSONArray array = new JSONArray(columns);
          for (int i = 0; i < array.length(); i++) inputs.add(array.get(i).toString());
        } else if (showModelsWithoutMatching) {
          list.add(obj);
        }
        if (worksheetId != null && !worksheetId.isEmpty()) {
          inputs.retainAll(worksheetcolumns);
          obj.put("inputColumns", inputs.size());
        } else obj.put("inputColumns", 0);
        if (!inputs.isEmpty() || (worksheetId == null || worksheetId.trim().isEmpty()))
          list.add(obj);
      }

      Collections.sort(
          list,
          new Comparator<JSONObject>() {

            @Override
            public int compare(JSONObject a, JSONObject b) {
              return b.getInt("inputColumns") - a.getInt("inputColumns");
            }
          });

      return new UpdateContainer(
          new AbstractUpdate() {
            @Override
            public void generateJson(String prefix, PrintWriter pw, VWorkspace vWorkspace) {
              try {
                JSONArray array = new JSONArray();
                for (JSONObject obj : list) {
                  array.put(obj);
                }
                pw.print(array.toString());
              } catch (Exception e) {
                logger.error("Error generating JSON!", e);
              }
            }
          });
    } catch (Exception e) {
      return new UpdateContainer(
          new ErrorUpdate("Unable to get mappings with metadata: " + e.getMessage()));
    }
  }
示例#16
0
  @Override
  public UpdateContainer doIt(Workspace workspace) {
    Worksheet worksheet = workspace.getWorksheet(worksheetId);
    SuperSelection selection = getSuperSelection(worksheet);
    String worksheetName = worksheet.getTitle();
    try {

      // preparing model file name
      final String modelFileName =
          workspace.getCommandPreferencesId() + worksheetId + "-" + worksheetName + "-model.ttl";
      final String modelFileLocalPath =
          ServletContextParameterMap.getParameterValue(ContextParameter.R2RML_PUBLISH_DIR)
              + modelFileName;

      File f = new File(modelFileLocalPath);

      // preparing the graphUri where the model is published in the triple store
      String graphName =
          worksheet
              .getMetadataContainer()
              .getWorksheetProperties()
              .getPropertyValue(Property.graphName);
      if (graphName == null || graphName.isEmpty()) {
        SimpleDateFormat sdf = new SimpleDateFormat("dd-MMM-yyyy-kkmmssS");
        String ts = sdf.format(Calendar.getInstance().getTime());
        graphName =
            "http://localhost/"
                + workspace.getCommandPreferencesId()
                + "/"
                + worksheetId
                + "/model/"
                + ts;
        worksheet
            .getMetadataContainer()
            .getWorksheetProperties()
            .setPropertyValue(Property.graphName, graphName);
      }

      // If the model is not published, publish it!
      if (!f.exists() || !f.isFile()) {
        GenerateR2RMLModelCommandFactory factory = new GenerateR2RMLModelCommandFactory();
        GenerateR2RMLModelCommand cmd =
            (GenerateR2RMLModelCommand)
                factory.createCommand(
                    workspace,
                    worksheetId,
                    TripleStoreUtil.defaultModelsRepoUrl,
                    graphName,
                    selection.getName());
        cmd.doIt(workspace);
      } else {
        // if the model was published 30 min ago, publish it again, just to be sure
        long diff = Calendar.getInstance().getTimeInMillis() - f.lastModified();
        if ((diff / 1000L / 60L) > 30) {
          f.delete();
          GenerateR2RMLModelCommandFactory factory = new GenerateR2RMLModelCommandFactory();
          GenerateR2RMLModelCommand cmd =
              (GenerateR2RMLModelCommand)
                  factory.createCommand(
                      workspace,
                      worksheetId,
                      TripleStoreUtil.defaultModelsRepoUrl,
                      graphName,
                      selection.getName());
          cmd.doIt(workspace);
        }
      }

      //			TripleStoreUtil tUtil = new TripleStoreUtil();
      StringBuffer query =
          new StringBuffer(
              "prefix rr: <http://www.w3.org/ns/r2rml#> prefix km-dev: <http://isi.edu/integration/karma/dev#> ");

      /* ****** this is the query for the list of columns.

      PREFIX km-dev: <http://isi.edu/integration/karma/dev#>
      PREFIX rr: <http://www.w3.org/ns/r2rml#>

      select distinct ?class where  {
        {
          ?x1 rr:subjectMap/km-dev:alignmentNodeId "------- The full url of the column/class --------".
          ?x1 rr:predicateObjectMap/rr:objectMap/rr:column ?column .
      	?x1 rr:subjectMap/rr:predicate ?class .
        }
        UNION
        {
          ?x1 rr:subjectMap/km-dev:alignmentNodeId "------- The full url of the column/class --------".
      	?x1 (rr:predicateObjectMap/rr:objectMap/rr:parentTriplesMap)* ?x2 .
      	?x2 rr:predicateObjectMap/rr:objectMap/rr:column ?column .
      	?x2 rr:predicateObjectMap/rr:predicate ?class .
        }
      }
      * */

      query.append("select distinct ?class ?column where { ");
      if (graphName != null && !graphName.trim().isEmpty()) {
        query.append(" graph  <" + graphName + "> { ");
      }
      query
          .append("{ ?x1 rr:subjectMap/km-dev:alignmentNodeId \"")
          .append(this.nodeId)
          .append(
              "\" . ?x1 rr:predicateObjectMap/rr:objectMap/rr:column ?column . ?x1 rr:subjectMap/rr:predicate ?class .")
          .append(" } UNION { ")
          .append("?x1 rr:subjectMap/km-dev:alignmentNodeId \"")
          .append(this.nodeId)
          .append("\" . ?x1 (rr:predicateObjectMap/rr:objectMap/rr:parentTriplesMap)* ?x2 .")
          .append(" ?x2 rr:predicateObjectMap ?x3 . ")
          .append(" ?x3 rr:objectMap/rr:column ?column . ?x3 rr:predicate ?class .")
          .append(" } }");
      if (graphName != null && !graphName.trim().isEmpty()) {
        query.append(" } ");
      }
      logger.info("Query: " + query.toString());
      String sData =
          TripleStoreUtil.invokeSparqlQuery(
              query.toString(), TripleStoreUtil.defaultModelsRepoUrl, "application/json", null);
      if (sData == null | sData.isEmpty()) {
        logger.error("Empty response object from query : " + query);
      }
      HashMap<String, String> cols = new HashMap<String, String>();
      try {
        JSONObject obj1 = new JSONObject(sData);
        JSONArray arr = obj1.getJSONObject("results").getJSONArray("bindings");
        for (int i = 0; i < arr.length(); i++) {
          String colName = arr.getJSONObject(i).getJSONObject("column").getString("value");
          String colValue = arr.getJSONObject(i).getJSONObject("class").getString("value");
          if (cols.containsKey(colName)) {
            logger.error("Duplicate Column <-> property mapping. " + colName + " <=> " + colValue);
          } else {
            cols.put(colName, colValue);
          }
        }
      } catch (Exception e2) {
        logger.error("Error in parsing json response", e2);
      }

      logger.info("Total Columns fetched : " + cols.size());
      final HashMap<String, String> columns = cols;
      return new UpdateContainer(
          new AbstractUpdate() {

            @Override
            public void generateJson(String prefix, PrintWriter pw, VWorkspace vWorkspace) {
              JSONObject obj = new JSONObject();
              try {
                Iterator<String> itr = columns.keySet().iterator();
                JSONArray colList = new JSONArray();
                while (itr.hasNext()) {
                  JSONObject o = new JSONObject();
                  String k = itr.next();
                  o.put("name", k);
                  o.put("url", columns.get(k));
                  colList.put(o);
                }
                obj.put("updateType", "FetchColumnUpdate");
                obj.put("columns", colList);
                obj.put("rootId", nodeId);
                pw.println(obj.toString());
              } catch (JSONException e) {
                logger.error("Error occurred while fetching worksheet properties!", e);
              }
            }
          });

    } catch (Exception e) {
      String msg = "Error occured while fetching columns!";
      logger.error(msg, e);
      return new UpdateContainer(new ErrorUpdate(msg));
    }
  }
  @Override
  public UpdateContainer doIt(Workspace workspace) throws CommandException {
    Worksheet wk = workspace.getWorksheet(worksheetId);
    SuperSelection selection = getSuperSelection(wk);
    String msg =
        String.format(
            "Gen rule start,Time,%d, Worksheet,%s", System.currentTimeMillis(), worksheetId);
    logger.info(msg);
    // Get the HNode
    HashMap<String, String> rows = new HashMap<String, String>();
    HashMap<String, Integer> amb = new HashMap<String, Integer>();
    HNodePath selectedPath = null;
    List<HNodePath> columnPaths = wk.getHeaders().getAllPaths();
    for (HNodePath path : columnPaths) {
      if (path.getLeaf().getId().equals(hNodeId)) {
        selectedPath = path;
      }
    }
    Collection<Node> nodes = new ArrayList<Node>();
    wk.getDataTable().collectNodes(selectedPath, nodes, selection);
    for (Node node : nodes) {
      String id = node.getId();
      if (!this.nodeIds.contains(id)) continue;
      String originalVal = node.getValue().asString();
      rows.put(id, originalVal);
      this.compResultString += originalVal + "\n";
      calAmbScore(id, originalVal, amb);
    }
    RamblerValueCollection vc = new RamblerValueCollection(rows);
    HashMap<String, Vector<String[]>> expFeData = new HashMap<String, Vector<String[]>>();
    inputs = new RamblerTransformationInputs(examples, vc);
    // generate the program
    boolean results = false;
    int iterNum = 0;
    RamblerTransformationOutput rtf = null;
    // initialize the vocabulary
    Iterator<String> iterx = inputs.getInputValues().getValues().iterator();
    Vector<String> v = new Vector<String>();
    int vb_cnt = 0;
    while (iterx.hasNext() && vb_cnt < 30) {
      String eString = iterx.next();
      v.add(eString);
      vb_cnt++;
    }
    Vector<String> vob = UtilTools.buildDict(v);
    inputs.setVocab(vob.toArray(new String[vob.size()]));
    while (iterNum < 1 && !results) // try to find an program within iterNum
    {
      rtf = new RamblerTransformationOutput(inputs);
      if (rtf.getTransformations().keySet().size() > 0) {
        results = true;
      }
      iterNum++;
    }
    Iterator<String> iter = rtf.getTransformations().keySet().iterator();
    // id:{org: tar: orgdis: tardis: }
    HashMap<String, HashMap<String, String>> resdata =
        new HashMap<String, HashMap<String, String>>();
    HashSet<String> keys = new HashSet<String>();
    while (iter.hasNext()) {
      String tpid = iter.next();
      ValueCollection rvco = rtf.getTransformedValues_debug(tpid);
      if (rvco == null) continue;
      // constructing displaying data
      HashMap<String, String[]> xyzHashMap = new HashMap<String, String[]>();
      for (String key : rvco.getNodeIDs()) {
        HashMap<String, String> dict = new HashMap<String, String>();
        // add to the example selection
        boolean isExp = false;
        String org = vc.getValue(key);
        String classLabel = rvco.getClass(key);
        String pretar = rvco.getValue(key);
        String dummyValue = pretar;
        if (pretar.indexOf("_FATAL_ERROR_") != -1) {
          dummyValue = org;
          // dummyValue = "#ERROR";
        }
        try {
          UtilTools.StringColorCode(org, dummyValue, dict);
        } catch (Exception ex) {
          logger.info(String.format("ColorCoding Exception%s, %s", org, dummyValue));
          // set dict
          dict.put("Org", org);
          dict.put("Tar", "ERROR");
          dict.put("Orgdis", org);
          dict.put("Tardis", "ERROR");
        }
        for (TransformationExample exp : examples) {
          if (exp.getNodeId().compareTo(key) == 0) {
            if (!expFeData.containsKey(classLabel)) {
              Vector<String[]> vstr = new Vector<String[]>();
              String[] texp = {dict.get("Org"), pretar};
              vstr.add(texp);
              expFeData.put(classLabel, vstr);
            } else {
              String[] texp = {dict.get("Org"), pretar};
              expFeData.get(classLabel).add(texp);
            }
            isExp = true;
          }
        }

        if (!isExp) {
          String[] pair = {dict.get("Org"), dict.get("Tar"), pretar, classLabel};
          xyzHashMap.put(key, pair);
        }
        resdata.put(key, dict);
      }
      if (!rtf.nullRule) keys.add(getBestExample(xyzHashMap, expFeData));
    }
    // find the best row
    String vars = "";
    String expstr = "";
    String recmd = "";
    for (TransformationExample x : examples) {
      expstr += String.format("%s|%s", x.getBefore(), x.getAfter());
    }
    expstr += "|";
    if (rtf.nullRule) {
      keys.clear();
      // keys.add("-2"); // "-2 indicates null rule"
    }
    if (!resdata.isEmpty() && !rtf.nullRule) {
      recmd = resdata.get(keys.iterator().next()).get("Org");
    } else {
      recmd = "";
    }
    msg =
        String.format(
            "Gen rule end, Time,%d, Worksheet,%s,Examples:%s,Recmd:%s",
            System.currentTimeMillis(), worksheetId, expstr, recmd);
    logger.info(msg);
    return new UpdateContainer(new CleaningResultUpdate(hNodeId, resdata, vars, keys));
  }
  @Override
  public UpdateContainer doIt(Workspace workspace) throws CommandException {
    final OntologyManager ontMgr = workspace.getOntologyManager();
    Set<LabeledLink> properties = new HashSet<>();

    logger.debug(
        "GetPropertiesCommand:"
            + propertiesRange
            + ":"
            + classURI
            + ","
            + domainURI
            + ", "
            + rangeURI);

    if (propertiesRange == INTERNAL_PROP_RANGE.allObjectProperties) {
      HashMap<String, Label> linkList = ontMgr.getObjectProperties();
      if (linkList != null) {
        for (Label label : linkList.values()) {
          properties.add(new DataPropertyLink(label.getUri(), label));
        }
      }
    } else if (propertiesRange == INTERNAL_PROP_RANGE.allDataProperties) {
      HashMap<String, Label> linkList = ontMgr.getDataProperties();
      for (Label label : linkList.values()) {
        properties.add(new DataPropertyLink(label.getUri(), label));
      }
    } else if (propertiesRange == INTERNAL_PROP_RANGE.propertiesWithDomainRange) {
      Map<String, Label> linkList =
          ontMgr.getObjectPropertiesByDomainRange(domainURI, rangeURI, true);
      for (Label label : linkList.values()) {
        properties.add(new DataPropertyLink(label.getUri(), label));
      }
    } else if (propertiesRange == INTERNAL_PROP_RANGE.dataPropertiesForClass) {
      Map<String, Label> linkList = ontMgr.getDataPropertiesByDomain(classURI, true);
      for (Label label : linkList.values()) {
        properties.add(new DataPropertyLink(label.getUri(), label));
      }
    } else if (propertiesRange == INTERNAL_PROP_RANGE.existingProperties) {
      Alignment alignment =
          AlignmentManager.Instance()
              .getAlignmentOrCreateIt(workspace.getId(), worksheetId, ontMgr);
      Set<String> steinerTreeNodeIds = new HashSet<String>();
      if (alignment != null && !alignment.isEmpty()) {
        DirectedWeightedMultigraph<Node, LabeledLink> steinerTree = alignment.getSteinerTree();
        for (Node node : steinerTree.vertexSet()) {
          if (node.getType() == NodeType.InternalNode) {
            steinerTreeNodeIds.add(node.getId());
          }
        }

        List<LabeledLink> specializedLinks = new ArrayList<LabeledLink>();
        Set<LabeledLink> temp = null;
        temp = alignment.getLinksByType(LinkType.DataPropertyLink);
        if (temp != null) specializedLinks.addAll(temp);
        for (LabeledLink link : steinerTree.edgeSet())
          if (link instanceof ObjectPropertyLink) specializedLinks.add(link);

        // Store the data property links for specialized edge link options
        properties.addAll(specializedLinks);
      }
    }

    logger.debug("Got back " + properties.size() + " results");
    final Set<LabeledLink> finalProperties = properties;

    UpdateContainer upd =
        new UpdateContainer(
            new AbstractUpdate() {
              @Override
              public void generateJson(String prefix, PrintWriter pw, VWorkspace vWorkspace) {
                JSONObject obj = new JSONObject();
                JSONArray resultArray = new JSONArray();

                try {
                  obj.put(JsonKeys.updateType.name(), "PropertyList");

                  for (LabeledLink link : finalProperties) {
                    Label linkLabel = link.getLabel();
                    String edgeLabelStr = linkLabel.getDisplayName();
                    JSONObject edgeObj = new JSONObject();
                    if (linkLabel.getUri() != null
                        && linkLabel.getNs() != null
                        && linkLabel.getUri().equalsIgnoreCase(linkLabel.getNs())) {
                      edgeLabelStr = linkLabel.getUri();
                    }

                    edgeObj.put(JsonKeys.label.name(), edgeLabelStr);
                    edgeObj.put(JsonKeys.uri.name(), linkLabel.getUri());
                    edgeObj.put(JsonKeys.id.name(), link.getId());
                    resultArray.put(edgeObj);
                  }

                  obj.put(JsonKeys.properties.name(), resultArray);
                  pw.println(obj.toString());
                } catch (Exception e) {
                  logger.error("Exception:", e);
                  e.printStackTrace();
                }
              }
            });
    return upd;
  }
示例#19
0
 /*
  * Pedro
  *
  * We should have an abstraction for Commands that operate on worksheets,
  * and this method should go there.
  */
 protected String formatWorsheetId(Workspace workspace, String worksheetId) {
   return worksheetId + " (" + workspace.getWorksheet(worksheetId).getTitle() + ")";
 }
  @Override
  public void generateJson(String prefix, PrintWriter pw, VWorkspace vWorkspace) {
    Workspace workspace = vWorkspace.getWorkspace();
    alignment = AlignmentManager.Instance().getAlignment(workspace.getId(), worksheetId);
    SemanticTypes types = worksheet.getSemanticTypes();
    Map<String, ColumnNode> hNodeIdTocolumnNodeMap = createColumnNodeMap();
    Map<String, SemanticTypeNode> hNodeIdToDomainNodeMap = createDomainNodeMap();

    JSONStringer jsonStr = new JSONStringer();
    try {
      JSONWriter writer = jsonStr.object();
      writer.key("worksheetId").value(worksheetId).key("updateType").value("SemanticTypesUpdate");

      writer.key(JsonKeys.Types.name());
      writer.array();
      // Iterate through all the columns
      for (HNodePath path : worksheet.getHeaders().getAllPaths()) {
        HNode node = path.getLeaf();
        String nodeId = node.getId();

        writer.object();

        // Check if a semantic type exists for the HNode
        SemanticType type = types.getSemanticTypeForHNodeId(nodeId);
        if (type != null && type.getConfidenceLevel() != SemanticType.ConfidenceLevel.Low) {
          writer
              .key(JsonKeys.HNodeId.name())
              .value(type.getHNodeId())
              .key(JsonKeys.SemanticTypesArray.name())
              .array();

          ColumnNode alignmentColumnNode = hNodeIdTocolumnNodeMap.get(type.getHNodeId());
          SemanticTypeNode domainNode = hNodeIdToDomainNodeMap.get(type.getHNodeId());

          if (alignmentColumnNode == null || domainNode == null) {
            logger.error(
                "Column node or domain node not found in alignment."
                    + " (This should not happen conceptually!):"
                    + type);
            continue;
          }

          // Add the primary semantic type
          writer
              .object()
              .key(JsonKeys.Origin.name())
              .value(type.getOrigin().name())
              .key(JsonKeys.ConfidenceLevel.name())
              .value(type.getConfidenceLevel().name())
              .key(JsonKeys.isPrimary.name())
              .value(true);

          // Add the RDF literal type to show in the text box
          String rdfLiteralType =
              alignmentColumnNode.getRdfLiteralType() == null
                  ? ""
                  : alignmentColumnNode.getRdfLiteralType().getDisplayName();
          String language =
              alignmentColumnNode.getLanguage() == null ? "" : alignmentColumnNode.getLanguage();
          writer.key(JsonKeys.rdfLiteralType.name()).value(rdfLiteralType);
          writer.key(JsonKeys.language.name()).value(language);

          //					String domainDisplayLabel = (domainNode.getLabel().getPrefix() != null &&
          // (!domainNode.getLabel().getPrefix().equals(""))) ?
          //							(domainNode.getLabel().getPrefix() + ":" + domainNode.getLocalId()) :
          // domainNode.getLocalId();
          if (!type.isClass()) {
            writer
                .key(JsonKeys.FullType.name())
                .value(type.getType().getUri())
                .key(JsonKeys.DisplayLabel.name())
                .value(type.getType().getDisplayName())
                .key(JsonKeys.DisplayRDFSLabel.name())
                .value(type.getType().getRdfsLabel())
                .key(JsonKeys.DisplayRDFSComment.name())
                .value(type.getType().getRdfsComment())
                .key(JsonKeys.DomainId.name())
                .value(domainNode.getId())
                .key(JsonKeys.DomainUri.name())
                .value(domainNode.getUri())
                .key(JsonKeys.DisplayDomainLabel.name())
                .value(domainNode.getDisplayId())
                .key(JsonKeys.DomainRDFSLabel.name())
                .value(domainNode.getRdfsLabel())
                .key(JsonKeys.DomainRDFSComment.name())
                .value(domainNode.getRdfsComment());
          } else {
            writer
                .key(JsonKeys.FullType.name())
                .value(domainNode.getId())
                .key(JsonKeys.DisplayLabel.name())
                .value(domainNode.getDisplayId())
                .key(JsonKeys.DisplayRDFSLabel.name())
                .value(domainNode.getRdfsLabel())
                .key(JsonKeys.DisplayRDFSComment.name())
                .value(domainNode.getRdfsComment())
                .key(JsonKeys.DomainId.name())
                .value("")
                .key(JsonKeys.DomainUri.name())
                .value("")
                .key(JsonKeys.DisplayDomainLabel.name())
                .value("")
                .key(JsonKeys.DomainRDFSLabel.name())
                .value("")
                .key(JsonKeys.DomainRDFSComment.name())
                .value("");
          }

          // Mark the special properties
          writer
              .key(JsonKeys.isMetaProperty.name())
              .value(isMetaProperty(type.getType(), alignmentColumnNode));

          writer.endObject();

          // Iterate through the synonym semantic types
          SynonymSemanticTypes synTypes = types.getSynonymTypesForHNodeId(nodeId);

          if (synTypes != null) {
            for (SemanticType synType : synTypes.getSynonyms()) {
              writer
                  .object()
                  .key(JsonKeys.HNodeId.name())
                  .value(synType.getHNodeId())
                  .key(JsonKeys.FullType.name())
                  .value(synType.getType().getUri())
                  .key(JsonKeys.Origin.name())
                  .value(synType.getOrigin().name())
                  .key(JsonKeys.ConfidenceLevel.name())
                  .value(synType.getConfidenceLevel().name())
                  .key(JsonKeys.DisplayLabel.name())
                  .value(synType.getType().getDisplayName())
                  .key(JsonKeys.DisplayRDFSLabel.name())
                  .value(synType.getType().getRdfsLabel())
                  .key(JsonKeys.DisplayRDFSComment.name())
                  .value(synType.getType().getRdfsComment())
                  .key(JsonKeys.isPrimary.name())
                  .value(false);
              if (!synType.isClass()) {
                writer
                    .key(JsonKeys.DomainUri.name())
                    .value(synType.getDomain().getUri())
                    .key(JsonKeys.DomainId.name())
                    .value("")
                    .key(JsonKeys.DisplayDomainLabel.name())
                    .value(synType.getDomain().getDisplayName())
                    .key(JsonKeys.DomainRDFSLabel.name())
                    .value(synType.getDomain().getRdfsLabel())
                    .key(JsonKeys.DomainRDFSComment.name())
                    .value(synType.getDomain().getRdfsComment());
              } else {
                writer
                    .key(JsonKeys.DomainId.name())
                    .value("")
                    .key(JsonKeys.DomainUri.name())
                    .value("")
                    .key(JsonKeys.DisplayDomainLabel.name())
                    .value("")
                    .key(JsonKeys.DomainRDFSLabel.name())
                    .value("")
                    .key(JsonKeys.DomainRDFSComment.name())
                    .value("");
              }
              writer.endObject();
            }
          }
          writer.endArray();
        } else {
          writer.key(JsonKeys.HNodeId.name()).value(nodeId);
          writer.key(JsonKeys.SemanticTypesArray.name()).array().endArray();
        }

        writer.endObject();
      }
      writer.endArray();
      writer.endObject();

      pw.print(writer.toString());
    } catch (JSONException e) {
      logger.error("Error occured while writing to JSON!", e);
    }
  }
 private boolean isSamehTableId(String hNodeId1, String hNodeId2, Workspace workspace) {
   HNode hNode1 = workspace.getFactory().getHNode(hNodeId1);
   HNode hNode2 = workspace.getFactory().getHNode(hNodeId2);
   if (hNode1 == null || hNode2 == null) return false;
   return hNode1.getHTableId().equals(hNode2.getHTableId());
 }
示例#22
0
 /*
  * Pedro
  *
  * Return an HNodeId in a nice format for printing on command logs.
  */
 protected String formatHNodeId(Workspace workspace, String hNodeId) {
   return hNodeId + " (" + workspace.getFactory().getColumnName(hNodeId) + ")";
 }
  @Override
  public UpdateContainer doIt(Workspace workspace) throws CommandException {
    Worksheet worksheet = workspace.getWorksheet(worksheetId);
    SuperSelection selection = getSuperSelection(worksheet);
    HNodePath selectedPath = null;
    List<HNodePath> columnPaths = worksheet.getHeaders().getAllPaths();
    for (HNodePath path : columnPaths) {
      if (path.getLeaf().getId().equals(hNodeId)) {
        selectedPath = path;
      }
    }
    Collection<Node> nodes = new ArrayList<Node>();
    workspace
        .getFactory()
        .getWorksheet(worksheetId)
        .getDataTable()
        .collectNodes(selectedPath, nodes, selection);

    try {
      JSONArray requestJsonArray = new JSONArray();
      for (Node node : nodes) {
        String id = node.getId();
        String originalVal = node.getValue().asString();
        JSONObject jsonRecord = new JSONObject();
        jsonRecord.put("id", id);
        originalVal = originalVal == null ? "" : originalVal;
        jsonRecord.put("value", originalVal);
        requestJsonArray.put(jsonRecord);
      }
      String jsonString = null;
      jsonString = requestJsonArray.toString();

      // String url =
      // "http://localhost:8080/cleaningService/IdentifyData";
      //			String url = "http://localhost:8070/myWS/IdentifyData";
      String url =
          ServletContextParameterMap.getParameterValue(ContextParameter.CLEANING_SERVICE_URL);

      HttpClient httpclient = new DefaultHttpClient();
      HttpPost httppost = null;
      HttpResponse response = null;
      HttpEntity entity;
      StringBuffer out = new StringBuffer();

      URI u = null;
      u = new URI(url);
      List<NameValuePair> formparams = new ArrayList<NameValuePair>();
      formparams.add(new BasicNameValuePair("json", jsonString));

      httppost = new HttpPost(u);
      httppost.setEntity(new UrlEncodedFormEntity(formparams, "UTF-8"));
      response = httpclient.execute(httppost);
      entity = response.getEntity();
      if (entity != null) {
        BufferedReader buf = new BufferedReader(new InputStreamReader(entity.getContent()));
        String line = buf.readLine();
        while (line != null) {
          out.append(line);
          line = buf.readLine();
        }
      }
      // logger.trace(out.toString());
      // logger.info("Connnection success : " + url + " Successful.");
      final JSONObject data1 = new JSONObject(out.toString());
      // logger.trace("Data--->" + data1);
      return new UpdateContainer(
          new AbstractUpdate() {

            @Override
            public void generateJson(String prefix, PrintWriter pw, VWorkspace vWorkspace) {
              JSONObject response = new JSONObject();
              // logger.trace("Reached here");
              try {
                response.put("updateType", "CleaningServiceOutput");
                response.put("chartData", data1);
                response.put("hNodeId", hNodeId);
                // logger.trace(response.toString(4));
              } catch (JSONException e) {
                pw.print("Error");
              }

              pw.print(response.toString());
            }
          });
    } catch (Exception e) {
      e.printStackTrace();
      return new UpdateContainer(new ErrorUpdate("Error!"));
    }
  }