public static UpdateContainer createSemanticTypesAndSVGAlignmentUpdates(
     String worksheetId, Workspace workspace, Alignment alignment) {
   Worksheet worksheet = workspace.getWorksheet(worksheetId);
   UpdateContainer c = new UpdateContainer();
   c.add(new SemanticTypesUpdate(worksheet, worksheetId, alignment));
   c.add(new AlignmentSVGVisualizationUpdate(worksheetId, alignment));
   return c;
 }
 public static void detectSelectionStatusChange(
     String worksheetId, Workspace workspace, Command command) {
   Worksheet worksheet = workspace.getWorksheet(worksheetId);
   for (Selection sel : worksheet.getSelectionManager().getAllDefinedSelection()) {
     Set<String> inputColumns = new HashSet<String>(sel.getInputColumns());
     inputColumns.retainAll(command.getOutputColumns());
     if (inputColumns.size() > 0) sel.invalidateSelection();
   }
 }
 @Override
 public UpdateContainer doIt(Workspace workspace) throws CommandException {
   UpdateContainer uc = new UpdateContainer();
   SuperSelection sel = this.getSuperSelection(workspace);
   for (int i = 0; i < updates.length(); i++) {
     String update = updates.getString(i);
     switch (update) {
       case "headers":
         uc.add(new WorksheetHeadersUpdate(worksheetId));
         break;
       case "list":
         uc.add(new WorksheetListUpdate());
         break;
       case "data":
         uc.add(new WorksheetDataUpdate(worksheetId, sel));
         break;
       case "alignment":
         {
           Alignment alignment =
               AlignmentManager.Instance()
                   .getAlignmentOrCreateIt(
                       workspace.getId(), worksheetId, workspace.getOntologyManager());
           uc.add(new AlignmentSVGVisualizationUpdate(worksheetId, alignment));
           break;
         }
       case "semanticTypes":
         {
           Alignment alignment =
               AlignmentManager.Instance()
                   .getAlignmentOrCreateIt(
                       workspace.getId(), worksheetId, workspace.getOntologyManager());
           uc.add(
               new SemanticTypesUpdate(
                   workspace.getWorksheet(worksheetId), worksheetId, alignment));
           break;
         }
       case "regenerate":
         uc.add(new RegenerateWorksheetUpdate(worksheetId));
         break;
       case "all":
         uc = WorksheetUpdateFactory.createRegenerateWorksheetUpdates(worksheetId, sel);
         break;
       case "cleaning":
         uc.add(new WorksheetCleaningUpdate(worksheetId, false, sel));
         break;
     }
   }
   return uc;
 }
  @Override
  public UpdateContainer doIt(Workspace workspace) throws CommandException {
    Worksheet worksheet = workspace.getWorksheet(worksheetId);
    RepFactory factory = workspace.getFactory();
    SuperSelection superSel = getSuperSelection(worksheet);
    HTable hTable = factory.getHTable(factory.getHNode(hNodeId).getHTableId());
    Selection currentSel = superSel.getSelection(hTable.getId());
    if (currentSel != null) {
      currentSel.updateSelection();
    }
    CommandHistory history = workspace.getCommandHistory();
    List<Command> tmp =
        gatherAllOperateSelectionCommands(
            history.getCommandsFromWorksheetId(worksheetId), workspace);
    if (tmp.size() > 0) {
      JSONArray inputJSON = new JSONArray();
      inputJSON.put(
          CommandInputJSONUtil.createJsonObject(
              "worksheetId", worksheetId, ParameterType.worksheetId));
      inputJSON.put(
          CommandInputJSONUtil.createJsonObject("hNodeId", hNodeId, ParameterType.hNodeId));
      inputJSON.put(
          CommandInputJSONUtil.createJsonObject(
              "operation", Operation.Intersect.name(), ParameterType.other));
      inputJSON.put(
          CommandInputJSONUtil.createJsonObject(
              "pythonCode", SelectionManager.defaultCode, ParameterType.other));
      inputJSON.put(CommandInputJSONUtil.createJsonObject("onError", "false", ParameterType.other));
      inputJSON.put(
          CommandInputJSONUtil.createJsonObject(
              "selectionName", superSel.getName(), ParameterType.other));
      Command t = null;
      try {
        t = new OperateSelectionCommandFactory().createCommand(inputJSON, workspace);
      } catch (Exception e) {

      }
      if (t != null) history._getHistory().add(t);
      history._getHistory().addAll(tmp);
    }
    UpdateContainer uc =
        WorksheetUpdateFactory.createWorksheetHierarchicalAndCleaningResultsUpdates(
            worksheetId, superSel);
    uc.add(new HistoryUpdate(history));
    return uc;
  }
 @Override
 public UpdateContainer doIt(Workspace workspace) throws CommandException {
   Worksheet wk = workspace.getWorksheet(worksheetId);
   SuperSelection selection = getSuperSelection(wk);
   String Msg =
       String.format("begin, Time,%d, Worksheet,%s", System.currentTimeMillis(), worksheetId);
   logger.info(Msg);
   // Get the HNode
   HashMap<String, HashMap<String, String>> rows = new HashMap<String, HashMap<String, String>>();
   HNodePath selectedPath = null;
   List<HNodePath> columnPaths = wk.getHeaders().getAllPaths();
   for (HNodePath path : columnPaths) {
     if (path.getLeaf().getId().equals(hNodeId)) {
       selectedPath = path;
     }
   }
   // random nodes
   Collection<Node> nodes = new ArrayList<Node>();
   wk.getDataTable().collectNodes(selectedPath, nodes, selection);
   HashSet<Integer> indSet = this.obtainIndexs(nodes.size());
   int index = 0;
   for (Iterator<Node> iterator = nodes.iterator(); iterator.hasNext(); ) {
     Node node = iterator.next();
     if (indSet.contains(index)) {
       String id = node.getId();
       String originalVal = node.getValue().asString();
       HashMap<String, String> x = new HashMap<String, String>();
       x.put("Org", originalVal);
       x.put("Tar", originalVal);
       x.put("Orgdis", originalVal);
       x.put("Tardis", originalVal);
       rows.put(id, x);
     }
     index++;
   }
   Msg = String.format("end, Time,%d, Worksheet,%s", System.currentTimeMillis(), worksheetId);
   logger.info(Msg);
   return new UpdateContainer(new FetchResultUpdate(hNodeId, rows));
 }
  @Override
  public UpdateContainer doIt(Workspace workspace) throws CommandException {
    Worksheet worksheet = workspace.getWorksheet(worksheetId);

    CSVFileExport csvFileExport = new CSVFileExport(worksheet);

    try {

      final String fileName = csvFileExport.publishCSV();
      if (fileName == null)
        return new UpdateContainer(
            new ErrorUpdate("No data to export! Have you aligned the worksheet?"));
      return new UpdateContainer(
          new AbstractUpdate() {
            @Override
            public void generateJson(String prefix, PrintWriter pw, VWorkspace vWorkspace) {
              JSONObject outputObject = new JSONObject();
              try {
                outputObject.put(JsonKeys.updateType.name(), "PublishCSVUpdate");
                outputObject.put(JsonKeys.fileUrl.name(), fileName);
                outputObject.put(JsonKeys.worksheetId.name(), worksheetId);
                pw.println(outputObject.toString(4));

              } catch (JSONException e) {
                logger.error("Error occured while generating JSON!");
              }
            }
          });
    } catch (FileNotFoundException e) {
      logger.error("CSV folder not found!", e);
      return new UpdateContainer(new ErrorUpdate("Error occurred while exporting CSV file!"));
    } catch (Exception e) {
      logger.error("CSV Export Error", e);
      return new UpdateContainer(new ErrorUpdate("Error occurred while exporting CSV file!"));
    }
  }
Example #7
0
 /*
  * Pedro
  *
  * We should have an abstraction for Commands that operate on worksheets,
  * and this method should go there.
  */
 protected String formatWorsheetId(Workspace workspace, String worksheetId) {
   return worksheetId + " (" + workspace.getWorksheet(worksheetId).getTitle() + ")";
 }
  @Override
  public UpdateContainer doIt(Workspace workspace) throws CommandException {
    Worksheet wk = workspace.getWorksheet(worksheetId);
    SuperSelection selection = getSuperSelection(wk);
    String msg =
        String.format(
            "Gen rule start,Time,%d, Worksheet,%s", System.currentTimeMillis(), worksheetId);
    logger.info(msg);
    // Get the HNode
    HashMap<String, String> rows = new HashMap<String, String>();
    HashMap<String, Integer> amb = new HashMap<String, Integer>();
    HNodePath selectedPath = null;
    List<HNodePath> columnPaths = wk.getHeaders().getAllPaths();
    for (HNodePath path : columnPaths) {
      if (path.getLeaf().getId().equals(hNodeId)) {
        selectedPath = path;
      }
    }
    Collection<Node> nodes = new ArrayList<Node>();
    wk.getDataTable().collectNodes(selectedPath, nodes, selection);
    for (Node node : nodes) {
      String id = node.getId();
      if (!this.nodeIds.contains(id)) continue;
      String originalVal = node.getValue().asString();
      rows.put(id, originalVal);
      this.compResultString += originalVal + "\n";
      calAmbScore(id, originalVal, amb);
    }
    RamblerValueCollection vc = new RamblerValueCollection(rows);
    HashMap<String, Vector<String[]>> expFeData = new HashMap<String, Vector<String[]>>();
    inputs = new RamblerTransformationInputs(examples, vc);
    // generate the program
    boolean results = false;
    int iterNum = 0;
    RamblerTransformationOutput rtf = null;
    // initialize the vocabulary
    Iterator<String> iterx = inputs.getInputValues().getValues().iterator();
    Vector<String> v = new Vector<String>();
    int vb_cnt = 0;
    while (iterx.hasNext() && vb_cnt < 30) {
      String eString = iterx.next();
      v.add(eString);
      vb_cnt++;
    }
    Vector<String> vob = UtilTools.buildDict(v);
    inputs.setVocab(vob.toArray(new String[vob.size()]));
    while (iterNum < 1 && !results) // try to find an program within iterNum
    {
      rtf = new RamblerTransformationOutput(inputs);
      if (rtf.getTransformations().keySet().size() > 0) {
        results = true;
      }
      iterNum++;
    }
    Iterator<String> iter = rtf.getTransformations().keySet().iterator();
    // id:{org: tar: orgdis: tardis: }
    HashMap<String, HashMap<String, String>> resdata =
        new HashMap<String, HashMap<String, String>>();
    HashSet<String> keys = new HashSet<String>();
    while (iter.hasNext()) {
      String tpid = iter.next();
      ValueCollection rvco = rtf.getTransformedValues_debug(tpid);
      if (rvco == null) continue;
      // constructing displaying data
      HashMap<String, String[]> xyzHashMap = new HashMap<String, String[]>();
      for (String key : rvco.getNodeIDs()) {
        HashMap<String, String> dict = new HashMap<String, String>();
        // add to the example selection
        boolean isExp = false;
        String org = vc.getValue(key);
        String classLabel = rvco.getClass(key);
        String pretar = rvco.getValue(key);
        String dummyValue = pretar;
        if (pretar.indexOf("_FATAL_ERROR_") != -1) {
          dummyValue = org;
          // dummyValue = "#ERROR";
        }
        try {
          UtilTools.StringColorCode(org, dummyValue, dict);
        } catch (Exception ex) {
          logger.info(String.format("ColorCoding Exception%s, %s", org, dummyValue));
          // set dict
          dict.put("Org", org);
          dict.put("Tar", "ERROR");
          dict.put("Orgdis", org);
          dict.put("Tardis", "ERROR");
        }
        for (TransformationExample exp : examples) {
          if (exp.getNodeId().compareTo(key) == 0) {
            if (!expFeData.containsKey(classLabel)) {
              Vector<String[]> vstr = new Vector<String[]>();
              String[] texp = {dict.get("Org"), pretar};
              vstr.add(texp);
              expFeData.put(classLabel, vstr);
            } else {
              String[] texp = {dict.get("Org"), pretar};
              expFeData.get(classLabel).add(texp);
            }
            isExp = true;
          }
        }

        if (!isExp) {
          String[] pair = {dict.get("Org"), dict.get("Tar"), pretar, classLabel};
          xyzHashMap.put(key, pair);
        }
        resdata.put(key, dict);
      }
      if (!rtf.nullRule) keys.add(getBestExample(xyzHashMap, expFeData));
    }
    // find the best row
    String vars = "";
    String expstr = "";
    String recmd = "";
    for (TransformationExample x : examples) {
      expstr += String.format("%s|%s", x.getBefore(), x.getAfter());
    }
    expstr += "|";
    if (rtf.nullRule) {
      keys.clear();
      // keys.add("-2"); // "-2 indicates null rule"
    }
    if (!resdata.isEmpty() && !rtf.nullRule) {
      recmd = resdata.get(keys.iterator().next()).get("Org");
    } else {
      recmd = "";
    }
    msg =
        String.format(
            "Gen rule end, Time,%d, Worksheet,%s,Examples:%s,Recmd:%s",
            System.currentTimeMillis(), worksheetId, expstr, recmd);
    logger.info(msg);
    return new UpdateContainer(new CleaningResultUpdate(hNodeId, resdata, vars, keys));
  }
  @Override
  public UpdateContainer doIt(Workspace workspace) {
    Worksheet worksheet = workspace.getWorksheet(worksheetId);
    SuperSelection selection = getSuperSelection(worksheet);
    String worksheetName = worksheet.getTitle();
    try {

      // preparing model file name
      final String modelFileName =
          workspace.getCommandPreferencesId() + worksheetId + "-" + worksheetName + "-model.ttl";
      final String modelFileLocalPath =
          ServletContextParameterMap.getParameterValue(ContextParameter.R2RML_PUBLISH_DIR)
              + modelFileName;

      File f = new File(modelFileLocalPath);

      // preparing the graphUri where the model is published in the triple store
      String graphName =
          worksheet
              .getMetadataContainer()
              .getWorksheetProperties()
              .getPropertyValue(Property.graphName);
      if (graphName == null || graphName.isEmpty()) {
        SimpleDateFormat sdf = new SimpleDateFormat("dd-MMM-yyyy-kkmmssS");
        String ts = sdf.format(Calendar.getInstance().getTime());
        graphName =
            "http://localhost/"
                + workspace.getCommandPreferencesId()
                + "/"
                + worksheetId
                + "/model/"
                + ts;
        worksheet
            .getMetadataContainer()
            .getWorksheetProperties()
            .setPropertyValue(Property.graphName, graphName);
      }

      // If the model is not published, publish it!
      if (!f.exists() || !f.isFile()) {
        GenerateR2RMLModelCommandFactory factory = new GenerateR2RMLModelCommandFactory();
        GenerateR2RMLModelCommand cmd =
            (GenerateR2RMLModelCommand)
                factory.createCommand(
                    workspace,
                    worksheetId,
                    TripleStoreUtil.defaultModelsRepoUrl,
                    graphName,
                    selection.getName());
        cmd.doIt(workspace);
      } else {
        // if the model was published 30 min ago, publish it again, just to be sure
        long diff = Calendar.getInstance().getTimeInMillis() - f.lastModified();
        if ((diff / 1000L / 60L) > 30) {
          f.delete();
          GenerateR2RMLModelCommandFactory factory = new GenerateR2RMLModelCommandFactory();
          GenerateR2RMLModelCommand cmd =
              (GenerateR2RMLModelCommand)
                  factory.createCommand(
                      workspace,
                      worksheetId,
                      TripleStoreUtil.defaultModelsRepoUrl,
                      graphName,
                      selection.getName());
          cmd.doIt(workspace);
        }
      }

      //			TripleStoreUtil tUtil = new TripleStoreUtil();
      StringBuffer query =
          new StringBuffer(
              "prefix rr: <http://www.w3.org/ns/r2rml#> prefix km-dev: <http://isi.edu/integration/karma/dev#> ");

      /* ****** this is the query for the list of columns.

      PREFIX km-dev: <http://isi.edu/integration/karma/dev#>
      PREFIX rr: <http://www.w3.org/ns/r2rml#>

      select distinct ?class where  {
        {
          ?x1 rr:subjectMap/km-dev:alignmentNodeId "------- The full url of the column/class --------".
          ?x1 rr:predicateObjectMap/rr:objectMap/rr:column ?column .
      	?x1 rr:subjectMap/rr:predicate ?class .
        }
        UNION
        {
          ?x1 rr:subjectMap/km-dev:alignmentNodeId "------- The full url of the column/class --------".
      	?x1 (rr:predicateObjectMap/rr:objectMap/rr:parentTriplesMap)* ?x2 .
      	?x2 rr:predicateObjectMap/rr:objectMap/rr:column ?column .
      	?x2 rr:predicateObjectMap/rr:predicate ?class .
        }
      }
      * */

      query.append("select distinct ?class ?column where { ");
      if (graphName != null && !graphName.trim().isEmpty()) {
        query.append(" graph  <" + graphName + "> { ");
      }
      query
          .append("{ ?x1 rr:subjectMap/km-dev:alignmentNodeId \"")
          .append(this.nodeId)
          .append(
              "\" . ?x1 rr:predicateObjectMap/rr:objectMap/rr:column ?column . ?x1 rr:subjectMap/rr:predicate ?class .")
          .append(" } UNION { ")
          .append("?x1 rr:subjectMap/km-dev:alignmentNodeId \"")
          .append(this.nodeId)
          .append("\" . ?x1 (rr:predicateObjectMap/rr:objectMap/rr:parentTriplesMap)* ?x2 .")
          .append(" ?x2 rr:predicateObjectMap ?x3 . ")
          .append(" ?x3 rr:objectMap/rr:column ?column . ?x3 rr:predicate ?class .")
          .append(" } }");
      if (graphName != null && !graphName.trim().isEmpty()) {
        query.append(" } ");
      }
      logger.info("Query: " + query.toString());
      String sData =
          TripleStoreUtil.invokeSparqlQuery(
              query.toString(), TripleStoreUtil.defaultModelsRepoUrl, "application/json", null);
      if (sData == null | sData.isEmpty()) {
        logger.error("Empty response object from query : " + query);
      }
      HashMap<String, String> cols = new HashMap<String, String>();
      try {
        JSONObject obj1 = new JSONObject(sData);
        JSONArray arr = obj1.getJSONObject("results").getJSONArray("bindings");
        for (int i = 0; i < arr.length(); i++) {
          String colName = arr.getJSONObject(i).getJSONObject("column").getString("value");
          String colValue = arr.getJSONObject(i).getJSONObject("class").getString("value");
          if (cols.containsKey(colName)) {
            logger.error("Duplicate Column <-> property mapping. " + colName + " <=> " + colValue);
          } else {
            cols.put(colName, colValue);
          }
        }
      } catch (Exception e2) {
        logger.error("Error in parsing json response", e2);
      }

      logger.info("Total Columns fetched : " + cols.size());
      final HashMap<String, String> columns = cols;
      return new UpdateContainer(
          new AbstractUpdate() {

            @Override
            public void generateJson(String prefix, PrintWriter pw, VWorkspace vWorkspace) {
              JSONObject obj = new JSONObject();
              try {
                Iterator<String> itr = columns.keySet().iterator();
                JSONArray colList = new JSONArray();
                while (itr.hasNext()) {
                  JSONObject o = new JSONObject();
                  String k = itr.next();
                  o.put("name", k);
                  o.put("url", columns.get(k));
                  colList.put(o);
                }
                obj.put("updateType", "FetchColumnUpdate");
                obj.put("columns", colList);
                obj.put("rootId", nodeId);
                pw.println(obj.toString());
              } catch (JSONException e) {
                logger.error("Error occurred while fetching worksheet properties!", e);
              }
            }
          });

    } catch (Exception e) {
      String msg = "Error occured while fetching columns!";
      logger.error(msg, e);
      return new UpdateContainer(new ErrorUpdate(msg));
    }
  }
  @Override
  public UpdateContainer doIt(Workspace workspace) throws CommandException {
    Worksheet worksheet = workspace.getWorksheet(worksheetId);
    SuperSelection selection = getSuperSelection(worksheet);
    HNodePath selectedPath = null;
    List<HNodePath> columnPaths = worksheet.getHeaders().getAllPaths();
    for (HNodePath path : columnPaths) {
      if (path.getLeaf().getId().equals(hNodeId)) {
        selectedPath = path;
      }
    }
    Collection<Node> nodes = new ArrayList<Node>();
    workspace
        .getFactory()
        .getWorksheet(worksheetId)
        .getDataTable()
        .collectNodes(selectedPath, nodes, selection);

    try {
      JSONArray requestJsonArray = new JSONArray();
      for (Node node : nodes) {
        String id = node.getId();
        String originalVal = node.getValue().asString();
        JSONObject jsonRecord = new JSONObject();
        jsonRecord.put("id", id);
        originalVal = originalVal == null ? "" : originalVal;
        jsonRecord.put("value", originalVal);
        requestJsonArray.put(jsonRecord);
      }
      String jsonString = null;
      jsonString = requestJsonArray.toString();

      // String url =
      // "http://localhost:8080/cleaningService/IdentifyData";
      //			String url = "http://localhost:8070/myWS/IdentifyData";
      String url =
          ServletContextParameterMap.getParameterValue(ContextParameter.CLEANING_SERVICE_URL);

      HttpClient httpclient = new DefaultHttpClient();
      HttpPost httppost = null;
      HttpResponse response = null;
      HttpEntity entity;
      StringBuffer out = new StringBuffer();

      URI u = null;
      u = new URI(url);
      List<NameValuePair> formparams = new ArrayList<NameValuePair>();
      formparams.add(new BasicNameValuePair("json", jsonString));

      httppost = new HttpPost(u);
      httppost.setEntity(new UrlEncodedFormEntity(formparams, "UTF-8"));
      response = httpclient.execute(httppost);
      entity = response.getEntity();
      if (entity != null) {
        BufferedReader buf = new BufferedReader(new InputStreamReader(entity.getContent()));
        String line = buf.readLine();
        while (line != null) {
          out.append(line);
          line = buf.readLine();
        }
      }
      // logger.trace(out.toString());
      // logger.info("Connnection success : " + url + " Successful.");
      final JSONObject data1 = new JSONObject(out.toString());
      // logger.trace("Data--->" + data1);
      return new UpdateContainer(
          new AbstractUpdate() {

            @Override
            public void generateJson(String prefix, PrintWriter pw, VWorkspace vWorkspace) {
              JSONObject response = new JSONObject();
              // logger.trace("Reached here");
              try {
                response.put("updateType", "CleaningServiceOutput");
                response.put("chartData", data1);
                response.put("hNodeId", hNodeId);
                // logger.trace(response.toString(4));
              } catch (JSONException e) {
                pw.print("Error");
              }

              pw.print(response.toString());
            }
          });
    } catch (Exception e) {
      e.printStackTrace();
      return new UpdateContainer(new ErrorUpdate("Error!"));
    }
  }