private static void getStatistics2(List<ColumnNode> columnNodes) {

    if (columnNodes == null) return;

    int numberOfAttributesWhoseTypeIsFirstCRFType = 0;
    int numberOfAttributesWhoseTypeIsInCRFTypes = 0;
    for (ColumnNode cn : columnNodes) {
      List<SemanticType> userSemanticTypes = cn.getUserSemanticTypes();
      List<SemanticType> top4Suggestions = cn.getTopKLearnedSemanticTypes(4);

      for (int i = 0; i < top4Suggestions.size(); i++) {
        SemanticType st = top4Suggestions.get(i);
        if (userSemanticTypes != null) {
          for (SemanticType t : userSemanticTypes) {
            if (st.getModelLabelString().equalsIgnoreCase(t.getModelLabelString())) {
              if (i == 0) numberOfAttributesWhoseTypeIsFirstCRFType++;
              numberOfAttributesWhoseTypeIsInCRFTypes++;
              i = top4Suggestions.size();
              break;
            }
          }
        }
      }
    }

    //		System.out.println(columnNodes.size() + "\t" + numberOfAttributesWhoseTypeIsInCRFTypes +
    // "\t" + numberOfAttributesWhoseTypeIsFirstCRFType);

    System.out.println("totalNumberOfAttributes: " + columnNodes.size());
    System.out.println(
        "numberOfAttributesWhoseTypeIsInCRFTypes: " + numberOfAttributesWhoseTypeIsInCRFTypes);
    System.out.println(
        "numberOfAttributesWhoseTypeIsFirstCRFType:" + numberOfAttributesWhoseTypeIsFirstCRFType);
  }
  private void saveSemanticTypesInformation(
      Worksheet worksheet, VWorkspace vWorkspace, Collection<SemanticType> semanticTypes)
      throws JSONException {
    JSONArray typesArray = new JSONArray();

    // Add the vworksheet information
    JSONObject vwIDJObj = new JSONObject();
    vwIDJObj.put(ClientJsonKeys.name.name(), ParameterType.vWorksheetId.name());
    vwIDJObj.put(ClientJsonKeys.type.name(), ParameterType.vWorksheetId.name());
    vwIDJObj.put(ClientJsonKeys.value.name(), vWorksheetId);
    typesArray.put(vwIDJObj);

    // Add the check history information
    JSONObject chIDJObj = new JSONObject();
    chIDJObj.put(ClientJsonKeys.name.name(), ParameterType.checkHistory.name());
    chIDJObj.put(ClientJsonKeys.type.name(), ParameterType.other.name());
    chIDJObj.put(ClientJsonKeys.value.name(), false);
    typesArray.put(chIDJObj);

    for (SemanticType type : semanticTypes) {
      // Add the hNode information
      JSONObject hNodeJObj = new JSONObject();
      hNodeJObj.put(ClientJsonKeys.name.name(), ParameterType.hNodeId.name());
      hNodeJObj.put(ClientJsonKeys.type.name(), ParameterType.hNodeId.name());
      hNodeJObj.put(ClientJsonKeys.value.name(), type.getHNodeId());
      typesArray.put(hNodeJObj);

      // Add the semantic type information
      JSONObject typeJObj = new JSONObject();
      typeJObj.put(ClientJsonKeys.name.name(), ClientJsonKeys.SemanticType.name());
      typeJObj.put(ClientJsonKeys.type.name(), ParameterType.other.name());
      typeJObj.put(ClientJsonKeys.value.name(), type.getJSONArrayRepresentation());
      typesArray.put(typeJObj);
    }
    setInputParameterJson(typesArray.toString(4));
  }
  @Override
  public void generateJson(String prefix, PrintWriter pw, VWorkspace vWorkspace) {
    Workspace workspace = vWorkspace.getWorkspace();
    alignment = AlignmentManager.Instance().getAlignment(workspace.getId(), worksheetId);
    SemanticTypes types = worksheet.getSemanticTypes();
    Map<String, ColumnNode> hNodeIdTocolumnNodeMap = createColumnNodeMap();
    Map<String, SemanticTypeNode> hNodeIdToDomainNodeMap = createDomainNodeMap();

    JSONStringer jsonStr = new JSONStringer();
    try {
      JSONWriter writer = jsonStr.object();
      writer.key("worksheetId").value(worksheetId).key("updateType").value("SemanticTypesUpdate");

      writer.key(JsonKeys.Types.name());
      writer.array();
      // Iterate through all the columns
      for (HNodePath path : worksheet.getHeaders().getAllPaths()) {
        HNode node = path.getLeaf();
        String nodeId = node.getId();

        writer.object();

        // Check if a semantic type exists for the HNode
        SemanticType type = types.getSemanticTypeForHNodeId(nodeId);
        if (type != null && type.getConfidenceLevel() != SemanticType.ConfidenceLevel.Low) {
          writer
              .key(JsonKeys.HNodeId.name())
              .value(type.getHNodeId())
              .key(JsonKeys.SemanticTypesArray.name())
              .array();

          ColumnNode alignmentColumnNode = hNodeIdTocolumnNodeMap.get(type.getHNodeId());
          SemanticTypeNode domainNode = hNodeIdToDomainNodeMap.get(type.getHNodeId());

          if (alignmentColumnNode == null || domainNode == null) {
            logger.error(
                "Column node or domain node not found in alignment."
                    + " (This should not happen conceptually!):"
                    + type);
            continue;
          }

          // Add the primary semantic type
          writer
              .object()
              .key(JsonKeys.Origin.name())
              .value(type.getOrigin().name())
              .key(JsonKeys.ConfidenceLevel.name())
              .value(type.getConfidenceLevel().name())
              .key(JsonKeys.isPrimary.name())
              .value(true);

          // Add the RDF literal type to show in the text box
          String rdfLiteralType =
              alignmentColumnNode.getRdfLiteralType() == null
                  ? ""
                  : alignmentColumnNode.getRdfLiteralType().getDisplayName();
          String language =
              alignmentColumnNode.getLanguage() == null ? "" : alignmentColumnNode.getLanguage();
          writer.key(JsonKeys.rdfLiteralType.name()).value(rdfLiteralType);
          writer.key(JsonKeys.language.name()).value(language);

          //					String domainDisplayLabel = (domainNode.getLabel().getPrefix() != null &&
          // (!domainNode.getLabel().getPrefix().equals(""))) ?
          //							(domainNode.getLabel().getPrefix() + ":" + domainNode.getLocalId()) :
          // domainNode.getLocalId();
          if (!type.isClass()) {
            writer
                .key(JsonKeys.FullType.name())
                .value(type.getType().getUri())
                .key(JsonKeys.DisplayLabel.name())
                .value(type.getType().getDisplayName())
                .key(JsonKeys.DisplayRDFSLabel.name())
                .value(type.getType().getRdfsLabel())
                .key(JsonKeys.DisplayRDFSComment.name())
                .value(type.getType().getRdfsComment())
                .key(JsonKeys.DomainId.name())
                .value(domainNode.getId())
                .key(JsonKeys.DomainUri.name())
                .value(domainNode.getUri())
                .key(JsonKeys.DisplayDomainLabel.name())
                .value(domainNode.getDisplayId())
                .key(JsonKeys.DomainRDFSLabel.name())
                .value(domainNode.getRdfsLabel())
                .key(JsonKeys.DomainRDFSComment.name())
                .value(domainNode.getRdfsComment());
          } else {
            writer
                .key(JsonKeys.FullType.name())
                .value(domainNode.getId())
                .key(JsonKeys.DisplayLabel.name())
                .value(domainNode.getDisplayId())
                .key(JsonKeys.DisplayRDFSLabel.name())
                .value(domainNode.getRdfsLabel())
                .key(JsonKeys.DisplayRDFSComment.name())
                .value(domainNode.getRdfsComment())
                .key(JsonKeys.DomainId.name())
                .value("")
                .key(JsonKeys.DomainUri.name())
                .value("")
                .key(JsonKeys.DisplayDomainLabel.name())
                .value("")
                .key(JsonKeys.DomainRDFSLabel.name())
                .value("")
                .key(JsonKeys.DomainRDFSComment.name())
                .value("");
          }

          // Mark the special properties
          writer
              .key(JsonKeys.isMetaProperty.name())
              .value(isMetaProperty(type.getType(), alignmentColumnNode));

          writer.endObject();

          // Iterate through the synonym semantic types
          SynonymSemanticTypes synTypes = types.getSynonymTypesForHNodeId(nodeId);

          if (synTypes != null) {
            for (SemanticType synType : synTypes.getSynonyms()) {
              writer
                  .object()
                  .key(JsonKeys.HNodeId.name())
                  .value(synType.getHNodeId())
                  .key(JsonKeys.FullType.name())
                  .value(synType.getType().getUri())
                  .key(JsonKeys.Origin.name())
                  .value(synType.getOrigin().name())
                  .key(JsonKeys.ConfidenceLevel.name())
                  .value(synType.getConfidenceLevel().name())
                  .key(JsonKeys.DisplayLabel.name())
                  .value(synType.getType().getDisplayName())
                  .key(JsonKeys.DisplayRDFSLabel.name())
                  .value(synType.getType().getRdfsLabel())
                  .key(JsonKeys.DisplayRDFSComment.name())
                  .value(synType.getType().getRdfsComment())
                  .key(JsonKeys.isPrimary.name())
                  .value(false);
              if (!synType.isClass()) {
                writer
                    .key(JsonKeys.DomainUri.name())
                    .value(synType.getDomain().getUri())
                    .key(JsonKeys.DomainId.name())
                    .value("")
                    .key(JsonKeys.DisplayDomainLabel.name())
                    .value(synType.getDomain().getDisplayName())
                    .key(JsonKeys.DomainRDFSLabel.name())
                    .value(synType.getDomain().getRdfsLabel())
                    .key(JsonKeys.DomainRDFSComment.name())
                    .value(synType.getDomain().getRdfsComment());
              } else {
                writer
                    .key(JsonKeys.DomainId.name())
                    .value("")
                    .key(JsonKeys.DomainUri.name())
                    .value("")
                    .key(JsonKeys.DisplayDomainLabel.name())
                    .value("")
                    .key(JsonKeys.DomainRDFSLabel.name())
                    .value("")
                    .key(JsonKeys.DomainRDFSComment.name())
                    .value("");
              }
              writer.endObject();
            }
          }
          writer.endArray();
        } else {
          writer.key(JsonKeys.HNodeId.name()).value(nodeId);
          writer.key(JsonKeys.SemanticTypesArray.name()).array().endArray();
        }

        writer.endObject();
      }
      writer.endArray();
      writer.endObject();

      pw.print(writer.toString());
    } catch (JSONException e) {
      logger.error("Error occured while writing to JSON!", e);
    }
  }
  private SemanticTypeMapping addSemanticTypeStruct(
      ColumnNode sourceColumn, SemanticType semanticType, Set<Node> addedNodes) {

    logger.debug("adding semantic type to the graph ... ");

    if (addedNodes == null) addedNodes = new HashSet<Node>();

    if (semanticType == null) {
      logger.error("semantic type is null.");
      return null;
    }
    if (semanticType.getDomain() == null) {
      logger.error("semantic type does not have any domain");
      return null;
    }

    if (semanticType.getType() == null) {
      logger.error("semantic type does not have any link");
      return null;
    }

    String domainUri = semanticType.getDomain().getUri();
    String propertyUri = semanticType.getType().getUri();
    Double confidence = semanticType.getConfidenceScore();
    Origin origin = semanticType.getOrigin();

    if (domainUri == null || domainUri.isEmpty()) {
      logger.error("semantic type does not have any domain");
      return null;
    }

    if (propertyUri == null || propertyUri.isEmpty()) {
      logger.error("semantic type does not have any link");
      return null;
    }

    logger.debug(
        "semantic type: " + domainUri + "|" + propertyUri + "|" + confidence + "|" + origin);

    InternalNode source = null;
    String nodeId;

    nodeId = nodeIdFactory.getNodeId(domainUri);
    source = new InternalNode(nodeId, new Label(domainUri));
    if (!this.graphBuilder.addNodeAndUpdate(source, addedNodes)) return null;

    nodeId = new RandomGUID().toString();
    ColumnNode target = new ColumnNode(nodeId, nodeId, sourceColumn.getColumnName(), null);
    if (!this.graphBuilder.addNode(target)) return null;
    addedNodes.add(target);

    String linkId = LinkIdFactory.getLinkId(propertyUri, source.getId(), target.getId());
    LabeledLink link;
    if (propertyUri.equalsIgnoreCase(ClassInstanceLink.getFixedLabel().getUri()))
      link = new ClassInstanceLink(linkId);
    else {
      Label label = this.ontologyManager.getUriLabel(propertyUri);
      link = new DataPropertyLink(linkId, label);
    }
    if (!this.graphBuilder.addLink(source, target, link)) return null;

    SemanticTypeMapping mappingStruct =
        new SemanticTypeMapping(sourceColumn, semanticType, source, link, target);

    return mappingStruct;
  }
  private Set<SemanticTypeMapping> findSemanticTypeInGraph(
      ColumnNode sourceColumn,
      SemanticType semanticType,
      HashMap<String, Integer> semanticTypesCount,
      Set<Node> addedNodes) {

    logger.debug("finding matches for semantic type in the graph ... ");

    if (addedNodes == null) addedNodes = new HashSet<Node>();

    Set<SemanticTypeMapping> mappings = new HashSet<SemanticTypeMapping>();

    if (semanticType == null) {
      logger.error("semantic type is null.");
      return mappings;
    }
    if (semanticType.getDomain() == null) {
      logger.error("semantic type does not have any domain");
      return mappings;
    }

    if (semanticType.getType() == null) {
      logger.error("semantic type does not have any link");
      return mappings;
    }

    String domainUri = semanticType.getDomain().getUri();
    String propertyUri = semanticType.getType().getUri();
    Double confidence = semanticType.getConfidenceScore();
    Origin origin = semanticType.getOrigin();

    Integer countOfSemanticType = semanticTypesCount.get(domainUri + propertyUri);
    if (countOfSemanticType == null) {
      logger.error("count of semantic type should not be null or zero");
      return mappings;
    }

    if (domainUri == null || domainUri.isEmpty()) {
      logger.error("semantic type does not have any domain");
      return mappings;
    }

    if (propertyUri == null || propertyUri.isEmpty()) {
      logger.error("semantic type does not have any link");
      return mappings;
    }

    logger.debug(
        "semantic type: " + domainUri + "|" + propertyUri + "|" + confidence + "|" + origin);

    // add dataproperty to existing classes if sl is a data node mapping
    //		Set<Node> foundInternalNodes = new HashSet<Node>();
    Set<SemanticTypeMapping> semanticTypeMatches =
        this.graphBuilder.getSemanticTypeMatches().get(domainUri + propertyUri);
    if (semanticTypeMatches != null) {
      for (SemanticTypeMapping stm : semanticTypeMatches) {

        SemanticTypeMapping mp =
            new SemanticTypeMapping(
                sourceColumn, semanticType, stm.getSource(), stm.getLink(), stm.getTarget());
        mappings.add(mp);
        //				foundInternalNodes.add(stm.getSource());
      }
    }

    logger.debug("adding data property to the found internal nodes ...");

    Integer count;
    boolean allowMultipleSamePropertiesPerNode =
        ModelingConfigurationRegistry.getInstance()
            .getModelingConfiguration(
                ContextParametersRegistry.getInstance()
                    .getContextParameters(ontologyManager.getContextId())
                    .getKarmaHome())
            .isMultipleSamePropertyPerNode();
    Set<Node> nodesWithSameUriOfDomain = this.graphBuilder.getUriToNodesMap().get(domainUri);
    if (nodesWithSameUriOfDomain != null) {
      for (Node source : nodesWithSameUriOfDomain) {
        count = this.graphBuilder.getNodeDataPropertyCount().get(source.getId() + propertyUri);

        if (count != null) {
          if (allowMultipleSamePropertiesPerNode) {
            if (count >= countOfSemanticType.intValue()) continue;
          } else {
            if (count >= 1) continue;
          }
        }

        String nodeId = new RandomGUID().toString();
        ColumnNode target = new ColumnNode(nodeId, nodeId, sourceColumn.getColumnName(), null);
        if (!this.graphBuilder.addNode(target)) continue;
        ;
        addedNodes.add(target);

        String linkId = LinkIdFactory.getLinkId(propertyUri, source.getId(), target.getId());
        LabeledLink link = new DataPropertyLink(linkId, new Label(propertyUri));
        if (!this.graphBuilder.addLink(source, target, link)) continue;
        ;

        SemanticTypeMapping mp =
            new SemanticTypeMapping(
                sourceColumn, semanticType, (InternalNode) source, link, target);
        mappings.add(mp);
      }
    }

    return mappings;
  }
  private CandidateSteinerSets getCandidateSteinerSets(
      List<ColumnNode> columnNodes,
      boolean useCorrectTypes,
      int numberOfCRFCandidates,
      Set<Node> addedNodes) {

    if (columnNodes == null || columnNodes.isEmpty()) return null;

    int maxNumberOfSteinerNodes = columnNodes.size() * 2;
    CandidateSteinerSets candidateSteinerSets =
        new CandidateSteinerSets(maxNumberOfSteinerNodes, ontologyManager.getContextId());

    if (addedNodes == null) addedNodes = new HashSet<Node>();

    Set<SemanticTypeMapping> tempSemanticTypeMappings;
    HashMap<ColumnNode, List<SemanticType>> columnSemanticTypes =
        new HashMap<ColumnNode, List<SemanticType>>();
    HashMap<String, Integer> semanticTypesCount = new HashMap<String, Integer>();
    List<SemanticType> candidateSemanticTypes;
    String domainUri = "", propertyUri = "";

    for (ColumnNode n : columnNodes) {

      candidateSemanticTypes = n.getTopKLearnedSemanticTypes(numberOfCRFCandidates);
      columnSemanticTypes.put(n, candidateSemanticTypes);

      for (SemanticType semanticType : candidateSemanticTypes) {

        if (semanticType == null
            || semanticType.getDomain() == null
            || semanticType.getType() == null) continue;

        domainUri = semanticType.getDomain().getUri();
        propertyUri = semanticType.getType().getUri();

        Integer count = semanticTypesCount.get(domainUri + propertyUri);
        if (count == null) semanticTypesCount.put(domainUri + propertyUri, 1);
        else semanticTypesCount.put(domainUri + propertyUri, count.intValue() + 1);
      }
    }

    int numOfMappings = 1;
    for (ColumnNode n : columnNodes) {

      candidateSemanticTypes = columnSemanticTypes.get(n);
      if (candidateSemanticTypes == null) continue;

      logger.info("===== Column: " + n.getColumnName());

      Set<SemanticTypeMapping> semanticTypeMappings = new HashSet<SemanticTypeMapping>();
      for (SemanticType semanticType : candidateSemanticTypes) {

        logger.info("\t===== Semantic Type: " + semanticType.getModelLabelString());

        if (semanticType == null
            || semanticType.getDomain() == null
            || semanticType.getType() == null) continue;

        domainUri = semanticType.getDomain().getUri();
        propertyUri = semanticType.getType().getUri();
        Integer countOfSemanticType = semanticTypesCount.get(domainUri + propertyUri);
        //				logger.info("count of semantic type: " +  countOfSemanticType);

        tempSemanticTypeMappings =
            findSemanticTypeInGraph(n, semanticType, semanticTypesCount, addedNodes);
        //				logger.info("number of matches for semantic type: " +
        //					 + (tempSemanticTypeMappings == null ? 0 : tempSemanticTypeMappings.size()));

        if (tempSemanticTypeMappings != null) semanticTypeMappings.addAll(tempSemanticTypeMappings);

        int countOfMatches = tempSemanticTypeMappings == null ? 0 : tempSemanticTypeMappings.size();
        if (countOfMatches
            < countOfSemanticType) // No struct in graph is matched with the semantic type, we add a
                                   // new struct to the graph
        {
          for (int i = 0; i < countOfSemanticType - countOfMatches; i++) {
            SemanticTypeMapping mp = addSemanticTypeStruct(n, semanticType, addedNodes);
            if (mp != null) semanticTypeMappings.add(mp);
          }
        }
      }
      //			System.out.println("number of matches for column " + n.getColumnName() +
      //					": " + (semanticTypeMappings == null ? 0 : semanticTypeMappings.size()));
      logger.info(
          "number of matches for column "
              + n.getColumnName()
              + ": "
              + (semanticTypeMappings == null ? 0 : semanticTypeMappings.size()));
      numOfMappings *=
          semanticTypeMappings == null || semanticTypeMappings.isEmpty()
              ? 1
              : semanticTypeMappings.size();

      candidateSteinerSets.updateSteinerSets(semanticTypeMappings);
    }

    //		System.out.println("number of possible mappings: " + numOfMappings);
    logger.info("number of possible mappings: " + numOfMappings);

    return candidateSteinerSets;
  }
  public void run() {
    long start = System.currentTimeMillis();
    // Find the corresponding hNodePath. Used to find examples for training the CRF Model.
    HNodePath currentColumnPath = null;
    List<HNodePath> paths = worksheet.getHeaders().getAllPaths();
    for (HNodePath path : paths) {
      if (path.getLeaf().getId().equals(newType.getHNodeId())) {
        currentColumnPath = path;
        break;
      }
    }

    Map<ColumnFeature, Collection<String>> columnFeatures =
        new HashMap<ColumnFeature, Collection<String>>();

    // Prepare the column name for training
    String columnName = currentColumnPath.getLeaf().getColumnName();
    Collection<String> columnNameList = new ArrayList<String>();
    columnNameList.add(columnName);
    columnFeatures.put(ColumnFeature.ColumnHeaderName, columnNameList);

    // Train the model with the new type
    ArrayList<String> trainingExamples =
        SemanticTypeUtil.getTrainingExamples(worksheet, currentColumnPath);
    boolean trainingResult = false;
    String newTypeString =
        (newType.getDomain() == null)
            ? newType.getType().getUri()
            : newType.getDomain().getUri() + "|" + newType.getType().getUri();

    trainingResult =
        crfModelHandler.addOrUpdateLabel(newTypeString, trainingExamples, columnFeatures);

    if (!trainingResult) {
      logger.error("Error occured while training CRF Model.");
    }
    //		logger.debug("Using type:" + newType.getDomain().getUri() + "|" +
    // newType.getType().getUri());

    // Add the new CRF column model for this column
    ArrayList<String> labels = new ArrayList<String>();
    ArrayList<Double> scores = new ArrayList<Double>();
    trainingResult =
        crfModelHandler.predictLabelForExamples(
            trainingExamples, 4, labels, scores, null, columnFeatures);
    if (!trainingResult) {
      logger.error("Error occured while predicting labels");
    }
    CRFColumnModel newModel = new CRFColumnModel(labels, scores);
    worksheet.getCrfModel().addColumnModel(newType.getHNodeId(), newModel);

    long elapsedTimeMillis = System.currentTimeMillis() - start;
    float elapsedTimeSec = elapsedTimeMillis / 1000F;
    logger.info("Time required for training the semantic type: " + elapsedTimeSec);

    //		long t2 = System.currentTimeMillis();

    // Identify the outliers for the column
    //		SemanticTypeUtil.identifyOutliers(worksheet, newTypeString,currentColumnPath,
    // vWorkspace.getWorkspace().getTagsContainer()
    //				.getTag(TagName.Outlier), columnFeatures, crfModelHandler);

    //		long t3 = System.currentTimeMillis();
    //		logger.info("Identify outliers: "+ (t3-t2));
  }