Пример #1
0
  public static ConnectionCosts build(String filename) throws IOException {
    FileInputStream inputStream = new FileInputStream(filename);
    InputStreamReader streamReader = new InputStreamReader(inputStream);
    LineNumberReader lineReader = new LineNumberReader(streamReader);

    String line = lineReader.readLine();
    String[] dimensions = line.split("\\s+");

    assert dimensions.length == 3;

    int forwardSize = Integer.parseInt(dimensions[0]);
    int backwardSize = Integer.parseInt(dimensions[1]);

    assert forwardSize > 0 && backwardSize > 0;

    ConnectionCosts costs = new ConnectionCosts(forwardSize, backwardSize);

    while ((line = lineReader.readLine()) != null) {
      String[] fields = line.split("\\s+");

      assert fields.length == 3;

      short forwardId = Short.parseShort(fields[0]);
      short backwardId = Short.parseShort(fields[1]);
      short cost = Short.parseShort(fields[2]);

      costs.add(forwardId, backwardId, cost);
    }
    return costs;
  }
Пример #2
0
  private void updateNode(ViterbiNode[] viterbiNodes, ViterbiNode node) {
    int backwardConnectionId = node.getLeftId();
    int wordCost = node.getWordCost();
    int leastPathCost = DEFAULT_COST;

    for (ViterbiNode leftNode : viterbiNodes) {
      // If array doesn't contain any more ViterbiNodes, continue to next index
      if (leftNode == null) {
        return;
      } else {
        // cost = [total cost from BOS to previous node] + [connection cost between previous node
        // and current node] + [word cost]
        int pathCost =
            leftNode.getPathCost()
                + costs.get(leftNode.getRightId(), backwardConnectionId)
                + wordCost;

        // Add extra cost for long nodes in "Search mode".
        if (mode == TokenizerBase.Mode.SEARCH || mode == TokenizerBase.Mode.EXTENDED) {
          pathCost += getPenaltyCost(node);
        }

        // If total cost is lower than before, set current previous node as best left node (previous
        // means left).
        if (pathCost < leastPathCost) {
          leastPathCost = pathCost;
          node.setPathCost(leastPathCost);
          node.setLeftNode(leftNode);
        }
      }
    }
  }