/** * Ester2 version. * * @param parameter the parameter * @throws DiarizationException the diarization exception * @throws Exception the exception */ public void ester2Version(Parameter parameter) throws DiarizationException, Exception { // ** Caution this system is developed using Sphinx MFCC computed with legacy mode ClusterSet referenceClusterSet = null; if (!parameter.getParameterSegmentationInputFile2().getMask().equals("")) { referenceClusterSet = MainTools.readTheSecondClusterSet(parameter); } ClusterSet uemClusterSet = null; if (!parameter.getParameterSegmentationInputFile3().getMask().equals("")) { referenceClusterSet = MainTools.readThe3rdClusterSet(parameter); } ParameterBNDiarization parameterDiarization = parameter.getParameterDiarization(); // ** mask for the output of the segmentation file ClusterSet clusterSet = initialize(parameter); // ** load the features, sphinx format (13 MFCC with C0) or compute it form a wave file AudioFeatureSet featureSet = loadFeature( parameter, clusterSet, parameter.getParameterInputFeature().getFeaturesDescriptorAsString()); featureSet.setCurrentShow(parameter.show); int nbFeatures = featureSet.getNumberOfFeatures(); if (parameter.getParameterDiarization().isLoadInputSegmentation() == false) { clusterSet.getFirstCluster().firstSegment().setLength(nbFeatures); } // clusterSet.debug(3); ClusterSet clustersSegInit = sanityCheck(clusterSet, featureSet, parameter); ClusterSet clustersSeg = segmentation("GLR", "FULL", clustersSegInit, featureSet, parameter); // Seg IRIT // ClusterSet clustersSegInit = clusterSet; // ClusterSet clustersSeg = clusterSet; // Seg IRIT ClusterSet clustersLClust = clusteringLinear( parameterDiarization.getThreshold("l"), clustersSeg, featureSet, parameter); ClusterSet clustersHClust = clustering(parameterDiarization.getThreshold("h"), clustersLClust, featureSet, parameter); // MainTools.writeClusterSet(parameter, clustersHClust, false); ClusterSet clustersDClust = decode(8, parameterDiarization.getThreshold("d"), clustersHClust, featureSet, parameter); ClusterSet clustersSplitClust = speech("10,10,50", clusterSet, clustersSegInit, clustersDClust, featureSet, parameter); ClusterSet clustersGender = gender(clusterSet, clustersSplitClust, featureSet, parameter); if (parameter.getParameterDiarization().isCEClustering()) { ClusterSet clustersCLR = speakerClustering( parameterDiarization.getThreshold("c"), "ce", clustersSegInit, clustersGender, featureSet, parameter); MainTools.writeClusterSet(parameter, clustersCLR, false); if (referenceClusterSet != null) { DiarizationError computeError = new DiarizationError(referenceClusterSet, uemClusterSet); computeError.scoreOfMatchedSpeakers(clustersCLR); } } else { MainTools.writeClusterSet(parameter, clustersGender, false); } }
/** * Tun ester2 diarization. * * @param parameter the parameter * @param clusterSet the cluster set * @return the tree map * @throws DiarizationException the diarization exception * @throws Exception the exception */ public TreeMap<String, DiarizationResultList> tunEster2Diarization( Parameter parameter, ClusterSet clusterSet) throws DiarizationException, Exception { TreeMap<String, DiarizationResultList> result = new TreeMap<String, DiarizationResultList>(); // double paramThr = parameter.getParameterClustering().getThreshold(); lMin = parameter.getParameterDiarization().getThreshold("l"); lMax = parameter.getParameterDiarization().getMaxThreshold("l"); hMin = parameter.getParameterDiarization().getThreshold("h"); hMax = parameter.getParameterDiarization().getMaxThreshold("h"); dMin = parameter.getParameterDiarization().getThreshold("d"); dMax = parameter.getParameterDiarization().getMaxThreshold("d"); cMin = parameter.getParameterDiarization().getThreshold("c"); cMax = parameter.getParameterDiarization().getMaxThreshold("c"); String featureDesc = parameter.getParameterInputFeature().getFeaturesDescriptorAsString(); AudioFeatureSet featureSet = null; ClusterSet clustersSegInit = null; if (parameter.getParameterDiarization().isLoadInputSegmentation() == false) { featureSet = loadFeature(parameter, clusterSet, featureDesc); featureSet.setCurrentShow(parameter.show); int nbFeatures = featureSet.getNumberOfFeatures(); clusterSet.getFirstCluster().firstSegment().setLength(nbFeatures); clustersSegInit = sanityCheck(clusterSet, featureSet, parameter); } else { featureSet = loadFeature(parameter, clusterSet, featureDesc); featureSet.setCurrentShow(parameter.show); clustersSegInit = sanityCheck(clusterSet, featureSet, parameter); featureSet = loadFeature(parameter, clustersSegInit, featureDesc); featureSet.setCurrentShow(parameter.show); } // seg IRIT // ClusterSet clustersSegSave = clustersSegInit; // seg IRIT ClusterSet referenceClusterSet = MainTools.readTheSecondClusterSet(parameter); ClusterSet uemClusterSet = MainTools.readThe3rdClusterSet(parameter); if (parameter.getParameterDiarization().isLastStepOnly()) { String key = "l=" + lMin + " h=" + hMin + " d=" + dMin; DiarizationResultList values = null; if (parameter.getParameterDiarization().isCEClustering() == false) { logger.warning(" nothing to do isCEClustering == false"); } else { values = tunEster2SpeakerCLRClustering( referenceClusterSet, uemClusterSet, key, "ce", clusterSet, clusterSet, featureSet, parameter); } result.put(key, values); return result; } ClusterSet clustersSegSave = segmentation("GLR", "FULL", clustersSegInit, featureSet, parameter); for (double l = lMin; l <= lMax; l += 0.5) { ClusterSet clustersSeg = clustersSegSave.clone(); logger.finest("clustering l=" + l); ClusterSet clustersLClust = clusteringLinear(l, clustersSeg, featureSet, parameter); // ---- Begin NEW v 1.14 --- for (double h = hMin; h <= hMax; h += 0.5) { // for (double h = hMin; h <= hMax; h += 0.2) { // ---- end NEW v 1.14 --- // if (h > l) { ClusterSet clustersHClust = clustering(h, clustersLClust, featureSet, parameter); for (double d = dMin; d <= dMax; d += 50) { ClusterSet clustersDClust = decode(8, d, clustersHClust, featureSet, parameter); // double error = DiarizationError.scoreOfMatchedSpeakers(referenceClusterSet, // clustersDClust); ClusterSet clustersSplitClust = speech( "10,10,50", clusterSet, clustersSegInit, clustersDClust, featureSet, parameter); ClusterSet clustersGender = gender(clusterSet, clustersSplitClust, featureSet, parameter); String key = "l=" + l + " h=" + h + " d=" + d; DiarizationResultList values = null; if (parameter.getParameterDiarization().isCEClustering() == false) { values = new DiarizationResultList(0, 0, 1); DiarizationError computeError = new DiarizationError(referenceClusterSet, uemClusterSet); DiarizationResult error = computeError.scoreOfMatchedSpeakers(clustersGender); values.setResult(0, 0, error); logger.finer(parameter.show + " key=" + key + " resultat du fichier"); values.log("partial result: " + parameter.show + " " + key); } else { // V4.19 = CLUST_H_BIC_GMM_MAP // values = tunEster2SpeakerCLRClustering(referenceClusterSet, key, "bicgmmmap", // clustersGender, clustersGender, featureSet, parameter); // V5.16 = ce_d // values = tunEster2SpeakerCLRClustering(referenceClusterSet, key, "ce_d", // clustersGender, clustersGender, featureSet, parameter); values = tunEster2SpeakerCLRClustering( referenceClusterSet, uemClusterSet, key, "ce", clustersGender, clustersGender, featureSet, parameter); } if (result.containsKey(key)) { result.get(key).addResultArray(values); } else { result.put(key, values); } } // } } } return result; }
public ClusterSet make( AudioFeatureSet featureSet, ClusterSet clusterSet, GMMArrayList gmmList, GMMArrayList gmmTopList, Parameter parameter) throws DiarizationException, IOException { logger.info("Compute Score"); int size = gmmList.size(); logger.finer("GMM size:" + size); ArrayList<String> genderString = new ArrayList<String>(); ArrayList<String> bandwidthString = new ArrayList<String>(); for (int i = 0; i < size; i++) { String gmmName = gmmList.get(i).getName(); if (parameter.getParameterScore().isGender() == true) { if (gmmName.equals("MS")) { genderString.add(Cluster.genderStrings[1]); bandwidthString.add(Segment.bandwidthStrings[2]); } else if (gmmName.equals("FS")) { genderString.add(Cluster.genderStrings[2]); bandwidthString.add(Segment.bandwidthStrings[2]); } else if (gmmName.equals("MT")) { genderString.add(Cluster.genderStrings[1]); bandwidthString.add(Segment.bandwidthStrings[1]); } else if (gmmName.equals("FT")) { genderString.add(Cluster.genderStrings[2]); bandwidthString.add(Segment.bandwidthStrings[1]); } else { genderString.add(Cluster.genderStrings[0]); bandwidthString.add(Segment.bandwidthStrings[0]); } } else { genderString.add(Cluster.genderStrings[0]); bandwidthString.add(Segment.bandwidthStrings[0]); } } ClusterSet clusterSetResult = new ClusterSet(); for (Cluster cluster : clusterSet.clusterSetValue()) { double[] sumScoreVector = new double[size]; int[] sumLenghtVector = new int[size]; double ubmScore = 0.0; GMM gmmTop = null; if (parameter.getParameterTopGaussian().getScoreNTop() >= 0) { gmmTop = gmmTopList.get(0); } Arrays.fill(sumScoreVector, 0.0); Arrays.fill(sumLenghtVector, 0); for (Segment currantSegment : cluster) { Segment segment = (currantSegment.clone()); int end = segment.getStart() + segment.getLength(); featureSet.setCurrentShow(segment.getShowName()); double[] scoreVector = new double[size]; double maxScore = 0.0; int idxMaxScore = 0; for (int i = 0; i < size; i++) { gmmList.get(i).score_initialize(); } for (int start = segment.getStart(); start < end; start++) { for (int i = 0; i < size; i++) { GMM gmm = gmmList.get(i); if (parameter.getParameterTopGaussian().getScoreNTop() >= 0) { if (i == 0) { gmmTop.score_getAndAccumulateAndFindTopComponents( featureSet, start, parameter.getParameterTopGaussian().getScoreNTop()); } gmm.score_getAndAccumulateForComponentSubset( featureSet, start, gmmTop.getTopGaussianVector()); } else { gmm.score_getAndAccumulate(featureSet, start); } } } if (parameter.getParameterTopGaussian().getScoreNTop() >= 0) { ubmScore = gmmTop.score_getMeanLog(); gmmTop.score_getSumLog(); gmmTop.score_getCount(); gmmTop.score_reset(); } for (int i = 0; i < size; i++) { GMM gmm = gmmList.get(i); scoreVector[i] = gmm.score_getMeanLog(); sumLenghtVector[i] += gmm.score_getCount(); sumScoreVector[i] += gmm.score_getSumLog(); if (i == 0) { maxScore = scoreVector[0]; idxMaxScore = 0; } else { double value = scoreVector[i]; if (maxScore < value) { maxScore = value; idxMaxScore = i; } } gmm.score_reset(); } if (parameter.getParameterScore().isTNorm()) { double sumScore = 0; double sum2Score = 0; for (int i = 0; i < size; i++) { sumScore += scoreVector[i]; sum2Score += (scoreVector[i] * scoreVector[i]); } for (int i = 0; i < size; i++) { double value = scoreVector[i]; double mean = (sumScore - value) / (size - 1); double et = Math.sqrt(((sum2Score - (value * value)) / (size - 1)) - (mean * mean)); scoreVector[i] = (value - mean) / et; } } if (parameter.getParameterScore().isGender() == true) { segment.setBandwidth(bandwidthString.get(idxMaxScore)); segment.setInformation("segmentGender", genderString.get(idxMaxScore)); } if (parameter.getParameterScore().isBySegment()) { for (int k = 0; k < size; k++) { double score = scoreVector[k]; GMM gmm = gmmList.get(k); segment.setInformation("score:" + gmm.getName(), score); currantSegment.setInformation("score:" + gmm.getName(), score); } if (parameter.getParameterTopGaussian().getScoreNTop() >= 0) { segment.setInformation("score:" + "UBM", ubmScore); currantSegment.setInformation("score:" + "UBM", ubmScore); } } String newName = cluster.getName(); if (parameter.getParameterScore().isByCluster() == false) { if ((scoreVector[idxMaxScore] > parameter.getParameterSegmentation().getThreshold()) && (parameter.getParameterScore().getLabel() != ParameterScore.LabelType.LABEL_TYPE_NONE.ordinal())) { if (parameter.getParameterScore().getLabel() == ParameterScore.LabelType.LABEL_TYPE_ADD.ordinal()) { newName += "_"; newName += gmmList.get(idxMaxScore).getName(); } else { newName = gmmList.get(idxMaxScore).getName(); } } Cluster temporaryCluster = clusterSetResult.getOrCreateANewCluster(newName); temporaryCluster.setGender(cluster.getGender()); if (parameter.getParameterScore().isGender() == true) { temporaryCluster.setGender(genderString.get(idxMaxScore)); } temporaryCluster.addSegment(segment); } } if (parameter.getParameterScore().isByCluster()) { for (int i = 0; i < size; i++) { sumScoreVector[i] /= sumLenghtVector[i]; } if (parameter.getParameterScore().isTNorm()) { double sumScore = 0; double sum2Score = 0; for (int i = 0; i < size; i++) { sumScore += sumScoreVector[i]; sum2Score += (sumScoreVector[i] * sumScoreVector[i]); } for (int i = 0; i < size; i++) { double value = sumScoreVector[i]; double mean = (sumScore - value) / (size - 1); double et = Math.sqrt(((sum2Score - (value * value)) / (size - 1)) - (mean * mean)); sumScoreVector[i] = (value - mean) / et; } } double maxScore = sumScoreVector[0]; int idxMaxScore = 0; for (int i = 1; i < size; i++) { double s = sumScoreVector[i]; if (maxScore < s) { maxScore = s; idxMaxScore = i; } } String newName = cluster.getName(); if ((sumScoreVector[idxMaxScore] > parameter.getParameterSegmentation().getThreshold()) && (parameter.getParameterScore().getLabel() != ParameterScore.LabelType.LABEL_TYPE_NONE.ordinal())) { if (parameter.getParameterScore().getLabel() == ParameterScore.LabelType.LABEL_TYPE_ADD.ordinal()) { newName += "_"; newName += gmmList.get(idxMaxScore).getName(); } else { newName = gmmList.get(idxMaxScore).getName(); } // logger.finer("cluster name=" + cluster.getName() + " new_name=" + newName); } Cluster tempororaryCluster = clusterSetResult.getOrCreateANewCluster(newName); tempororaryCluster.setGender(cluster.getGender()); if (parameter.getParameterScore().isGender() == true) { tempororaryCluster.setGender(genderString.get(idxMaxScore)); } tempororaryCluster.setName(newName); for (Segment currantSegment : cluster) { Segment segment = (currantSegment.clone()); if (parameter.getParameterScore().isGender() == true) { segment.setBandwidth(bandwidthString.get(idxMaxScore)); } tempororaryCluster.addSegment(segment); } for (int k = 0; k < size; k++) { double score = sumScoreVector[k]; GMM gmm = gmmList.get(k); // logger.finer("****clustername = " + newName + " name=" + gmm.getName() + " =" + score+" // k="+k); // logger.log(Level.SEVERE, "****clustername = " + newName + " name=" + gmm.getName() + " // =" + score); tempororaryCluster.setInformation("score:" + gmm.getName(), score); ClusterResult cr = new ClusterResult(); cr.setName(newName); cr.getValue().put(score, gmm.getName()); System.out.println( "------ clusterResultSet.putValue(newName, gmm.getName(), score)=----------------"); System.out.println(newName + " " + gmm.getName() + " " + score); if (isName(gmm.getName())) { clusterResultSet.putValue(newName, gmm.getName(), score); } else { System.out.println("*****************" + gmm.getName() + " Non nome valido "); } } if (parameter.getParameterTopGaussian().getScoreNTop() >= 0) { // tempororaryCluster.putInformation("score:" + "length", ubmSumLen); // tempororaryCluster.putInformation("score:" + "UBM", ubmSumScore / ubmSumLen); } } } this.clusterSetResult = clusterSetResult; return clusterSetResult; }