public DoubleMatrix getScoreMatrix(File file) {
Counter<String> docWords = new Counter<String>();
try {
LineIterator iter = FileUtils.lineIterator(file);
while (iter.hasNext()) {
Tokenizer t =
tokenizerFactory.create((new InputHomogenization(iter.nextLine()).transform()));
while (t.hasMoreTokens()) {
docWords.incrementCount(t.nextToken(), 1.0);
}
}
iter.close();
} catch (IOException e) {
throw new IllegalStateException("Unable to read file", e);
}
DoubleMatrix ret = new DoubleMatrix(1, currVocab.size());
for (int i = 0; i < currVocab.size(); i++) {
if (docWords.getCount(currVocab.get(i).toString()) > 0) {
ret.put(i, wordScores.getCount(currVocab.get(i).toString()));
}
}
return ret;
}
@Override
public double mse() {
DoubleMatrix reconstructed = reconstruct(input);
DoubleMatrix diff = reconstructed.sub(input);
double sum = 0.5 * MatrixFunctions.pow(diff, 2).columnSums().sum() / input.rows;
return sum;
}
@Override
public NeuralNetwork clone() {
try {
Constructor<?> c = Dl4jReflection.getEmptyConstructor(getClass());
c.setAccessible(true);
NeuralNetwork ret = (NeuralNetwork) c.newInstance();
ret.setMomentumAfter(momentumAfter);
ret.setResetAdaGradIterations(resetAdaGradIterations);
ret.setHbiasAdaGrad(hBiasAdaGrad);
ret.setVBiasAdaGrad(vBiasAdaGrad);
ret.sethBias(hBias.dup());
ret.setvBias(vBias.dup());
ret.setnHidden(getnHidden());
ret.setnVisible(getnVisible());
ret.setW(W.dup());
ret.setL2(l2);
ret.setMomentum(momentum);
ret.setRenderEpochs(getRenderIterations());
ret.setSparsity(sparsity);
ret.setRng(getRng());
ret.setDist(getDist());
ret.setAdaGrad(wAdaGrad);
ret.setLossFunction(lossFunction);
ret.setConstrainGradientToUnitNorm(constrainGradientToUnitNorm);
ret.setOptimizationAlgorithm(optimizationAlgo);
return ret;
} catch (Exception e) {
throw new RuntimeException(e);
}
}
public DoubleMatrix free_energy(DoubleMatrix v_sample) {
DoubleMatrix wv_hb =
weights.mmul(v_sample.transpose()).addi(this.hbias.repmat(v_sample.rows, 1).transpose());
DoubleMatrix vb = v_sample.mmul(this.vbias.transpose());
DoubleMatrix hi = MatrixMath.sum(MatrixMath.log(MatrixMath.exp(wv_hb).addi(1.0)), 1);
return hi.mul(-1.0).subi(vb);
}
public static void DFS(
DoubleMatrix A, DoubleMatrix d, DoubleMatrix g, ArrayList[] V, int vertNum) {
int krawedzie = 0;
for (int i = 0; i <= vertNum; i++) {
krawedzie += V[i].size();
}
krawedzie /= 2;
// System.out.println("A wymiar: " + A.columns + " rows: "+A.rows);
int t = 0;
VertexState state[] = new VertexState[(d.rows - 1) * d.rows / 2]; // tyle stanów
for (int i = 0; i < d.rows - 1; i++) {
for (int j = i + 1; j < d.columns; j++) {
for (int x = 0; x < state.length; x++) {
state[x] = VertexState.White; // na początku białe
}
g.put(t, 0, d.get(i, j));
runDFS(i, j, state, d, V, vertNum);
for (int ii = 0; ii < krawedzie; ii++) {
if (state[ii] == VertexState.Black) {
A.put(t, ii, 1);
}
}
t++;
}
}
System.out.println(g);
System.out.println("t:" + t);
return;
}
public static void main(String argv[]) {
modshogun.init_shogun_with_defaults();
double width = 1.4;
int size_cache = 10;
DoubleMatrix traindata_real = Load.load_numbers("../data/fm_train_real.dat");
DoubleMatrix testdata_real = Load.load_numbers("../data/fm_test_real.dat");
RealFeatures feats_train = new RealFeatures(traindata_real);
RealFeatures feats_test = new RealFeatures(testdata_real);
NormOne preproc = new NormOne();
preproc.init(feats_train);
feats_train.add_preprocessor(preproc);
feats_train.apply_preprocessor();
feats_test.add_preprocessor(preproc);
feats_test.apply_preprocessor();
Chi2Kernel kernel = new Chi2Kernel(feats_train, feats_train, width, size_cache);
DoubleMatrix km_train = kernel.get_kernel_matrix();
kernel.init(feats_train, feats_test);
DoubleMatrix km_test = kernel.get_kernel_matrix();
System.out.println(km_train.toString());
System.out.println(km_test.toString());
}
private static void modi(DoubleMatrix syndrome, int radix) {
for (int i = 0; i < syndrome.getRows(); i++) {
for (int j = 0; j < syndrome.getColumns(); j++) {
syndrome.put(i, j, ((int) syndrome.get(i, j)) % radix);
}
}
}
/** Compute y <- alpha*op(a)*x + beta * y (general matrix vector multiplication) */
public static DoubleMatrix gemv(
double alpha, DoubleMatrix a, DoubleMatrix x, double beta, DoubleMatrix y) {
if (false) {
NativeBlas.dgemv(
'N', a.rows, a.columns, alpha, a.data, 0, a.rows, x.data, 0, 1, beta, y.data, 0, 1);
} else {
if (beta == 0.0) {
for (int i = 0; i < y.length; i++) y.data[i] = 0.0;
for (int j = 0; j < a.columns; j++) {
double xj = x.get(j);
if (xj != 0.0) {
for (int i = 0; i < a.rows; i++) y.data[i] += a.get(i, j) * xj;
}
}
} else {
for (int j = 0; j < a.columns; j++) {
double byj = beta * y.data[j];
double xj = x.get(j);
for (int i = 0; i < a.rows; i++) y.data[j] = a.get(i, j) * xj + byj;
}
}
}
return y;
}
/**
* 对角线增加值
*
* @param matrix
* @throws Exception
*/
private void addPositiveValue(DoubleMatrix matrix) {
int m = matrix.rows;
if (matrix.columns != m) return;
for (int i = 0; i < m; i++) {
matrix.put(i, i, matrix.get(i, i) + Math.random());
}
}
protected void applyDropOutIfNecessary(DoubleMatrix input) {
if (dropOut > 0) {
this.doMask = DoubleMatrix.rand(input.rows, input.columns).gt(dropOut);
} else this.doMask = DoubleMatrix.ones(input.rows, input.columns);
// actually apply drop out
input.muli(doMask);
}
// Returns value of approximated function
public double approximateFunction(double x, final int NUMBER_OF_EQUATIONS) {
DoubleMatrix X = solveLinearEquation(NUMBER_OF_EQUATIONS);
double result = 0.0;
for (int i = 0; i < NUMBER_OF_EQUATIONS; i++) {
result += X.get(i) * Math.pow(x, i);
}
return result;
}
/**
* 求逆矩阵
*
* @param mt
* @return
*/
public DoubleMatrix inverse(DoubleMatrix mt) {
int m = mt.rows;
DoubleMatrix E = DoubleMatrix.zeros(m, m);
for (int i = 0; i < m; i++) {
E.put(i, i, 1);
}
return Solve.solve(mt, E);
}
public ClassifierTheta(int FeatureLength, int CatSize) {
CatSize--;
this.FeatureLength = FeatureLength;
this.CatSize = CatSize;
W = DoubleMatrix.rand(FeatureLength, CatSize).subi(0.5);
b = DoubleMatrix.rand(CatSize, 1).subi(0.5);
Theta = new double[FeatureLength * CatSize + CatSize];
flatten();
}
/**
* Initialize weights. This includes steps for doing a random initialization of W as well as the
* vbias and hbias
*/
protected void initWeights() {
if (this.nVisible < 1)
throw new IllegalStateException("Number of visible can not be less than 1");
if (this.nHidden < 1)
throw new IllegalStateException("Number of hidden can not be less than 1");
if (this.dist == null)
dist =
new NormalDistribution(rng, 0, .01, NormalDistribution.DEFAULT_INVERSE_ABSOLUTE_ACCURACY);
/*
* Initialize based on the number of visible units..
* The lower bound is called the fan in
* The outer bound is called the fan out.
*
* Below's advice works for Denoising AutoEncoders and other
* neural networks you will use due to the same baseline guiding principles for
* both RBMs and Denoising Autoencoders.
*
* Hinton's Guide to practical RBMs:
* The weights are typically initialized to small random values chosen from a zero-mean Gaussian with
* a standard deviation of about 0.01. Using larger random values can speed the initial learning, but
* it may lead to a slightly worse final model. Care should be taken to ensure that the initial weight
* values do not allow typical visible vectors to drive the hidden unit probabilities very close to 1 or 0
* as this significantly slows the learning.
*/
if (this.W == null) {
this.W = DoubleMatrix.zeros(nVisible, nHidden);
for (int i = 0; i < this.W.rows; i++)
this.W.putRow(i, new DoubleMatrix(dist.sample(this.W.columns)));
}
this.wAdaGrad = new AdaGrad(this.W.rows, this.W.columns);
if (this.hBias == null) {
this.hBias = DoubleMatrix.zeros(nHidden);
/*
* Encourage sparsity.
* See Hinton's Practical guide to RBMs
*/
// this.hBias.subi(4);
}
this.hBiasAdaGrad = new AdaGrad(hBias.rows, hBias.columns);
if (this.vBias == null) {
if (this.input != null) {
this.vBias = DoubleMatrix.zeros(nVisible);
} else this.vBias = DoubleMatrix.zeros(nVisible);
}
this.vBiasAdaGrad = new AdaGrad(vBias.rows, vBias.columns);
}
public void weight_contribution(
DoubleMatrix h0, DoubleMatrix v0, DoubleMatrix h1, DoubleMatrix v1) {
this.w1
.addi(h0.transpose().mmul(v0).subi(h1.transpose().mmul(v1)))
.muli(learningRate / v_data.rows)
.subi(weights.mul(weightCost));
this.vb1.addi(v0.sub(v1).muli(learningRate / v_data.rows).columnMeans());
this.hb1.addi(h0.sub(h1).muli(learningRate / v_data.rows).columnMeans());
}
@Override
public double squaredLoss() {
DoubleMatrix squaredDiff = pow(reconstruct(input).sub(input), 2);
double loss = squaredDiff.columnSums().sum() / input.rows;
if (this.useRegularization) {
loss += 0.5 * l2 * MatrixFunctions.pow(W, 2).sum();
}
return loss;
}
public static void main(String argv[]) {
modshogun.init_shogun_with_defaults();
int dim = 7;
DoubleMatrix data = DoubleMatrix.rand(dim, dim);
RealFeatures feats = new RealFeatures(data);
DoubleMatrix data_T = data.transpose();
DoubleMatrix symdata = data.add(data_T);
int cols = (1 + dim) * dim / 2;
DoubleMatrix lowertriangle = DoubleMatrix.zeros(1, cols);
int count = 0;
for (int i = 0; i < dim; i++) {
for (int j = 0; j < dim; j++) {
if (j <= i) {
lowertriangle.put(0, count++, symdata.get(i, j));
}
}
}
CustomKernel kernel = new CustomKernel();
kernel.set_triangle_kernel_matrix_from_triangle(lowertriangle);
DoubleMatrix km_triangletriangle = kernel.get_kernel_matrix();
kernel.set_triangle_kernel_matrix_from_full(symdata);
DoubleMatrix km_fulltriangle = kernel.get_kernel_matrix();
kernel.set_full_kernel_matrix_from_full(data);
DoubleMatrix km_fullfull = kernel.get_kernel_matrix();
modshogun.exit_shogun();
}
/**
* Negative log likelihood of the current input given the corruption level
*
* @return the negative log likelihood of the auto encoder given the corruption level
*/
public double negativeLoglikelihood(DoubleMatrix input) {
DoubleMatrix z = this.reconstruct(input);
if (this.useRegularization) {
double reg = (2 / l2) * MatrixFunctions.pow(this.W, 2).sum();
return -input.mul(log(z)).add(oneMinus(input).mul(log(oneMinus(z)))).columnSums().mean()
+ reg;
}
return -input.mul(log(z)).add(oneMinus(input).mul(log(oneMinus(z)))).columnSums().mean();
}
/**
* Transforms the matrix
*
* @param text
* @return
*/
@Override
public DoubleMatrix transform(String text) {
Tokenizer tokenizer = tokenizerFactory.create(text);
List<String> tokens = tokenizer.getTokens();
DoubleMatrix input = new DoubleMatrix(1, vocab.size());
for (int i = 0; i < tokens.size(); i++) {
int idx = vocab.indexOf(tokens.get(i));
if (vocab.indexOf(tokens.get(i)) >= 0) input.put(idx, wordCounts.getCount(tokens.get(i)));
}
return input;
}
@Test
public void extractFeature_simpleExpression_correctAnswer() {
// Arrange
setCustomFeatureExpression(" 8 + 14 ");
FormattedSegments segments = getFormattedSegments();
// Act
DoubleMatrix features = featureExtractor.extractFeatures(segments);
// Assert
assertEquals(22, features.get(0, 0), errorMargin);
}
/**
* 求广义逆矩阵,使用正交投影方法
*
* @param ht
* @return
*/
public DoubleMatrix getMPMatrixByOP(DoubleMatrix H) {
DoubleMatrix M;
try {
M = H.mmul(H.transpose()); // HHt
addPositiveValue(M);
return H.transpose().mmul(inverse(M));
} catch (LapackException e) {
M = H.transpose().mmul(H); // HtH
addPositiveValue(M);
return inverse(M).mmul(H.transpose());
}
}
/**
* Vectorizes the passed in text treating it as one document
*
* @param text the text to vectorize
* @param label the label of the text
* @return a dataset with a applyTransformToDestination of weights(relative to impl; could be word
* counts or tfidf scores)
*/
@Override
public DataSet vectorize(String text, String label) {
Tokenizer tokenizer = tokenizerFactory.create(text);
List<String> tokens = tokenizer.getTokens();
DoubleMatrix input = new DoubleMatrix(1, vocab.size());
for (int i = 0; i < tokens.size(); i++) {
int idx = vocab.indexOf(tokens.get(i));
if (vocab.indexOf(tokens.get(i)) >= 0) input.put(idx, wordCounts.getCount(tokens.get(i)));
}
DoubleMatrix labelMatrix = MatrixUtil.toOutcomeVector(labels.indexOf(label), labels.size());
return new DataSet(input, labelMatrix);
}
@Test
public void extractFeature_gpsSpeedVariableExpression_correctAnswer() {
// Arrange
setCustomFeatureExpression(" 8+ speed1 ");
double[][] gpsData = new double[][] {{0.5}};
FormattedSegments segments = getFormattedSegments(gpsData);
// Act
DoubleMatrix features = featureExtractor.extractFeatures(segments);
// Assert
assertEquals(8.5, features.get(0, 0), errorMargin);
}
public static void main(String[] args) throws IOException {
try (BufferedReader reader = new BufferedReader(new FileReader("matrix_test"))) {
// read table from file
List<String[]> rows =
reader.lines().map(line -> line.split(" ")).collect(Collectors.toList());
double[][] values = new double[rows.size()][rows.get(0).length];
for (int i = 0; i < values.length; i++) {
String[] row = rows.get(i);
for (int j = 0; j < values[0].length; j++) {
values[i][j] = Double.parseDouble(row[j]);
}
}
DoubleMatrix matrix = new DoubleMatrix(values);
Vector coordVector = new Vector(matrix.getRows()); // 000..00 (n-times)
while (coordVector.hasNextVector()) {
coordVector.nextVector(); // get next vector (eg 000..00 -> 000..01)
DoubleMatrix coord = coordVector.row();
DoubleMatrix codeWord = coord.mmul(matrix); // m * G
modi(codeWord, 2);
String prefix = "0 0 0 0 1 ";
String word = MatrixUtils.toString(codeWord);
if (word.startsWith(prefix)) {
System.out.println(word);
}
}
/*// find syndromes
Syndrome syndrome = new Syndrome(new DoubleMatrix(values));
syndrome.build();
// generate xls file with table
HSSFWorkbook wb = new HSSFWorkbook();
HSSFSheet sheet = wb.createSheet("Syndrome table");
int rowNumber = 0;
Vector syndromeVector = new Vector(values.length);
putValue(sheet, rowNumber++, syndromeVector.toString(),
syndrome.getErrorVector(syndromeVector.toString()));
while (syndromeVector.hasNextVector()) {
syndromeVector.nextVector();
putValue(sheet, rowNumber++, syndromeVector.toString(),
syndrome.getErrorVector(syndromeVector.toString()));
}
try (FileOutputStream fileOut = new FileOutputStream("syndrome_table.xls")) {
wb.write(fileOut);
}*/
}
}
@Test
public void extractFeature_multipleSegmentsExpression_2ndHasCorrectAnswer() {
// Arrange
setCustomFeatureExpression(" 8+ speed1");
double[][] gpsData = new double[][] {{0.5}, {11.0}};
double[][] xData = new double[][] {{0}, {0}};
FormattedSegments segments = getFormattedSegments(xData, xData, xData, gpsData);
// Act
DoubleMatrix features = featureExtractor.extractFeatures(segments);
// Assert
assertEquals(19, features.get(1, 0), errorMargin);
}
private void train(CharText ctext, double lr) {
Map<Integer, String> indexChar = ctext.getIndexChar();
Map<String, DoubleMatrix> charVector = ctext.getCharVector();
List<String> sequence = ctext.getSequence();
for (int i = 0; i < 100; i++) {
double error = 0;
double num = 0;
double start = System.currentTimeMillis();
for (int s = 0; s < sequence.size(); s++) {
String seq = sequence.get(s);
if (seq.length() < 3) {
continue;
}
Map<String, DoubleMatrix> acts = new HashMap<>();
// forward pass
System.out.print(String.valueOf(seq.charAt(0)));
for (int t = 0; t < seq.length() - 1; t++) {
DoubleMatrix xt = charVector.get(String.valueOf(seq.charAt(t)));
acts.put("x" + t, xt);
gru.active(t, acts);
DoubleMatrix predcitYt = gru.decode(acts.get("h" + t));
acts.put("py" + t, predcitYt);
DoubleMatrix trueYt = charVector.get(String.valueOf(seq.charAt(t + 1)));
acts.put("y" + t, trueYt);
System.out.print(indexChar.get(predcitYt.argmax()));
error += LossFunction.getMeanCategoricalCrossEntropy(predcitYt, trueYt);
}
System.out.println();
// bptt
gru.bptt(acts, seq.length() - 2, lr);
num += seq.length();
}
System.out.println(
"Iter = "
+ i
+ ", error = "
+ error / num
+ ", time = "
+ (System.currentTimeMillis() - start) / 1000
+ "s");
}
}
/**
* Applies sparsity to the passed in hbias gradient
*
* @param hBiasGradient the hbias gradient to apply to
* @param learningRate the learning rate used
*/
protected void applySparsity(DoubleMatrix hBiasGradient, double learningRate) {
if (useAdaGrad) {
DoubleMatrix change =
this.hBiasAdaGrad
.getLearningRates(hBias)
.neg()
.mul(sparsity)
.mul(hBiasGradient.mul(sparsity));
hBiasGradient.addi(change);
} else {
DoubleMatrix change = hBiasGradient.mul(sparsity).mul(-learningRate * sparsity);
hBiasGradient.addi(change);
}
}
/**
* Get Result String
*
* @return result string
*/
public String printComparableResults(boolean newline) {
if (!(resultVector != null)) log.error("Attempted to access null vector.");
String outString = "";
int i = 0;
for (TypeQ question : sortedQuestions) {
Double tempResult = resultVector.get(i);
// if(tempResult.intValue() == 0)
// continue;
if (groundTruthVector != null) {
Double evalInd = groundTruthVector.getSecond().get(i);
if (evalInd.intValue() == 0) {
++i;
continue;
}
}
++i;
if (!newline) outString += "\n";
outString += question + " " + tempResult.intValue();
newline = false;
}
if (log.isDebugEnabled()) {
log.debug(outString);
}
return outString;
}
public ClassifierTheta(double[] t, int FeatureLength, int CatSize) {
CatSize--;
Theta = t.clone();
this.FeatureLength = FeatureLength;
this.CatSize = CatSize;
W = DoubleMatrix.zeros(FeatureLength, CatSize);
b = DoubleMatrix.zeros(CatSize, 1);
if (Theta.length != CatSize * (1 + FeatureLength))
System.err.println(
"ClassifierTheta : Size Mismatch : "
+ Theta.length
+ " != "
+ (CatSize * (1 + FeatureLength)));
build();
}
@Test
public void extractFeatures_3points_correctResult() {
// Arrange
DoubleMatrix x, y, z, gpsSpeed;
x = new DoubleMatrix(new double[][] {{1, 1, 1}});
y = new DoubleMatrix(new double[][] {{1, 1, 1}});
z = new DoubleMatrix(new double[][] {{7, 3, 5}});
gpsSpeed = new DoubleMatrix(x.rows, 1);
FormattedSegments input = createFormattedSegments(x, y, z, gpsSpeed);
// Act
DoubleMatrix features = featureExtractor.extractFeatures(input);
// Assert
assertEquals(3, features.get(0, 0), errorMargin);
}