private void computeTotalGradient( Gradients totalGradients, Gradients partialGradients, SetOfIOPairs trainingSet) { // na zaciatku sa inicializuju gradienty (total) totalGradients.resetGradients(); // partialGradients.resetGradients(); // Gradients totalGradients = new Gradients(this); // Gradients partialGradients = new Gradients(this); /***/ for (SetOfIOPairs.IOPair pair : trainingSet.pairs) { // pre kazdy par trenovacej mnoziny // partialGradients = computeGradient(pair.inputs, pair.outputs); computeGradient(partialGradients, pair.inputs, pair.outputs); for (int il = this.numberOfLayers() - 1; il >= 1; il--) { // pre vsetky vrstvy okrem poslednej NeuralLayer currentLayer = this.getLayer(il); for (int in = 0; in < currentLayer.numberOfNeurons(); in++) { // pre vsetky neurony na currentLayer // upravime gradient prahov : totalGradients.incrementThreshold(il, in, partialGradients.getThreshold(il, in)); for (int ii = 0; ii < currentLayer.lowerLayer().numberOfNeurons(); ii++) { // pre vsetky vstupy totalGradients.incrementWeight(il, in, ii, partialGradients.getWeight(il, in, ii)); } } } // end for layer } // end foreach // return totalGradients; } // end method
public void connectLayers(NeuralLayer connectFrom, NeuralLayer connectTo) { for (Neuron layer1Neuron : connectFrom.getNeurons()) { for (Neuron layer2Neuron : connectTo.getNeurons()) { layer2Neuron.addInputConnection(layer1Neuron, 0.0); } } }
private void setBiasNeurons(int startIndex) throws DuplicateNeuronID_Exception { InputNeuron biasNeuron; for (NeuralLayer layer : hiddenLayers) { biasNeuron = new InputNeuron(startIndex); layer.setBiasNeuron(biasNeuron); mapNeuronToID(biasNeuron); startIndex++; } biasNeuron = new InputNeuron(startIndex); outputLayer.setBiasNeuron(biasNeuron); mapNeuronToID(biasNeuron); }
private void updateAllNeuronsMap() throws DuplicateNeuronID_Exception { allNeurons.clear(); for (Neuron neuron : inputLayer.getNeurons()) { mapNeuronToID(neuron); } for (NeuralLayer layer : hiddenLayers) { for (Neuron neuron : layer.getNeurons()) { mapNeuronToID(neuron); } } for (Neuron neuron : outputLayer.getNeurons()) { mapNeuronToID(neuron); } }
public void update() throws DuplicateNeuronID_Exception { updateAllNeuronsMap(); for (Neuron neuron : inputLayer.getNeurons()) { neuron.update(); } for (NeuralLayer layer : hiddenLayers) { for (Neuron neuron : layer.getNeurons()) { neuron.update(); } } for (Neuron neuron : outputLayer.getNeurons()) { neuron.update(); } }
public List<Double> getOutputs() { List<Double> output = new ArrayList<Double>(); for (Neuron neuron : outputLayer.getNeurons()) { output.add(neuron.getOutputValue()); } return output; }
public NeuralNetwork( int inputLayerSize, int outputLayerSize, int numberOfhidden, int hiddenLayerSizes) { try { int count = 0; inputLayer = new NeuralLayer(0); for (int i = 0; i < inputLayerSize; i++) { InputNeuron neuron = new InputNeuron(count); inputLayer.getNeurons().add(neuron); mapNeuronToID(neuron); count++; } hiddenLayers = new ArrayList<NeuralLayer>(); for (int i = 0; i < numberOfhidden; i++) { NeuralLayer layer = new NeuralLayer(1 + i); for (int j = 0; j < hiddenLayerSizes; j++) { SigmoidNeuron neuron = new SigmoidNeuron(count); layer.getNeurons().add(neuron); mapNeuronToID(neuron); count++; } hiddenLayers.add(layer); } outputLayer = new NeuralLayer(numberOfhidden); for (int i = 0; i < outputLayerSize; i++) { SigmoidNeuron neuron = new SigmoidNeuron(count); outputLayer.getNeurons().add(neuron); mapNeuronToID(neuron); count++; } setBiasNeurons(count); if (hiddenLayers.size() == 0) { connectLayers(inputLayer, outputLayer); } else { connectAllLayers(null); } updateAllNeuronsMap(); } catch (DuplicateNeuronID_Exception e) { e.printStackTrace(); } }
public NeuralNetwork(NeuralLayer in, List<NeuralLayer> hidden, NeuralLayer out) throws DuplicateNeuronID_Exception { inputLayer = in; outputLayer = out; hiddenLayers = hidden; for (Neuron neuron : inputLayer.getNeurons()) { mapNeuronToID(neuron); } for (NeuralLayer layer : hiddenLayers) { for (Neuron neuron : layer.getNeurons()) { mapNeuronToID(neuron); } } for (Neuron neuron : outputLayer.getNeurons()) { mapNeuronToID(neuron); } connectAllLayers(null); update(); }
public static PMML generateSimpleNeuralNetwork( String modelName, String[] inputfieldNames, String[] outputfieldNames, double[] inputMeans, double[] inputStds, double[] outputMeans, double[] outputStds, int hiddenSize, double[] weights) { int counter = 0; int wtsIndex = 0; PMML pmml = new PMML(); pmml.setVersion("4.0"); Header header = new Header(); Application app = new Application(); app.setName("Drools PMML Generator"); app.setVersion("0.01 Alpha"); header.setApplication(app); header.setCopyright("BSD"); header.setDescription(" Smart Vent Model "); Timestamp ts = new Timestamp(); ts.getContent().add(new java.util.Date().toString()); header.setTimestamp(ts); pmml.setHeader(header); DataDictionary dic = new DataDictionary(); dic.setNumberOfFields(BigInteger.valueOf(inputfieldNames.length + outputfieldNames.length)); for (String ifld : inputfieldNames) { DataField dataField = new DataField(); dataField.setName(ifld); dataField.setDataType(DATATYPE.DOUBLE); dataField.setDisplayName(ifld); dataField.setOptype(OPTYPE.CONTINUOUS); dic.getDataFields().add(dataField); } for (String ofld : outputfieldNames) { DataField dataField = new DataField(); dataField.setName(ofld); dataField.setDataType(DATATYPE.DOUBLE); dataField.setDisplayName(ofld); dataField.setOptype(OPTYPE.CONTINUOUS); dic.getDataFields().add(dataField); } pmml.setDataDictionary(dic); NeuralNetwork nnet = new NeuralNetwork(); nnet.setActivationFunction(ACTIVATIONFUNCTION.LOGISTIC); nnet.setFunctionName(MININGFUNCTION.REGRESSION); nnet.setNormalizationMethod(NNNORMALIZATIONMETHOD.NONE); nnet.setModelName(modelName); MiningSchema miningSchema = new MiningSchema(); for (String ifld : inputfieldNames) { MiningField mfld = new MiningField(); mfld.setName(ifld); mfld.setOptype(OPTYPE.CONTINUOUS); mfld.setUsageType(FIELDUSAGETYPE.ACTIVE); miningSchema.getMiningFields().add(mfld); } for (String ofld : outputfieldNames) { MiningField mfld = new MiningField(); mfld.setName(ofld); mfld.setOptype(OPTYPE.CONTINUOUS); mfld.setUsageType(FIELDUSAGETYPE.PREDICTED); miningSchema.getMiningFields().add(mfld); } nnet.getExtensionsAndNeuralLayersAndNeuralInputs().add(miningSchema); Output outputs = new Output(); for (String ofld : outputfieldNames) { OutputField outFld = new OutputField(); outFld.setName("Out_" + ofld); outFld.setTargetField(ofld); outputs.getOutputFields().add(outFld); } nnet.getExtensionsAndNeuralLayersAndNeuralInputs().add(outputs); NeuralInputs nins = new NeuralInputs(); nins.setNumberOfInputs(BigInteger.valueOf(inputfieldNames.length)); for (int j = 0; j < inputfieldNames.length; j++) { String ifld = inputfieldNames[j]; NeuralInput nin = new NeuralInput(); nin.setId("" + counter++); DerivedField der = new DerivedField(); der.setDataType(DATATYPE.DOUBLE); der.setOptype(OPTYPE.CONTINUOUS); NormContinuous nc = new NormContinuous(); nc.setField(ifld); nc.setOutliers(OUTLIERTREATMENTMETHOD.AS_IS); LinearNorm lin1 = new LinearNorm(); lin1.setOrig(0); lin1.setNorm(-inputMeans[j] / inputStds[j]); nc.getLinearNorms().add(lin1); LinearNorm lin2 = new LinearNorm(); lin2.setOrig(inputMeans[j]); lin2.setNorm(0); nc.getLinearNorms().add(lin2); der.setNormContinuous(nc); nin.setDerivedField(der); nins.getNeuralInputs().add(nin); } nnet.getExtensionsAndNeuralLayersAndNeuralInputs().add(nins); NeuralLayer hidden = new NeuralLayer(); hidden.setNumberOfNeurons(BigInteger.valueOf(hiddenSize)); for (int j = 0; j < hiddenSize; j++) { Neuron n = new Neuron(); n.setId("" + counter++); n.setBias(weights[wtsIndex++]); for (int k = 0; k < inputfieldNames.length; k++) { Synapse con = new Synapse(); con.setFrom("" + k); con.setWeight(weights[wtsIndex++]); n.getCons().add(con); } hidden.getNeurons().add(n); } nnet.getExtensionsAndNeuralLayersAndNeuralInputs().add(hidden); NeuralLayer outer = new NeuralLayer(); outer.setActivationFunction(ACTIVATIONFUNCTION.IDENTITY); outer.setNumberOfNeurons(BigInteger.valueOf(outputfieldNames.length)); for (int j = 0; j < outputfieldNames.length; j++) { Neuron n = new Neuron(); n.setId("" + counter++); n.setBias(weights[wtsIndex++]); for (int k = 0; k < hiddenSize; k++) { Synapse con = new Synapse(); con.setFrom("" + (k + inputfieldNames.length)); con.setWeight(weights[wtsIndex++]); n.getCons().add(con); } outer.getNeurons().add(n); } nnet.getExtensionsAndNeuralLayersAndNeuralInputs().add(outer); NeuralOutputs finalOuts = new NeuralOutputs(); finalOuts.setNumberOfOutputs(BigInteger.valueOf(outputfieldNames.length)); for (int j = 0; j < outputfieldNames.length; j++) { NeuralOutput output = new NeuralOutput(); output.setOutputNeuron("" + (j + inputfieldNames.length + hiddenSize)); DerivedField der = new DerivedField(); der.setDataType(DATATYPE.DOUBLE); der.setOptype(OPTYPE.CONTINUOUS); NormContinuous nc = new NormContinuous(); nc.setField(outputfieldNames[j]); nc.setOutliers(OUTLIERTREATMENTMETHOD.AS_IS); LinearNorm lin1 = new LinearNorm(); lin1.setOrig(0); lin1.setNorm(-outputMeans[j] / outputStds[j]); nc.getLinearNorms().add(lin1); LinearNorm lin2 = new LinearNorm(); lin2.setOrig(outputMeans[j]); lin2.setNorm(0); nc.getLinearNorms().add(lin2); der.setNormContinuous(nc); output.setDerivedField(der); finalOuts.getNeuralOutputs().add(output); } nnet.getExtensionsAndNeuralLayersAndNeuralInputs().add(finalOuts); pmml.getAssociationModelsAndBaselineModelsAndClusteringModels().add(nnet); return pmml; }
private void adaptation( SetOfIOPairs trainingSet, int maxK, double eps, double lambda, double micro) { // trainingSet : trenovacia mnozina // maxK : maximalny pocet iteracii // eps : pozadovana presnost normovanej dlzky gradientu // lambda : rychlost ucenia (0.1) // micro : momentovy clen double delta; Gradients deltaGradients = new Gradients(this); Gradients totalGradients = new Gradients(this); Gradients partialGradients = new Gradients(this); System.out.println("setting up random weights and thresholds ..."); // prahy a vahy neuronovej siete nastavime na nahodne hodnoty, delta-gradienty vynulujeme (oni // sa nuluju uz pri init) for (int il = this.numberOfLayers() - 1; il >= 1; il--) { // iteracia cez vsetky vrstvy nadol okrem poslednej NeuralLayer currentLayer = this.getLayer(il); for (int in = 0; in < currentLayer.numberOfNeurons(); in++) { // pre kazdy neuron na vrstve Neuron currentNeuron = currentLayer.getNeuron(in); currentNeuron.threshold = 2 * this.random() - 1; // deltaGradients.setThreshold(il,in,0.0); for (int ii = 0; ii < currentNeuron.numberOfInputs(); ii++) { currentNeuron.getInput(ii).weight = 2 * this.random() - 1; // deltaGradients.setWeight(il,in,ii,0.0); } // end ii } // end in } // end il int currK = 0; // citac iteracii double currE = Double.POSITIVE_INFINITY; // pociatocna aktualna presnost bude nekonecna (tendencia // znizovania) System.out.println("entering adaptation loop ... (maxK = " + maxK + ")"); while (currK < maxK && currE > eps) { computeTotalGradient(totalGradients, partialGradients, trainingSet); for (int il = this.numberOfLayers() - 1; il >= 1; il--) { // iteracia cez vsetky vrstvy nadol okrem poslednej NeuralLayer currentLayer = this.getLayer(il); for (int in = 0; in < currentLayer.numberOfNeurons(); in++) { // pre kazdy neuron na vrstve Neuron currentNeuron = currentLayer.getNeuron(in); delta = -lambda * totalGradients.getThreshold(il, in) + micro * deltaGradients.getThreshold(il, in); currentNeuron.threshold += delta; deltaGradients.setThreshold(il, in, delta); } // end for ii 1 for (int in = 0; in < currentLayer.numberOfNeurons(); in++) { // pre kazdy neuron na vrstve Neuron currentNeuron = currentLayer.getNeuron(in); for (int ii = 0; ii < currentNeuron.numberOfInputs(); ii++) { // a pre kazdy vstup neuronu delta = -lambda * totalGradients.getWeight(il, in, ii) + micro * deltaGradients.getWeight(il, in, ii); currentNeuron.getInput(ii).weight += delta; deltaGradients.setWeight(il, in, ii, delta); } // end for ii } // end for in 2 } // end for il currE = totalGradients.getGradientAbs(); currK++; if (currK % 25 == 0) System.out.println("currK=" + currK + " currE=" + currE); } // end while }
private void computeGradient( Gradients gradients, Vector<Double> inputs, Vector<Double> requiredOutputs) { // Gradients gradients = new Gradients(this); activities(inputs); for (int il = this.numberOfLayers() - 1; il >= 1; il--) { // backpropagation cez vsetky vrstvy okrem poslednej NeuralLayer currentLayer = this.getLayer(il); if (currentLayer.isLayerTop()) { // ak sa jedna o najvyssiu vrstvu // pridame gradient prahov pre danu vrstvu do odpovedajuceho vektora a tento gradient // pocitame cez neurony : // gradients.thresholds.add(il, new Vector<Double>()); for (int in = 0; in < currentLayer.numberOfNeurons(); in++) { // pre vsetky neurony na vrstve Neuron currentNeuron = currentLayer.getNeuron(in); gradients.setThreshold( il, in, currentNeuron.output * (1 - currentNeuron.output) * (currentNeuron.output - requiredOutputs.elementAt(in))); } // end for each neuron for (int in = 0; in < currentLayer.numberOfNeurons(); in++) { // for each neuron Neuron currentNeuron = currentLayer.getNeuron(in); for (int ii = 0; ii < currentNeuron.numberOfInputs(); ii++) { // for each neuron's input NeuralInput currentInput = currentNeuron.getInput(ii); gradients.setWeight( il, in, ii, gradients.getThreshold(il, in) * currentLayer.lowerLayer().getNeuron(ii).output); } // end for each input } // end for each neuron } else { // ak sa jedna o spodnejsie vrstvy (najnizsiu vrstvu nepocitame, ideme len po 1.) // pocitame gradient prahov : // gradients.thresholds.add(il, new Vector<Double>()); for (int in = 0; in < currentLayer.numberOfNeurons(); in++) { // for each neuron double aux = 0; // iterujeme cez vsetky axony neuronu (resp. synapsie neuronov na vyssej vrstve) for (int ia = 0; ia < currentLayer.upperLayer().numberOfNeurons(); ia++) { aux += gradients.getThreshold(il + 1, ia) * currentLayer.upperLayer().getNeuron(ia).getInput(in).weight; } gradients.setThreshold( il, in, currentLayer.getNeuron(in).output * (1 - currentLayer.getNeuron(in).output) * aux); } // end for each neuron // pocitame gradienty vah : for (int in = 0; in < currentLayer.numberOfNeurons(); in++) { // for each neuron Neuron currentNeuron = currentLayer.getNeuron(in); for (int ii = 0; ii < currentNeuron.numberOfInputs(); ii++) { // for each neuron's input NeuralInput currentInput = currentNeuron.getInput(ii); gradients.setWeight( il, in, ii, gradients.getThreshold(il, in) * currentLayer.lowerLayer().getNeuron(ii).output); } // end for each input } // end for each neuron } // end layer IF } // end backgropagation for each layer // return gradients; }
public void setBiaInputs(double value) { for (NeuralLayer layer : hiddenLayers) { layer.getBiasNeuron().setOutputValue(value); } outputLayer.getBiasNeuron().setOutputValue(value); }