protected void recalcQa(Sign tl, int pos, Node destination, boolean light, Sign tl_new, int pos_new, PosMov[] posMovs) { float newQvalue=0; int size = tl.getLane().getCompleteLength()-1; int R; int tlId = tl.getId(); int desId = destination.getId(); float Va; for(; size>=0; size--) { PEntry P = new PEntry(tl, pos, destination, light, tl, size); int p_index = p_table[tlId][pos][desId].indexOf(P); if(p_index>=0) { try { P = (PEntry) p_table[tlId][pos][desId].elementAt(p_index); Va = va_table[tlId][size][desId]; R = rewardFunction(tl_new, pos_new, posMovs); newQvalue += P.getValue() *(((float)R) + gamma * Va); } catch (Exception e) { System.out.println("Error in recalc Q'"); } } } try { qa_table[tl.getId()][pos][destination.getId()][light?green_index:red_index] = newQvalue; } catch (Exception e) { System.out.println("ERROR, Zwaluw is not found"); } }
public void updateRoaduserMove( Roaduser ru, Drivelane prevlane, Sign prevsign, int prevpos, Drivelane dlanenow, Sign signnow, int posnow, PosMov[] posMovs, Drivelane desired) { // When a roaduser leaves the city; this will if (dlanenow == null || signnow == null) { dlanenow = prevlane; signnow = prevsign; posnow = -1; return; // ?? is recalculation is not necessary ?? } // This ordening is important for the execution of the algorithm! if (prevsign.getType() == Sign.TRAFFICLIGHT && (signnow.getType() == Sign.TRAFFICLIGHT || signnow.getType() == Sign.NO_SIGN)) { Node dest = ru.getDestNode(); recalcQ( prevsign, prevpos, dest, prevsign.mayDrive(), signnow, posnow, signnow.mayDrive(), posMovs); } }
/** * The constructor for TL controllers * * @param The model being used. */ public SL1TLC(Infrastructure infra) throws InfraException { super(infra); Node[] nodes = infra.getAllNodes(); // Moet Edge zijn eigenlijk, alleen testSimModel knalt er dan op int num_nodes = nodes.length; count = new Vector(); int numSigns = infra.getAllInboundLanes().size(); q_table = new float[numSigns + 1][][][]; int num_specialnodes = infra.getNumSpecialNodes(); for (int i = 0; i < nodes.length; i++) { Node n = nodes[i]; Drivelane[] dls = n.getInboundLanes(); for (int j = 0; j < dls.length; j++) { Drivelane d = dls[j]; Sign s = d.getSign(); int id = s.getId(); int num_pos_on_dl = d.getCompleteLength(); q_table[id] = new float[num_pos_on_dl][][]; for (int k = 0; k < num_pos_on_dl; k++) { q_table[id][k] = new float[num_specialnodes][]; for (int l = 0; l < q_table[id][k].length; l++) { q_table[id][k][l] = new float[2]; q_table[id][k][l][0] = 0.0f; q_table[id][k][l][1] = 0.0f; } } } } System.out.println("Startet med Alpha = " + alpha); random_number = new Random(); }
public float getColearnValue(Sign sign_new, Sign sign, Node destination, int pos) { int Ktl = sign.getLane().getNumRoadusersWaiting(); int tlId = sign.getId(); int desId = destination.getId(); // Calculate the colearning value float newCovalue=0; int size = sign.getLane().getCompleteLength()-1; for(; size>=0; size--) { float V; PKtlEntry P = new PKtlEntry(sign, 0, destination, green, sign_new, size, Ktl); int p_index = pKtl_table[tlId][pos][desId].indexOf(P); if(p_index>=0) { try { P = (PKtlEntry) pKtl_table[tlId][pos][desId].elementAt(p_index); V = v_table[tlId][size][desId]; newCovalue += P.getValue() * V; } catch (Exception e) { System.out.println("Error"); } } } return newCovalue; }
protected void recalcQ( Sign tl, int pos, Node destination, boolean light, Sign tl_new, int pos_new, boolean light_new, PosMov[] posMovs) { /* Recalculate the Q values, only one PEntry has changed, meaning also only 1 QEntry has to change */ int R; float oldQvalue = 0; float Qmark = 0; float newQvalue = 0; R = rewardFunction(tl_new, pos_new, posMovs); try { oldQvalue = q_table[tl.getId()][pos][destination.getId()][light ? green_index : red_index]; Qmark = q_table[tl_new.getId()][pos_new][destination.getId()][ light_new ? green_index : red_index]; // Q( [ tl' , p' ] , L') } catch (Exception e) { System.out.println("ERROR"); System.out.println("tl: " + tl.getId()); System.out.println("pos:" + pos); System.out.println("des:" + destination.getId()); } newQvalue = oldQvalue + alpha * (R + gamma * Qmark - oldQvalue); q_table[tl.getId()][pos][destination.getId()][light ? green_index : red_index] = newQvalue; }
protected void recalcQ(Sign tl, int pos, Node destination, boolean light, Sign tl_new, int pos_new, PosMov[] posMovs, int Ktl) { /* The calculation of the Q values in TC-3 */ float newQvalue = qa_table[tl.getId()][pos][destination.getId()][light?green_index:red_index]; float V=0; // Waarom splitst TC2 wel op rood/groen, en TC3 niet?? CountEntry currentsituation = new CountEntry (tl, pos, destination, light, tl_new, pos_new, Ktl); Enumeration e = pKtl_table[tl.getId()][pos][destination.getId()].elements(); while(e.hasMoreElements()) { PKtlEntry P = (PKtlEntry) e.nextElement(); if(P.sameSourceKtl(currentsituation) != -1.0f) { try { V = v_table[P.tl_new.getId()][P.pos_new][destination.getId()]; } catch (Exception excep) { System.out.println("ERROR in q"); } // Moet er hier geen reward functie?? newQvalue += P.getValue() *gamma * V; } } q_table[tl.getId()][pos][destination.getId()][light?green_index:red_index] = newQvalue; //sign, pos, des, color (red=0, green=1) }
/** * Calculates how every traffic light should be switched Per node, per sign the waiting roadusers * are passed and per each roaduser the gain is calculated. * * @param The TLDecision is a tuple consisting of a traffic light and a reward (Q) value, for it * to be green * @see gld.algo.tlc.TLDecision */ public TLDecision[][] decideTLs() { int num_dec; int num_tld = tld.length; // Determine wheter it should be random or not boolean do_this_random = false; if (random_number.nextFloat() < random_chance) do_this_random = true; for (int i = 0; i < num_tld; i++) { num_dec = tld[i].length; for (int j = 0; j < num_dec; j++) { Sign currenttl = tld[i][j].getTL(); float gain = 0; Drivelane currentlane = currenttl.getLane(); int waitingsize = currentlane.getNumRoadusersWaiting(); ListIterator queue = currentlane.getQueue().listIterator(); if (!do_this_random) { for (; waitingsize > 0; waitingsize--) { Roaduser ru = (Roaduser) queue.next(); int pos = ru.getPosition(); Node destination = ru.getDestNode(); gain += q_table[currenttl.getId()][pos][destination.getId()][1] - q_table[currenttl.getId()][pos][destination.getId()][0]; // red - green } float q = gain; } else gain = random_number.nextFloat(); tld[i][j].setGain(gain); } } return tld; }
protected void recalcV(Sign tl, int pos, Node destination, boolean light, int Ktl) { /* The calculation of the V values in TC-3 */ float newVvalue; float tempSumGreen=0, tempSumRed=0; float V; int[] amount = count(tl, pos, destination); int tlId = tl.getId(); int desId = destination.getId(); float total = (float) amount[green_index] + (float) amount[red_index]; newVvalue = va_table[tl.getId()][pos][destination.getId()]; CountEntry currentsituation_green = new CountEntry (tl, pos, destination, green, tl, pos, Ktl); CountEntry currentsituation_red = new CountEntry (tl, pos, destination, red, tl, pos, Ktl); Enumeration e = pKtl_table[tlId][pos][desId].elements(); while(e.hasMoreElements()) { //Green part PKtlEntry P = (PKtlEntry) e.nextElement(); if(P.sameSourceKtl(currentsituation_green) != -1) { try { V = v_table[P.tl_new.getId()][P.pos_new][destination.getId()]; tempSumGreen += P.getValue() *gamma * V; } catch (Exception excep) { System.out.println(excep+""); excep.printStackTrace(); } } //Red Part if(P.sameSourceKtl(currentsituation_red) != -1) { try { V = v_table[P.tl_new.getId()][P.pos_new][destination.getId()]; tempSumRed += P.getValue() *gamma * V; } catch (Exception excep) { System.out.println("ERROR in recalc V2"); System.out.println(excep+""); excep.printStackTrace(); } } } newVvalue += ((float)amount[green_index]/ (float)total) * tempSumGreen + ((float)amount[red_index]/ (float)total) * tempSumRed; try { v_table[tl.getId()][pos][destination.getId()] = newVvalue; } catch (Exception excep) { System.out.println("Error in v"); } }
public XMLElement saveSelf () throws XMLCannotSaveException { XMLElement result=new XMLElement("pval"); result.addAttribute(new XMLAttribute("tl-id",tl.getId())); result.addAttribute(new XMLAttribute("pos",pos)); result.addAttribute(new XMLAttribute("destination",destination.getId())); result.addAttribute(new XMLAttribute("light",light)); result.addAttribute(new XMLAttribute("newtl-id",tl_new.getId())); result.addAttribute(new XMLAttribute("new-pos",pos_new)); result.addAttribute(new XMLAttribute("value",value)); return result; }
protected int[] count(Sign tl, int pos, Node destination) { int tlId = tl.getId(); int desId = destination.getId(); int[] counters; counters = new int[2]; //See the green_index definitions above !!!! counters[green_index] = 0; counters[red_index] = 0; //Calcs the number of entries in the table matching the given characteristics, and returns the count int psize = p_table[tlId][pos][desId].size()-1; for(; psize>=0; psize--) { PEntry candidate = (PEntry) p_table[tlId][pos][desId].elementAt(psize); if(candidate.tl.getId() == tlId && candidate.pos == pos && candidate.destination.getId() == desId) { if(candidate.light == green) { counters[green_index]++; } else { counters[red_index]++; } } } return counters; }
/** * Calculates how every traffic light should be switched * Per node, per sign the waiting roadusers are passed and per each roaduser the gain is calculated. * @param The TLDecision is a tuple consisting of a traffic light and a reward (Q) value, for it to be green * @see gld.algo.tlc.TLDecision */ public TLDecision[][] decideTLs() { int num_dec, currenttlID, waitingsize, pos, destID; float gain =0, passenger_factor; Sign currenttl; Drivelane currentlane; ListIterator queue; Roaduser ru; //Determine wheter it should be random or not boolean randomrun = false; if (random_number.nextFloat() < random_chance) randomrun = true; for (int i=0;i<num_nodes;i++) { num_dec = tld[i].length; for(int j=0;j<num_dec;j++) { currenttl = tld[i][j].getTL(); currenttlID = currenttl.getId(); currentlane = currenttl.getLane(); waitingsize = currentlane.getNumRoadusersWaiting(); queue = currentlane.getCompleteQueue().listIterator(); gain = 0; for(; waitingsize>0; waitingsize--) { ru = (Roaduser) queue.next(); pos = ru.getPosition(); destID = ru.getDestNode().getId(); passenger_factor = ru.getNumPassengers(); gain += passenger_factor * (q_table[currenttlID][pos][destID][red_index] - q_table[currenttlID][pos][destID][green_index]); //red - green } if(trackNode!=-1 && i==trackNode) { Drivelane currentlane2 = tld[i][j].getTL().getLane(); boolean[] targets = currentlane2.getTargets(); System.out.println("node: "+i+" light: "+j+" gain: "+gain+" "+targets[0]+" "+targets[1]+" "+targets[2]+" "+currentlane2.getNumRoadusersWaiting()); } if(randomrun) gain = random_number.nextFloat(); tld[i][j].setGain(gain); } } return tld; }
protected void recalcVa(Sign tl, int pos, Node destination) { float newWvalue; float qa_red = qa_table[tl.getId()][pos][destination.getId()][red_index]; float qa_green = qa_table[tl.getId()][pos][destination.getId()][green_index]; int[] amount = count(tl, pos, destination); float total = (float) amount[green_index] + (float) amount[red_index]; newWvalue = ((float)amount[green_index]/(float)total)*qa_green + ((float)amount[red_index]/(float)total)*qa_red; try { va_table[tl.getId()][pos][destination.getId()] = newWvalue; } catch (Exception e) { System.out.println("Error in recalc W"); } }
public XMLElement saveSelf () throws XMLCannotSaveException { XMLElement result=new XMLElement("count"); result.addAttribute(new XMLAttribute("tl-id",tl.getId())); result.addAttribute(new XMLAttribute("pos",pos)); result.addAttribute(new XMLAttribute("destination",destination.getId())); result.addAttribute(new XMLAttribute("light",light)); result.addAttribute(new XMLAttribute("newtl-id",tl_new.getId())); result.addAttribute(new XMLAttribute("new-pos",pos_new)); result.addAttribute(new XMLAttribute("ktl",Ktl)); result.addAttribute(new XMLAttribute("value",value)); if ( ! infrastructure.laneDictionary.containsKey (new Integer (tl.getId()))) { System.out.println ("WARNING : Unknown Trafficlight ID "+tl.getId()+ " in TC3$CountEntry. Loading will go wrong"); } return result; }
public float getVValue(Sign sign, Node des, int pos) { try { return v_table[sign.getId()][pos][des.getId()]; } catch (Exception e) { System.out.print("Error in v_table"); return 0; } }
public void updateRoaduserMove(Roaduser ru, Drivelane prevlane, Sign prevsign, int prevpos, Drivelane dlanenow, Sign signnow, int posnow, PosMov[] posMovs, Drivelane desired) { // Roaduser has just left the building if(dlanenow == null || signnow == null) { return; } //This ordening is important for the execution of the algorithm! int Ktl = dlanenow.getNumRoadusersWaiting(); if(prevsign.getType()==Sign.TRAFFICLIGHT && (signnow.getType()==Sign.TRAFFICLIGHT || signnow.getType()==Sign.NO_SIGN)) { boolean light = prevsign.mayDrive(); Node dest = ru.getDestNode(); recalcP(prevsign, prevpos, dest, light, signnow, posnow, Ktl); recalcVa(prevsign, prevpos, dest); recalcV(prevsign, prevpos, dest, light, Ktl); recalcQa(prevsign, prevpos, dest, light, signnow, posnow, posMovs); recalcQ(prevsign, prevpos, dest, light, signnow, posnow, posMovs, Ktl); } }
protected int rewardFunction(Sign tl_new, int pos_new, PosMov[] posMovs) { //Ok, the reward function is actually very simple; it searches for the tuple (tl_new, pos_new) in the given set int size = posMovs.length; for(int i=0; i<size; i++) if( posMovs[i].tlId==tl_new.getId()) if(posMovs[i].pos != pos_new) return 0; return 1; }
protected int rewardFunction(Sign tl_new, int pos_new, PosMov[] posMovs) { // Ok, the reward function is actually very simple; it searches for the tuple (tl_new, pos_new) // in the given set int size = posMovs.length; for (int i = 0; i < size; i++) { if (posMovs[i].tlId == tl_new.getId() && posMovs[i].pos == pos_new) return 0; } /*int size = possiblelanes.length; for(int i=0; i<size; i++) { if( possiblelanes[i].equals(tl_new.getLane()) ) { if(ranges[i].x < pos_new) { if(ranges[i].y > pos_new) { return 0; } } } }*/ return 1; }
public XMLElement saveSelf () throws XMLCannotSaveException { XMLElement result=new XMLElement("target"); result.addAttribute(new XMLAttribute("tl-id",tl.getId())); result.addAttribute(new XMLAttribute("pos",pos)); return result; }
protected void recalcP(Sign tl, int pos, Node destination, boolean light, Sign tl_new, int pos_new, int Ktl) { int tlId = tl.getId(); int desId = destination.getId(); //Update the count table CountEntry currentsituation = new CountEntry(tl, pos, destination, light, tl_new, pos_new, Ktl); int count_index = count[tlId][pos][desId].indexOf(currentsituation); if (count_index>=0) { currentsituation = (CountEntry) count[tlId][pos][desId].elementAt(count_index); currentsituation.incrementValue(); } else { count[tlId][pos][desId].add(currentsituation); } //Update the p_table PEntry currentchance = new PEntry(tl, pos, destination, light, tl_new, pos_new); int dest=0, source=0; Enumeration enum = count[tlId][pos][desId].elements(); while(enum.hasMoreElements()) { CountEntry current = (CountEntry) enum.nextElement(); dest += current.sameSourceDifferentKtl(currentsituation); source += current.sameSource(currentsituation); } if(source == 0) currentchance.setValue(0); else currentchance.setValue((float)dest/(float)source); int p_index = p_table[tlId][pos][desId].indexOf(currentchance); if(p_index>=0) p_table[tlId][pos][desId].setElementAt(currentchance, p_index); else { p_table[tlId][pos][desId].add(currentchance); p_index = p_table[tlId][pos][desId].indexOf(currentchance); } // Change the rest of the p_table, Also check the other chances for updates int size = p_table[tlId][pos][desId].size()-1; for(; size>=0; size--) { PEntry P = (PEntry) p_table[tlId][pos][desId].elementAt(size); float pvalue = P.sameSource(currentsituation); if(pvalue > -1.0f) { if(size != p_index) P.setValue(pvalue * (float)(source-1) / (float)source); } } //update the p'_table ...... PKtlEntry currentchance2 = new PKtlEntry(tl, pos, destination, light, tl_new, pos_new, Ktl); source=0; enum = count[tlId][pos][desId].elements(); while(enum.hasMoreElements()) { source += ((CountEntry) enum.nextElement()).sameSourceWithKtl(currentsituation); } dest = currentsituation.getValue(); if(source == 0) currentchance2.setValue(0); else currentchance2.setValue((float)dest/(float)source); p_index = pKtl_table[tlId][pos][desId].indexOf(currentchance2); if(p_index>=0) pKtl_table[tlId][pos][desId].setElementAt(currentchance2, p_index); else { pKtl_table[tlId][pos][desId].add(currentchance2); p_index = pKtl_table[tlId][pos][desId].indexOf(currentchance2); } // Change the rest of the pKtl_table, Also check the other chances for updates size = pKtl_table[tlId][pos][desId].size()-1; for(; size>=0; size--) { PKtlEntry P = (PKtlEntry) pKtl_table[tlId][pos][desId].elementAt(size); float pvalue = P.sameSource(currentsituation); if(pvalue > -1) { if(size != p_index) { P.setValue(pvalue * (float)(source-1) / (float)source); } } } if(currentchance.getValue() >1 ||currentchance2.getValue() >1 || currentchance.getValue() <0 ||currentchance2.getValue() <0 ) System.out.println("Serious error !!!!!!!!!1"); }