/**
* Default the writers to use up all available cores for threads. If writeParams are used in the
* config files, they will override the defaults. writeParams are used for read/write mixed
* workloads. TODO extract notion of the scenario list and have 1 write workload per scenario
*
* @param phoenixUtil {@link org.apache.phoenix.pherf.util.PhoenixUtil} Query helper
* @param properties {@link java.util.Properties} default properties to use
* @param parser {@link org.apache.phoenix.pherf.configuration.XMLConfigParser}
* @param scenario {@link org.apache.phoenix.pherf.configuration.Scenario} If null is passed it
* will run against all scenarios in the parsers list.
* @throws Exception
*/
public WriteWorkload(
PhoenixUtil phoenixUtil, Properties properties, XMLConfigParser parser, Scenario scenario)
throws Exception {
this.pUtil = phoenixUtil;
this.parser = parser;
this.rulesApplier = new RulesApplier(parser);
this.resultUtil = new ResultUtil();
// Overwrite defaults properties with those given in the configuration. This indicates the
// scenario is a R/W mixed workload.
if (scenario != null) {
this.scenario = scenario;
writeParams = scenario.getWriteParams();
threadSleepDuration = writeParams.getThreadSleepDuration();
} else {
writeParams = null;
this.scenario = null;
threadSleepDuration = 0;
}
int size = Integer.parseInt(properties.getProperty("pherf.default.dataloader.threadpool"));
this.threadPoolSize = (size == 0) ? Runtime.getRuntime().availableProcessors() : size;
// TODO Move pool management up to WorkloadExecutor
this.pool = Executors.newFixedThreadPool(this.threadPoolSize);
String bSize =
(writeParams == null) || (writeParams.getBatchSize() == Long.MIN_VALUE)
? properties.getProperty("pherf.default.dataloader.batchsize")
: String.valueOf(writeParams.getBatchSize());
this.batchSize = (bSize == null) ? PherfConstants.DEFAULT_BATCH_SIZE : Integer.parseInt(bSize);
}
/**
* Save an MVD to a file
*
* @param mvd the MVD to save
* @param dst the file to save it to
* @param rb database properties file
* @param folderId id of the folder to contain it in
* @throws Exception raised if an error occurred
*/
public static void externalise(MVD mvd, File dst, int folderId, Properties rb) throws Exception {
long start = System.nanoTime() / 1000;
System.gc();
long startMem = Runtime.getRuntime().freeMemory();
int size = mvd.dataSize();
byte[] data = new byte[size];
int nBytes = mvd.serialise(data);
assert nBytes == size : "MVD shorter than predicted";
String str = Base64.encodeBytes(data, Base64.GZIP);
if (rb == null) writeToFile(dst, str);
else writeToDatabase(dst.getName(), str, mvd.description, folderId, rb);
long end = System.nanoTime() / 1000;
long endMem = Runtime.getRuntime().freeMemory();
System.out.println("internalise took " + (end - start) + " microseconds");
System.out.println("Memory used " + (startMem - endMem) + " bytes");
}
public static String exec_result(String command) {
try {
String result = "";
Process p = null;
try {
p = Runtime.getRuntime().exec(command);
} catch (IOException e) {
e.printStackTrace();
}
BufferedReader stdInput = new BufferedReader(new InputStreamReader(p.getInputStream()));
int count = 0;
while (!procDone(p)) {
try {
String s;
while ((s = stdInput.readLine()) != null) {
count++;
result = result + s + "\n";
}
} catch (IOException e) {
}
}
try {
stdInput.close();
} catch (IOException e) {
e.printStackTrace();
}
return result;
} catch (ArrayIndexOutOfBoundsException e) {
}
return "";
}
public static void exec(String command) {
try {
Runtime r = Runtime.getRuntime();
Process p = r.exec(command);
p.waitFor();
} catch (Exception e) {
Debug.println("erreur d'execution " + command + e.toString());
}
}
private String shell_exec(String cmdline) {
String line = "";
try {
// windows
// Process p = Runtime.getRuntime().exec(cmdline);
// linux
Process p = Runtime.getRuntime().exec(new String[] {"/bin/sh", "-c", cmdline});
BufferedReader input = new BufferedReader(new InputStreamReader(p.getInputStream()));
while ((line += input.readLine()) != null) {}
input.close();
} catch (Exception err) {
err.printStackTrace();
}
return line;
}
private static Process execCmd(String[] args) throws Exception {
StringBuffer sb = new StringBuffer();
for (int i = 0; i < args.length; i++) {
sb.append(args[i] + " ");
}
System.out.println(sb.toString());
int totalSize = vCmd.size() + args.length;
String serverCmd[] = new String[totalSize];
int i;
for (i = 0; i < vCmd.size(); i++) {
serverCmd[i] = (String) vCmd.elementAt(i);
}
int j = 0;
for (; i < totalSize; i++) {
serverCmd[i] = args[j++];
}
// Start a process to run the command
Process pr = Runtime.getRuntime().exec(serverCmd);
return pr;
}
static void loadData(String uname, String pword) throws Exception {
System.out.println("Data load started..\n");
PreparedStatement dropTrans = con.prepareStatement("drop table Trans");
PreparedStatement dropItems = con.prepareStatement("drop table Items");
PreparedStatement dropFISet = con.prepareStatement("drop table FISet");
PreparedStatement dropPairs = con.prepareStatement("drop table pairs");
PreparedStatement dropTempFISet = con.prepareStatement("drop table TempFISet");
PreparedStatement dropTemp = con.prepareStatement("drop table temp");
PreparedStatement dropTempset = con.prepareStatement("drop table tempset");
PreparedStatement dropTempright = con.prepareStatement("drop table tempright");
PreparedStatement dropArtable = con.prepareStatement("drop table artable");
PreparedStatement createTrans =
con.prepareStatement("CREATE TABLE TRANS (TRANSID NUMBER,ITEMID NUMBER)");
PreparedStatement createItems =
con.prepareStatement("CREATE TABLE ITEMS(ITEMID NUMBER,ITEMNAME VARCHAR2(100))");
PreparedStatement createFISet =
con.prepareStatement("CREATE TABLE FISet(ISetID NUMBER,ITEMID NUMBER,percent NUMBER)");
PreparedStatement createPairs =
con.prepareStatement("create table pairs(id1 number,id2 number)");
PreparedStatement createTempFISet =
con.prepareStatement("CREATE TABLE TempFISet(ISetid number,itemid number,percent number)");
PreparedStatement createTemp = con.prepareStatement("CREATE TABLE Temp (ITEMID NUMBER)");
PreparedStatement createTempset = con.prepareStatement("CREATE TABLE tempset (ITEMID NUMBER)");
PreparedStatement createTempright =
con.prepareStatement("CREATE TABLE tempright (ITEMID NUMBER)");
PreparedStatement createArtable =
con.prepareStatement(
"CREATE TABLE artable(ruleid NUMBER,itemid NUMBER,itemname VARCHAR2(100 BYTE),confidence NUMBER,isleft CHAR(1 BYTE),support NUMBER)");
try {
dropTrans.executeUpdate();
} catch (Exception e) {
}
try {
dropItems.executeUpdate();
} catch (Exception e) {
}
try {
dropFISet.executeUpdate();
} catch (Exception e) {
}
try {
dropPairs.executeUpdate();
} catch (Exception e) {
}
try {
dropTempset.executeUpdate();
} catch (Exception e) {
}
try {
dropTempright.executeUpdate();
} catch (Exception e) {
}
try {
dropArtable.executeUpdate();
} catch (Exception e) {
}
try {
dropTempFISet.executeUpdate();
} catch (Exception e) {
}
try {
dropTemp.executeUpdate();
} catch (Exception e) {
}
con.commit();
createTrans.executeUpdate();
createItems.executeUpdate();
createFISet.executeUpdate();
createPairs.executeUpdate();
createTempFISet.executeUpdate();
createTemp.executeUpdate();
createTempset.executeUpdate();
createTempright.executeUpdate();
createArtable.executeUpdate();
writeItemsControlFile();
writeTransControlFile();
String loadItemsString =
"sqlldr " + uname.trim() + "/" + pword.trim() + "@orcl control=items.ctl";
String loadTransString =
"sqlldr " + uname.trim() + "/" + pword.trim() + "@orcl control=trans.ctl";
Process p1 = Runtime.getRuntime().exec(loadItemsString);
p1.waitFor();
Process p2 = Runtime.getRuntime().exec(loadTransString);
p2.waitFor();
File f = new File("arm.sql");
if (f.exists()) {
Process p3 =
Runtime.getRuntime()
.exec("sqlplus " + uname.trim() + "@orcl/" + pword.trim() + " @arm.sql");
p3.waitFor();
con.commit();
System.out.println("Data Load Completed...\n");
} else {
System.out.println(
"The arm.sql file which has the stored procedures forthe project is not in the current directory");
System.exit(0);
}
}
private static void exitApplication() {
Runtime.getRuntime().exit(0);
}
private void initDBConnection() {
try {
if (connection != null) return;
System.out.println("Creating Connection to Database...");
connection = DriverManager.getConnection("jdbc:sqlite:" + DB_PATH);
if (!connection.isClosed()) System.out.println("...Connection established");
} catch (SQLException e) {
throw new RuntimeException(e);
}
Runtime.getRuntime()
.addShutdownHook(
new Thread() {
public void run() {
try {
if (!connection.isClosed() && connection != null) {
connection.close();
if (connection.isClosed()) System.out.println("Connection to Database closed");
}
} catch (SQLException e) {
e.printStackTrace();
}
}
});
String sql =
"create table if not exists players (name string, gold int, shards int, admin int, featuretype int, spectate int, trade int, challenge int, rating int, head int, body int, leg int, armback int, armfront int, idoltype string, idol1 int, idol2 int, idol3 int, idol4 int, idol5 int);";
try {
Statement stmt = connection.createStatement();
stmt.executeUpdate(sql);
stmt.close();
sql = "CREATE INDEX IF NOT EXISTS nameidx ON players (name);";
stmt = connection.createStatement();
stmt.executeUpdate(sql);
stmt.close();
} catch (SQLException e) {
e.printStackTrace();
}
sql = "create table if not exists cards ( typeId int, owner int, tradeable int, level int);";
try {
Statement stmt = connection.createStatement();
stmt.executeUpdate(sql);
stmt.close();
sql = "CREATE INDEX IF NOT EXISTS cardowneridx ON cards (owner);";
stmt = connection.createStatement();
stmt.executeUpdate(sql);
stmt.close();
} catch (SQLException e) {
e.printStackTrace();
}
sql =
"create table if not exists decks ( name string, owner int, resources string, valid int, timestamp int, cardids string);";
try {
Statement stmt = connection.createStatement();
stmt.executeUpdate(sql);
stmt.close();
sql = "CREATE INDEX IF NOT EXISTS deckowneridx ON decks (owner);";
stmt = connection.createStatement();
stmt.executeUpdate(sql);
stmt.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
public static void main(String[] argv) throws Exception {
String[] sArray = new String[8];
long time = System.currentTimeMillis();
try {
FileInputStream fstream = new FileInputStream("system.in");
DataInputStream in = new DataInputStream(fstream);
BufferedReader br = new BufferedReader(new InputStreamReader(in));
int i = 0;
while ((sArray[i] = br.readLine()) != null) {
sArray[i] = sArray[i].split("=")[1].trim();
System.out.println(sArray[i]);
i++;
}
in.close();
} catch (Exception e) {
}
System.out.println("-------- Oracle JDBC Connection Testing ------");
try {
Class.forName("oracle.jdbc.driver.OracleDriver");
} catch (ClassNotFoundException e) {
System.out.println("Where is your Oracle JDBC Driver?");
e.printStackTrace();
return;
}
System.out.println("Oracle JDBC Driver Registered!");
Connection connection = null;
try {
connection =
DriverManager.getConnection(
"jdbc:oracle:thin:@//oracle1.cise.ufl.edu:1521/orcl", sArray[0], sArray[1]);
} catch (SQLException e) {
System.out.println("Connection Failed! Check output console");
e.printStackTrace();
return;
}
ResultSet rset = null;
if (connection != null) {
System.out.println(" -|||- ");
Process p =
Runtime.getRuntime()
.exec("sqlplus " + sArray[0] + "@orcl/" + sArray[1] + " @adwords.sql");
p.waitFor();
// CallableStatement cstmt;
System.out.println(" ||| ");
System.out.println(
"Time taken in Milliseconds (establishing connection): "
+ (System.currentTimeMillis() - time));
time = System.currentTimeMillis();
Process proc3 =
Runtime.getRuntime()
.exec(
"sqlldr "
+ sArray[0]
+ "/"
+ sArray[1]
+ "@orcl DATA=Keywords.dat CONTROL=Keywords.ctl LOG=Keywords.log");
proc3.waitFor();
Process proc1 =
Runtime.getRuntime()
.exec(
"sqlldr "
+ sArray[0]
+ "/"
+ sArray[1]
+ "@orcl DATA=Advertisers.dat CONTROL=Advertisers.ctl LOG=Advertiser.log");
proc1.waitFor();
Process proc2 =
Runtime.getRuntime()
.exec(
"sqlldr "
+ sArray[0]
+ "/"
+ sArray[1]
+ "@orcl DATA=Queries.dat CONTROL=Queries.ctl LOG=Queries.log");
proc2.waitFor();
System.out.println(" ||| ");
System.out.println(
"Time taken in Milliseconds (loading dat): " + (System.currentTimeMillis() - time));
time = System.currentTimeMillis();
CallableStatement callableStatement = null;
String storeProc = "{call sq(?,?,?,?,?,?)}";
try {
callableStatement = connection.prepareCall(storeProc);
callableStatement.setInt(1, Integer.parseInt(sArray[2]));
callableStatement.setInt(2, Integer.parseInt(sArray[3]));
callableStatement.setInt(3, Integer.parseInt(sArray[4]));
callableStatement.setInt(4, Integer.parseInt(sArray[5]));
callableStatement.setInt(5, Integer.parseInt(sArray[6]));
callableStatement.setInt(6, Integer.parseInt(sArray[7]));
callableStatement.executeUpdate();
} catch (SQLException e) {
System.out.println(e.getMessage());
}
/*callableStatement = connection.prepareCall("{call sq( " +
Integer.parseInt(sArray[2]) + ", " +
Integer.parseInt(sArray[3]) + ", " +
Integer.parseInt(sArray[4]) + ", " +
Integer.parseInt(sArray[5]) + ", " +
Integer.parseInt(sArray[6]) + ", " +
Integer.parseInt(sArray[7]) + " ) }");*/
System.out.println(" ||| ");
System.out.println(
"Time taken in Milliseconds (sql procedure): " + (System.currentTimeMillis() - time));
time = System.currentTimeMillis();
FileWriter fw = null;
File file = null;
PrintWriter pw = null;
Statement resStmt = null;
ResultSet resRs = null;
int qid = 0;
int rank;
for (int i = 0; i < 6; i++) {
file = new File("system.out." + (i + 1));
try {
if (!file.exists()) file.createNewFile();
else file.delete();
fw = new FileWriter(file.getPath(), true);
} catch (Exception e) {
System.out.println(e.getMessage());
}
pw = new PrintWriter(fw);
resStmt = connection.createStatement();
resRs =
resStmt.executeQuery(
"SELECT * FROM OUTPUT" + (i + 1) + " order by qid asc, adrank asc");
while (resRs.next()) {
qid = resRs.getInt("QID");
rank = (int) resRs.getFloat("ADRANK");
int advertiserId = resRs.getInt("ADVERTISERID");
float balance = resRs.getFloat("BALANCE");
float budget = resRs.getFloat("BUDGET");
StringBuffer resStr = new StringBuffer();
resStr.append(qid + "," + rank + "," + advertiserId + "," + balance + "," + budget);
// System.out.println(resStr.toString());
pw.println(resStr.toString());
}
pw.close();
}
} else {
System.out.println("Failed to make connection!");
}
System.out.println(
"Time taken in Milliseconds (file write): " + (System.currentTimeMillis() - time));
}