private static Long latestVersion(Hashtable<String, String> config, dbutil db_util)
throws Exception {
if (!config.get("timestamp_stop").equals(Integer.toString(Integer.MAX_VALUE))) {
return new Long(config.get("timestamp_stop"));
}
String rowName = config.get("file_id") + config.get("run_id") + "_";
if (config.get("task_id") != "") {
try {
rowName = rowName + String.format("%04d", new Integer(config.get("task_id")));
} catch (NumberFormatException E) {
rowName = rowName + config.get("task_id");
}
}
Get timestampGet = new Get(rowName.getBytes());
timestampGet.addColumn("d".getBytes(), "update".getBytes());
Result timestampResult = db_util.doGet(config.get("db_name_updates"), timestampGet);
KeyValue tsKv = timestampResult.getColumnLatest("d".getBytes(), "update".getBytes());
if (tsKv == null) {
rowName = config.get("file_id") + "_";
timestampGet = new Get(rowName.getBytes());
timestampGet.addColumn("d".getBytes(), "update".getBytes());
timestampResult = db_util.doGet(config.get("db_name_updates"), timestampGet);
tsKv = timestampResult.getColumnLatest("d".getBytes(), "update".getBytes());
}
if (tsKv == null) {
return new Long(Integer.MAX_VALUE);
}
Long latestVersion = new Long(tsKv.getTimestamp());
return latestVersion;
}
public String getLegend(String[] cityMap, int[] POIs) {
char[] ans = new char[POIs.length];
Hashtable hash = new Hashtable();
for (int i = 0; i < cityMap.length; i++) {
for (int j = 0; j < cityMap[i].length(); j++) {
char cur = cityMap[i].charAt(j);
int k = 1;
if (!hash.containsKey(cur)) {
hash.put(cur, k);
} else {
int prev = (int) hash.get(cur);
hash.remove(cur);
hash.put(cur, prev + 1);
}
}
}
Enumeration vals = hash.keys();
while (vals.hasMoreElements()) {
char c = (char) vals.nextElement();
for (int i = 0; i < POIs.length; i++) {
if (hash.get(c) == POIs[i]) {
ans[i] = c;
}
}
}
String str = new String(ans);
return str;
}
private static toFrom checkArgs(Hashtable config) {
if (config.get("type").equals("l2r")) {
return toFrom.LOCAL2REMOTE;
} else if (config.get("type").equals("r2l")) {
return toFrom.REMOTE2LOCAL;
} else {
return toFrom.ERROR;
}
}
/** Sets up configuration based on params */
private static boolean setup(Hashtable<String, String> curConf, Configuration argConf) {
if (argConf.get("file") == null) {
logger.fatal("Missing file parameter");
System.exit(1);
}
if (argConf.get("hdfs_base_path") == null) {
logger.fatal("Missing HDFS base path, check gestore-conf.xml");
System.exit(1);
}
if (argConf.get("hdfs_temp_path") == null) {
logger.fatal("Missing HDFS temp path, check gestore-conf.xml");
System.exit(1);
}
if (argConf.get("local_temp_path") == null) {
logger.fatal("Missing local temp path, check gestore-conf.xml");
System.exit(1);
}
// Input paramaters
curConf.put("run_id", argConf.get("run", ""));
curConf.put("task_id", argConf.get("task", ""));
curConf.put("file_id", argConf.get("file"));
curConf.put("local_path", argConf.get("path", ""));
curConf.put("type", argConf.get("type", "l2r"));
curConf.put("timestamp_start", argConf.get("timestamp_start", "1"));
curConf.put(
"timestamp_stop", argConf.get("timestamp_stop", Integer.toString(Integer.MAX_VALUE)));
curConf.put("delimiter", argConf.get("regex", "ID=.*"));
curConf.put("taxon", argConf.get("taxon", "all"));
curConf.put("intermediate", argConf.get("full_run", "false"));
curConf.put("quick_add", argConf.get("quick_add", "false"));
Boolean full_run = curConf.get("intermediate").matches("(?i).*true.*");
curConf.put("format", argConf.get("format", "unknown"));
curConf.put("split", argConf.get("split", "1"));
curConf.put("copy", argConf.get("copy", "true"));
// Constants
curConf.put("base_path", argConf.get("hdfs_base_path"));
curConf.put("temp_path", argConf.get("hdfs_temp_path"));
curConf.put("local_temp_path", argConf.get("local_temp_path"));
curConf.put("db_name_files", argConf.get("hbase_file_table"));
curConf.put("db_name_runs", argConf.get("hbase_run_table"));
curConf.put("db_name_updates", argConf.get("hbase_db_update_table"));
// Timestamps
Date currentTime = new Date();
Date endDate = new Date(new Long(curConf.get("timestamp_stop")));
curConf.put("timestamp_real", Long.toString(currentTime.getTime()));
return true;
}
/**
* Implements CallListener.callEnded. Stops sounds that are playing at the moment if there're any.
* Removes the call panel and disables the hangup button.
*/
public void callEnded(CallEvent event) {
Call sourceCall = event.getSourceCall();
NotificationManager.stopSound(NotificationManager.BUSY_CALL);
NotificationManager.stopSound(NotificationManager.INCOMING_CALL);
NotificationManager.stopSound(NotificationManager.OUTGOING_CALL);
if (activeCalls.get(sourceCall) != null) {
CallPanel callPanel = (CallPanel) activeCalls.get(sourceCall);
this.removeCallPanelWait(callPanel);
}
}
public void setParameters(Hashtable params) {
setName("TiniBogusLocationResource");
setScalingFactor((long) Constants.Geophysical.DEGTOBILLIONTHS);
attrtable = params;
if (params != null) {
if (params.get("Latitude") != null) {
String pstr = (String) params.get("Latitude");
Double temp = new Double(pstr);
latitude = temp.doubleValue();
}
if (params.get("Longitude") != null) {
String pstr = (String) params.get("Longitude");
Double temp = new Double(pstr);
longitude = temp.doubleValue();
}
if (params.get("Heading") != null) {
String pstr = (String) params.get("Heading");
Double temp = new Double(pstr);
heading = temp.doubleValue();
}
if (params.get("Altitude") != null) {
String pstr = (String) params.get("Altitude");
Double temp = new Double(pstr);
altitude = temp.doubleValue();
}
}
}
/**
* Filters a PMData's measurements by the modifier of each measurement... the format for the data
* should be such for each modifier, there is parameter name/value pairs for each parameter name
* supplied. the resulting list is a list of String arrays with the parameters
*
* @param data PMData: the PM data object
* @param params String[]: the PM parameters
* @return Hashtable: a table of string arrays, with the modifier as the key (or "?" if no
* modifier)
*/
public static Hashtable process(PMData data, String[] params) {
if (data != null) {
// table for temporary storage of param lists...
Hashtable ht = new Hashtable();
// for each param...
for (int i = 0; i < params.length; i++) {
PmMeasurement[] m = data.getMeasurementList();
for (int j = 0; j < m.length; j++) {
if (m[j].getParameterName().equals(params[i])) {
String mod = m[j].getModifier();
if (mod == null) {
mod = "?";
}
String[] datum = (String[]) ht.get(mod);
if (datum == null) {
datum = new String[params.length];
for (int k = 0; k < params.length; k++) {
datum[k] = "";
}
}
datum[i] = m[j].getParameterValue();
ht.put(mod, datum);
}
}
}
return ht;
}
return null;
}
/**
* Filters a PMData's measurements by the modifier of each measurement... the format for the data
* should be such for each modifier, there is parameter name/value pairs for each parameter name
* supplied. the resulting list is a list of String arrays with the parameters
*
* @param data PMData: the PM data object
* @param params String[]: the PM parameters
* @param includeMod boolean: include modifier as first value if true
* @return ArrayList: a list of string arrays
*/
public static ArrayList processByMod(PMData data, String[] params, boolean includeMod) {
if (data != null) {
// table for temporary storage of param lists...
Hashtable ht = new Hashtable();
// for each param...
for (int i = 0; i < params.length; i++) {
PmMeasurement[] m = data.getMeasurementList();
for (int j = 0; j < m.length; j++) {
if (m[j].getParameterName().equals(params[i])) {
String mod = m[j].getModifier();
if (mod == null) {
mod = "?";
}
String[] datum = (String[]) ht.get(mod);
int offset = (includeMod ? 1 : 0);
if (datum == null) {
if (includeMod) {
datum = new String[params.length + 1];
datum[0] = mod;
} else {
datum = new String[params.length];
}
for (int k = 0; k < params.length; k++) {
datum[k + offset] = "";
}
}
datum[i + offset] = m[j].getParameterValue();
ht.put(mod, datum);
}
}
}
return new ArrayList(ht.values());
}
return null;
}
/**
* Filters a PMData's measurements by the modifier of each measurement... the format for the data
* should be such for each modifier, there is parameter name/value pairs for each parameter name
* supplied. the resulting list is a list of String arrays with the parameters
*
* @param data PMData: the PM data object
* @param params String[]: the PM parameters
* @return Hashtable: a table of string arrays, with the modifier as the key (or "?" if no
* modifier)
*/
public static Hashtable processInclude(PMData data, String[] params) {
if (data != null) {
Hashtable ht = new Hashtable();
// for each measurement...
PmMeasurement[] m = data.getMeasurementList();
for (int i = 0; i < m.length; i++) {
if (m[i] == null) {
continue;
}
String mod = m[i].getModifier();
if (mod == null) {
mod = "?";
}
String[] datum = (String[]) ht.get(mod);
if (datum == null) {
datum = new String[params.length];
for (int j = 0; j < params.length; j++) {
datum[j] = "";
}
}
for (int j = 0; j < params.length; j++) {
if (m[i].getParameterName().equals(params[j])) {
datum[j] = m[i].getParameterValue();
}
}
ht.put(mod, datum);
}
return ht;
}
return null;
}
public synchronized void messageReceived(int to, Message m) {
DrainMsg mhMsg = (DrainMsg) m;
log.debug(
"incoming: localDest: "
+ to
+ " type:"
+ mhMsg.get_type()
+ " hops:"
+ (16 - mhMsg.get_ttl())
+ " seqNo:"
+ mhMsg.get_seqNo()
+ " source:"
+ mhMsg.get_source()
+ " finalDest:"
+ mhMsg.get_dest());
// lets assume that the network cannot buffer more than 25 drain msgs from a single source at a
// time (should be more than reasonable)
if (seqNos.containsKey(new Integer(mhMsg.get_source()))) {
int oldSeqNo = ((Integer) seqNos.get(new Integer(mhMsg.get_source()))).intValue();
int upperBound = mhMsg.get_seqNo() + 25;
int wrappedUpperBound = 25 - (255 - mhMsg.get_seqNo());
if ((oldSeqNo >= mhMsg.get_seqNo() && oldSeqNo < upperBound)
|| (oldSeqNo >= 0 && oldSeqNo < wrappedUpperBound)) {
log.debug(
"Dropping message from "
+ mhMsg.get_source()
+ " with duplicate seqNo: "
+ mhMsg.get_seqNo());
return;
}
}
seqNos.put(new Integer(mhMsg.get_source()), new Integer(mhMsg.get_seqNo()));
if (to != spAddr && to != MoteIF.TOS_BCAST_ADDR && to != TOS_UART_ADDR) {
log.debug("Dropping message not for me.");
return;
}
HashSet promiscuousSet = (HashSet) idTable.get(new Integer(BCAST_ID));
HashSet listenerSet = (HashSet) idTable.get(new Integer(mhMsg.get_type()));
if (listenerSet != null && promiscuousSet != null) {
listenerSet.addAll(promiscuousSet);
} else if (listenerSet == null && promiscuousSet != null) {
listenerSet = promiscuousSet;
}
if (listenerSet == null) {
log.debug("No Listener for type: " + mhMsg.get_type());
return;
}
for (Iterator it = listenerSet.iterator(); it.hasNext(); ) {
MessageListener ml = (MessageListener) it.next();
ml.messageReceived(to, mhMsg);
}
}
// Look up an older packet for responses
// Return true if the packet is unique; false if we have seen it before
private boolean rememberPacket(GnutellaPacket pkt) {
GnutellaConnection gc = (GnutellaConnection) packetTable.get(pkt.getGUID());
if (gc != null) return false;
if (DEBUG) System.err.println("**** REMEMBERING: " + pkt + " from " + pkt.getConnection());
packetTable.put(pkt.getGUID(), pkt.getConnection());
return true;
}
// Forward an incoming packet to the corresponding source
private void forwardPacket(GnutellaPacket pkt) {
GnutellaConnection gc;
gc = (GnutellaConnection) packetTable.get(pkt.getGUID());
if (gc == null) {
if (VERBOSE) System.err.println("-- Received reply with no request: " + pkt);
return;
}
if (DEBUG) System.err.println("**** REPLYING: " + pkt + " to " + gc);
if ((pkt.ttl == 0) || (--pkt.ttl == 0)) {
if (VERBOSE) System.err.println("-- Dropping packet, TTL expired: " + pkt);
}
pkt.hops++;
gc.enqueue_lossy(pkt);
}
/** replaces the font (PS name) if necessary and returns the new name */
private static String replacePSFont(String font) {
String result;
result = font;
// do we have to replace it? -> same style, size
if (m_PSFontReplacement.containsKey(font)) {
result = m_PSFontReplacement.get(font).toString();
if (DEBUG)
System.out.println(
Messages.getInstance().getString("PostscriptGraphics_ReplacePSFont_Text_First")
+ font
+ Messages.getInstance().getString("PostscriptGraphics_ReplacePSFont_Text_Second")
+ result
+ Messages.getInstance().getString("PostscriptGraphics_ReplacePSFont_Text_Third"));
}
return result;
}
public static String format(String message, Object[] args) {
MessageFormat mf;
String msg;
try {
mf = (MessageFormat) _formats.get(message);
if (mf == null) {
try {
msg = _messages.getString(message);
} catch (MissingResourceException except) {
return message;
}
mf = new MessageFormat(msg);
_formats.put(message, mf);
}
return mf.format(args);
} catch (Exception except) {
return "An internal error occured while processing message " + message;
}
}
/**
* Reads the children of an XML element and matches them to properties of a bean.
*
* @param ob The bean to receive the values
* @param element The element the corresponds to the bean
* @throws IOException If there is an error reading the document
*/
public void readObject(Object ob, Element element) throws IOException {
// If the object is null, skip the element
if (ob == null) {
return;
}
try {
BeanInfo info = (BeanInfo) beanCache.get(ob.getClass());
if (info == null) {
// Get the bean info for the object
info = Introspector.getBeanInfo(ob.getClass(), Object.class);
beanCache.put(ob.getClass(), info);
}
// Get the object's properties
PropertyDescriptor[] props = info.getPropertyDescriptors();
// Get the attributes of the node
NamedNodeMap attrs = element.getAttributes();
// Get the children of the XML element
NodeList nodes = element.getChildNodes();
int numNodes = nodes.getLength();
for (int i = 0; i < props.length; i++) {
// Treat indexed properties a little differently
if (props[i] instanceof IndexedPropertyDescriptor) {
readIndexedProperty(ob, (IndexedPropertyDescriptor) props[i], nodes, attrs);
} else {
readProperty(ob, props[i], nodes, attrs);
}
}
} catch (IntrospectionException exc) {
throw new IOException(
"Error getting bean info for " + ob.getClass().getName() + ": " + exc.toString());
}
}
static void writeUsersToFile() {
userWriter = createCSVFile("Users.dat");
// write the list of users to file
System.out.println("Users that are both seller and buyer BUT different ratings: ");
Enumeration<String> keys = userList.keys();
while (keys.hasMoreElements()) {
String curUserID = keys.nextElement();
String curUser[] = userList.get(curUserID);
String userRow =
wrapQuotations(curUser[0])
+ ","
+ curUser[1]
+ ","
+ curUser[2]
+ ","
+ wrapQuotations(curUser[3])
+ ","
+ wrapQuotations(curUser[4]);
writeLine(userWriter, userRow);
// if(curUser[1] != null && curUser[2] != null && !curUser[1].equals(curUser[2]))
// System.out.println(curUser[0]);
}
System.out.println("----end different ratings----");
// System.out.println("The users that are both seller and buyer: ");
// Enumeration<String> both = bothSellerAndBuyer.elements();
// while(both.hasMoreElements()) {
// System.out.println(both.nextElement());
// }
// System.out.println("----end both----");
try {
userWriter.flush();
userWriter.close();
} catch (IOException e) {
e.printStackTrace();
}
}
static UplInfo getInfo(String fName) {
UplInfo info = (UplInfo) uploadTable.get(fName);
return info;
}
/**
* Handles the <tt>ActionEvent</tt> generated when user presses one of the buttons in this panel.
*/
public void actionPerformed(ActionEvent evt) {
JButton button = (JButton) evt.getSource();
String buttonName = button.getName();
if (buttonName.equals("call")) {
Component selectedPanel = mainFrame.getSelectedTab();
// call button is pressed over an already open call panel
if (selectedPanel != null
&& selectedPanel instanceof CallPanel
&& ((CallPanel) selectedPanel).getCall().getCallState()
== CallState.CALL_INITIALIZATION) {
NotificationManager.stopSound(NotificationManager.BUSY_CALL);
NotificationManager.stopSound(NotificationManager.INCOMING_CALL);
CallPanel callPanel = (CallPanel) selectedPanel;
Iterator participantPanels = callPanel.getParticipantsPanels();
while (participantPanels.hasNext()) {
CallParticipantPanel panel = (CallParticipantPanel) participantPanels.next();
panel.setState("Connecting");
}
Call call = callPanel.getCall();
answerCall(call);
}
// call button is pressed over the call list
else if (selectedPanel != null
&& selectedPanel instanceof CallListPanel
&& ((CallListPanel) selectedPanel).getCallList().getSelectedIndex() != -1) {
CallListPanel callListPanel = (CallListPanel) selectedPanel;
GuiCallParticipantRecord callRecord =
(GuiCallParticipantRecord) callListPanel.getCallList().getSelectedValue();
String stringContact = callRecord.getParticipantName();
createCall(stringContact);
}
// call button is pressed over the contact list
else if (selectedPanel != null && selectedPanel instanceof ContactListPanel) {
// call button is pressed when a meta contact is selected
if (isCallMetaContact) {
Object[] selectedContacts =
mainFrame.getContactListPanel().getContactList().getSelectedValues();
Vector telephonyContacts = new Vector();
for (int i = 0; i < selectedContacts.length; i++) {
Object o = selectedContacts[i];
if (o instanceof MetaContact) {
Contact contact =
((MetaContact) o).getDefaultContact(OperationSetBasicTelephony.class);
if (contact != null) telephonyContacts.add(contact);
else {
new ErrorDialog(
this.mainFrame,
Messages.getI18NString("warning").getText(),
Messages.getI18NString(
"contactNotSupportingTelephony",
new String[] {((MetaContact) o).getDisplayName()})
.getText())
.showDialog();
}
}
}
if (telephonyContacts.size() > 0) createCall(telephonyContacts);
} else if (!phoneNumberCombo.isComboFieldEmpty()) {
// if no contact is selected checks if the user has chosen
// or has
// writen something in the phone combo box
String stringContact = phoneNumberCombo.getEditor().getItem().toString();
createCall(stringContact);
}
} else if (selectedPanel != null && selectedPanel instanceof DialPanel) {
String stringContact = phoneNumberCombo.getEditor().getItem().toString();
createCall(stringContact);
}
} else if (buttonName.equalsIgnoreCase("hangup")) {
Component selectedPanel = this.mainFrame.getSelectedTab();
if (selectedPanel != null && selectedPanel instanceof CallPanel) {
NotificationManager.stopSound(NotificationManager.BUSY_CALL);
NotificationManager.stopSound(NotificationManager.INCOMING_CALL);
NotificationManager.stopSound(NotificationManager.OUTGOING_CALL);
CallPanel callPanel = (CallPanel) selectedPanel;
Call call = callPanel.getCall();
if (removeCallTimers.containsKey(callPanel)) {
((Timer) removeCallTimers.get(callPanel)).stop();
removeCallTimers.remove(callPanel);
}
removeCallPanel(callPanel);
if (call != null) {
ProtocolProviderService pps = call.getProtocolProvider();
OperationSetBasicTelephony telephony = mainFrame.getTelephonyOpSet(pps);
Iterator participants = call.getCallParticipants();
while (participants.hasNext()) {
try {
// now we hang up the first call participant in the
// call
telephony.hangupCallParticipant((CallParticipant) participants.next());
} catch (OperationFailedException e) {
logger.error("Hang up was not successful: " + e);
}
}
}
}
} else if (buttonName.equalsIgnoreCase("minimize")) {
JCheckBoxMenuItem hideCallPanelItem =
mainFrame.getMainMenu().getViewMenu().getHideCallPanelItem();
if (!hideCallPanelItem.isSelected()) hideCallPanelItem.setSelected(true);
this.setCallPanelVisible(false);
} else if (buttonName.equalsIgnoreCase("restore")) {
JCheckBoxMenuItem hideCallPanelItem =
mainFrame.getMainMenu().getViewMenu().getHideCallPanelItem();
if (hideCallPanelItem.isSelected()) hideCallPanelItem.setSelected(false);
this.setCallPanelVisible(true);
}
}
/* Broken:
* http://sohowww.nascom.nasa.gov/data/synoptic/gif/000123/nanc_164Mz_fd_2023_1046.gif
* http://sohowww.nascom.nasa.gov/data/synoptic/gif/000123/nanc_164Mz_fd_2023_0916.gif
* http://sohowww.nascom.nasa.gov/data/synoptic/gif/000123/meud_halph_fd_2023_1356.gif
* http://sohowww.nascom.nasa.gov/data/synoptic/gif/000213/nanc_164Mz_fd_2113_0919.gif
*
* Ok:
* http://sohowww.nascom.nasa.gov/data/synoptic/gif/000212/pdmo_cogha_fd_20000212_0814.gif
* http://sohowww.nascom.nasa.gov/data/synoptic/gif/000212/pdmo_cogha_fd_20000212_0945.gif
*/
private static void determineAttributes_(Hashtable h, String url) {
int idxl = url.lastIndexOf('/') + 1;
int idxr = url.indexOf('_');
String tmp = null;
try {
// Messages.debug(2, "Soho:DeterminAttrib1 idxl=%1 idxr=%2 url=%3", String.valueOf(idxl),
// String.valueOf(idxr), url);
tmp = (String) INSTITUTE_CODES_.get(url.substring(idxl, idxr));
h.put("U_INSTITUTE_CODE", (tmp == null) ? "unknown" : EtcUtil.makeURLish(tmp));
h.put("INSTITUTE_CODE", tmp);
idxl = idxr + 1;
idxr = url.indexOf('_', idxl);
// Messages.debug(2, "Soho:DeterminAttrib2 idxl=%1 idxr=%2", String.valueOf(idxl),
// String.valueOf(idxr));
tmp = (String) TYPE_CODES_.get(url.substring(idxl, idxr));
h.put("U_TYPE_CODE", (tmp == null) ? "unknown" : EtcUtil.makeURLish(tmp));
h.put("TYPE_CODE", tmp);
idxl = idxr + 1;
idxr = url.indexOf('_', idxl);
// Messages.debug(2, "Soho:DeterminAttrib3 idxl=%1 idxr=%2", String.valueOf(idxl),
// String.valueOf(idxr));
tmp = (String) INFO_CODES_.get(url.substring(idxl, idxr));
h.put("U_INFO_CODE", (tmp == null) ? "unknown" : EtcUtil.makeURLish(tmp));
idxl = idxr + 1;
idxr = url.indexOf('_', idxl);
// Messages.debug(2, "Soho:DeterminAttrib4 idxl=%1 idxr=%2", String.valueOf(idxl),
// String.valueOf(idxr));
tmp = url.substring(idxl, idxr);
idxl = idxr + 1;
idxr = url.indexOf('.', idxl);
// Messages.debug(2, "Soho:DeterminAttrib5 idxl=%1 idxr=%2", String.valueOf(idxl),
// String.valueOf(idxr));
tmp += url.substring(idxl, idxr);
Date d = null;
if (tmp.length() == 12) {
// Messages.debug(1, "Soho::determineAttributes before parse");
d = dateFormatter2_.parse(tmp);
tmp = RandomDate.format(d);
} else if (tmp.length() == 8) tmp = "date not parsed";
h.put("U_DATETIME", EtcUtil.makeURLish(tmp));
h.put("DATETIME", tmp);
idxl = url.lastIndexOf(".");
// Messages.debug(2, "Soho:DeterminAttrib6 idxl=%1", String.valueOf(idxl));
tmp = (String) FORMAT_CODES_.get(url.substring(idxl + 1));
// Messages.debug(1, "Soho::determineAttributes formatcode=%1", tmp);
h.put("U_FORMAT_CODE", (tmp == null) ? "unknown" : EtcUtil.makeURLish(tmp));
// Messages.debug(3, SOH02_, url);
} catch (Exception e) {
Messages.warn(0, SOH01_, url);
// Messages.debug(1, "Soho::exception during parsing (%1)", e);
// e.printStackTrace();
if (h.get("INSTITUTE_CODE") == null) h.put("INSTITUTE_CODE", "unknown");
if (h.get("U_INSTITUTE_CODE") == null) h.put("U_INSTITUTE_CODE", "unknown");
if (h.get("TYPE_CODE") == null) h.put("TYPE_CODE", "unknown");
if (h.get("U_TYPE_CODE") == null) h.put("U_TYPE_CODE", "unknown");
if (h.get("U_INFO_CODE") == null) h.put("U_INFO_CODE", "unknown");
if (h.get("DATE_TIME") == null) h.put("DATE_TIME", "unknown");
if (h.get("U_DATE_TIME") == null) h.put("U_DATE_TIME", "unknown");
if (h.get("U_FORMAT_CODE") == null) h.put("U_FORMAT_CODE", "unknown");
}
}
public void chefUpdated(Chef chef) {
Integer row = (Integer) (_chefsByName.get(chef.getName()));
int index = row.intValue();
fireTableRowsUpdated(index, index);
}
// Information needed to get a single file:
// BASE_PATH, FILE_ID, TIMESTAMP_START, TIMESTAMP_STOP, SOURCE, FILESYSTEM
private static Vector<Path> getFile(
FileSystem fs, Hashtable<String, String> config, dbutil db_util) throws Exception {
Long latestVersion = latestVersion(config, db_util);
try {
config.put("timestamp_start", config.get("timestamp_start"));
config.put("timestamp_real", latestVersion.toString());
config.put("timestamp_stop", latestVersion.toString());
} catch (Exception E) {
logger.error("Tryign to get file that is impossible to generate: " + getFullPath(config));
return null;
}
if (Integer.parseInt(config.get("timestamp_start"))
> Integer.parseInt(config.get("timestamp_stop"))) {
return null;
}
logger.debug(
"Getting DB for timestamp "
+ config.get("timestamp_start")
+ " to "
+ config.get("timestamp_stop"));
String final_result = getFullPath(config);
String temp_path_base =
config.get("local_temp_path")
+ "_"
+ config.get("task_id")
+ "_"
+ config.get("run_id")
+ "/";
Path newPath = new Path(final_result + "*");
Vector<Path> ret_path = new Vector<Path>();
String lockName = lock(final_result.replaceAll("/", "_"));
if (fs.globStatus(newPath).length != 0) {
ret_path.add(newPath);
unlock(lockName);
config.put("full_file_name", final_result);
return ret_path;
} else {
if (!config.get("source").equals("local")) {
config.put("temp_path_base", temp_path_base);
config.put("timestamp_start", config.get("timestamp_start"));
config.put("timestamp_real", latestVersion.toString());
config.put("timestamp_stop", latestVersion.toString());
Class<?> sourceClass =
Class.forName("org.gestore.plugin.source." + config.get("source") + "Source");
Method process_data = sourceClass.getMethod("process", Hashtable.class, FileSystem.class);
Object processor = sourceClass.newInstance();
Object retVal;
try {
retVal = process_data.invoke(processor, config, fs);
} catch (InvocationTargetException E) {
Throwable exception = E.getTargetException();
logger.error("Unable to call method in child class: " + exception.toString());
exception.printStackTrace(System.out);
unlock(lockName);
return null;
}
FileStatus[] files = (FileStatus[]) retVal;
if (files == null) {
logger.error("Error getting files, no files returned");
return null;
}
for (FileStatus file : files) {
Path cur_file = file.getPath();
Path cur_local_path = new Path(temp_path_base + config.get("file_id"));
String suffix = getSuffix(config.get("file_id"), cur_file.getName());
cur_local_path = cur_local_path.suffix(suffix);
Path res_path = new Path(new String(final_result + suffix));
logger.debug("Moving file" + cur_file.toString() + " to " + res_path.toString());
if (config.get("copy").equals("true")) {
fs.moveFromLocalFile(cur_file, res_path);
} else {
fs.rename(cur_file, res_path);
}
}
config.put("full_file_name", final_result);
}
}
unlock(lockName);
return ret_path;
}
public int run(String[] args) throws Exception {
// printUsage();
/*
* SETUP
*/
Configuration argConf = getConf();
Hashtable<String, String> confArg = new Hashtable<String, String>();
setup(confArg, argConf);
Date currentTime = new Date();
Date endDate = new Date(new Long(confArg.get("timestamp_stop")));
Boolean full_run = confArg.get("intermediate").matches("(?i).*true.*");
Boolean quick_add = confArg.get("quick_add").matches("(?i).*true.*");
logger.info("Running GeStore");
// ZooKeeper setup
Configuration config = HBaseConfiguration.create();
zkWatcher = new ZooKeeperWatcher(config, "Testing", new HBaseAdmin(config));
zkInstance =
new ZooKeeper(
ZKConfig.getZKQuorumServersString(config),
config.getInt("zookeeper.session.timeout", -1),
zkWatcher);
if (!confArg.get("task_id").isEmpty()) {
confArg.put("temp_path", confArg.get("temp_path") + confArg.get("task_id"));
}
String lockRequest = confArg.get("file_id");
if (!confArg.get("run_id").isEmpty())
lockRequest = lockRequest + "_" + confArg.get("run_id") + "_";
if (!confArg.get("task_id").isEmpty())
lockRequest = lockRequest + "_" + confArg.get("task_id") + "_";
// Get type of movement
toFrom type_move = checkArgs(confArg);
if (type_move == toFrom.LOCAL2REMOTE && !confArg.get("format").equals("unknown")) {
List<String> arguments = new ArrayList<String>();
arguments.add("-Dinput=" + confArg.get("local_path"));
arguments.add("-Dtable=" + confArg.get("file_id"));
arguments.add("-Dtimestamp=" + confArg.get("timestamp_stop"));
arguments.add("-Dtype=" + confArg.get("format"));
arguments.add("-Dtarget_dir=" + confArg.get("base_path") + "_" + confArg.get("file_id"));
arguments.add("-Dtemp_hdfs_path=" + confArg.get("temp_path"));
arguments.add("-Drun_id=" + confArg.get("run_id"));
if (!confArg.get("run_id").isEmpty()) arguments.add("-Drun_id=" + confArg.get("run_id"));
if (!confArg.get("task_id").isEmpty()) arguments.add("-Dtask_id=" + confArg.get("task_id"));
if (quick_add) arguments.add("-Dquick_add=" + confArg.get("quick_add"));
String lockName = lock(lockRequest);
String[] argumentString = arguments.toArray(new String[arguments.size()]);
adddb.main(argumentString);
unlock(lockName);
System.exit(0);
}
// Database registration
dbutil db_util = new dbutil(config);
db_util.register_database(confArg.get("db_name_files"), true);
db_util.register_database(confArg.get("db_name_runs"), true);
db_util.register_database(confArg.get("db_name_updates"), true);
FileSystem hdfs = FileSystem.get(config);
FileSystem localFS = FileSystem.getLocal(config);
// Get source type
confArg.put("source", getSource(db_util, confArg.get("db_name_files"), confArg.get("file_id")));
confArg.put(
"database", isDatabase(db_util, confArg.get("db_name_files"), confArg.get("file_id")));
if (!confArg.get("source").equals("local")
&& type_move == toFrom.REMOTE2LOCAL
&& !confArg.get("timestamp_stop").equals(Integer.toString(Integer.MAX_VALUE))) {
confArg.put("timestamp_stop", Long.toString(latestVersion(confArg, db_util)));
}
/*
* Get previous timestamp
*/
Get run_id_get = new Get(confArg.get("run_id").getBytes());
Result run_get = db_util.doGet(confArg.get("db_name_runs"), run_id_get);
KeyValue run_file_prev =
run_get.getColumnLatest(
"d".getBytes(), (confArg.get("file_id") + "_db_timestamp").getBytes());
String last_timestamp = new String("0");
if (null != run_file_prev && !confArg.get("source").equals("local")) {
long last_timestamp_real = run_file_prev.getTimestamp();
Long current_timestamp = new Long(confArg.get("timestamp_real"));
if ((current_timestamp - last_timestamp_real) > 36000) {
last_timestamp = new String(run_file_prev.getValue());
Integer lastTimestamp = new Integer(last_timestamp);
lastTimestamp += 1;
last_timestamp = lastTimestamp.toString();
logger.info("Last timestamp: " + last_timestamp + " End data: " + endDate);
Date last_run = new Date(run_file_prev.getTimestamp());
if (last_run.before(endDate) && !full_run) {
confArg.put("timestamp_start", last_timestamp);
}
}
}
Integer tse = new Integer(confArg.get("timestamp_stop"));
Integer tss = new Integer(confArg.get("timestamp_start"));
if (tss > tse) {
logger.info("No new version of requested file.");
return 0;
}
/*
* Generate file
*/
String lockName = lock(lockRequest);
Get file_id_get = new Get(confArg.get("file_id").getBytes());
Result file_get = db_util.doGet(confArg.get("db_name_files"), file_id_get);
if (!file_get.isEmpty()) {
boolean found =
hasFile(
db_util,
hdfs,
confArg.get("db_name_files"),
confArg.get("file_id"),
getFullPath(confArg));
if (confArg.get("source").equals("fullfile")) {
found = false;
}
String filenames_put =
getFileNames(
db_util, confArg.get("db_name_files"), confArg.get("file_id"), getFullPath(confArg));
// Filename not found in file database
if (!found && type_move == toFrom.REMOTE2LOCAL) {
if (!confArg.get("source").equals("local")) {
// Generate intermediate file
if (getFile(hdfs, confArg, db_util) == null) {
unlock(lockName);
return 1;
}
// Put generated file into file database
if (!confArg.get("format").equals("fullfile")) {
putFileEntry(
db_util,
hdfs,
confArg.get("db_name_files"),
confArg.get("file_id"),
confArg.get("full_file_name"),
confArg.get("source"));
}
} else {
logger.warn("Remote file not found, and cannot be generated! File: " + confArg);
unlock(lockName);
return 1;
}
}
} else {
if (type_move == toFrom.REMOTE2LOCAL) {
logger.warn("Remote file not found, and cannot be generated.");
unlock(lockName);
return 1;
}
}
/*
* Copy file
* Update tables
*/
if (type_move == toFrom.LOCAL2REMOTE) {
if (!confArg.get("format").equals("fullfile")) {
putFileEntry(
db_util,
hdfs,
confArg.get("db_name_files"),
confArg.get("file_id"),
getFullPath(confArg),
confArg.get("source"));
}
putRunEntry(
db_util,
confArg.get("db_name_runs"),
confArg.get("run_id"),
confArg.get("file_id"),
confArg.get("type"),
confArg.get("timestamp_real"),
confArg.get("timestamp_stop"),
getFullPath(confArg),
confArg.get("delimiter"));
hdfs.copyFromLocalFile(new Path(confArg.get("local_path")), new Path(getFullPath(confArg)));
} else if (type_move == toFrom.REMOTE2LOCAL) {
FileStatus[] files = hdfs.globStatus(new Path(getFullPath(confArg) + "*"));
putRunEntry(
db_util,
confArg.get("db_name_runs"),
confArg.get("run_id"),
confArg.get("file_id"),
confArg.get("type"),
confArg.get("timestamp_real"),
confArg.get("timestamp_stop"),
getFullPath(confArg),
confArg.get("delimiter"));
unlock(lockName);
for (FileStatus file : files) {
Path cur_file = file.getPath();
Path cur_local_path =
new Path(new String(confArg.get("local_path") + confArg.get("file_id")));
String suffix = getSuffix(getFileName(confArg), cur_file.getName());
if (suffix.length() > 0) {
cur_local_path = cur_local_path.suffix(new String("." + suffix));
}
if (confArg.get("copy").equals("true")) {
String crc = hdfs.getFileChecksum(cur_file).toString();
if (checksumLocalTest(cur_local_path, crc)) {
continue;
} else {
hdfs.copyToLocalFile(cur_file, cur_local_path);
writeChecksum(cur_local_path, crc);
}
} else {
System.out.println(cur_local_path + "\t" + cur_file);
}
}
}
unlock(lockName);
return 0;
}
static void processItem(Element curItem) {
String itemRow = "";
// get the item columns
String itemID = getAttributeText(curItem, "ItemID");
itemRow += itemID + ",";
String name = getElementTextByTagNameNR(curItem, "Name");
itemRow += wrapQuotations(name) + ",";
// get Buy_Price, just put in the "" if it doesn't exist.
String buyPrice = strip(getElementTextByTagNameNR(curItem, "Buy_Price"));
if (!buyPrice.equals("")) itemRow += buyPrice + ",";
else itemRow += "NULL,";
// get First_Bid, just put in the "" if it doesn't exist.
String firstBid = strip(getElementTextByTagNameNR(curItem, "First_Bid"));
itemRow += firstBid + ",";
String started = convertToSQLTime(getElementTextByTagNameNR(curItem, "Started"));
itemRow += wrapQuotations(started) + ",";
String ends = convertToSQLTime(getElementTextByTagNameNR(curItem, "Ends"));
itemRow += wrapQuotations(ends) + ",";
Element locationElem = getElementByTagNameNR(curItem, "Location");
String latitude = getAttributeText(locationElem, "Latitude");
itemRow += wrapQuotations(latitude) + ",";
String longitude = getAttributeText(locationElem, "Longitude");
itemRow += wrapQuotations(longitude) + ",";
String location = getElementText(locationElem);
itemRow += wrapQuotations(location) + ",";
String country = getElementTextByTagNameNR(curItem, "Country");
itemRow += wrapQuotations(country) + ",";
String description = getElementTextByTagNameNR(curItem, "Description");
description = description.substring(0, Math.min(description.length(), 4000));
itemRow += wrapQuotations(description) + ",";
Element sellerElem = getElementByTagNameNR(curItem, "Seller");
String sellerID = getAttributeText(sellerElem, "UserID");
itemRow += wrapQuotations(sellerID);
String sellerRating = getAttributeText(sellerElem, "Rating");
// check if this seller already exists in our hashtable
String sellerRow[] = userList.get(sellerID);
if (sellerRow == null) {
sellerRow = new String[5];
sellerRow[0] = sellerID;
sellerRow[1] = "NULL";
sellerRow[2] = sellerRating;
sellerRow[3] = "NULL";
sellerRow[4] = "NULL";
userList.put(sellerID, sellerRow);
} else {
sellerRow[2] = sellerRating;
// if(sellerRow[1] != null)
// bothSellerAndBuyer.add(sellerID);
}
// String sellerRow = wrapQuotations(seller) + "," + wrapQuotations(sellerRating);
// System.out.println("itemid: " + itemID);
// System.out.println("name: " + name);
// System.out.println("buy_Price: " + buyPrice);
// System.out.println("started: " + started);
// System.out.println("ends: " + ends);
// System.out.println("location: " + location);
// System.out.println("latitude: " + latitude);
// System.out.println("longitude: " + longitude);
// System.out.println("country: " + country);
// System.out.println("description: " + description);
// System.out.println("seller ID: " + seller);
// System.out.println("SQL line: " + itemRow);
// System.out.println("seller rating: " + sellerRating);
// for each itemCategory
Element[] categories = getElementsByTagNameNR(curItem, "Category");
for (Element category : categories) {
String categoryName = getElementText(category);
String categoryRow = wrapQuotations(itemID) + "," + wrapQuotations(categoryName);
writeLine(itemCategoriesWriter, categoryRow);
}
// for each bid
Element bidsElem = getElementByTagNameNR(curItem, "Bids");
Element[] bids = getElementsByTagNameNR(bidsElem, "Bid");
for (Element bid : bids) {
// process the user (bidder) first
Element bidderElem = getElementByTagNameNR(bid, "Bidder");
String bidderID = getAttributeText(bidderElem, "UserID");
String bidderRating = getAttributeText(bidderElem, "Rating");
String bidderLocation = getElementTextByTagNameNR(bidderElem, "Location");
String bidderCountry = getElementTextByTagNameNR(bidderElem, "Country");
String bidderRow[] = userList.get(bidderID);
if (bidderRow == null) {
bidderRow = new String[5];
bidderRow[0] = bidderID;
bidderRow[1] = bidderRating;
bidderRow[2] = "NULL";
bidderRow[3] = bidderLocation;
bidderRow[4] = bidderCountry;
userList.put(bidderID, bidderRow);
} else {
bidderRow[1] = bidderRating;
bidderRow[3] = bidderLocation;
bidderRow[4] = bidderCountry;
// if(!bidderRow[2].equals("NULL"))
// bothSellerAndBuyer.add(bidderID);
}
String bidTime = convertToSQLTime(getElementTextByTagNameNR(bid, "Time"));
String amount = strip(getElementTextByTagNameNR(bid, "Amount"));
String bidRow =
wrapQuotations(bidderID)
+ ","
+ itemID
+ ","
+ wrapQuotations(bidTime)
+ ","
+ wrapQuotations(amount);
writeLine(bidWriter, bidRow);
}
writeLine(itemWriter, itemRow);
// System.out.println("--------END ITEM PROCESS---------");
}
private static String getFullPath(Hashtable<String, String> config) {
return new String(config.get("base_path") + "_" + getFileName(config));
}
private static String getFileName(Hashtable<String, String> config) {
String retString =
config.get("file_id")
+ "_"
+ config.get("timestamp_start")
+ "_"
+ config.get("timestamp_stop")
+ "_"
+ config.get("delimiter").hashCode()
+ "_"
+ config.get("taxon")
+ "_"
+ config.get("split");
if (config.get("database").equals("y")) {
return retString;
} else {
if (config.get("task_id").isEmpty()) {
return retString + "_" + config.get("run_id") + "_";
} else {
return retString + "_" + config.get("run_id") + "_" + config.get("task_id") + "_";
}
}
}
