public static void main(String[] args) throws IOException {
Path baseDir = null;
String localPath = null;
String preservePath = null;
String sIgnoreTablesFilename = null;
String sNoPreserveFilename = null;
String sDateString = null;
long size = 0;
// UNIX dates for right now
long now = new java.util.Date().getTime() / 1000;
long maxDate = now;
for (int i = 0; i < args.length; i++) {
if (args[i].equals("--hdfs-path")) {
baseDir = new Path(args[++i]);
continue;
}
if (args[i].equals("--local-path")) {
localPath = args[++i];
continue;
}
if (args[i].equals("--preserve-path")) {
preservePath = args[++i];
continue;
}
if (args[i].equals("--no-preserve")) {
sNoPreserveFilename = args[++i];
continue;
}
if (args[i].equals("--ignore-tables")) {
sIgnoreTablesFilename = args[++i];
continue;
}
if (args[i].equals("--sleep")) {
try {
m_nSleepSeconds = Integer.parseInt(args[++i]);
} catch (Exception e) {
System.err.println("ERROR: " + e.toString() + "\n");
usage();
}
continue;
}
if (args[i].equals("--dry-run")) {
m_bDryRun = true;
continue;
}
if (args[i].equals("--date")) {
sDateString = args[++i];
continue;
}
if (args[i].equals("--max-date")) {
maxDate = Long.parseLong(args[++i]);
continue;
}
if (args[i].equals("--max-bytes")) {
size = Long.parseLong(args[++i]);
continue;
}
System.err.println("ERROR: unknown arg " + args[i]);
usage();
}
if (baseDir == null || localPath == null || preservePath == null || sDateString == null) {
usage();
}
long minDate;
if ("yesterday".equals(sDateString)) {
// figure out yesterday's dates
Calendar cal = Calendar.getInstance();
cal.roll(Calendar.DAY_OF_YEAR, -1);
// yesterday midnight
cal.set(Calendar.HOUR_OF_DAY, 0);
cal.set(Calendar.MINUTE, 0);
cal.set(Calendar.SECOND, 0);
cal.set(Calendar.MILLISECOND, 0);
minDate = cal.getTimeInMillis() / 1000;
// yesterday end of day
cal.set(Calendar.HOUR_OF_DAY, 23);
cal.set(Calendar.MINUTE, 59);
cal.set(Calendar.SECOND, 59);
cal.set(Calendar.MILLISECOND, 999);
maxDate = cal.getTimeInMillis() / 1000;
} else if ("last-week".equals(sDateString)) {
minDate = maxDate - (7 * 24 * 60 * 60);
} else if ("last-day".equals(sDateString)) {
minDate = maxDate - (24 * 60 * 60);
} else {
// UNIX date since epoch of last backup
minDate = Long.parseLong(sDateString);
}
long tmpDate = 0;
BackupHdfs bak = new BackupHdfs();
// initialize the list of tables to ignore
if (sIgnoreTablesFilename != null) {
bak.initializeTablesToIgnore(sIgnoreTablesFilename);
}
// initialize list of files to not preserve
if (sNoPreserveFilename != null) {
bak.initializeNoPreserve(sNoPreserveFilename);
}
ArrayList<Path> pathList = new ArrayList<Path>(2000);
HashMap<Path, Long> hmTimestamps = new HashMap<Path, Long>();
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
// If the HDFS path is a dir continue
if (fs.getFileStatus(baseDir).isDir()) {
Calendar cal = Calendar.getInstance();
System.err.println("");
cal.setTimeInMillis(minDate * 1000);
System.err.println("min date = " + cal.getTime().toString());
cal.setTimeInMillis(maxDate * 1000);
System.err.println("max date = " + cal.getTime().toString());
System.err.println("");
System.err.println("Searching filesystem: " + baseDir.toUri().getPath());
bak.checkDir(fs, minDate, maxDate, baseDir, pathList, hmTimestamps);
System.err.println("");
System.err.println("Skipped " + m_nIgnoredTables + " files due to ignored tables");
System.err.println("");
System.err.println("Number of files to backup = " + pathList.size());
System.err.println("Total bytes to backup = " + prettyPrintBytes(m_nTotalBytes));
System.err.println("");
System.err.println("sorting list of files...");
Collections.sort(pathList, new DateComparator(hmTimestamps));
System.err.println("done");
System.err.println("");
System.err.println("starting backup...");
tmpDate = bak.backupFiles(localPath, preservePath, fs, pathList, size);
bak.closeFiles();
System.err.println("");
System.err.println("backup completed...");
}
if (tmpDate == 0) {
// If not size limit reached print out date for right now
System.out.println(maxDate);
} else {
// Print out date for last file backed up
System.err.println("Size limit reached.");
System.out.println(tmpDate);
}
System.exit(0);
}
/**
* Parses the given text as an integer, avoiding runtime exceptions.
*
* @param aText the text to parse as an integer, can be <code>null</code> or empty;
* @param aDefault the default value to return in case parsing failed.
* @return the numeric representation of the given text, or the given default if the text could
* not be parsed correctly as integer.
* @see Long#parseLong(String)
*/
public static long safeParseLong(final String aText, final long aDefault) {
try {
return Long.parseLong(aText);
} catch (NumberFormatException exception) {
return aDefault;
}
}
private long[] parse(String[] as) {
long ans[] = new long[as.length];
int index = 0;
for (String cur : as) ans[index++] = Long.parseLong(cur, 2);
Arrays.sort(ans);
return ans;
}
public String _long2date(String args[]) {
try {
return new Date(Long.parseLong(args[1])).toString();
} catch (Exception e) {
e.printStackTrace();
}
return "not a valid long";
}
/**
* Ensure that the supplied importance is a valid value.
*
* @param imps String representation of the importance.
* @throws IllegalArgumentException if the importance is not valid.
*/
protected static long validateImportance(String imps) throws IllegalArgumentException {
long imp;
try {
imp = Long.parseLong(imps);
} catch (NumberFormatException nfe) {
throw new IllegalArgumentException("importance value " + imps + " is not legal");
}
if (imp < 0)
throw new IllegalArgumentException(
"importance value " + imps + " is not legal (must be positive)");
return (imp);
}
public String _tstamp(String args[]) {
String format = "yyyyMMddHHmm";
long now = System.currentTimeMillis();
TimeZone tz = TimeZone.getTimeZone("UTC");
if (args.length > 1) {
format = args[1];
}
if (args.length > 2) {
tz = TimeZone.getTimeZone(args[2]);
}
if (args.length > 3) {
now = Long.parseLong(args[3]);
}
if (args.length > 4) {
domain.warning("Too many arguments for tstamp: " + Arrays.toString(args));
}
SimpleDateFormat sdf = new SimpleDateFormat(format);
sdf.setTimeZone(tz);
return sdf.format(new Date(now));
}
protected boolean checkConditionValue(String value, String comparator, String target)
throws IllegalArgumentException {
// Check to see if we're talking about times, and if so, de-time them.
if (comparator.contains("<") || comparator.contains(">") || comparator.contains("%")) {
// We're doing maths. Are they times? Times have colons.
if (value.contains(":")) {
String[] s = value.split(":");
if (s.length == 2)
try {
value = String.valueOf(Long.parseLong(s[0]) * 60 + Long.parseLong(s[1]));
} catch (NumberFormatException nfE) {
}
}
if (target.contains(":")) {
String[] s = target.split(":");
if (s.length == 2)
try {
target = String.valueOf(Long.parseLong(s[0]) * 60 + Long.parseLong(s[1]));
} catch (NumberFormatException nfE) {
}
}
}
if ("=".equals(comparator)) {
return value.equals(target);
} else if ("!=".equals(comparator)) {
return !value.equals(target);
} else if ("<".equals(comparator)) {
try {
return (Long.parseLong(value) < Long.parseLong(target));
} catch (NumberFormatException nfE) {
}
try {
return (Double.parseDouble(value) < Double.parseDouble(target));
} catch (NumberFormatException nfE) {
}
return (value.compareTo(target) < 0);
} else if ("<=".equals(comparator)) {
try {
return (Long.parseLong(value) <= Long.parseLong(target));
} catch (NumberFormatException nfE) {
}
try {
return (Double.parseDouble(value) <= Double.parseDouble(target));
} catch (NumberFormatException nfE) {
}
return (value.compareTo(target) <= 0);
} else if (">".equals(comparator)) {
try {
return (Long.parseLong(value) > Long.parseLong(target));
} catch (NumberFormatException nfE) {
}
try {
return (Double.parseDouble(value) > Double.parseDouble(target));
} catch (NumberFormatException nfE) {
}
return (value.compareTo(target) > 0);
} else if (">=".equals(comparator)) {
try {
return (Long.parseLong(value) >= Long.parseLong(target));
} catch (NumberFormatException nfE) {
}
try {
return (Double.parseDouble(value) >= Double.parseDouble(target));
} catch (NumberFormatException nfE) {
}
return (value.compareTo(target) >= 0);
} else if ("%".equals(comparator)) {
try {
return (0 == (Long.parseLong(value) % Long.parseLong(target)));
} catch (NumberFormatException nfE) {
return false;
} catch (ArithmeticException aE) { // most likely target == 0, % by 0 is invalid
return false;
}
}
throw new IllegalArgumentException("Invalid comparator : " + comparator);
}
public long nextLong() {
return Long.parseLong(next());
}
/* (non-Javadoc)
* @see org.nees.rbnb.RBNBBase#setArgs(org.apache.commons.cli.CommandLine)
*/
protected boolean setArgs(CommandLine cmd) {
if (!setBaseArgs(cmd)) return false;
if (cmd.hasOption('k')) {
String a = cmd.getOptionValue('k');
if (a != null) sinkName = a;
}
if (cmd.hasOption('c')) {
String a = cmd.getOptionValue('c');
if (a != null) channelPathPattern = a;
}
if (cmd.hasOption('C')) {
String a = cmd.getOptionValue('C');
if (a != null) channelPathListString = a;
}
if (cmd.hasOption('W')) {
String a = cmd.getOptionValue('W');
if (a != null) {
try {
long value = Long.parseLong(a);
durationTime = value;
} catch (Exception ex) {
System.out.println("Failed to parse Collection Time (" + a + "): " + ex);
}
}
}
if (cmd.hasOption('t')) {
String a = cmd.getOptionValue('t');
if (a != null) endTimeString = a;
}
if (cmd.hasOption('x')) {
String a = cmd.getOptionValue('x');
if (a != null) {
if (a.equals("true")) includeHidden = true;
if (a.equals("false")) includeHidden = false;
}
}
if (cmd.hasOption('v')) {
String a = cmd.getOptionValue('v');
if (a != null) videoChannelPath = a;
}
if (cmd.hasOption('d')) {
String a = cmd.getOptionValue('d');
if (a != null) archiveDirectory = a;
}
if (cmd.hasOption('M')) {
String a = cmd.getOptionValue('M');
if (a != null) movieFileName = a;
}
if (cmd.hasOption('D')) {
String a = cmd.getOptionValue('D');
if (a != null) dataFileName = a;
}
if (cmd.hasOption('f')) {
String a = cmd.getOptionValue('f');
if (a != null) {
try {
float value = Float.parseFloat(a);
userGivenFrameRate = value;
} catch (Exception ex) {
System.out.println("Failed to parse Frame Rate (" + a + "): " + ex);
}
}
}
if (cmd.hasOption('i')) {
String a = cmd.getOptionValue('i');
if (a != null) {
try {
int value = Integer.parseInt(a);
itemsToSkip = value;
} catch (Exception ex) {
System.out.println("Failed to parse Frame Rate (" + a + "): " + ex);
}
}
}
if (cmd.hasOption('Z')) {
repeatMissingValue = false;
}
System.out.println("User Supplied parameter (or relivent default value) are:");
System.out.println("sink name = " + sinkName);
System.out.println("data channel path pattern = " + channelPathPattern);
System.out.println("data channel list = " + channelPathListString);
System.out.println("include hidden channel value = " + includeHidden);
System.out.println("zero for missing data = " + !repeatMissingValue);
System.out.println("video source path = " + videoChannelPath);
System.out.println("archive directory = " + archiveDirectory);
System.out.println("movie file name = " + movieFileName);
System.out.println("data file name = " + dataFileName);
System.out.println("frame rate of final movie = " + userGivenFrameRate);
System.out.println("items to skip = " + itemsToSkip);
// System.out.println("wait time between cycles = " + waitTime);
System.out.println("duration = " + durationTime);
System.out.println("end time string " + endTimeString);
System.out.println("");
System.out.println("Use DataVideoGather -h to see parameters");
System.out.println("");
setupChannelList();
System.out.println("Starting DataVideoGather on " + getServer() + " as " + sinkName);
System.out.println("Gatering data from: ");
for (int i = 0; i < channelPathArray.length; i++) {
System.out.println(" " + channelPathArray[i]);
}
if ((videoChannelPath != null) && (!videoChannelPath.equals(""))) {
System.out.println("Gatering video from: " + videoChannelPath);
System.out.println("Frame rate = " + userGivenFrameRate + " frames per second.");
System.out.println("Data frames to skip between video frame = " + itemsToSkip);
}
return true;
}
/**
* Reads the data from a given reader.
*
* @param aProject the project to read the settings to;
* @param aReader the reader to read the data from, cannot be <code>null</code>.
* @throws IOException in case of I/O problems.
*/
@SuppressWarnings("boxing")
public static void read(final StubDataSet aDataSet, final Reader aReader) throws IOException {
int size = -1;
Integer rate = null, channels = null, enabledChannels = null;
long triggerPos = -1L;
long absLen = -1L;
// assume 'new' file format is in use, don't support uncompressed ones...
boolean compressed = true;
final BufferedReader br = new BufferedReader(aReader);
final List<String[]> dataValues = new ArrayList<String[]>();
String line;
while ((line = br.readLine()) != null) {
// Determine whether the line is an instruction, or data...
final Matcher instructionMatcher = OLS_INSTRUCTION_PATTERN.matcher(line);
final Matcher dataMatcher = OLS_DATA_PATTERN.matcher(line);
if (dataMatcher.matches()) {
final String[] dataPair = new String[] {dataMatcher.group(1), dataMatcher.group(2)};
dataValues.add(dataPair);
} else if (instructionMatcher.matches()) {
// Ok; found an instruction...
final String instrKey = instructionMatcher.group(1);
final String instrValue = instructionMatcher.group(2);
if ("Size".equals(instrKey)) {
size = safeParseInt(instrValue);
} else if ("Rate".equals(instrKey)) {
rate = safeParseInt(instrValue);
} else if ("Channels".equals(instrKey)) {
channels = safeParseInt(instrValue);
} else if ("TriggerPosition".equals(instrKey)) {
triggerPos = Long.parseLong(instrValue);
} else if ("EnabledChannels".equals(instrKey)) {
enabledChannels = safeParseInt(instrValue);
} else if ("CursorEnabled".equals(instrKey)) {
aDataSet.setCursorsEnabled(Boolean.parseBoolean(instrValue));
} else if ("Compressed".equals(instrKey)) {
compressed = Boolean.parseBoolean(instrValue);
} else if ("AbsoluteLength".equals(instrKey)) {
absLen = Long.parseLong(instrValue);
} else if ("CursorA".equals(instrKey)) {
final long value = safeParseLong(instrValue);
if (value > Long.MIN_VALUE) {
aDataSet.getCursor(0).setTimestamp(value);
}
} else if ("CursorB".equals(instrKey)) {
final long value = safeParseLong(instrValue);
if (value > Long.MIN_VALUE) {
aDataSet.getCursor(1).setTimestamp(value);
}
} else if (instrKey.startsWith("Cursor")) {
final int idx = safeParseInt(instrKey.substring(6));
final long pos = Long.parseLong(instrValue);
if (pos > Long.MIN_VALUE) {
aDataSet.getCursor(idx).setTimestamp(pos);
}
}
}
}
// Perform some sanity checks, make it not possible to import invalid
// data...
if (dataValues.isEmpty()) {
throw new IOException("Data file does not contain any sample data!");
}
if (!compressed) {
throw new IOException(
"Uncompressed data file found! Please send this file to the OLS developers!");
}
// In case the size is not provided (as of 0.9.4 no longer mandatory),
// take the length of the data values as size indicator...
if (size < 0) {
size = dataValues.size();
}
if (size != dataValues.size()) {
throw new IOException("Data file is corrupt?! Data size does not match sample count!");
}
if (rate == null) {
throw new IOException("Data file is corrupt?! Sample rate is not provided!");
}
if ((channels == null) || (channels <= 0) || (channels > 32)) {
throw new IOException("Data file is corrupt?! Channel count is not provided!");
}
// Make sure the enabled channels are defined...
if (enabledChannels == null) {
enabledChannels = NumberUtils.getBitMask(channels);
}
int[] values = new int[size];
long[] timestamps = new long[size];
try {
for (int i = 0; i < size; i++) {
final String[] dataPair = dataValues.get(i);
values[i] = (int) Long.parseLong(dataPair[0], 16);
timestamps[i] = Long.parseLong(dataPair[1], 10) & Long.MAX_VALUE;
}
} catch (final NumberFormatException exception) {
throw new IOException("Invalid data encountered.", exception);
}
// Allow the absolute length to be undefined, in which case the last
// time stamp is used (+ some margin to be able to see the last
// sample)...
long absoluteLength = Math.max(absLen, timestamps[size - 1]);
// Finally set the captured data, and notify all event listeners...
aDataSet.setCapturedData(
new CapturedData(
values, timestamps, triggerPos, rate, channels, enabledChannels, absoluteLength));
}
private static boolean handleURL(String address) {
Main.status(String.format("Processing page \"%s\".", address));
try {
NodeList posts = getPosts(address);
if (posts.toNodeArray().length == 0) {
return false;
}
for (Node post_node : posts.toNodeArray()) {
if (post_node instanceof TagNode) {
TagNode post = (TagNode) post_node;
Post new_post = new Post(Long.parseLong(post.getAttribute("id").substring(5)));
if (!Main.post_post_hash.containsKey(new_post)) {
NodeList photo_posts = getPhotoPosts(post.getChildren());
NodeList remarks = getRemarks(photo_posts);
for (Node node : remarks.toNodeArray()) {
Matcher matcher = lores.matcher(node.getText());
String media_url = "";
if (matcher.find()) {
media_url = matcher.group();
media_url = media_url.substring(17, media_url.length() - 1);
}
String thumb =
media_url.replace(
media_url.substring(media_url.lastIndexOf("_"), media_url.lastIndexOf(".")),
"_75sq");
URL thumb_url = new URL(thumb);
new_post.pictures.add(new Picture(new URL(media_url), thumb_url));
}
NodeList photoset_posts = getPhotosetPosts(post.getChildren());
NodeList iframes = getIFrames(photoset_posts);
for (Node node : iframes.toNodeArray()) {
if (node instanceof TagNode) {
String iframe_url = ((TagNode) node).getAttribute("src");
Parser parser2 = new Parser(iframe_url);
NodeList a_list = parser2.extractAllNodesThatMatch(new TagNameFilter("a"));
Node[] a_array = a_list.toNodeArray();
Node[] img_array =
a_list.extractAllNodesThatMatch(new TagNameFilter("img"), true).toNodeArray();
String media_url;
for (int i = 0; i < a_array.length; i++) {
media_url = ((TagNode) img_array[i]).getAttribute("src");
String thumb =
media_url.replace(
media_url.substring(
media_url.lastIndexOf("_"), media_url.lastIndexOf(".")),
"_75sq");
URL thumb_url = new URL(thumb);
new_post.pictures.add(new Picture(new URL(media_url), thumb_url));
}
}
}
Main.handlePost(new_post);
} else {
new_post = post_post_hash.get(new_post);
handleNonDownloadPost(new_post);
}
}
}
} catch (Exception ex) {
ex.printStackTrace();
Main.status("Error handling post.");
}
return true;
}
@SuppressWarnings("unchecked")
public static Object directBind(
String name, Annotation[] annotations, String value, Class<?> clazz, Type type)
throws Exception {
Logger.trace(
"directBind: value ["
+ value
+ "] annotation ["
+ Utils.join(annotations, " ")
+ "] Class ["
+ clazz
+ "]");
boolean nullOrEmpty = value == null || value.trim().length() == 0;
if (annotations != null) {
for (Annotation annotation : annotations) {
if (annotation.annotationType().equals(As.class)) {
Class<? extends TypeBinder<?>> toInstanciate = ((As) annotation).binder();
if (!(toInstanciate.equals(As.DEFAULT.class))) {
// Instantiate the binder
TypeBinder<?> myInstance = toInstanciate.newInstance();
return myInstance.bind(name, annotations, value, clazz, type);
}
}
}
}
// custom types
for (Class<?> c : supportedTypes.keySet()) {
Logger.trace("directBind: value [" + value + "] c [" + c + "] Class [" + clazz + "]");
if (c.isAssignableFrom(clazz)) {
Logger.trace("directBind: isAssignableFrom is true");
return supportedTypes.get(c).bind(name, annotations, value, clazz, type);
}
}
// application custom types
for (Class<TypeBinder<?>> c : Play.classloader.getAssignableClasses(TypeBinder.class)) {
if (c.isAnnotationPresent(Global.class)) {
Class<?> forType =
(Class) ((ParameterizedType) c.getGenericInterfaces()[0]).getActualTypeArguments()[0];
if (forType.isAssignableFrom(clazz)) {
return c.newInstance().bind(name, annotations, value, clazz, type);
}
}
}
// raw String
if (clazz.equals(String.class)) {
return value;
}
// Enums
if (Enum.class.isAssignableFrom(clazz)) {
if (nullOrEmpty) {
return null;
}
return Enum.valueOf((Class<Enum>) clazz, value);
}
// int or Integer binding
if (clazz.getName().equals("int") || clazz.equals(Integer.class)) {
if (nullOrEmpty) {
return clazz.isPrimitive() ? 0 : null;
}
return Integer.parseInt(value.contains(".") ? value.substring(0, value.indexOf(".")) : value);
}
// long or Long binding
if (clazz.getName().equals("long") || clazz.equals(Long.class)) {
if (nullOrEmpty) {
return clazz.isPrimitive() ? 0l : null;
}
return Long.parseLong(value.contains(".") ? value.substring(0, value.indexOf(".")) : value);
}
// byte or Byte binding
if (clazz.getName().equals("byte") || clazz.equals(Byte.class)) {
if (nullOrEmpty) {
return clazz.isPrimitive() ? (byte) 0 : null;
}
return Byte.parseByte(value.contains(".") ? value.substring(0, value.indexOf(".")) : value);
}
// short or Short binding
if (clazz.getName().equals("short") || clazz.equals(Short.class)) {
if (nullOrEmpty) {
return clazz.isPrimitive() ? (short) 0 : null;
}
return Short.parseShort(value.contains(".") ? value.substring(0, value.indexOf(".")) : value);
}
// float or Float binding
if (clazz.getName().equals("float") || clazz.equals(Float.class)) {
if (nullOrEmpty) {
return clazz.isPrimitive() ? 0f : null;
}
return Float.parseFloat(value);
}
// double or Double binding
if (clazz.getName().equals("double") || clazz.equals(Double.class)) {
if (nullOrEmpty) {
return clazz.isPrimitive() ? 0d : null;
}
return Double.parseDouble(value);
}
// BigDecimal binding
if (clazz.equals(BigDecimal.class)) {
if (nullOrEmpty) {
return null;
}
return new BigDecimal(value);
}
// boolean or Boolean binding
if (clazz.getName().equals("boolean") || clazz.equals(Boolean.class)) {
if (nullOrEmpty) {
return clazz.isPrimitive() ? false : null;
}
if (value.equals("1")
|| value.toLowerCase().equals("on")
|| value.toLowerCase().equals("yes")) {
return true;
}
return Boolean.parseBoolean(value);
}
return null;
}