public static String getGoalText(
final ChallengeGoalData goal, final ChallengeData data, final ChallengeDataModel model) {
String rawText = goal.getFormattedString();
if (rawText == null || rawText.length() == 0) {
return null;
}
rawText =
(String)
ChallengeFormatter.extractValue(
rawText, ChallengeFormatter.Type.STRING, model.getParams());
for (Matcher match = ChallengeGoalView.GOAL_BREED_PATTERN.matcher(rawText);
match.find();
match = ChallengeGoalView.GOAL_BREED_PATTERN.matcher(rawText)) {
final String name =
WakfuTranslator.getInstance()
.getString(7, PrimitiveConverter.getInteger(match.group(1)), new Object[0]);
rawText = match.replaceFirst(name);
}
for (Matcher match = ChallengeGoalView.GOAL_RESOURCE_PATTERN.matcher(rawText);
match.find();
match = ChallengeGoalView.GOAL_RESOURCE_PATTERN.matcher(rawText)) {
final String name =
WakfuTranslator.getInstance()
.getString(12, PrimitiveConverter.getInteger(match.group(1)), new Object[0]);
rawText = match.replaceFirst(name);
}
for (Matcher match = ChallengeGoalView.GOAL_VAR_PATTERN.matcher(rawText);
match.find();
match = ChallengeGoalView.GOAL_VAR_PATTERN.matcher(rawText)) {
final Long value = data.getVarValue(match.group().substring(1, match.group().length() - 1));
rawText = match.replaceFirst(value.toString());
}
return rawText;
}
public static void main(String[] args) throws IOException {
Path baseDir = null;
String localPath = null;
String preservePath = null;
String sIgnoreTablesFilename = null;
String sNoPreserveFilename = null;
String sDateString = null;
long size = 0;
// UNIX dates for right now
long now = new java.util.Date().getTime() / 1000;
long maxDate = now;
for (int i = 0; i < args.length; i++) {
if (args[i].equals("--hdfs-path")) {
baseDir = new Path(args[++i]);
continue;
}
if (args[i].equals("--local-path")) {
localPath = args[++i];
continue;
}
if (args[i].equals("--preserve-path")) {
preservePath = args[++i];
continue;
}
if (args[i].equals("--no-preserve")) {
sNoPreserveFilename = args[++i];
continue;
}
if (args[i].equals("--ignore-tables")) {
sIgnoreTablesFilename = args[++i];
continue;
}
if (args[i].equals("--sleep")) {
try {
m_nSleepSeconds = Integer.parseInt(args[++i]);
} catch (Exception e) {
System.err.println("ERROR: " + e.toString() + "\n");
usage();
}
continue;
}
if (args[i].equals("--dry-run")) {
m_bDryRun = true;
continue;
}
if (args[i].equals("--date")) {
sDateString = args[++i];
continue;
}
if (args[i].equals("--max-date")) {
maxDate = Long.parseLong(args[++i]);
continue;
}
if (args[i].equals("--max-bytes")) {
size = Long.parseLong(args[++i]);
continue;
}
System.err.println("ERROR: unknown arg " + args[i]);
usage();
}
if (baseDir == null || localPath == null || preservePath == null || sDateString == null) {
usage();
}
long minDate;
if ("yesterday".equals(sDateString)) {
// figure out yesterday's dates
Calendar cal = Calendar.getInstance();
cal.roll(Calendar.DAY_OF_YEAR, -1);
// yesterday midnight
cal.set(Calendar.HOUR_OF_DAY, 0);
cal.set(Calendar.MINUTE, 0);
cal.set(Calendar.SECOND, 0);
cal.set(Calendar.MILLISECOND, 0);
minDate = cal.getTimeInMillis() / 1000;
// yesterday end of day
cal.set(Calendar.HOUR_OF_DAY, 23);
cal.set(Calendar.MINUTE, 59);
cal.set(Calendar.SECOND, 59);
cal.set(Calendar.MILLISECOND, 999);
maxDate = cal.getTimeInMillis() / 1000;
} else if ("last-week".equals(sDateString)) {
minDate = maxDate - (7 * 24 * 60 * 60);
} else if ("last-day".equals(sDateString)) {
minDate = maxDate - (24 * 60 * 60);
} else {
// UNIX date since epoch of last backup
minDate = Long.parseLong(sDateString);
}
long tmpDate = 0;
BackupHdfs bak = new BackupHdfs();
// initialize the list of tables to ignore
if (sIgnoreTablesFilename != null) {
bak.initializeTablesToIgnore(sIgnoreTablesFilename);
}
// initialize list of files to not preserve
if (sNoPreserveFilename != null) {
bak.initializeNoPreserve(sNoPreserveFilename);
}
ArrayList<Path> pathList = new ArrayList<Path>(2000);
HashMap<Path, Long> hmTimestamps = new HashMap<Path, Long>();
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
// If the HDFS path is a dir continue
if (fs.getFileStatus(baseDir).isDir()) {
Calendar cal = Calendar.getInstance();
System.err.println("");
cal.setTimeInMillis(minDate * 1000);
System.err.println("min date = " + cal.getTime().toString());
cal.setTimeInMillis(maxDate * 1000);
System.err.println("max date = " + cal.getTime().toString());
System.err.println("");
System.err.println("Searching filesystem: " + baseDir.toUri().getPath());
bak.checkDir(fs, minDate, maxDate, baseDir, pathList, hmTimestamps);
System.err.println("");
System.err.println("Skipped " + m_nIgnoredTables + " files due to ignored tables");
System.err.println("");
System.err.println("Number of files to backup = " + pathList.size());
System.err.println("Total bytes to backup = " + prettyPrintBytes(m_nTotalBytes));
System.err.println("");
System.err.println("sorting list of files...");
Collections.sort(pathList, new DateComparator(hmTimestamps));
System.err.println("done");
System.err.println("");
System.err.println("starting backup...");
tmpDate = bak.backupFiles(localPath, preservePath, fs, pathList, size);
bak.closeFiles();
System.err.println("");
System.err.println("backup completed...");
}
if (tmpDate == 0) {
// If not size limit reached print out date for right now
System.out.println(maxDate);
} else {
// Print out date for last file backed up
System.err.println("Size limit reached.");
System.out.println(tmpDate);
}
System.exit(0);
}
private long[] parse(String[] as) {
long ans[] = new long[as.length];
int index = 0;
for (String cur : as) ans[index++] = Long.parseLong(cur, 2);
Arrays.sort(ans);
return ans;
}
/**
* Parses the given text as an integer, avoiding runtime exceptions.
*
* @param aText the text to parse as an integer, can be <code>null</code> or empty;
* @param aDefault the default value to return in case parsing failed.
* @return the numeric representation of the given text, or the given default if the text could
* not be parsed correctly as integer.
* @see Long#parseLong(String)
*/
public static long safeParseLong(final String aText, final long aDefault) {
try {
return Long.parseLong(aText);
} catch (NumberFormatException exception) {
return aDefault;
}
}
public String _long2date(String args[]) {
try {
return new Date(Long.parseLong(args[1])).toString();
} catch (Exception e) {
e.printStackTrace();
}
return "not a valid long";
}
/**
* Ensure that the supplied importance is a valid value.
*
* @param imps String representation of the importance.
* @throws IllegalArgumentException if the importance is not valid.
*/
protected static long validateImportance(String imps) throws IllegalArgumentException {
long imp;
try {
imp = Long.parseLong(imps);
} catch (NumberFormatException nfe) {
throw new IllegalArgumentException("importance value " + imps + " is not legal");
}
if (imp < 0)
throw new IllegalArgumentException(
"importance value " + imps + " is not legal (must be positive)");
return (imp);
}
public String _tstamp(String args[]) {
String format = "yyyyMMddHHmm";
long now = System.currentTimeMillis();
TimeZone tz = TimeZone.getTimeZone("UTC");
if (args.length > 1) {
format = args[1];
}
if (args.length > 2) {
tz = TimeZone.getTimeZone(args[2]);
}
if (args.length > 3) {
now = Long.parseLong(args[3]);
}
if (args.length > 4) {
domain.warning("Too many arguments for tstamp: " + Arrays.toString(args));
}
SimpleDateFormat sdf = new SimpleDateFormat(format);
sdf.setTimeZone(tz);
return sdf.format(new Date(now));
}
/**
* Metoda ustawia nagłówki o cacheowaniu. Określa dokument jako trwały przez wskazaną liczbę
* sekund
*
* @param response odpowiedź jaka zostanie wysłana z serwera do klienta
* @param sec ilość sekund ważności dokumentu
*/
public static void setCachingFor(javax.servlet.http.HttpServletResponse response, long sec) {
java.util.Date d = new java.util.Date((new java.util.Date()).getTime() + sec * 1000);
// java.text.SimpleDateFormat d1 = new java.text.SimpleDateFormat("dd MM yyyy hh:mm:ss");
response.setHeader("Expires", d.toString()); // d1.format(d));
response.setHeader("Cache-Control", "max-age = " + Long.toString(sec));
}
/*
* Given a double value, create a hexadecimal floating-point
* string via an intermediate long hex string.
*/
static String doubleToHexString(double d) {
return hexLongStringtoHexDoubleString(Long.toHexString(Double.doubleToLongBits(d)));
}
protected boolean checkConditionValue(String value, String comparator, String target)
throws IllegalArgumentException {
// Check to see if we're talking about times, and if so, de-time them.
if (comparator.contains("<") || comparator.contains(">") || comparator.contains("%")) {
// We're doing maths. Are they times? Times have colons.
if (value.contains(":")) {
String[] s = value.split(":");
if (s.length == 2)
try {
value = String.valueOf(Long.parseLong(s[0]) * 60 + Long.parseLong(s[1]));
} catch (NumberFormatException nfE) {
}
}
if (target.contains(":")) {
String[] s = target.split(":");
if (s.length == 2)
try {
target = String.valueOf(Long.parseLong(s[0]) * 60 + Long.parseLong(s[1]));
} catch (NumberFormatException nfE) {
}
}
}
if ("=".equals(comparator)) {
return value.equals(target);
} else if ("!=".equals(comparator)) {
return !value.equals(target);
} else if ("<".equals(comparator)) {
try {
return (Long.parseLong(value) < Long.parseLong(target));
} catch (NumberFormatException nfE) {
}
try {
return (Double.parseDouble(value) < Double.parseDouble(target));
} catch (NumberFormatException nfE) {
}
return (value.compareTo(target) < 0);
} else if ("<=".equals(comparator)) {
try {
return (Long.parseLong(value) <= Long.parseLong(target));
} catch (NumberFormatException nfE) {
}
try {
return (Double.parseDouble(value) <= Double.parseDouble(target));
} catch (NumberFormatException nfE) {
}
return (value.compareTo(target) <= 0);
} else if (">".equals(comparator)) {
try {
return (Long.parseLong(value) > Long.parseLong(target));
} catch (NumberFormatException nfE) {
}
try {
return (Double.parseDouble(value) > Double.parseDouble(target));
} catch (NumberFormatException nfE) {
}
return (value.compareTo(target) > 0);
} else if (">=".equals(comparator)) {
try {
return (Long.parseLong(value) >= Long.parseLong(target));
} catch (NumberFormatException nfE) {
}
try {
return (Double.parseDouble(value) >= Double.parseDouble(target));
} catch (NumberFormatException nfE) {
}
return (value.compareTo(target) >= 0);
} else if ("%".equals(comparator)) {
try {
return (0 == (Long.parseLong(value) % Long.parseLong(target)));
} catch (NumberFormatException nfE) {
return false;
} catch (ArithmeticException aE) { // most likely target == 0, % by 0 is invalid
return false;
}
}
throw new IllegalArgumentException("Invalid comparator : " + comparator);
}
@SuppressWarnings("unchecked")
public static Object directBind(
String name, Annotation[] annotations, String value, Class<?> clazz, Type type)
throws Exception {
Logger.trace(
"directBind: value ["
+ value
+ "] annotation ["
+ Utils.join(annotations, " ")
+ "] Class ["
+ clazz
+ "]");
boolean nullOrEmpty = value == null || value.trim().length() == 0;
if (annotations != null) {
for (Annotation annotation : annotations) {
if (annotation.annotationType().equals(As.class)) {
Class<? extends TypeBinder<?>> toInstanciate = ((As) annotation).binder();
if (!(toInstanciate.equals(As.DEFAULT.class))) {
// Instantiate the binder
TypeBinder<?> myInstance = toInstanciate.newInstance();
return myInstance.bind(name, annotations, value, clazz, type);
}
}
}
}
// custom types
for (Class<?> c : supportedTypes.keySet()) {
Logger.trace("directBind: value [" + value + "] c [" + c + "] Class [" + clazz + "]");
if (c.isAssignableFrom(clazz)) {
Logger.trace("directBind: isAssignableFrom is true");
return supportedTypes.get(c).bind(name, annotations, value, clazz, type);
}
}
// application custom types
for (Class<TypeBinder<?>> c : Play.classloader.getAssignableClasses(TypeBinder.class)) {
if (c.isAnnotationPresent(Global.class)) {
Class<?> forType =
(Class) ((ParameterizedType) c.getGenericInterfaces()[0]).getActualTypeArguments()[0];
if (forType.isAssignableFrom(clazz)) {
return c.newInstance().bind(name, annotations, value, clazz, type);
}
}
}
// raw String
if (clazz.equals(String.class)) {
return value;
}
// Enums
if (Enum.class.isAssignableFrom(clazz)) {
if (nullOrEmpty) {
return null;
}
return Enum.valueOf((Class<Enum>) clazz, value);
}
// int or Integer binding
if (clazz.getName().equals("int") || clazz.equals(Integer.class)) {
if (nullOrEmpty) {
return clazz.isPrimitive() ? 0 : null;
}
return Integer.parseInt(value.contains(".") ? value.substring(0, value.indexOf(".")) : value);
}
// long or Long binding
if (clazz.getName().equals("long") || clazz.equals(Long.class)) {
if (nullOrEmpty) {
return clazz.isPrimitive() ? 0l : null;
}
return Long.parseLong(value.contains(".") ? value.substring(0, value.indexOf(".")) : value);
}
// byte or Byte binding
if (clazz.getName().equals("byte") || clazz.equals(Byte.class)) {
if (nullOrEmpty) {
return clazz.isPrimitive() ? (byte) 0 : null;
}
return Byte.parseByte(value.contains(".") ? value.substring(0, value.indexOf(".")) : value);
}
// short or Short binding
if (clazz.getName().equals("short") || clazz.equals(Short.class)) {
if (nullOrEmpty) {
return clazz.isPrimitive() ? (short) 0 : null;
}
return Short.parseShort(value.contains(".") ? value.substring(0, value.indexOf(".")) : value);
}
// float or Float binding
if (clazz.getName().equals("float") || clazz.equals(Float.class)) {
if (nullOrEmpty) {
return clazz.isPrimitive() ? 0f : null;
}
return Float.parseFloat(value);
}
// double or Double binding
if (clazz.getName().equals("double") || clazz.equals(Double.class)) {
if (nullOrEmpty) {
return clazz.isPrimitive() ? 0d : null;
}
return Double.parseDouble(value);
}
// BigDecimal binding
if (clazz.equals(BigDecimal.class)) {
if (nullOrEmpty) {
return null;
}
return new BigDecimal(value);
}
// boolean or Boolean binding
if (clazz.getName().equals("boolean") || clazz.equals(Boolean.class)) {
if (nullOrEmpty) {
return clazz.isPrimitive() ? false : null;
}
if (value.equals("1")
|| value.toLowerCase().equals("on")
|| value.toLowerCase().equals("yes")) {
return true;
}
return Boolean.parseBoolean(value);
}
return null;
}
/**
* Reads the data from a given reader.
*
* @param aProject the project to read the settings to;
* @param aReader the reader to read the data from, cannot be <code>null</code>.
* @throws IOException in case of I/O problems.
*/
@SuppressWarnings("boxing")
public static void read(final StubDataSet aDataSet, final Reader aReader) throws IOException {
int size = -1;
Integer rate = null, channels = null, enabledChannels = null;
long triggerPos = -1L;
long absLen = -1L;
// assume 'new' file format is in use, don't support uncompressed ones...
boolean compressed = true;
final BufferedReader br = new BufferedReader(aReader);
final List<String[]> dataValues = new ArrayList<String[]>();
String line;
while ((line = br.readLine()) != null) {
// Determine whether the line is an instruction, or data...
final Matcher instructionMatcher = OLS_INSTRUCTION_PATTERN.matcher(line);
final Matcher dataMatcher = OLS_DATA_PATTERN.matcher(line);
if (dataMatcher.matches()) {
final String[] dataPair = new String[] {dataMatcher.group(1), dataMatcher.group(2)};
dataValues.add(dataPair);
} else if (instructionMatcher.matches()) {
// Ok; found an instruction...
final String instrKey = instructionMatcher.group(1);
final String instrValue = instructionMatcher.group(2);
if ("Size".equals(instrKey)) {
size = safeParseInt(instrValue);
} else if ("Rate".equals(instrKey)) {
rate = safeParseInt(instrValue);
} else if ("Channels".equals(instrKey)) {
channels = safeParseInt(instrValue);
} else if ("TriggerPosition".equals(instrKey)) {
triggerPos = Long.parseLong(instrValue);
} else if ("EnabledChannels".equals(instrKey)) {
enabledChannels = safeParseInt(instrValue);
} else if ("CursorEnabled".equals(instrKey)) {
aDataSet.setCursorsEnabled(Boolean.parseBoolean(instrValue));
} else if ("Compressed".equals(instrKey)) {
compressed = Boolean.parseBoolean(instrValue);
} else if ("AbsoluteLength".equals(instrKey)) {
absLen = Long.parseLong(instrValue);
} else if ("CursorA".equals(instrKey)) {
final long value = safeParseLong(instrValue);
if (value > Long.MIN_VALUE) {
aDataSet.getCursor(0).setTimestamp(value);
}
} else if ("CursorB".equals(instrKey)) {
final long value = safeParseLong(instrValue);
if (value > Long.MIN_VALUE) {
aDataSet.getCursor(1).setTimestamp(value);
}
} else if (instrKey.startsWith("Cursor")) {
final int idx = safeParseInt(instrKey.substring(6));
final long pos = Long.parseLong(instrValue);
if (pos > Long.MIN_VALUE) {
aDataSet.getCursor(idx).setTimestamp(pos);
}
}
}
}
// Perform some sanity checks, make it not possible to import invalid
// data...
if (dataValues.isEmpty()) {
throw new IOException("Data file does not contain any sample data!");
}
if (!compressed) {
throw new IOException(
"Uncompressed data file found! Please send this file to the OLS developers!");
}
// In case the size is not provided (as of 0.9.4 no longer mandatory),
// take the length of the data values as size indicator...
if (size < 0) {
size = dataValues.size();
}
if (size != dataValues.size()) {
throw new IOException("Data file is corrupt?! Data size does not match sample count!");
}
if (rate == null) {
throw new IOException("Data file is corrupt?! Sample rate is not provided!");
}
if ((channels == null) || (channels <= 0) || (channels > 32)) {
throw new IOException("Data file is corrupt?! Channel count is not provided!");
}
// Make sure the enabled channels are defined...
if (enabledChannels == null) {
enabledChannels = NumberUtils.getBitMask(channels);
}
int[] values = new int[size];
long[] timestamps = new long[size];
try {
for (int i = 0; i < size; i++) {
final String[] dataPair = dataValues.get(i);
values[i] = (int) Long.parseLong(dataPair[0], 16);
timestamps[i] = Long.parseLong(dataPair[1], 10) & Long.MAX_VALUE;
}
} catch (final NumberFormatException exception) {
throw new IOException("Invalid data encountered.", exception);
}
// Allow the absolute length to be undefined, in which case the last
// time stamp is used (+ some margin to be able to see the last
// sample)...
long absoluteLength = Math.max(absLen, timestamps[size - 1]);
// Finally set the captured data, and notify all event listeners...
aDataSet.setCapturedData(
new CapturedData(
values, timestamps, triggerPos, rate, channels, enabledChannels, absoluteLength));
}
/**
* Writes the data to the given writer.
*
* @param aDataSet the project to write the settings for, cannot be <code>null</code> ;
* @param aWriter the writer to write the data to, cannot be <code>null</code>.
* @throws IOException in case of I/O problems.
*/
public static void write(final StubDataSet aDataSet, final Writer aWriter) throws IOException {
final BufferedWriter bw = new BufferedWriter(aWriter);
final AcquisitionResult capturedData = aDataSet.getCapturedData();
final Cursor[] cursors = aDataSet.getCursors();
final boolean cursorsEnabled = aDataSet.isCursorsEnabled();
try {
final int[] values = capturedData.getValues();
final long[] timestamps = capturedData.getTimestamps();
bw.write(";Size: ");
bw.write(Integer.toString(values.length));
bw.newLine();
bw.write(";Rate: ");
bw.write(Integer.toString(capturedData.getSampleRate()));
bw.newLine();
bw.write(";Channels: ");
bw.write(Integer.toString(capturedData.getChannels()));
bw.newLine();
bw.write(";EnabledChannels: ");
bw.write(Integer.toString(capturedData.getEnabledChannels()));
bw.newLine();
if (capturedData.hasTriggerData()) {
bw.write(";TriggerPosition: ");
bw.write(Long.toString(capturedData.getTriggerPosition()));
bw.newLine();
}
bw.write(";Compressed: ");
bw.write(Boolean.toString(true));
bw.newLine();
bw.write(";AbsoluteLength: ");
bw.write(Long.toString(capturedData.getAbsoluteLength()));
bw.newLine();
bw.write(";CursorEnabled: ");
bw.write(Boolean.toString(cursorsEnabled));
bw.newLine();
for (int i = 0; cursorsEnabled && (i < cursors.length); i++) {
if (cursors[i].isDefined()) {
bw.write(String.format(";Cursor%d: ", Integer.valueOf(i)));
bw.write(Long.toString(cursors[i].getTimestamp()));
bw.newLine();
}
}
for (int i = 0; i < values.length; i++) {
bw.write(formatSample(values[i], timestamps[i]));
bw.newLine();
}
} finally {
bw.flush();
}
}
private static boolean handleURL(String address) {
Main.status(String.format("Processing page \"%s\".", address));
try {
NodeList posts = getPosts(address);
if (posts.toNodeArray().length == 0) {
return false;
}
for (Node post_node : posts.toNodeArray()) {
if (post_node instanceof TagNode) {
TagNode post = (TagNode) post_node;
Post new_post = new Post(Long.parseLong(post.getAttribute("id").substring(5)));
if (!Main.post_post_hash.containsKey(new_post)) {
NodeList photo_posts = getPhotoPosts(post.getChildren());
NodeList remarks = getRemarks(photo_posts);
for (Node node : remarks.toNodeArray()) {
Matcher matcher = lores.matcher(node.getText());
String media_url = "";
if (matcher.find()) {
media_url = matcher.group();
media_url = media_url.substring(17, media_url.length() - 1);
}
String thumb =
media_url.replace(
media_url.substring(media_url.lastIndexOf("_"), media_url.lastIndexOf(".")),
"_75sq");
URL thumb_url = new URL(thumb);
new_post.pictures.add(new Picture(new URL(media_url), thumb_url));
}
NodeList photoset_posts = getPhotosetPosts(post.getChildren());
NodeList iframes = getIFrames(photoset_posts);
for (Node node : iframes.toNodeArray()) {
if (node instanceof TagNode) {
String iframe_url = ((TagNode) node).getAttribute("src");
Parser parser2 = new Parser(iframe_url);
NodeList a_list = parser2.extractAllNodesThatMatch(new TagNameFilter("a"));
Node[] a_array = a_list.toNodeArray();
Node[] img_array =
a_list.extractAllNodesThatMatch(new TagNameFilter("img"), true).toNodeArray();
String media_url;
for (int i = 0; i < a_array.length; i++) {
media_url = ((TagNode) img_array[i]).getAttribute("src");
String thumb =
media_url.replace(
media_url.substring(
media_url.lastIndexOf("_"), media_url.lastIndexOf(".")),
"_75sq");
URL thumb_url = new URL(thumb);
new_post.pictures.add(new Picture(new URL(media_url), thumb_url));
}
}
}
Main.handlePost(new_post);
} else {
new_post = post_post_hash.get(new_post);
handleNonDownloadPost(new_post);
}
}
}
} catch (Exception ex) {
ex.printStackTrace();
Main.status("Error handling post.");
}
return true;
}
/**
* Method to go though the HDFS filesystem in a DFS to find all files
*
* <p>fs:FileSystem object from HDFS minDate: Oldest date for files to be backed up maxDate:Newest
* date for files to be backed up p:Path in HDFS to look for files pathList:Will be filled with
* all files in p hmTimestamps: hashmap of timestamps for later sorting
*/
public void checkDir(
FileSystem fs,
long minDate,
long maxDate,
Path p,
ArrayList<Path> pathList,
HashMap<Path, Long> hmTimestamps) {
long tmpDate;
FileStatus[] fStat;
try {
String sPath = p.toUri().getPath();
// If this is a directory
if (fs.getFileStatus(p).isDir()) {
// ignore certain directories
if ("dfstmp".equals(p.getName())
|| "tmp".equals(p.getName())
|| "jobtracker".equals(p.getName())
|| sPath.startsWith("/mapred")
|| "ops".equals(p.getName())
|| p.getName().startsWith("_distcp_logs")) {
return;
}
// dump the mkdir and chmod commands for this
// directory -- skip root directory only
{
FileStatus stat = fs.getFileStatus(p);
if (!sPath.equals("/")) {
m_wrMkdirs.println("hadoop fs -mkdir " + sPath);
}
m_wrChmods.println(
"hadoop fs -chown " + stat.getOwner() + ":" + stat.getGroup() + " " + sPath);
Short sh = new Short(stat.getPermission().toShort());
m_wrChmods.println(
"hadoop fs -chmod " + Long.toOctalString(sh.longValue()) + " " + sPath);
}
fStat = fs.listStatus(p);
// Do a recursive call to all elements
for (int i = 0; i < fStat.length; i++) {
checkDir(fs, minDate, maxDate, fStat[i].getPath(), pathList, hmTimestamps);
}
} else {
// If not a directory then we've found a file
// ignore crc files
if (p.getName().endsWith(".crc")) {
return;
}
// ignore other files
if (sPath.startsWith("/user/oozie/etl/workflows/")) {
return;
}
// try to get the table name from the path. There are
// various types of tables, from those replicated from
// another database to regular hive tables to
// partitioned hive tables. We use table names to
// both exclude some from the backup, and for the rest
// to dump out the schema and partition name.
if (m_ignoreTables != null && m_ignoreTables.doIgnoreFile(sPath)) {
m_nIgnoredTables++;
if (m_nIgnoredTables < 5) {
System.out.println("Skipping ignore-table file: " + sPath);
} else if (m_nIgnoredTables == 5) {
System.out.println("(...not showing other skipped tables...)");
}
return;
}
FileStatus stat = fs.getFileStatus(p);
tmpDate = stat.getModificationTime() / 1000;
// store the chmods/chowns for all files
m_wrChmods.println(
"hadoop fs -chown " + stat.getOwner() + ":" + stat.getGroup() + " " + sPath);
m_wrChmods.println("hadoop fs -chmod " + stat.getPermission().toShort() + " " + sPath);
// check dates. is it too young?
if (tmpDate < minDate) {
return;
}
// is the file too recent?
if (tmpDate > maxDate) {
// System.out.println("file too recent: " + sPath);
return;
}
// file timestamp is ok
pathList.add(p);
hmTimestamps.put(p, new Long(tmpDate));
// store info about total bytes neeed to backup
m_nTotalBytes += fs.getContentSummary(p).getLength();
}
} catch (IOException e) {
System.err.println("ERROR: could not open " + p + ": " + e);
// System.exit(1) ;
}
}
public String _currenttime(@SuppressWarnings("unused") String args[]) {
return Long.toString(System.currentTimeMillis());
}
/* (non-Javadoc)
* @see org.nees.rbnb.RBNBBase#setArgs(org.apache.commons.cli.CommandLine)
*/
protected boolean setArgs(CommandLine cmd) {
if (!setBaseArgs(cmd)) return false;
if (cmd.hasOption('k')) {
String a = cmd.getOptionValue('k');
if (a != null) sinkName = a;
}
if (cmd.hasOption('c')) {
String a = cmd.getOptionValue('c');
if (a != null) channelPathPattern = a;
}
if (cmd.hasOption('C')) {
String a = cmd.getOptionValue('C');
if (a != null) channelPathListString = a;
}
if (cmd.hasOption('W')) {
String a = cmd.getOptionValue('W');
if (a != null) {
try {
long value = Long.parseLong(a);
durationTime = value;
} catch (Exception ex) {
System.out.println("Failed to parse Collection Time (" + a + "): " + ex);
}
}
}
if (cmd.hasOption('t')) {
String a = cmd.getOptionValue('t');
if (a != null) endTimeString = a;
}
if (cmd.hasOption('x')) {
String a = cmd.getOptionValue('x');
if (a != null) {
if (a.equals("true")) includeHidden = true;
if (a.equals("false")) includeHidden = false;
}
}
if (cmd.hasOption('v')) {
String a = cmd.getOptionValue('v');
if (a != null) videoChannelPath = a;
}
if (cmd.hasOption('d')) {
String a = cmd.getOptionValue('d');
if (a != null) archiveDirectory = a;
}
if (cmd.hasOption('M')) {
String a = cmd.getOptionValue('M');
if (a != null) movieFileName = a;
}
if (cmd.hasOption('D')) {
String a = cmd.getOptionValue('D');
if (a != null) dataFileName = a;
}
if (cmd.hasOption('f')) {
String a = cmd.getOptionValue('f');
if (a != null) {
try {
float value = Float.parseFloat(a);
userGivenFrameRate = value;
} catch (Exception ex) {
System.out.println("Failed to parse Frame Rate (" + a + "): " + ex);
}
}
}
if (cmd.hasOption('i')) {
String a = cmd.getOptionValue('i');
if (a != null) {
try {
int value = Integer.parseInt(a);
itemsToSkip = value;
} catch (Exception ex) {
System.out.println("Failed to parse Frame Rate (" + a + "): " + ex);
}
}
}
if (cmd.hasOption('Z')) {
repeatMissingValue = false;
}
System.out.println("User Supplied parameter (or relivent default value) are:");
System.out.println("sink name = " + sinkName);
System.out.println("data channel path pattern = " + channelPathPattern);
System.out.println("data channel list = " + channelPathListString);
System.out.println("include hidden channel value = " + includeHidden);
System.out.println("zero for missing data = " + !repeatMissingValue);
System.out.println("video source path = " + videoChannelPath);
System.out.println("archive directory = " + archiveDirectory);
System.out.println("movie file name = " + movieFileName);
System.out.println("data file name = " + dataFileName);
System.out.println("frame rate of final movie = " + userGivenFrameRate);
System.out.println("items to skip = " + itemsToSkip);
// System.out.println("wait time between cycles = " + waitTime);
System.out.println("duration = " + durationTime);
System.out.println("end time string " + endTimeString);
System.out.println("");
System.out.println("Use DataVideoGather -h to see parameters");
System.out.println("");
setupChannelList();
System.out.println("Starting DataVideoGather on " + getServer() + " as " + sinkName);
System.out.println("Gatering data from: ");
for (int i = 0; i < channelPathArray.length; i++) {
System.out.println(" " + channelPathArray[i]);
}
if ((videoChannelPath != null) && (!videoChannelPath.equals(""))) {
System.out.println("Gatering video from: " + videoChannelPath);
System.out.println("Frame rate = " + userGivenFrameRate + " frames per second.");
System.out.println("Data frames to skip between video frame = " + itemsToSkip);
}
return true;
}
public long nextLong() {
return Long.parseLong(next());
}
private TypedValues guessUntypedValue(String name, String value) {
TypedValues list = new TypedValues(name);
if (value.startsWith("java:")) { // possible java static reference
int dot;
if (value.equals("java:null")) {
list.add(new TypedValue(Object.class, null));
} else if (value.charAt(5) == '$') { // reference to a local object (assemble @id)
ElementInfo element = _local.get(value.substring(6));
if (element != null) {
list.add(new TypedValue(element.type, element.data));
}
} else if ((dot = value.lastIndexOf('.')) > 5) {
try {
// public static refernce? (don't override access control)
_logger.fine("public static refernce? " + value);
Object object =
Class.forName(value.substring(5, dot)).getField(value.substring(dot + 1)).get(null);
Class<?> type = object.getClass();
list.add(new TypedValue(type, object));
// automatic upscaling to larger types
if (type == Byte.class) {
object = new Short(((Byte) object).byteValue());
list.add(new TypedValue(Short.class, object));
type = Short.class;
}
if (type == Short.class) {
object = new Integer(((Short) object).shortValue());
list.add(new TypedValue(Integer.class, object));
type = Integer.class;
}
if (type == Integer.class) {
object = new Long(((Integer) object).intValue());
list.add(new TypedValue(Long.class, object));
type = Long.class;
}
if (type == Long.class) {
object = new Float(((Long) object).longValue());
list.add(new TypedValue(Float.class, object));
type = Float.class;
}
if (type == Float.class) {
object = new Double(((Float) object).floatValue());
list.add(new TypedValue(Double.class, object));
}
} catch (Exception x) {
_logger.fine(value + " looked like a static reference but is not: " + x.getMessage());
// class reference?
guessClassReference(list, value.substring(5));
}
} else { // no '.' at all
// class reference?
guessClassReference(list, value.substring(5));
}
} else if (value.equals("true")) {
list.add(new TypedValue(Boolean.class, Boolean.TRUE));
} else if (value.equals("false")) {
list.add(new TypedValue(Boolean.class, Boolean.FALSE));
} else {
// numbers? multiple possibilities
try {
list.add(new TypedValue(Integer.class, Integer.valueOf(value)));
} catch (Exception x) {
}
try {
list.add(new TypedValue(Long.class, Long.valueOf(value)));
} catch (Exception x) {
}
try {
list.add(new TypedValue(Float.class, Float.valueOf(value)));
} catch (Exception x) {
}
try {
list.add(new TypedValue(Double.class, Double.valueOf(value)));
} catch (Exception x) {
}
}
// always try String as the last resort
list.add(new TypedValue(String.class, value));
return list;
}