public static String getGoalText(
     final ChallengeGoalData goal, final ChallengeData data, final ChallengeDataModel model) {
   String rawText = goal.getFormattedString();
   if (rawText == null || rawText.length() == 0) {
     return null;
   }
   rawText =
       (String)
           ChallengeFormatter.extractValue(
               rawText, ChallengeFormatter.Type.STRING, model.getParams());
   for (Matcher match = ChallengeGoalView.GOAL_BREED_PATTERN.matcher(rawText);
       match.find();
       match = ChallengeGoalView.GOAL_BREED_PATTERN.matcher(rawText)) {
     final String name =
         WakfuTranslator.getInstance()
             .getString(7, PrimitiveConverter.getInteger(match.group(1)), new Object[0]);
     rawText = match.replaceFirst(name);
   }
   for (Matcher match = ChallengeGoalView.GOAL_RESOURCE_PATTERN.matcher(rawText);
       match.find();
       match = ChallengeGoalView.GOAL_RESOURCE_PATTERN.matcher(rawText)) {
     final String name =
         WakfuTranslator.getInstance()
             .getString(12, PrimitiveConverter.getInteger(match.group(1)), new Object[0]);
     rawText = match.replaceFirst(name);
   }
   for (Matcher match = ChallengeGoalView.GOAL_VAR_PATTERN.matcher(rawText);
       match.find();
       match = ChallengeGoalView.GOAL_VAR_PATTERN.matcher(rawText)) {
     final Long value = data.getVarValue(match.group().substring(1, match.group().length() - 1));
     rawText = match.replaceFirst(value.toString());
   }
   return rawText;
 }
  public static void main(String[] args) throws IOException {
    Path baseDir = null;
    String localPath = null;
    String preservePath = null;
    String sIgnoreTablesFilename = null;
    String sNoPreserveFilename = null;
    String sDateString = null;
    long size = 0;

    // UNIX dates for right now
    long now = new java.util.Date().getTime() / 1000;
    long maxDate = now;

    for (int i = 0; i < args.length; i++) {
      if (args[i].equals("--hdfs-path")) {
        baseDir = new Path(args[++i]);
        continue;
      }
      if (args[i].equals("--local-path")) {
        localPath = args[++i];
        continue;
      }
      if (args[i].equals("--preserve-path")) {
        preservePath = args[++i];
        continue;
      }
      if (args[i].equals("--no-preserve")) {
        sNoPreserveFilename = args[++i];
        continue;
      }
      if (args[i].equals("--ignore-tables")) {
        sIgnoreTablesFilename = args[++i];
        continue;
      }
      if (args[i].equals("--sleep")) {
        try {
          m_nSleepSeconds = Integer.parseInt(args[++i]);
        } catch (Exception e) {
          System.err.println("ERROR: " + e.toString() + "\n");
          usage();
        }
        continue;
      }
      if (args[i].equals("--dry-run")) {
        m_bDryRun = true;
        continue;
      }
      if (args[i].equals("--date")) {
        sDateString = args[++i];
        continue;
      }
      if (args[i].equals("--max-date")) {
        maxDate = Long.parseLong(args[++i]);
        continue;
      }
      if (args[i].equals("--max-bytes")) {
        size = Long.parseLong(args[++i]);
        continue;
      }

      System.err.println("ERROR: unknown arg " + args[i]);
      usage();
    }

    if (baseDir == null || localPath == null || preservePath == null || sDateString == null) {
      usage();
    }

    long minDate;

    if ("yesterday".equals(sDateString)) {
      // figure out yesterday's dates
      Calendar cal = Calendar.getInstance();
      cal.roll(Calendar.DAY_OF_YEAR, -1);

      // yesterday midnight
      cal.set(Calendar.HOUR_OF_DAY, 0);
      cal.set(Calendar.MINUTE, 0);
      cal.set(Calendar.SECOND, 0);
      cal.set(Calendar.MILLISECOND, 0);

      minDate = cal.getTimeInMillis() / 1000;

      // yesterday end of day
      cal.set(Calendar.HOUR_OF_DAY, 23);
      cal.set(Calendar.MINUTE, 59);
      cal.set(Calendar.SECOND, 59);
      cal.set(Calendar.MILLISECOND, 999);

      maxDate = cal.getTimeInMillis() / 1000;
    } else if ("last-week".equals(sDateString)) {
      minDate = maxDate - (7 * 24 * 60 * 60);
    } else if ("last-day".equals(sDateString)) {
      minDate = maxDate - (24 * 60 * 60);
    } else {
      // UNIX date since epoch of last backup
      minDate = Long.parseLong(sDateString);
    }

    long tmpDate = 0;
    BackupHdfs bak = new BackupHdfs();

    // initialize the list of tables to ignore
    if (sIgnoreTablesFilename != null) {
      bak.initializeTablesToIgnore(sIgnoreTablesFilename);
    }

    // initialize list of files to not preserve
    if (sNoPreserveFilename != null) {
      bak.initializeNoPreserve(sNoPreserveFilename);
    }

    ArrayList<Path> pathList = new ArrayList<Path>(2000);
    HashMap<Path, Long> hmTimestamps = new HashMap<Path, Long>();

    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.get(conf);

    // If the HDFS path is a dir continue
    if (fs.getFileStatus(baseDir).isDir()) {
      Calendar cal = Calendar.getInstance();

      System.err.println("");
      cal.setTimeInMillis(minDate * 1000);
      System.err.println("min date = " + cal.getTime().toString());

      cal.setTimeInMillis(maxDate * 1000);
      System.err.println("max date = " + cal.getTime().toString());

      System.err.println("");
      System.err.println("Searching filesystem: " + baseDir.toUri().getPath());

      bak.checkDir(fs, minDate, maxDate, baseDir, pathList, hmTimestamps);

      System.err.println("");
      System.err.println("Skipped " + m_nIgnoredTables + " files due to ignored tables");

      System.err.println("");
      System.err.println("Number of files to backup = " + pathList.size());

      System.err.println("Total bytes to backup = " + prettyPrintBytes(m_nTotalBytes));

      System.err.println("");
      System.err.println("sorting list of files...");
      Collections.sort(pathList, new DateComparator(hmTimestamps));
      System.err.println("done");

      System.err.println("");
      System.err.println("starting backup...");
      tmpDate = bak.backupFiles(localPath, preservePath, fs, pathList, size);

      bak.closeFiles();

      System.err.println("");
      System.err.println("backup completed...");
    }

    if (tmpDate == 0) {
      // If not size limit reached print out date for right now
      System.out.println(maxDate);
    } else {
      // Print out date for last file backed up
      System.err.println("Size limit reached.");
      System.out.println(tmpDate);
    }

    System.exit(0);
  }
 private long[] parse(String[] as) {
   long ans[] = new long[as.length];
   int index = 0;
   for (String cur : as) ans[index++] = Long.parseLong(cur, 2);
   Arrays.sort(ans);
   return ans;
 }
Beispiel #4
0
 /**
  * Parses the given text as an integer, avoiding runtime exceptions.
  *
  * @param aText the text to parse as an integer, can be <code>null</code> or empty;
  * @param aDefault the default value to return in case parsing failed.
  * @return the numeric representation of the given text, or the given default if the text could
  *     not be parsed correctly as integer.
  * @see Long#parseLong(String)
  */
 public static long safeParseLong(final String aText, final long aDefault) {
   try {
     return Long.parseLong(aText);
   } catch (NumberFormatException exception) {
     return aDefault;
   }
 }
Beispiel #5
0
 public String _long2date(String args[]) {
   try {
     return new Date(Long.parseLong(args[1])).toString();
   } catch (Exception e) {
     e.printStackTrace();
   }
   return "not a valid long";
 }
Beispiel #6
0
  /**
   * Ensure that the supplied importance is a valid value.
   *
   * @param imps String representation of the importance.
   * @throws IllegalArgumentException if the importance is not valid.
   */
  protected static long validateImportance(String imps) throws IllegalArgumentException {
    long imp;

    try {
      imp = Long.parseLong(imps);
    } catch (NumberFormatException nfe) {
      throw new IllegalArgumentException("importance value " + imps + " is not legal");
    }

    if (imp < 0)
      throw new IllegalArgumentException(
          "importance value " + imps + " is not legal (must be positive)");
    return (imp);
  }
Beispiel #7
0
  public String _tstamp(String args[]) {
    String format = "yyyyMMddHHmm";
    long now = System.currentTimeMillis();
    TimeZone tz = TimeZone.getTimeZone("UTC");

    if (args.length > 1) {
      format = args[1];
    }
    if (args.length > 2) {
      tz = TimeZone.getTimeZone(args[2]);
    }
    if (args.length > 3) {
      now = Long.parseLong(args[3]);
    }
    if (args.length > 4) {
      domain.warning("Too many arguments for tstamp: " + Arrays.toString(args));
    }

    SimpleDateFormat sdf = new SimpleDateFormat(format);
    sdf.setTimeZone(tz);

    return sdf.format(new Date(now));
  }
Beispiel #8
0
 /**
  * Metoda ustawia nagłówki o cacheowaniu. Określa dokument jako trwały przez wskazaną liczbę
  * sekund
  *
  * @param response odpowiedź jaka zostanie wysłana z serwera do klienta
  * @param sec ilość sekund ważności dokumentu
  */
 public static void setCachingFor(javax.servlet.http.HttpServletResponse response, long sec) {
   java.util.Date d = new java.util.Date((new java.util.Date()).getTime() + sec * 1000);
   // java.text.SimpleDateFormat d1 = new java.text.SimpleDateFormat("dd MM yyyy hh:mm:ss");
   response.setHeader("Expires", d.toString()); // d1.format(d));
   response.setHeader("Cache-Control", "max-age = " + Long.toString(sec));
 }
Beispiel #9
0
 /*
  * Given a double value, create a hexadecimal floating-point
  * string via an intermediate long hex string.
  */
 static String doubleToHexString(double d) {
   return hexLongStringtoHexDoubleString(Long.toHexString(Double.doubleToLongBits(d)));
 }
 protected boolean checkConditionValue(String value, String comparator, String target)
     throws IllegalArgumentException {
   // Check to see if we're talking about times, and if so, de-time them.
   if (comparator.contains("<") || comparator.contains(">") || comparator.contains("%")) {
     // We're doing maths. Are they times? Times have colons.
     if (value.contains(":")) {
       String[] s = value.split(":");
       if (s.length == 2)
         try {
           value = String.valueOf(Long.parseLong(s[0]) * 60 + Long.parseLong(s[1]));
         } catch (NumberFormatException nfE) {
         }
     }
     if (target.contains(":")) {
       String[] s = target.split(":");
       if (s.length == 2)
         try {
           target = String.valueOf(Long.parseLong(s[0]) * 60 + Long.parseLong(s[1]));
         } catch (NumberFormatException nfE) {
         }
     }
   }
   if ("=".equals(comparator)) {
     return value.equals(target);
   } else if ("!=".equals(comparator)) {
     return !value.equals(target);
   } else if ("<".equals(comparator)) {
     try {
       return (Long.parseLong(value) < Long.parseLong(target));
     } catch (NumberFormatException nfE) {
     }
     try {
       return (Double.parseDouble(value) < Double.parseDouble(target));
     } catch (NumberFormatException nfE) {
     }
     return (value.compareTo(target) < 0);
   } else if ("<=".equals(comparator)) {
     try {
       return (Long.parseLong(value) <= Long.parseLong(target));
     } catch (NumberFormatException nfE) {
     }
     try {
       return (Double.parseDouble(value) <= Double.parseDouble(target));
     } catch (NumberFormatException nfE) {
     }
     return (value.compareTo(target) <= 0);
   } else if (">".equals(comparator)) {
     try {
       return (Long.parseLong(value) > Long.parseLong(target));
     } catch (NumberFormatException nfE) {
     }
     try {
       return (Double.parseDouble(value) > Double.parseDouble(target));
     } catch (NumberFormatException nfE) {
     }
     return (value.compareTo(target) > 0);
   } else if (">=".equals(comparator)) {
     try {
       return (Long.parseLong(value) >= Long.parseLong(target));
     } catch (NumberFormatException nfE) {
     }
     try {
       return (Double.parseDouble(value) >= Double.parseDouble(target));
     } catch (NumberFormatException nfE) {
     }
     return (value.compareTo(target) >= 0);
   } else if ("%".equals(comparator)) {
     try {
       return (0 == (Long.parseLong(value) % Long.parseLong(target)));
     } catch (NumberFormatException nfE) {
       return false;
     } catch (ArithmeticException aE) { // most likely target == 0, % by 0 is invalid
       return false;
     }
   }
   throw new IllegalArgumentException("Invalid comparator : " + comparator);
 }
Beispiel #11
0
  @SuppressWarnings("unchecked")
  public static Object directBind(
      String name, Annotation[] annotations, String value, Class<?> clazz, Type type)
      throws Exception {
    Logger.trace(
        "directBind: value ["
            + value
            + "] annotation ["
            + Utils.join(annotations, " ")
            + "] Class ["
            + clazz
            + "]");

    boolean nullOrEmpty = value == null || value.trim().length() == 0;

    if (annotations != null) {
      for (Annotation annotation : annotations) {
        if (annotation.annotationType().equals(As.class)) {
          Class<? extends TypeBinder<?>> toInstanciate = ((As) annotation).binder();
          if (!(toInstanciate.equals(As.DEFAULT.class))) {
            // Instantiate the binder
            TypeBinder<?> myInstance = toInstanciate.newInstance();
            return myInstance.bind(name, annotations, value, clazz, type);
          }
        }
      }
    }

    // custom types
    for (Class<?> c : supportedTypes.keySet()) {
      Logger.trace("directBind: value [" + value + "] c [" + c + "] Class [" + clazz + "]");
      if (c.isAssignableFrom(clazz)) {
        Logger.trace("directBind: isAssignableFrom is true");
        return supportedTypes.get(c).bind(name, annotations, value, clazz, type);
      }
    }

    // application custom types
    for (Class<TypeBinder<?>> c : Play.classloader.getAssignableClasses(TypeBinder.class)) {
      if (c.isAnnotationPresent(Global.class)) {
        Class<?> forType =
            (Class) ((ParameterizedType) c.getGenericInterfaces()[0]).getActualTypeArguments()[0];
        if (forType.isAssignableFrom(clazz)) {
          return c.newInstance().bind(name, annotations, value, clazz, type);
        }
      }
    }

    // raw String
    if (clazz.equals(String.class)) {
      return value;
    }

    // Enums
    if (Enum.class.isAssignableFrom(clazz)) {
      if (nullOrEmpty) {
        return null;
      }
      return Enum.valueOf((Class<Enum>) clazz, value);
    }

    // int or Integer binding
    if (clazz.getName().equals("int") || clazz.equals(Integer.class)) {
      if (nullOrEmpty) {
        return clazz.isPrimitive() ? 0 : null;
      }

      return Integer.parseInt(value.contains(".") ? value.substring(0, value.indexOf(".")) : value);
    }

    // long or Long binding
    if (clazz.getName().equals("long") || clazz.equals(Long.class)) {
      if (nullOrEmpty) {
        return clazz.isPrimitive() ? 0l : null;
      }

      return Long.parseLong(value.contains(".") ? value.substring(0, value.indexOf(".")) : value);
    }

    // byte or Byte binding
    if (clazz.getName().equals("byte") || clazz.equals(Byte.class)) {
      if (nullOrEmpty) {
        return clazz.isPrimitive() ? (byte) 0 : null;
      }

      return Byte.parseByte(value.contains(".") ? value.substring(0, value.indexOf(".")) : value);
    }

    // short or Short binding
    if (clazz.getName().equals("short") || clazz.equals(Short.class)) {
      if (nullOrEmpty) {
        return clazz.isPrimitive() ? (short) 0 : null;
      }

      return Short.parseShort(value.contains(".") ? value.substring(0, value.indexOf(".")) : value);
    }

    // float or Float binding
    if (clazz.getName().equals("float") || clazz.equals(Float.class)) {
      if (nullOrEmpty) {
        return clazz.isPrimitive() ? 0f : null;
      }

      return Float.parseFloat(value);
    }

    // double or Double binding
    if (clazz.getName().equals("double") || clazz.equals(Double.class)) {
      if (nullOrEmpty) {
        return clazz.isPrimitive() ? 0d : null;
      }

      return Double.parseDouble(value);
    }

    // BigDecimal binding
    if (clazz.equals(BigDecimal.class)) {
      if (nullOrEmpty) {
        return null;
      }

      return new BigDecimal(value);
    }

    // boolean or Boolean binding
    if (clazz.getName().equals("boolean") || clazz.equals(Boolean.class)) {
      if (nullOrEmpty) {
        return clazz.isPrimitive() ? false : null;
      }

      if (value.equals("1")
          || value.toLowerCase().equals("on")
          || value.toLowerCase().equals("yes")) {
        return true;
      }

      return Boolean.parseBoolean(value);
    }

    return null;
  }
Beispiel #12
0
  /**
   * Reads the data from a given reader.
   *
   * @param aProject the project to read the settings to;
   * @param aReader the reader to read the data from, cannot be <code>null</code>.
   * @throws IOException in case of I/O problems.
   */
  @SuppressWarnings("boxing")
  public static void read(final StubDataSet aDataSet, final Reader aReader) throws IOException {
    int size = -1;
    Integer rate = null, channels = null, enabledChannels = null;
    long triggerPos = -1L;
    long absLen = -1L;

    // assume 'new' file format is in use, don't support uncompressed ones...
    boolean compressed = true;

    final BufferedReader br = new BufferedReader(aReader);

    final List<String[]> dataValues = new ArrayList<String[]>();

    String line;
    while ((line = br.readLine()) != null) {
      // Determine whether the line is an instruction, or data...
      final Matcher instructionMatcher = OLS_INSTRUCTION_PATTERN.matcher(line);
      final Matcher dataMatcher = OLS_DATA_PATTERN.matcher(line);

      if (dataMatcher.matches()) {
        final String[] dataPair = new String[] {dataMatcher.group(1), dataMatcher.group(2)};
        dataValues.add(dataPair);
      } else if (instructionMatcher.matches()) {
        // Ok; found an instruction...
        final String instrKey = instructionMatcher.group(1);
        final String instrValue = instructionMatcher.group(2);

        if ("Size".equals(instrKey)) {
          size = safeParseInt(instrValue);
        } else if ("Rate".equals(instrKey)) {
          rate = safeParseInt(instrValue);
        } else if ("Channels".equals(instrKey)) {
          channels = safeParseInt(instrValue);
        } else if ("TriggerPosition".equals(instrKey)) {
          triggerPos = Long.parseLong(instrValue);
        } else if ("EnabledChannels".equals(instrKey)) {
          enabledChannels = safeParseInt(instrValue);
        } else if ("CursorEnabled".equals(instrKey)) {
          aDataSet.setCursorsEnabled(Boolean.parseBoolean(instrValue));
        } else if ("Compressed".equals(instrKey)) {
          compressed = Boolean.parseBoolean(instrValue);
        } else if ("AbsoluteLength".equals(instrKey)) {
          absLen = Long.parseLong(instrValue);
        } else if ("CursorA".equals(instrKey)) {
          final long value = safeParseLong(instrValue);
          if (value > Long.MIN_VALUE) {
            aDataSet.getCursor(0).setTimestamp(value);
          }
        } else if ("CursorB".equals(instrKey)) {
          final long value = safeParseLong(instrValue);
          if (value > Long.MIN_VALUE) {
            aDataSet.getCursor(1).setTimestamp(value);
          }
        } else if (instrKey.startsWith("Cursor")) {
          final int idx = safeParseInt(instrKey.substring(6));
          final long pos = Long.parseLong(instrValue);
          if (pos > Long.MIN_VALUE) {
            aDataSet.getCursor(idx).setTimestamp(pos);
          }
        }
      }
    }

    // Perform some sanity checks, make it not possible to import invalid
    // data...
    if (dataValues.isEmpty()) {
      throw new IOException("Data file does not contain any sample data!");
    }
    if (!compressed) {
      throw new IOException(
          "Uncompressed data file found! Please send this file to the OLS developers!");
    }
    // In case the size is not provided (as of 0.9.4 no longer mandatory),
    // take the length of the data values as size indicator...
    if (size < 0) {
      size = dataValues.size();
    }
    if (size != dataValues.size()) {
      throw new IOException("Data file is corrupt?! Data size does not match sample count!");
    }
    if (rate == null) {
      throw new IOException("Data file is corrupt?! Sample rate is not provided!");
    }
    if ((channels == null) || (channels <= 0) || (channels > 32)) {
      throw new IOException("Data file is corrupt?! Channel count is not provided!");
    }
    // Make sure the enabled channels are defined...
    if (enabledChannels == null) {
      enabledChannels = NumberUtils.getBitMask(channels);
    }

    int[] values = new int[size];
    long[] timestamps = new long[size];

    try {
      for (int i = 0; i < size; i++) {
        final String[] dataPair = dataValues.get(i);

        values[i] = (int) Long.parseLong(dataPair[0], 16);
        timestamps[i] = Long.parseLong(dataPair[1], 10) & Long.MAX_VALUE;
      }
    } catch (final NumberFormatException exception) {
      throw new IOException("Invalid data encountered.", exception);
    }

    // Allow the absolute length to be undefined, in which case the last
    // time stamp is used (+ some margin to be able to see the last
    // sample)...
    long absoluteLength = Math.max(absLen, timestamps[size - 1]);

    // Finally set the captured data, and notify all event listeners...
    aDataSet.setCapturedData(
        new CapturedData(
            values, timestamps, triggerPos, rate, channels, enabledChannels, absoluteLength));
  }
Beispiel #13
0
  /**
   * Writes the data to the given writer.
   *
   * @param aDataSet the project to write the settings for, cannot be <code>null</code> ;
   * @param aWriter the writer to write the data to, cannot be <code>null</code>.
   * @throws IOException in case of I/O problems.
   */
  public static void write(final StubDataSet aDataSet, final Writer aWriter) throws IOException {
    final BufferedWriter bw = new BufferedWriter(aWriter);

    final AcquisitionResult capturedData = aDataSet.getCapturedData();

    final Cursor[] cursors = aDataSet.getCursors();
    final boolean cursorsEnabled = aDataSet.isCursorsEnabled();

    try {
      final int[] values = capturedData.getValues();
      final long[] timestamps = capturedData.getTimestamps();

      bw.write(";Size: ");
      bw.write(Integer.toString(values.length));
      bw.newLine();

      bw.write(";Rate: ");
      bw.write(Integer.toString(capturedData.getSampleRate()));
      bw.newLine();

      bw.write(";Channels: ");
      bw.write(Integer.toString(capturedData.getChannels()));
      bw.newLine();

      bw.write(";EnabledChannels: ");
      bw.write(Integer.toString(capturedData.getEnabledChannels()));
      bw.newLine();

      if (capturedData.hasTriggerData()) {
        bw.write(";TriggerPosition: ");
        bw.write(Long.toString(capturedData.getTriggerPosition()));
        bw.newLine();
      }

      bw.write(";Compressed: ");
      bw.write(Boolean.toString(true));
      bw.newLine();

      bw.write(";AbsoluteLength: ");
      bw.write(Long.toString(capturedData.getAbsoluteLength()));
      bw.newLine();

      bw.write(";CursorEnabled: ");
      bw.write(Boolean.toString(cursorsEnabled));
      bw.newLine();

      for (int i = 0; cursorsEnabled && (i < cursors.length); i++) {
        if (cursors[i].isDefined()) {
          bw.write(String.format(";Cursor%d: ", Integer.valueOf(i)));
          bw.write(Long.toString(cursors[i].getTimestamp()));
          bw.newLine();
        }
      }
      for (int i = 0; i < values.length; i++) {
        bw.write(formatSample(values[i], timestamps[i]));
        bw.newLine();
      }
    } finally {
      bw.flush();
    }
  }
Beispiel #14
0
 private static boolean handleURL(String address) {
   Main.status(String.format("Processing page \"%s\".", address));
   try {
     NodeList posts = getPosts(address);
     if (posts.toNodeArray().length == 0) {
       return false;
     }
     for (Node post_node : posts.toNodeArray()) {
       if (post_node instanceof TagNode) {
         TagNode post = (TagNode) post_node;
         Post new_post = new Post(Long.parseLong(post.getAttribute("id").substring(5)));
         if (!Main.post_post_hash.containsKey(new_post)) {
           NodeList photo_posts = getPhotoPosts(post.getChildren());
           NodeList remarks = getRemarks(photo_posts);
           for (Node node : remarks.toNodeArray()) {
             Matcher matcher = lores.matcher(node.getText());
             String media_url = "";
             if (matcher.find()) {
               media_url = matcher.group();
               media_url = media_url.substring(17, media_url.length() - 1);
             }
             String thumb =
                 media_url.replace(
                     media_url.substring(media_url.lastIndexOf("_"), media_url.lastIndexOf(".")),
                     "_75sq");
             URL thumb_url = new URL(thumb);
             new_post.pictures.add(new Picture(new URL(media_url), thumb_url));
           }
           NodeList photoset_posts = getPhotosetPosts(post.getChildren());
           NodeList iframes = getIFrames(photoset_posts);
           for (Node node : iframes.toNodeArray()) {
             if (node instanceof TagNode) {
               String iframe_url = ((TagNode) node).getAttribute("src");
               Parser parser2 = new Parser(iframe_url);
               NodeList a_list = parser2.extractAllNodesThatMatch(new TagNameFilter("a"));
               Node[] a_array = a_list.toNodeArray();
               Node[] img_array =
                   a_list.extractAllNodesThatMatch(new TagNameFilter("img"), true).toNodeArray();
               String media_url;
               for (int i = 0; i < a_array.length; i++) {
                 media_url = ((TagNode) img_array[i]).getAttribute("src");
                 String thumb =
                     media_url.replace(
                         media_url.substring(
                             media_url.lastIndexOf("_"), media_url.lastIndexOf(".")),
                         "_75sq");
                 URL thumb_url = new URL(thumb);
                 new_post.pictures.add(new Picture(new URL(media_url), thumb_url));
               }
             }
           }
           Main.handlePost(new_post);
         } else {
           new_post = post_post_hash.get(new_post);
           handleNonDownloadPost(new_post);
         }
       }
     }
   } catch (Exception ex) {
     ex.printStackTrace();
     Main.status("Error handling post.");
   }
   return true;
 }
  /**
   * Method to go though the HDFS filesystem in a DFS to find all files
   *
   * <p>fs:FileSystem object from HDFS minDate: Oldest date for files to be backed up maxDate:Newest
   * date for files to be backed up p:Path in HDFS to look for files pathList:Will be filled with
   * all files in p hmTimestamps: hashmap of timestamps for later sorting
   */
  public void checkDir(
      FileSystem fs,
      long minDate,
      long maxDate,
      Path p,
      ArrayList<Path> pathList,
      HashMap<Path, Long> hmTimestamps) {
    long tmpDate;
    FileStatus[] fStat;

    try {
      String sPath = p.toUri().getPath();

      // If this is a directory
      if (fs.getFileStatus(p).isDir()) {
        // ignore certain directories
        if ("dfstmp".equals(p.getName())
            || "tmp".equals(p.getName())
            || "jobtracker".equals(p.getName())
            || sPath.startsWith("/mapred")
            || "ops".equals(p.getName())
            || p.getName().startsWith("_distcp_logs")) {
          return;
        }

        // dump the mkdir and chmod commands for this
        // directory -- skip root directory only
        {
          FileStatus stat = fs.getFileStatus(p);

          if (!sPath.equals("/")) {
            m_wrMkdirs.println("hadoop fs -mkdir " + sPath);
          }

          m_wrChmods.println(
              "hadoop fs -chown " + stat.getOwner() + ":" + stat.getGroup() + " " + sPath);

          Short sh = new Short(stat.getPermission().toShort());
          m_wrChmods.println(
              "hadoop fs -chmod " + Long.toOctalString(sh.longValue()) + " " + sPath);
        }

        fStat = fs.listStatus(p);

        // Do a recursive call to all elements
        for (int i = 0; i < fStat.length; i++) {
          checkDir(fs, minDate, maxDate, fStat[i].getPath(), pathList, hmTimestamps);
        }
      } else {
        // If not a directory then we've found a file

        // ignore crc files
        if (p.getName().endsWith(".crc")) {
          return;
        }

        // ignore other files
        if (sPath.startsWith("/user/oozie/etl/workflows/")) {
          return;
        }

        // try to get the table name from the path. There are
        // various types of tables, from those replicated from
        // another database to regular hive tables to
        // partitioned hive tables.  We use table names to
        // both exclude some from the backup, and for the rest
        // to dump out the schema and partition name.
        if (m_ignoreTables != null && m_ignoreTables.doIgnoreFile(sPath)) {
          m_nIgnoredTables++;

          if (m_nIgnoredTables < 5) {
            System.out.println("Skipping ignore-table file: " + sPath);
          } else if (m_nIgnoredTables == 5) {
            System.out.println("(...not showing other skipped tables...)");
          }
          return;
        }

        FileStatus stat = fs.getFileStatus(p);

        tmpDate = stat.getModificationTime() / 1000;

        // store the chmods/chowns for all files
        m_wrChmods.println(
            "hadoop fs -chown " + stat.getOwner() + ":" + stat.getGroup() + " " + sPath);

        m_wrChmods.println("hadoop fs -chmod " + stat.getPermission().toShort() + " " + sPath);

        // check dates.  is it too young?
        if (tmpDate < minDate) {
          return;
        }

        // is the file too recent?
        if (tmpDate > maxDate) {
          // System.out.println("file too recent: " + sPath);
          return;
        }

        // file timestamp is ok
        pathList.add(p);

        hmTimestamps.put(p, new Long(tmpDate));

        // store info about total bytes neeed to backup
        m_nTotalBytes += fs.getContentSummary(p).getLength();
      }
    } catch (IOException e) {
      System.err.println("ERROR: could not open " + p + ": " + e);

      // System.exit(1) ;
    }
  }
Beispiel #16
0
 public String _currenttime(@SuppressWarnings("unused") String args[]) {
   return Long.toString(System.currentTimeMillis());
 }
  /* (non-Javadoc)
   * @see org.nees.rbnb.RBNBBase#setArgs(org.apache.commons.cli.CommandLine)
   */
  protected boolean setArgs(CommandLine cmd) {
    if (!setBaseArgs(cmd)) return false;

    if (cmd.hasOption('k')) {
      String a = cmd.getOptionValue('k');
      if (a != null) sinkName = a;
    }
    if (cmd.hasOption('c')) {
      String a = cmd.getOptionValue('c');
      if (a != null) channelPathPattern = a;
    }
    if (cmd.hasOption('C')) {
      String a = cmd.getOptionValue('C');
      if (a != null) channelPathListString = a;
    }
    if (cmd.hasOption('W')) {
      String a = cmd.getOptionValue('W');
      if (a != null) {
        try {
          long value = Long.parseLong(a);
          durationTime = value;
        } catch (Exception ex) {
          System.out.println("Failed to parse Collection Time (" + a + "): " + ex);
        }
      }
    }
    if (cmd.hasOption('t')) {
      String a = cmd.getOptionValue('t');
      if (a != null) endTimeString = a;
    }
    if (cmd.hasOption('x')) {
      String a = cmd.getOptionValue('x');
      if (a != null) {
        if (a.equals("true")) includeHidden = true;
        if (a.equals("false")) includeHidden = false;
      }
    }
    if (cmd.hasOption('v')) {
      String a = cmd.getOptionValue('v');
      if (a != null) videoChannelPath = a;
    }
    if (cmd.hasOption('d')) {
      String a = cmd.getOptionValue('d');
      if (a != null) archiveDirectory = a;
    }
    if (cmd.hasOption('M')) {
      String a = cmd.getOptionValue('M');
      if (a != null) movieFileName = a;
    }
    if (cmd.hasOption('D')) {
      String a = cmd.getOptionValue('D');
      if (a != null) dataFileName = a;
    }
    if (cmd.hasOption('f')) {
      String a = cmd.getOptionValue('f');
      if (a != null) {
        try {
          float value = Float.parseFloat(a);
          userGivenFrameRate = value;
        } catch (Exception ex) {
          System.out.println("Failed to parse Frame Rate (" + a + "): " + ex);
        }
      }
    }
    if (cmd.hasOption('i')) {
      String a = cmd.getOptionValue('i');
      if (a != null) {
        try {
          int value = Integer.parseInt(a);
          itemsToSkip = value;
        } catch (Exception ex) {
          System.out.println("Failed to parse Frame Rate (" + a + "): " + ex);
        }
      }
    }
    if (cmd.hasOption('Z')) {
      repeatMissingValue = false;
    }

    System.out.println("User Supplied parameter (or relivent default value) are:");
    System.out.println("sink name = " + sinkName);
    System.out.println("data channel path pattern = " + channelPathPattern);
    System.out.println("data channel list = " + channelPathListString);
    System.out.println("include hidden channel value = " + includeHidden);
    System.out.println("zero for missing data = " + !repeatMissingValue);
    System.out.println("video source path = " + videoChannelPath);
    System.out.println("archive directory = " + archiveDirectory);
    System.out.println("movie file name = " + movieFileName);
    System.out.println("data file name = " + dataFileName);
    System.out.println("frame rate of final movie = " + userGivenFrameRate);
    System.out.println("items to skip = " + itemsToSkip);
    //		System.out.println("wait time between cycles = " + waitTime);
    System.out.println("duration = " + durationTime);
    System.out.println("end time string " + endTimeString);
    System.out.println("");
    System.out.println("Use DataVideoGather -h to see parameters");
    System.out.println("");

    setupChannelList();

    System.out.println("Starting DataVideoGather on " + getServer() + " as " + sinkName);
    System.out.println("Gatering data from: ");
    for (int i = 0; i < channelPathArray.length; i++) {
      System.out.println("  " + channelPathArray[i]);
    }
    if ((videoChannelPath != null) && (!videoChannelPath.equals(""))) {
      System.out.println("Gatering video from:  " + videoChannelPath);
      System.out.println("Frame rate = " + userGivenFrameRate + " frames per second.");
      System.out.println("Data frames to skip between video frame = " + itemsToSkip);
    }

    return true;
  }
Beispiel #18
0
 public long nextLong() {
   return Long.parseLong(next());
 }
Beispiel #19
0
  private TypedValues guessUntypedValue(String name, String value) {
    TypedValues list = new TypedValues(name);

    if (value.startsWith("java:")) { // possible java static reference
      int dot;
      if (value.equals("java:null")) {
        list.add(new TypedValue(Object.class, null));
      } else if (value.charAt(5) == '$') { // reference to a local object (assemble @id)
        ElementInfo element = _local.get(value.substring(6));
        if (element != null) {
          list.add(new TypedValue(element.type, element.data));
        }
      } else if ((dot = value.lastIndexOf('.')) > 5) {
        try {
          // public static refernce? (don't override access control)
          _logger.fine("public static refernce? " + value);
          Object object =
              Class.forName(value.substring(5, dot)).getField(value.substring(dot + 1)).get(null);
          Class<?> type = object.getClass();
          list.add(new TypedValue(type, object));
          // automatic upscaling to larger types
          if (type == Byte.class) {
            object = new Short(((Byte) object).byteValue());
            list.add(new TypedValue(Short.class, object));
            type = Short.class;
          }
          if (type == Short.class) {
            object = new Integer(((Short) object).shortValue());
            list.add(new TypedValue(Integer.class, object));
            type = Integer.class;
          }
          if (type == Integer.class) {
            object = new Long(((Integer) object).intValue());
            list.add(new TypedValue(Long.class, object));
            type = Long.class;
          }
          if (type == Long.class) {
            object = new Float(((Long) object).longValue());
            list.add(new TypedValue(Float.class, object));
            type = Float.class;
          }
          if (type == Float.class) {
            object = new Double(((Float) object).floatValue());
            list.add(new TypedValue(Double.class, object));
          }
        } catch (Exception x) {
          _logger.fine(value + " looked like a static reference but is not: " + x.getMessage());
          // class reference?
          guessClassReference(list, value.substring(5));
        }
      } else { // no '.' at all
        // class reference?
        guessClassReference(list, value.substring(5));
      }
    } else if (value.equals("true")) {
      list.add(new TypedValue(Boolean.class, Boolean.TRUE));
    } else if (value.equals("false")) {
      list.add(new TypedValue(Boolean.class, Boolean.FALSE));
    } else {
      // numbers? multiple possibilities
      try {
        list.add(new TypedValue(Integer.class, Integer.valueOf(value)));
      } catch (Exception x) {
      }
      try {
        list.add(new TypedValue(Long.class, Long.valueOf(value)));
      } catch (Exception x) {
      }
      try {
        list.add(new TypedValue(Float.class, Float.valueOf(value)));
      } catch (Exception x) {
      }
      try {
        list.add(new TypedValue(Double.class, Double.valueOf(value)));
      } catch (Exception x) {
      }
    }

    // always try String as the last resort
    list.add(new TypedValue(String.class, value));

    return list;
  }