Ejemplo n.º 1
0
  private static final class MessageFormatter {
    private static final Pattern variablePattern =
        Pattern.compile("(?i)\\$(\\{[a-z0-9_]{1,}\\}|[a-z0-9_]{1,})");
    private static final Pattern colorPattern = Pattern.compile("(?i)&[0-9A-FK-OR]");

    private final Map<String, String> variables;

    public MessageFormatter() {
      variables = new HashMap<String, String>();
    }

    public synchronized void setVariable(String variable, String value) {
      if (value == null) variables.remove(value);
      else variables.put(variable, value);
    }

    public synchronized String format(String message) {
      Matcher matcher = variablePattern.matcher(message);
      while (matcher.find()) {
        String variable = matcher.group();
        variable = variable.substring(1);
        if (variable.startsWith("{") && variable.endsWith("}"))
          variable = variable.substring(1, variable.length() - 1);
        String value = variables.get(variable);
        if (value == null) value = "";
        message =
            message.replaceFirst(Pattern.quote(matcher.group()), Matcher.quoteReplacement(value));
      }
      matcher = colorPattern.matcher(message);
      while (matcher.find())
        message =
            message.substring(0, matcher.start()) + "\247" + message.substring(matcher.end() - 1);
      return message;
    }
  }
Ejemplo n.º 2
0
 /**
  * Get the noun and verb <roots> (i.e. 'the highest' synsets in WordNet) from the particular
  * 'icfile' (Information Content) that you are applying. Store a noun <root>: a synset offset
  * number of type Integer in nounroots: an ArrayList<Integer> defined in the constructor of this
  * class. Store a verb <root>: a synset offset number of type Integer in verbroots: an
  * ArrayList<Integer> defined in the constructor of this class.
  *
  * <p>An example line in an 'icfile', showing a noun <root>: 1740n 128767 ROOT
  */
 private void getRoots() {
   Pattern pn = Pattern.compile("[0-9]+n [0-9]+ ROOT"); // find noun <root>
   Pattern pv = Pattern.compile("[0-9]+v [0-9]+ ROOT"); // find verb <root>
   Matcher m = null;
   String root = "";
   try {
     BufferedReader in = new BufferedReader(new FileReader(icfile));
     String line;
     while ((line = in.readLine()) != null) {
       // nouns
       m = pn.matcher(line);
       if (m.matches()) {
         root = (line.split("\\s")[0]).split("n")[0]; // !!! double split !!!
         nounroots.add(Integer.parseInt(root));
       }
       // verbs
       m = pv.matcher(line);
       if (m.matches()) {
         root = (line.split("\\s")[0]).split("v")[0]; // !!! double split !!!
         verbroots.add(Integer.parseInt(root));
       }
     }
     in.close();
   } catch (IOException e) {
     e.printStackTrace();
   }
 }
Ejemplo n.º 3
0
  // Chamando Robot
  public static void robot() throws Exception {
    Database db = new Database();
    db.connect();
    ResultSet rs = Page.findAll(db);

    Page p = null;
    while ((p = Page.next(rs)) != null) {
      String body = Robot.get(p.getUrl());

      // procurar por urls dentro do body
      // buscar por essas paginas

      // String expr = "href=\"([^\"]*)";
      String ereg = "href=\"https{0,1}:\\/\\/([^\"]*)\"";
      Pattern pt = Pattern.compile(ereg);
      Matcher m = pt.matcher(body);

      while (m.find()) {
        System.out.println(m.group());
        String[] _url = m.group().split("\"");
        Page.newUrl(_url[1]);
      }

      p.setBody(body);
      p.update();
    }

    db.close();
  }
Ejemplo n.º 4
0
  private static String expand(String str) {
    if (str == null) {
      return null;
    }

    StringBuilder result = new StringBuilder();
    Pattern re = Pattern.compile("^(.*?)\\$\\{([^}]*)\\}(.*)");
    while (true) {
      Matcher matcher = re.matcher(str);
      if (matcher.matches()) {
        result.append(matcher.group(1));
        String property = matcher.group(2);
        if (property.equals("/")) {
          property = "file.separator";
        }
        String value = System.getProperty(property);
        if (value != null) {
          result.append(value);
        }
        str = matcher.group(3);
      } else {
        result.append(str);
        break;
      }
    }
    return result.toString();
  }
Ejemplo n.º 5
0
Archivo: Macro.java Proyecto: bramk/bnd
  String filter(String[] args, boolean include) {
    verifyCommand(args, String.format(_filterHelp, args[0]), null, 3, 3);

    Collection<String> list = new ArrayList<String>(Processor.split(args[1]));
    Pattern pattern = Pattern.compile(args[2]);

    for (Iterator<String> i = list.iterator(); i.hasNext(); ) {
      if (pattern.matcher(i.next()).matches() == include) i.remove();
    }
    return Processor.join(list);
  }
Ejemplo n.º 6
0
  public static void loadPermissions(URL url) throws IOException, PermissionParseException {

    BufferedReader in = new BufferedReader(new InputStreamReader(url.openStream()));
    String line;
    Pattern ignore = Pattern.compile("^\\s*(//.*)?$");
    Pattern valid =
        Pattern.compile("^\\s*permission\\s+(\\S+)" + "(\\s+\"([^\"]*)\"(,\\s+\"([^\"]*)\")?)?;$");

    Set<Permission> perms = new HashSet<Permission>();

    while ((line = in.readLine()) != null) {
      if (ignore.matcher(line).matches()) {
        continue;
      }

      Matcher matcher = valid.matcher(line);
      if (!matcher.matches()) {
        throw new PermissionParseException("invalid syntax: " + line);
      }

      int nGroups = matcher.groupCount();
      String type = matcher.group(1);
      String name = expand(nGroups >= 3 ? matcher.group(3) : null);
      String actions = expand(nGroups >= 5 ? matcher.group(5) : null);

      try {
        Permission perm = getPermission(type, name, actions);
        perms.add(perm);
      } catch (Throwable e) {
        String message =
            String.format(
                "could not instantiate permission: " + "type=%s name=%s actions=",
                type, name, actions);
        throw new PermissionParseException(message, e);
      }
    }

    in.close();

    permSet.addAll(perms);
  }
Ejemplo n.º 7
0
 private void processResponse(InputStream response) throws IOException {
   Pattern p = Pattern.compile("<cdbId>\\s*(\\d+)");
   BufferedReader in = new BufferedReader(new InputStreamReader(response));
   String line = in.readLine();
   while (line != null) {
     System.out.println(line);
     Matcher m = p.matcher(line);
     if (m.find()) {
       String idText = m.group(1);
       if (idText != null) {
         cdbId = Integer.parseInt(idText);
       }
     }
     line = in.readLine();
   }
 }
Ejemplo n.º 8
0
 @Override
 @EventHandler
 public void onChatReceived(ChatReceivedEvent event) {
   super.onChatReceived(event);
   String message = Util.stripColors(event.getMessage());
   if (message.startsWith("Please register with \"/register")) {
     String password = Util.generateRandomString(10 + random.nextInt(6));
     bot.say("/register " + password + " " + password);
   } else if (message.contains("You are not member of any faction.")
       && spamMessage != null
       && createFaction) {
     String msg = "/f create " + Util.generateRandomString(7 + random.nextInt(4));
     bot.say(msg);
   }
   for (String s : captchaList) {
     Matcher captchaMatcher = Pattern.compile(s).matcher(message);
     if (captchaMatcher.matches()) bot.say(captchaMatcher.group(1));
   }
 }
Ejemplo n.º 9
0
 private static List<String> loadAccounts(String fileName) {
   List<String> accounts = new ArrayList<String>();
   try {
     Pattern pattern = Pattern.compile("[\\w]{1,16}");
     BufferedReader reader = new BufferedReader(new FileReader(new File(fileName)));
     String line;
     while ((line = reader.readLine()) != null) {
       Matcher matcher = pattern.matcher(line);
       if (!matcher.find()) continue;
       String username = matcher.group();
       if (!matcher.find()) continue;
       String password = matcher.group();
       accounts.add(username + ":" + password);
     }
     reader.close();
   } catch (Exception exception) {
     throw new RuntimeException(exception);
   }
   System.out.println("Loaded " + accounts.size() + " accounts.");
   return accounts;
 }
Ejemplo n.º 10
0
Archivo: ZCM.java Proyecto: ZeroCM/zcm
  /**
   * Subscribe to all channels whose name matches the regular expression. Note that to subscribe to
   * all channels, you must specify ".*", not "*".
   */
  public void subscribe(String regex, ZCMSubscriber sub) {
    if (this.closed) throw new IllegalStateException();
    SubscriptionRecord srec = new SubscriptionRecord();
    srec.regex = regex;
    srec.pat = Pattern.compile(regex);
    srec.lcsub = sub;

    synchronized (this) {
      zcmjni.subscribe(regex, this);
    }

    synchronized (subscriptions) {
      subscriptions.add(srec);

      for (String channel : subscriptionsMap.keySet()) {
        if (srec.pat.matcher(channel).matches()) {
          ArrayList<SubscriptionRecord> subs = subscriptionsMap.get(channel);
          subs.add(srec);
        }
      }
    }
  }
Ejemplo n.º 11
0
Archivo: Macro.java Proyecto: bramk/bnd
/**
 * Provide a macro processor. This processor can replace variables in strings based on a properties
 * and a domain. The domain can implement functions that start with a "_" and take args[], the names
 * of these functions are available as functions in the macro processor (without the _). Macros can
 * nest to any depth but may not contain loops. Add POSIX macros: ${#parameter} String length.
 * ${parameter%word} Remove smallest suffix pattern. ${parameter%%word} Remove largest suffix
 * pattern. ${parameter#word} Remove smallest prefix pattern. ${parameter##word} Remove largest
 * prefix pattern.
 */
public class Macro {
  Processor domain;
  Object targets[];
  boolean flattening;

  public Macro(Processor domain, Object... targets) {
    this.domain = domain;
    this.targets = targets;
    if (targets != null) {
      for (Object o : targets) {
        assert o != null;
      }
    }
  }

  public String process(String line, Processor source) {
    return process(line, new Link(source, null, line));
  }

  String process(String line, Link link) {
    StringBuilder sb = new StringBuilder();
    process(line, 0, '\u0000', '\u0000', sb, link);
    return sb.toString();
  }

  int process(CharSequence org, int index, char begin, char end, StringBuilder result, Link link) {
    StringBuilder line = new StringBuilder(org);
    int nesting = 1;

    StringBuilder variable = new StringBuilder();
    outer:
    while (index < line.length()) {
      char c1 = line.charAt(index++);
      if (c1 == end) {
        if (--nesting == 0) {
          result.append(replace(variable.toString(), link));
          return index;
        }
      } else if (c1 == begin) nesting++;
      else if (c1 == '\\' && index < line.length() - 1 && line.charAt(index) == '$') {
        // remove the escape backslash and interpret the dollar
        // as a
        // literal
        index++;
        variable.append('$');
        continue outer;
      } else if (c1 == '$' && index < line.length() - 2) {
        char c2 = line.charAt(index);
        char terminator = getTerminator(c2);
        if (terminator != 0) {
          index = process(line, index + 1, c2, terminator, variable, link);
          continue outer;
        }
      } else if (c1 == '.' && index < line.length() && line.charAt(index) == '/') {
        // Found the sequence ./
        if (index == 1 || Character.isWhitespace(line.charAt(index - 2))) {
          // make sure it is preceded by whitespace or starts at begin
          index++;
          variable.append(domain.getBase().getAbsolutePath());
          variable.append('/');
          continue outer;
        }
      }
      variable.append(c1);
    }
    result.append(variable);
    return index;
  }

  public static char getTerminator(char c) {
    switch (c) {
      case '(':
        return ')';
      case '[':
        return ']';
      case '{':
        return '}';
      case '<':
        return '>';
      case '\u00ab': // Guillemet double << >>
        return '\u00bb';
      case '\u2039': // Guillemet single
        return '\u203a';
    }
    return 0;
  }

  protected String replace(String key, Link link) {
    if (link != null && link.contains(key)) return "${infinite:" + link.toString() + "}";

    if (key != null) {
      key = key.trim();
      if (key.length() > 0) {
        Processor source = domain;
        String value = null;

        if (key.indexOf(';') < 0) {
          Instruction ins = new Instruction(key);
          if (!ins.isLiteral()) {
            SortedList<String> sortedList = SortedList.fromIterator(domain.iterator());
            StringBuilder sb = new StringBuilder();
            String del = "";
            for (String k : sortedList) {
              if (ins.matches(k)) {
                String v = replace(k, new Link(source, link, key));
                if (v != null) {
                  sb.append(del);
                  del = ",";
                  sb.append(v);
                }
              }
            }
            return sb.toString();
          }
        }
        while (value == null && source != null) {
          value = source.getProperties().getProperty(key);
          source = source.getParent();
        }

        if (value != null) return process(value, new Link(source, link, key));

        value = doCommands(key, link);
        if (value != null) return process(value, new Link(source, link, key));

        if (key != null && key.trim().length() > 0) {
          value = System.getProperty(key);
          if (value != null) return value;
        }
        if (!flattening && !key.equals("@"))
          domain.warning("No translation found for macro: " + key);
      } else {
        domain.warning("Found empty macro key");
      }
    } else {
      domain.warning("Found null macro key");
    }
    return "${" + key + "}";
  }

  /**
   * Parse the key as a command. A command consist of parameters separated by ':'.
   *
   * @param key
   * @return
   */
  static Pattern commands = Pattern.compile("(?<!\\\\);");

  private String doCommands(String key, Link source) {
    String[] args = commands.split(key);
    if (args == null || args.length == 0) return null;

    for (int i = 0; i < args.length; i++)
      if (args[i].indexOf('\\') >= 0) args[i] = args[i].replaceAll("\\\\;", ";");

    if (args[0].startsWith("^")) {
      String varname = args[0].substring(1).trim();

      Processor parent = source.start.getParent();
      if (parent != null) return parent.getProperty(varname);
      return null;
    }

    Processor rover = domain;
    while (rover != null) {
      String result = doCommand(rover, args[0], args);
      if (result != null) return result;

      rover = rover.getParent();
    }

    for (int i = 0; targets != null && i < targets.length; i++) {
      String result = doCommand(targets[i], args[0], args);
      if (result != null) return result;
    }

    return doCommand(this, args[0], args);
  }

  private String doCommand(Object target, String method, String[] args) {
    if (target == null) ; // System.err.println("Huh? Target should never be null " +
    // domain);
    else {
      String cname = "_" + method.replaceAll("-", "_");
      try {
        Method m = target.getClass().getMethod(cname, new Class[] {String[].class});
        return (String) m.invoke(target, new Object[] {args});
      } catch (NoSuchMethodException e) {
        // Ignore
      } catch (InvocationTargetException e) {
        if (e.getCause() instanceof IllegalArgumentException) {
          domain.error(
              "%s, for cmd: %s, arguments; %s",
              e.getCause().getMessage(), method, Arrays.toString(args));
        } else {
          domain.warning("Exception in replace: %s", e.getCause());
          e.getCause().printStackTrace();
        }
      } catch (Exception e) {
        domain.warning("Exception in replace: " + e + " method=" + method);
        e.printStackTrace();
      }
    }
    return null;
  }

  /**
   * Return a unique list where the duplicates are removed.
   *
   * @param args
   * @return
   */
  static String _uniqHelp = "${uniq;<list> ...}";

  public String _uniq(String args[]) {
    verifyCommand(args, _uniqHelp, null, 1, Integer.MAX_VALUE);
    Set<String> set = new LinkedHashSet<String>();
    for (int i = 1; i < args.length; i++) {
      Processor.split(args[i], set);
    }
    return Processor.join(set, ",");
  }

  public String _pathseparator(@SuppressWarnings("unused") String args[]) {
    return File.pathSeparator;
  }

  public String _separator(@SuppressWarnings("unused") String args[]) {
    return File.separator;
  }

  public String _filter(String args[]) {
    return filter(args, false);
  }

  public String _filterout(String args[]) {
    return filter(args, true);
  }

  static String _filterHelp = "${%s;<list>;<regex>}";

  String filter(String[] args, boolean include) {
    verifyCommand(args, String.format(_filterHelp, args[0]), null, 3, 3);

    Collection<String> list = new ArrayList<String>(Processor.split(args[1]));
    Pattern pattern = Pattern.compile(args[2]);

    for (Iterator<String> i = list.iterator(); i.hasNext(); ) {
      if (pattern.matcher(i.next()).matches() == include) i.remove();
    }
    return Processor.join(list);
  }

  static String _sortHelp = "${sort;<list>...}";

  public String _sort(String args[]) {
    verifyCommand(args, _sortHelp, null, 2, Integer.MAX_VALUE);

    List<String> result = new ArrayList<String>();
    for (int i = 1; i < args.length; i++) {
      Processor.split(args[i], result);
    }
    Collections.sort(result);
    return Processor.join(result);
  }

  static String _joinHelp = "${join;<list>...}";

  public String _join(String args[]) {

    verifyCommand(args, _joinHelp, null, 1, Integer.MAX_VALUE);

    List<String> result = new ArrayList<String>();
    for (int i = 1; i < args.length; i++) {
      Processor.split(args[i], result);
    }
    return Processor.join(result);
  }

  static String _ifHelp = "${if;<condition>;<iftrue> [;<iffalse>] }";

  public String _if(String args[]) {
    verifyCommand(args, _ifHelp, null, 3, 4);
    String condition = args[1].trim();
    if (!condition.equalsIgnoreCase("false")) if (condition.length() != 0) return args[2];

    if (args.length > 3) return args[3];
    return "";
  }

  public String _now(@SuppressWarnings("unused") String args[]) {
    return new Date().toString();
  }

  public static final String _fmodifiedHelp =
      "${fmodified;<list of filenames>...}, return latest modification date";

  public String _fmodified(String args[]) throws Exception {
    verifyCommand(args, _fmodifiedHelp, null, 2, Integer.MAX_VALUE);

    long time = 0;
    Collection<String> names = new ArrayList<String>();
    for (int i = 1; i < args.length; i++) {
      Processor.split(args[i], names);
    }
    for (String name : names) {
      File f = new File(name);
      if (f.exists() && f.lastModified() > time) time = f.lastModified();
    }
    return "" + time;
  }

  public String _long2date(String args[]) {
    try {
      return new Date(Long.parseLong(args[1])).toString();
    } catch (Exception e) {
      e.printStackTrace();
    }
    return "not a valid long";
  }

  public String _literal(String args[]) {
    if (args.length != 2)
      throw new RuntimeException("Need a value for the ${literal;<value>} macro");
    return "${" + args[1] + "}";
  }

  public String _def(String args[]) {
    if (args.length != 2) throw new RuntimeException("Need a value for the ${def;<value>} macro");

    return domain.getProperty(args[1], "");
  }

  /**
   * replace ; <list> ; regex ; replace
   *
   * @param args
   * @return
   */
  public String _replace(String args[]) {
    if (args.length != 4) {
      domain.warning("Invalid nr of arguments to replace " + Arrays.asList(args));
      return null;
    }

    String list[] = args[1].split("\\s*,\\s*");
    StringBuilder sb = new StringBuilder();
    String del = "";
    for (int i = 0; i < list.length; i++) {
      String element = list[i].trim();
      if (!element.equals("")) {
        sb.append(del);
        sb.append(element.replaceAll(args[2], args[3]));
        del = ", ";
      }
    }

    return sb.toString();
  }

  public String _warning(String args[]) {
    for (int i = 1; i < args.length; i++) {
      domain.warning(process(args[i]));
    }
    return "";
  }

  public String _error(String args[]) {
    for (int i = 1; i < args.length; i++) {
      domain.error(process(args[i]));
    }
    return "";
  }

  /**
   * toclassname ; <path>.class ( , <path>.class ) *
   *
   * @param args
   * @return
   */
  static String _toclassnameHelp =
      "${classname;<list of class names>}, convert class paths to FQN class names ";

  public String _toclassname(String args[]) {
    verifyCommand(args, _toclassnameHelp, null, 2, 2);
    Collection<String> paths = Processor.split(args[1]);

    List<String> names = new ArrayList<String>(paths.size());
    for (String path : paths) {
      if (path.endsWith(".class")) {
        String name = path.substring(0, path.length() - 6).replace('/', '.');
        names.add(name);
      } else if (path.endsWith(".java")) {
        String name = path.substring(0, path.length() - 5).replace('/', '.');
        names.add(name);
      } else {
        domain.warning(
            "in toclassname, "
                + args[1]
                + " is not a class path because it does not end in .class");
      }
    }
    return Processor.join(names, ",");
  }

  /**
   * toclassname ; <path>.class ( , <path>.class ) *
   *
   * @param args
   * @return
   */
  static String _toclasspathHelp =
      "${toclasspath;<list>[;boolean]}, convert a list of class names to paths";

  public String _toclasspath(String args[]) {
    verifyCommand(args, _toclasspathHelp, null, 2, 3);
    boolean cl = true;
    if (args.length > 2) cl = Boolean.valueOf(args[2]);

    Collection<String> names = Processor.split(args[1]);
    Collection<String> paths = new ArrayList<String>(names.size());
    for (String name : names) {
      String path = name.replace('.', '/') + (cl ? ".class" : "");
      paths.add(path);
    }
    return Processor.join(paths, ",");
  }

  public String _dir(String args[]) {
    if (args.length < 2) {
      domain.warning("Need at least one file name for ${dir;...}");
      return null;
    }
    String del = "";
    StringBuilder sb = new StringBuilder();
    for (int i = 1; i < args.length; i++) {
      File f = domain.getFile(args[i]);
      if (f.exists() && f.getParentFile().exists()) {
        sb.append(del);
        sb.append(f.getParentFile().getAbsolutePath());
        del = ",";
      }
    }
    return sb.toString();
  }

  public String _basename(String args[]) {
    if (args.length < 2) {
      domain.warning("Need at least one file name for ${basename;...}");
      return null;
    }
    String del = "";
    StringBuilder sb = new StringBuilder();
    for (int i = 1; i < args.length; i++) {
      File f = domain.getFile(args[i]);
      if (f.exists() && f.getParentFile().exists()) {
        sb.append(del);
        sb.append(f.getName());
        del = ",";
      }
    }
    return sb.toString();
  }

  public String _isfile(String args[]) {
    if (args.length < 2) {
      domain.warning("Need at least one file name for ${isfile;...}");
      return null;
    }
    boolean isfile = true;
    for (int i = 1; i < args.length; i++) {
      File f = new File(args[i]).getAbsoluteFile();
      isfile &= f.isFile();
    }
    return isfile ? "true" : "false";
  }

  public String _isdir(String args[]) {
    if (args.length < 2) {
      domain.warning("Need at least one file name for ${isdir;...}");
      return null;
    }
    boolean isdir = true;
    for (int i = 1; i < args.length; i++) {
      File f = new File(args[i]).getAbsoluteFile();
      isdir &= f.isDirectory();
    }
    return isdir ? "true" : "false";
  }

  public String _tstamp(String args[]) {
    String format = "yyyyMMddHHmm";
    long now = System.currentTimeMillis();
    TimeZone tz = TimeZone.getTimeZone("UTC");

    if (args.length > 1) {
      format = args[1];
    }
    if (args.length > 2) {
      tz = TimeZone.getTimeZone(args[2]);
    }
    if (args.length > 3) {
      now = Long.parseLong(args[3]);
    }
    if (args.length > 4) {
      domain.warning("Too many arguments for tstamp: " + Arrays.toString(args));
    }

    SimpleDateFormat sdf = new SimpleDateFormat(format);
    sdf.setTimeZone(tz);

    return sdf.format(new Date(now));
  }

  /**
   * Wildcard a directory. The lists can contain Instruction that are matched against the given
   * directory ${lsr;<dir>;<list>(;<list>)*} ${lsa;<dir>;<list>(;<list>)*}
   *
   * @author aqute
   */
  public String _lsr(String args[]) {
    return ls(args, true);
  }

  public String _lsa(String args[]) {
    return ls(args, false);
  }

  String ls(String args[], boolean relative) {
    if (args.length < 2)
      throw new IllegalArgumentException(
          "the ${ls} macro must at least have a directory as parameter");

    File dir = domain.getFile(args[1]);
    if (!dir.isAbsolute())
      throw new IllegalArgumentException(
          "the ${ls} macro directory parameter is not absolute: " + dir);

    if (!dir.exists())
      throw new IllegalArgumentException(
          "the ${ls} macro directory parameter does not exist: " + dir);

    if (!dir.isDirectory())
      throw new IllegalArgumentException(
          "the ${ls} macro directory parameter points to a file instead of a directory: " + dir);

    Collection<File> files = new ArrayList<File>(new SortedList<File>(dir.listFiles()));

    for (int i = 2; i < args.length; i++) {
      Instructions filters = new Instructions(args[i]);
      files = filters.select(files, true);
    }

    List<String> result = new ArrayList<String>();
    for (File file : files) result.add(relative ? file.getName() : file.getAbsolutePath());

    return Processor.join(result, ",");
  }

  public String _currenttime(@SuppressWarnings("unused") String args[]) {
    return Long.toString(System.currentTimeMillis());
  }

  /**
   * Modify a version to set a version policy. Thed policy is a mask that is mapped to a version.
   *
   * <pre>
   * +           increment
   * -           decrement
   * =           maintain
   * &tilde;           discard
   *
   * ==+      = maintain major, minor, increment micro, discard qualifier
   * &tilde;&tilde;&tilde;=     = just get the qualifier
   * version=&quot;[${version;==;${@}},${version;=+;${@}})&quot;
   * </pre>
   *
   * @param args
   * @return
   */
  static final String MASK_STRING = "[\\-+=~0123456789]{0,3}[=~]?";

  static final Pattern MASK = Pattern.compile(MASK_STRING);
  static final String _versionHelp =
      "${version;<mask>;<version>}, modify a version\n"
          + "<mask> ::= [ M [ M [ M [ MQ ]]]\n"
          + "M ::= '+' | '-' | MQ\n"
          + "MQ ::= '~' | '='";
  static final Pattern _versionPattern[] = new Pattern[] {null, null, MASK, Verifier.VERSION};

  public String _version(String args[]) {
    verifyCommand(args, _versionHelp, null, 2, 3);

    String mask = args[1];

    Version version = null;
    if (args.length >= 3) version = new Version(args[2]);

    return version(version, mask);
  }

  String version(Version version, String mask) {
    if (version == null) {
      String v = domain.getProperty("@");
      if (v == null) {
        domain.error(
            "No version specified for ${version} or ${range} and no implicit version ${@} either, mask=%s",
            mask);
        v = "0";
      }
      version = new Version(v);
    }

    StringBuilder sb = new StringBuilder();
    String del = "";

    for (int i = 0; i < mask.length(); i++) {
      char c = mask.charAt(i);
      String result = null;
      if (c != '~') {
        if (i == 3) {
          result = version.getQualifier();
        } else if (Character.isDigit(c)) {
          // Handle masks like +00, =+0
          result = String.valueOf(c);
        } else {
          int x = version.get(i);
          switch (c) {
            case '+':
              x++;
              break;
            case '-':
              x--;
              break;
            case '=':
              break;
          }
          result = Integer.toString(x);
        }
        if (result != null) {
          sb.append(del);
          del = ".";
          sb.append(result);
        }
      }
    }
    return sb.toString();
  }

  /**
   * Schortcut for version policy
   *
   * <pre>
   * -provide-policy : ${policy;[==,=+)}
   * -consume-policy : ${policy;[==,+)}
   * </pre>
   *
   * @param args
   * @return
   */
  static Pattern RANGE_MASK =
      Pattern.compile("(\\[|\\()(" + MASK_STRING + "),(" + MASK_STRING + ")(\\]|\\))");

  static String _rangeHelp =
      "${range;<mask>[;<version>]}, range for version, if version not specified lookyp ${@}\n"
          + "<mask> ::= [ M [ M [ M [ MQ ]]]\n"
          + "M ::= '+' | '-' | MQ\n"
          + "MQ ::= '~' | '='";
  static Pattern _rangePattern[] = new Pattern[] {null, RANGE_MASK};

  public String _range(String args[]) {
    verifyCommand(args, _rangeHelp, _rangePattern, 2, 3);
    Version version = null;
    if (args.length >= 3) version = new Version(args[2]);
    else {
      String v = domain.getProperty("@");
      if (v == null) return null;
      version = new Version(v);
    }
    String spec = args[1];

    Matcher m = RANGE_MASK.matcher(spec);
    m.matches();
    String floor = m.group(1);
    String floorMask = m.group(2);
    String ceilingMask = m.group(3);
    String ceiling = m.group(4);

    String left = version(version, floorMask);
    String right = version(version, ceilingMask);
    StringBuilder sb = new StringBuilder();
    sb.append(floor);
    sb.append(left);
    sb.append(",");
    sb.append(right);
    sb.append(ceiling);

    String s = sb.toString();
    VersionRange vr = new VersionRange(s);
    if (!(vr.includes(vr.getHigh()) || vr.includes(vr.getLow()))) {
      domain.error(
          "${range} macro created an invalid range %s from %s and mask %s", s, version, spec);
    }
    return sb.toString();
  }

  /**
   * System command. Execute a command and insert the result.
   *
   * @param args
   * @param help
   * @param patterns
   * @param low
   * @param high
   */
  public String system_internal(boolean allowFail, String args[]) throws Exception {
    verifyCommand(
        args,
        "${"
            + (allowFail ? "system-allow-fail" : "system")
            + ";<command>[;<in>]}, execute a system command",
        null,
        2,
        3);
    String command = args[1];
    String input = null;

    if (args.length > 2) {
      input = args[2];
    }

    Process process = Runtime.getRuntime().exec(command, null, domain.getBase());
    if (input != null) {
      process.getOutputStream().write(input.getBytes("UTF-8"));
    }
    process.getOutputStream().close();

    String s = IO.collect(process.getInputStream(), "UTF-8");
    int exitValue = process.waitFor();
    if (exitValue != 0) return exitValue + "";

    if (!allowFail && (exitValue != 0)) {
      domain.error("System command " + command + " failed with " + exitValue);
    }
    return s.trim();
  }

  public String _system(String args[]) throws Exception {
    return system_internal(false, args);
  }

  public String _system_allow_fail(String args[]) throws Exception {
    String result = "";
    try {
      result = system_internal(true, args);
    } catch (Throwable t) {
      /* ignore */
    }
    return result;
  }

  public String _env(String args[]) {
    verifyCommand(args, "${env;<name>}, get the environmet variable", null, 2, 2);

    try {
      return System.getenv(args[1]);
    } catch (Throwable t) {
      return null;
    }
  }

  /**
   * Get the contents of a file.
   *
   * @param in
   * @return
   * @throws IOException
   */
  public String _cat(String args[]) throws IOException {
    verifyCommand(args, "${cat;<in>}, get the content of a file", null, 2, 2);
    File f = domain.getFile(args[1]);
    if (f.isFile()) {
      return IO.collect(f);
    } else if (f.isDirectory()) {
      return Arrays.toString(f.list());
    } else {
      try {
        URL url = new URL(args[1]);
        return IO.collect(url, "UTF-8");
      } catch (MalformedURLException mfue) {
        // Ignore here
      }
      return null;
    }
  }

  public static void verifyCommand(
      String args[],
      @SuppressWarnings("unused") String help,
      Pattern[] patterns,
      int low,
      int high) {
    String message = "";
    if (args.length > high) {
      message = "too many arguments";
    } else if (args.length < low) {
      message = "too few arguments";
    } else {
      for (int i = 0; patterns != null && i < patterns.length && i < args.length; i++) {
        if (patterns[i] != null) {
          Matcher m = patterns[i].matcher(args[i]);
          if (!m.matches())
            message +=
                String.format(
                    "Argument %s (%s) does not match %s%n", i, args[i], patterns[i].pattern());
        }
      }
    }
    if (message.length() != 0) {
      StringBuilder sb = new StringBuilder();
      String del = "${";
      for (String arg : args) {
        sb.append(del);
        sb.append(arg);
        del = ";";
      }
      sb.append("}, is not understood. ");
      sb.append(message);
      throw new IllegalArgumentException(sb.toString());
    }
  }

  // Helper class to track expansion of variables
  // on the stack.
  static class Link {
    Link previous;
    String key;
    Processor start;

    public Link(Processor start, Link previous, String key) {
      this.previous = previous;
      this.key = key;
      this.start = start;
    }

    public boolean contains(String key) {
      if (this.key.equals(key)) return true;

      if (previous == null) return false;

      return previous.contains(key);
    }

    @Override
    public String toString() {
      StringBuilder sb = new StringBuilder();
      String del = "[";
      for (Link r = this; r != null; r = r.previous) {
        sb.append(del);
        sb.append(r.key);
        del = ",";
      }
      sb.append("]");
      return sb.toString();
    }
  }

  /**
   * Take all the properties and translate them to actual values. This method takes the set
   * properties and traverse them over all entries, including the default properties for that
   * properties. The values no longer contain macros.
   *
   * @return A new Properties with the flattened values
   */
  public Properties getFlattenedProperties() {
    // Some macros only work in a lower processor, so we
    // do not report unknown macros while flattening
    flattening = true;
    try {
      Properties flattened = new Properties();
      Properties source = domain.getProperties();
      for (Enumeration<?> e = source.propertyNames(); e.hasMoreElements(); ) {
        String key = (String) e.nextElement();
        if (!key.startsWith("_"))
          if (key.startsWith("-")) flattened.put(key, source.getProperty(key));
          else flattened.put(key, process(source.getProperty(key)));
      }
      return flattened;
    } finally {
      flattening = false;
    }
  }

  public static final String _fileHelp =
      "${file;<base>;<paths>...}, create correct OS dependent path";

  public String _osfile(String args[]) {
    verifyCommand(args, _fileHelp, null, 3, 3);
    File base = new File(args[1]);
    File f = Processor.getFile(base, args[2]);
    return f.getAbsolutePath();
  }

  public String _path(String args[]) {
    List<String> list = new ArrayList<String>();
    for (int i = 1; i < args.length; i++) {
      list.addAll(Processor.split(args[i]));
    }
    return Processor.join(list, File.pathSeparator);
  }

  public static Properties getParent(Properties p) {
    try {
      Field f = Properties.class.getDeclaredField("defaults");
      f.setAccessible(true);
      return (Properties) f.get(p);
    } catch (Exception e) {
      Field[] fields = Properties.class.getFields();
      System.err.println(Arrays.toString(fields));
      return null;
    }
  }

  public String process(String line) {
    return process(line, domain);
  }
}
Ejemplo n.º 12
0
/**
 * JPM is the Java package manager. It manages a local repository in the user global directory
 * and/or a global directory. This class is the main entry point for the command line. This program
 * maintains a repository, a list of installed commands, and a list of installed service. It
 * provides the commands to changes these resources. All information is kept in a platform specific
 * area. However, the layout of this area is standardized.
 *
 * <pre>
 * 	platform/
 *      check									check for write access
 *      repo/                              		repository
 *        &lt;bsn&gt;/                     		bsn directory
 *          &lt;bsn&gt;-&lt;version&gt;.jar		jar file
 *      service/                               All service
 *        &lt;service&gt;/                      A service
 *        	data								Service data (JSON)
 *          wdir/								Working dir
 *          lock								Lock file (if running, contains port)
 *      commands/
 *        &lt;command&gt;						Command data (JSON)
 * </pre>
 *
 * For each service, the platform must also have a user writable directory used for working dir,
 * lock, and logging.
 *
 * <pre>
 *   platform var/
 *       wdir/									Working dir
 *       lock									Lock file (exists only when running, contains UDP port)
 * </pre>
 */
public class JustAnotherPackageManager {
  private static final String SERVICE_JAR_FILE = "service.jar";
  public static final String SERVICE = "service";
  public static final String COMMANDS = "commands";
  public static final String LOCK = "lock";
  private static final String JPM_CACHE_LOCAL = "jpm.cache.local";
  private static final String JPM_CACHE_GLOBAL = "jpm.cache.global";
  static final String PERMISSION_ERROR =
      "No write acces, might require administrator or root privileges (sudo in *nix)";
  static JSONCodec codec = new JSONCodec();
  static Pattern BSN_P =
      Pattern.compile(
          "([-a-z0-9_]+(?:\\.[-a-z0-9_]+)+)(?:@([0-9]+(?:\\.[0-9]+(?:\\.[0-9]+(?:\\.[-_a-z0-9]+)?)?)?))?",
          Pattern.CASE_INSENSITIVE);
  static Pattern COORD_P =
      Pattern.compile(
          "([-a-z0-9_.]+):([-a-z0-9_.]+)(?::([-a-z0-9_.]+))?(?:@([-a-z0-9._]+))?",
          Pattern.CASE_INSENSITIVE);
  static Pattern URL_P = Pattern.compile("([a-z]{3,6}:/.*)", Pattern.CASE_INSENSITIVE);
  static Pattern CMD_P = Pattern.compile("([a-z_][a-z\\d_]*)", Pattern.CASE_INSENSITIVE);
  static Pattern SHA_P = Pattern.compile("(?:sha:)?([a-fA-F0-9]{40,40})", Pattern.CASE_INSENSITIVE);
  static Executor executor;

  final File homeDir;
  final File binDir;
  final File repoDir;
  final File commandDir;
  final File serviceDir;
  final File service;
  final Platform platform;
  final Reporter reporter;

  JpmRepo library;
  final List<Service> startedByDaemon = new ArrayList<Service>();
  boolean localInstall = false;
  private URLClient host;
  private boolean underTest;

  /**
   * Constructor
   *
   * @throws IOException
   */
  public JustAnotherPackageManager(Reporter reporter, Platform platform, File homeDir, File binDir)
      throws IOException {
    this.platform = platform;
    this.reporter = reporter;
    this.homeDir = homeDir;
    if (!homeDir.exists() && !homeDir.mkdirs())
      throw new IllegalArgumentException("Could not create directory " + homeDir);

    repoDir = IO.getFile(homeDir, "repo");
    if (!repoDir.exists() && !repoDir.mkdirs())
      throw new IllegalArgumentException("Could not create directory " + repoDir);

    commandDir = new File(homeDir, COMMANDS);
    serviceDir = new File(homeDir, SERVICE);
    commandDir.mkdir();
    serviceDir.mkdir();
    service = new File(repoDir, SERVICE_JAR_FILE);
    if (!service.isFile()) init();

    this.binDir = binDir;
    if (!binDir.exists() && !binDir.mkdirs())
      throw new IllegalArgumentException("Could not create bin directory " + binDir);
  }

  public String getArtifactIdFromCoord(String coord) {
    Matcher m = COORD_P.matcher(coord);
    if (m.matches()) {
      return m.group(2);
    } else {
      return null;
    }
  }

  public boolean hasAccess() {
    assert (binDir != null);
    assert (homeDir != null);

    return binDir.canWrite() && homeDir.canWrite();
  }

  public File getHomeDir() {
    return homeDir;
  }

  public File getRepoDir() {
    return repoDir;
  }

  public File getBinDir() {
    return binDir;
  }

  public List<ServiceData> getServices() throws Exception {
    return getServices(serviceDir);
  }

  public List<ServiceData> getServices(File serviceDir) throws Exception {
    List<ServiceData> result = new ArrayList<ServiceData>();

    if (!serviceDir.exists()) {
      return result;
    }

    for (File sdir : serviceDir.listFiles()) {
      File dataFile = new File(sdir, "data");
      ServiceData data = getData(ServiceData.class, dataFile);
      result.add(data);
    }
    return result;
  }

  public List<CommandData> getCommands() throws Exception {
    return getCommands(commandDir);
  }

  public List<CommandData> getCommands(File commandDir) throws Exception {
    List<CommandData> result = new ArrayList<CommandData>();

    if (!commandDir.exists()) {
      return result;
    }

    for (File f : commandDir.listFiles()) {
      CommandData data = getData(CommandData.class, f);
      if (data != null) result.add(data);
    }
    return result;
  }

  public CommandData getCommand(String name) throws Exception {
    File f = new File(commandDir, name);
    if (!f.isFile()) return null;

    return getData(CommandData.class, f);
  }

  /**
   * Garbage collect repository
   *
   * @throws Exception
   */
  public void gc() throws Exception {
    HashSet<byte[]> deps = new HashSet<byte[]>();

    // deps.add(SERVICE_JAR_FILE);

    for (File cmd : commandDir.listFiles()) {
      CommandData data = getData(CommandData.class, cmd);
      addDependencies(deps, data);
    }

    for (File service : serviceDir.listFiles()) {
      File dataFile = new File(service, "data");
      ServiceData data = getData(ServiceData.class, dataFile);
      addDependencies(deps, data);
    }

    int count = 0;
    for (File f : repoDir.listFiles()) {
      String name = f.getName();
      if (!deps.contains(name)) {
        if (!name.endsWith(".json")
            || !deps.contains(name.substring(0, name.length() - ".json".length()))) { // Remove
          // json
          // files
          // only
          // if
          // the
          // bin
          // is
          // going
          // as
          // well
          f.delete();
          count++;
        } else {

        }
      }
    }
    System.out.format("Garbage collection done (%d file(s) removed)%n", count);
  }

  private void addDependencies(HashSet<byte[]> deps, CommandData data) {
    for (byte[] dep : data.dependencies) {
      deps.add(dep);
    }
    for (byte[] dep : data.runbundles) {
      deps.add(dep);
    }
  }

  public void deinit(Appendable out, boolean force) throws Exception {
    Settings settings = new Settings(platform.getConfigFile());

    if (!force) {
      Justif justify = new Justif(80, 40);
      StringBuilder sb = new StringBuilder();
      Formatter f = new Formatter(sb);

      try {
        String list = listFiles(platform.getGlobal());
        if (list != null) {
          f.format("In global default environment:%n");
          f.format(list);
        }

        list = listFiles(platform.getLocal());
        if (list != null) {
          f.format("In local default environment:%n");
          f.format(list);
        }

        if (settings.containsKey(JPM_CACHE_GLOBAL)) {
          list = listFiles(IO.getFile(settings.get(JPM_CACHE_GLOBAL)));
          if (list != null) {
            f.format("In global configured environment:%n");
            f.format(list);
          }
        }

        if (settings.containsKey(JPM_CACHE_LOCAL)) {
          list = listFiles(IO.getFile(settings.get(JPM_CACHE_LOCAL)));
          if (list != null) {
            f.format("In local configured environment:%n");
            f.format(list);
          }
        }

        list = listSupportFiles();
        if (list != null) {
          f.format("jpm support files:%n");
          f.format(list);
        }

        f.format("%n%n");

        f.format(
            "All files listed above will be deleted if deinit is run with the force flag set"
                + " (\"jpm deinit -f\" or \"jpm deinit --force\"%n%n");
        f.flush();

        justify.wrap(sb);
        out.append(sb.toString());
      } finally {
        f.close();
      }
    } else { // i.e. if(force)
      int count = 0;
      File[] caches = {platform.getGlobal(), platform.getLocal(), null, null};
      if (settings.containsKey(JPM_CACHE_LOCAL)) {
        caches[2] = IO.getFile(settings.get(JPM_CACHE_LOCAL));
      }
      if (settings.containsKey(JPM_CACHE_GLOBAL)) {
        caches[3] = IO.getFile(settings.get(JPM_CACHE_GLOBAL));
      }
      ArrayList<File> toDelete = new ArrayList<File>();

      for (File cache : caches) {
        if (cache == null || !cache.exists()) {
          continue;
        }
        listFiles(cache, toDelete);
        if (toDelete.size() > count) {
          count = toDelete.size();
          if (!cache.canWrite()) {
            reporter.error(PERMISSION_ERROR + " (" + cache + ")");
            return;
          }
          toDelete.add(cache);
        }
      }
      listSupportFiles(toDelete);

      for (File f : toDelete) {
        if (f.exists() && !f.canWrite()) {
          reporter.error(PERMISSION_ERROR + " (" + f + ")");
        }
      }
      if (reporter.getErrors().size() > 0) {
        return;
      }

      for (File f : toDelete) {
        if (f.exists()) {
          IO.deleteWithException(f);
        }
      }
    }
  }

  // Adapter to list without planning to delete
  private String listFiles(final File cache) throws Exception {
    return listFiles(cache, null);
  }

  private String listFiles(final File cache, List<File> toDelete) throws Exception {
    boolean stopServices = false;
    if (toDelete == null) {
      toDelete = new ArrayList<File>();
    } else {
      stopServices = true;
    }
    int count = 0;
    Formatter f = new Formatter();

    f.format(" - Cache:%n    * %s%n", cache.getCanonicalPath());
    f.format(" - Commands:%n");
    for (CommandData cdata : getCommands(new File(cache, COMMANDS))) {
      f.format("    * %s \t0 handle for \"%s\"%n", cdata.bin, cdata.name);
      toDelete.add(new File(cdata.bin));
      count++;
    }
    f.format(" - Services:%n");
    for (ServiceData sdata : getServices(new File(cache, SERVICE))) {
      if (sdata != null) {
        f.format("    * %s \t0 service directory for \"%s\"%n", sdata.sdir, sdata.name);
        toDelete.add(new File(sdata.sdir));
        File initd = platform.getInitd(sdata);
        if (initd != null && initd.exists()) {
          f.format("    * %s \t0 init.d file for \"%s\"%n", initd.getCanonicalPath(), sdata.name);
          toDelete.add(initd);
        }
        if (stopServices) {
          Service s = getService(sdata);
          try {
            s.stop();
          } catch (Exception e) {
          }
        }
        count++;
      }
    }
    f.format("%n");

    String result = (count > 0) ? f.toString() : null;
    f.close();
    return result;
  }

  private String listSupportFiles() throws Exception { // Adapter to list
    // without planning
    // to delete
    return listSupportFiles(null);
  }

  private String listSupportFiles(List<File> toDelete) throws Exception {
    Formatter f = new Formatter();
    try {
      if (toDelete == null) {
        toDelete = new ArrayList<File>();
      }
      int precount = toDelete.size();
      File confFile = IO.getFile(platform.getConfigFile()).getCanonicalFile();
      if (confFile.exists()) {
        f.format("    * %s \t0 Config file%n", confFile);
        toDelete.add(confFile);
      }

      String result = (toDelete.size() > precount) ? f.toString() : null;
      return result;
    } finally {
      f.close();
    }
  }

  /**
   * @param data
   * @param target
   * @throws Exception
   * @throws IOException
   */
  public String createService(ServiceData data) throws Exception, IOException {

    File sdir = new File(serviceDir, data.name);
    if (!sdir.exists() && !sdir.mkdirs()) {
      throw new IOException("Could not create directory " + data.sdir);
    }
    data.sdir = sdir.getAbsolutePath();

    File lock = new File(data.sdir, LOCK);
    data.lock = lock.getAbsolutePath();

    if (data.work == null) data.work = new File(data.sdir, "work").getAbsolutePath();
    if (data.user == null) data.user = platform.user();

    if (data.user == null) data.user = "******";

    new File(data.work).mkdir();

    if (data.log == null) data.log = new File(data.sdir, "log").getAbsolutePath();

    // TODO
    // if (Data.validate(data) != null)
    // return "Invalid service data: " + Data.validate(data);

    if (service == null)
      throw new RuntimeException(
          "Missing biz.aQute.jpm.service in repo, should have been installed by init, try reiniting");

    data.serviceLib = service.getAbsolutePath();

    platform.chown(data.user, true, new File(data.sdir));

    String s = platform.createService(data, null, false);
    if (s == null) storeData(new File(data.sdir, "data"), data);
    return s;
  }

  /**
   * @param data
   * @param target
   * @throws Exception
   * @throws IOException
   */
  public String createCommand(CommandData data, boolean force) throws Exception, IOException {

    // TODO
    // if (Data.validate(data) != null)
    // return "Invalid command data: " + Data.validate(data);

    Map<String, String> map = null;
    if (data.trace) {
      map = new HashMap<String, String>();
      map.put("java.security.manager", "aQute.jpm.service.TraceSecurityManager");
      reporter.trace("tracing");
    }
    String s = platform.createCommand(data, map, force, service.getAbsolutePath());
    if (s == null) storeData(new File(commandDir, data.name), data);
    return s;
  }

  public void deleteCommand(String name) throws Exception {
    CommandData cmd = getCommand(name);
    if (cmd == null) throw new IllegalArgumentException("No such command " + name);

    platform.deleteCommand(cmd);
    File tobedel = new File(commandDir, name);
    IO.deleteWithException(tobedel);
  }

  public Service getService(String serviceName) throws Exception {
    File base = new File(serviceDir, serviceName);
    return getService(base);
  }

  public Service getService(ServiceData sdata) throws Exception {
    return getService(new File(sdata.sdir));
  }

  private Service getService(File base) throws Exception {
    File dataFile = new File(base, "data");
    if (!dataFile.isFile()) return null;

    ServiceData data = getData(ServiceData.class, dataFile);
    return new Service(this, data);
  }

  /**
   * Verify that the jar file is correct. This also verifies ok when there are no checksums or.
   *
   * @throws IOException
   */
  static Pattern MANIFEST_ENTRY = Pattern.compile("(META-INF/[^/]+)|(.*/)");

  public String verify(JarFile jar, String... algorithms) throws IOException {
    if (algorithms == null || algorithms.length == 0) algorithms = new String[] {"MD5", "SHA"};
    else if (algorithms.length == 1 && algorithms[0].equals("-")) return null;

    try {
      Manifest m = jar.getManifest();
      if (m.getEntries().isEmpty()) return "No name sections";

      for (Enumeration<JarEntry> e = jar.entries(); e.hasMoreElements(); ) {
        JarEntry je = e.nextElement();
        if (MANIFEST_ENTRY.matcher(je.getName()).matches()) continue;

        Attributes nameSection = m.getAttributes(je.getName());
        if (nameSection == null) return "No name section for " + je.getName();

        for (String algorithm : algorithms) {
          try {
            MessageDigest md = MessageDigest.getInstance(algorithm);
            String expected = nameSection.getValue(algorithm + "-Digest");
            if (expected != null) {
              byte digest[] = Base64.decodeBase64(expected);
              copy(jar.getInputStream(je), md);
              if (!Arrays.equals(digest, md.digest()))
                return "Invalid digest for "
                    + je.getName()
                    + ", "
                    + expected
                    + " != "
                    + Base64.encodeBase64(md.digest());
            } else reporter.error("could not find digest for " + algorithm + "-Digest");
          } catch (NoSuchAlgorithmException nsae) {
            return "Missing digest algorithm " + algorithm;
          }
        }
      }
    } catch (Exception e) {
      return "Failed to verify due to exception: " + e.getMessage();
    }
    return null;
  }

  /**
   * @param clazz
   * @param dataFile
   * @return
   * @throws Exception
   */
  private <T> T getData(Class<T> clazz, File dataFile) throws Exception {
    try {
      return codec.dec().from(dataFile).get(clazz);
    } catch (Exception e) {
      // e.printStackTrace();
      // System.out.println("Cannot read data file "+dataFile+": " +
      // IO.collect(dataFile));
      return null;
    }
  }

  private void storeData(File dataFile, Object o) throws Exception {
    codec.enc().to(dataFile).put(o);
  }

  /**
   * This is called when JPM runs in the background to start jobs
   *
   * @throws Exception
   */
  public void daemon() throws Exception {
    Runtime.getRuntime()
        .addShutdownHook(
            new Thread("Daemon shutdown") {
              public void run() {

                for (Service service : startedByDaemon) {
                  try {
                    reporter.error("Stopping " + service);
                    service.stop();
                    reporter.error("Stopped " + service);
                  } catch (Exception e) {
                    // Ignore
                  }
                }
              }
            });
    List<ServiceData> services = getServices();
    Map<String, ServiceData> map = new HashMap<String, ServiceData>();
    for (ServiceData d : services) {
      map.put(d.name, d);
    }
    List<ServiceData> start = new ArrayList<ServiceData>();
    Set<ServiceData> set = new HashSet<ServiceData>();
    for (ServiceData sd : services) {
      checkStartup(map, start, sd, set);
    }

    if (start.isEmpty()) reporter.warning("No services to start");

    for (ServiceData sd : start) {
      try {
        Service service = getService(sd.name);
        reporter.trace("Starting " + service);
        String result = service.start();
        if (result != null) reporter.error("Started error " + result);
        else startedByDaemon.add(service);
        reporter.trace("Started " + service);
      } catch (Exception e) {
        reporter.error("Cannot start daemon %s, due to %s", sd.name, e);
      }
    }

    while (true) {
      for (Service sd : startedByDaemon) {
        try {
          if (!sd.isRunning()) {
            reporter.error("Starting due to failure " + sd);
            String result = sd.start();
            if (result != null) reporter.error("Started error " + result);
          }
        } catch (Exception e) {
          reporter.error("Cannot start daemon %s, due to %s", sd, e);
        }
      }
      Thread.sleep(10000);
    }
  }

  private void checkStartup(
      Map<String, ServiceData> map,
      List<ServiceData> start,
      ServiceData sd,
      Set<ServiceData> cyclic) {
    if (sd.after.isEmpty() || start.contains(sd)) return;

    if (cyclic.contains(sd)) {
      reporter.error("Cyclic dependency for " + sd.name);
      return;
    }

    cyclic.add(sd);

    for (String dependsOn : sd.after) {
      if (dependsOn.equals("boot")) continue;

      ServiceData deps = map.get(dependsOn);
      if (deps == null) {
        reporter.error("No such service " + dependsOn + " but " + sd.name + " depends on it");
      } else {
        checkStartup(map, start, deps, cyclic);
      }
    }
    start.add(sd);
  }

  public void register(boolean user) throws Exception {
    platform.installDaemon(user);
  }

  public ArtifactData putAsync(final URI uri) {
    final ArtifactData data = new ArtifactData();
    data.busy = true;
    Runnable r =
        new Runnable() {

          public void run() {
            try {
              put(uri, data);
            } catch (Throwable e) {
              e.printStackTrace();
              data.error = e.toString();
            } finally {
              reporter.trace("done downloading %s", uri);
              data.done();
            }
          }
        };
    getExecutor().execute(r);
    return data;
  }

  public ArtifactData put(final URI uri) throws Exception {
    final ArtifactData data = new ArtifactData();
    put(uri, data);
    return data;
  }

  void put(final URI uri, ArtifactData data) throws Exception {
    reporter.trace("put %s %s", uri, data);
    File tmp = createTempFile(repoDir, "mtp", ".whatever");
    tmp.deleteOnExit();
    try {
      copy(uri.toURL(), tmp);
      byte[] sha = SHA1.digest(tmp).digest();
      reporter.trace("SHA %s %s", uri, Hex.toHexString(sha));
      ArtifactData existing = get(sha);
      if (existing != null) {
        reporter.trace("existing");
        xcopy(existing, data);
        return;
      }
      File meta = new File(repoDir, Hex.toHexString(sha) + ".json");
      File file = new File(repoDir, Hex.toHexString(sha));
      rename(tmp, file);
      reporter.trace("file %s", file);
      data.file = file.getAbsolutePath();
      data.sha = sha;
      data.busy = false;
      CommandData cmddata = parseCommandData(data);
      if (cmddata.bsn != null) {
        data.name = cmddata.bsn + "-" + cmddata.version;
      } else data.name = Strings.display(cmddata.title, cmddata.bsn, cmddata.name, uri);
      codec.enc().to(meta).put(data);
      reporter.trace("TD = " + data);
    } finally {
      tmp.delete();
      reporter.trace("puted %s %s", uri, data);
    }
  }

  public ArtifactData get(byte[] sha) throws Exception {
    String name = Hex.toHexString(sha);
    File data = IO.getFile(repoDir, name + ".json");
    reporter.trace("artifact data file %s", data);
    if (data.isFile()) { // Bin + metadata
      ArtifactData artifact = codec.dec().from(data).get(ArtifactData.class);
      artifact.file = IO.getFile(repoDir, name).getAbsolutePath();
      return artifact;
    }
    File bin = IO.getFile(repoDir, name);
    if (bin.exists()) { // Only bin
      ArtifactData artifact = new ArtifactData();
      artifact.file = bin.getAbsolutePath();
      artifact.sha = sha;
      return artifact;
    }

    return null;
  }

  public List<Revision> filter(Collection<Revision> list, EnumSet<Library.Phase> phases) {
    List<Revision> filtered = new ArrayList<Library.Revision>();
    for (Revision r : list) if (phases.contains(r.phase)) filtered.add(r);

    return filtered;
  }

  public Map<String, Revision> latest(Collection<Revision> list) {
    Map<String, Revision> programs = new HashMap<String, Library.Revision>();

    for (Revision r : list) {
      String coordinates = r.groupId + ":" + r.artifactId;
      if (r.classifier != null) coordinates += ":" + r.classifier;

      if (r.groupId.equals(Library.SHA_GROUP)) continue;

      Revision current = programs.get(coordinates);
      if (current == null) programs.put(coordinates, r);
      else {
        // who is better?
        if (compare(r, current) >= 0) programs.put(coordinates, r);
      }
    }
    return programs;
  }

  private int compare(Revision a, Revision b) {
    if (Arrays.equals(a._id, b._id)) return 0;

    Version va = getVersion(a);
    Version vb = getVersion(b);
    int n = va.compareTo(vb);
    if (n != 0) return n;

    if (a.created != b.created) return a.created > b.created ? 1 : -1;

    for (int i = 0; i < a._id.length; i++)
      if (a._id[i] != b._id[i]) return a._id[i] > b._id[i] ? 1 : -1;

    return 0;
  }

  private Version getVersion(Revision a) {
    if (a.qualifier != null) return new Version(a.baseline + "." + a.qualifier);
    return new Version(a.baseline);
  }

  public String getCoordinates(Revision r) {
    StringBuilder sb = new StringBuilder(r.groupId).append(":").append(r.artifactId);
    if (r.classifier != null) sb.append(":").append(r.classifier);
    sb.append("@").append(r.version);

    return sb.toString();
  }

  public String getCoordinates(RevisionRef r) {
    StringBuilder sb = new StringBuilder(r.groupId).append(":").append(r.artifactId).append(":");
    if (r.classifier != null) sb.append(r.classifier).append("@");
    sb.append(r.version);

    return sb.toString();
  }

  public ArtifactData getCandidate(String key) throws Exception {
    ArtifactData data = getCandidateAsync(key);
    if (data != null) {
      data.sync();
    }
    return data;
  }

  public ArtifactData getCandidateAsync(String arg) throws Exception {
    reporter.trace("coordinate %s", arg);
    if (isUrl(arg))
      try {
        ArtifactData data = putAsync(new URI(arg));
        data.local = true;
        return data;
      } catch (Exception e) {
        reporter.trace("hmm, not a valid url %s, will try the server", arg);
      }

    Coordinate c = new Coordinate(arg);

    if (c.isSha()) {
      ArtifactData r = get(c.getSha());
      if (r != null) return r;
    }

    Revision revision = library.getRevisionByCoordinate(c);
    if (revision == null) return null;

    reporter.trace("revision %s", Hex.toHexString(revision._id));

    ArtifactData ad = get(revision._id);
    if (ad != null) {
      reporter.trace("found in cache");
      return ad;
    }

    URI url = revision.urls.iterator().next();
    ArtifactData artifactData = putAsync(url);
    artifactData.coordinate = c;
    return artifactData;
  }

  public static Executor getExecutor() {
    if (executor == null) executor = Executors.newFixedThreadPool(4);
    return executor;
  }

  public static void setExecutor(Executor executor) {
    JustAnotherPackageManager.executor = executor;
  }

  public void setLibrary(URI url) throws Exception {
    if (url == null) url = new URI("http://repo.jpm4j.org/");

    this.host = new URLClient(url.toString());
    host.setReporter(reporter);
    library = JSONRPCProxy.createRPC(JpmRepo.class, host, "jpm");
  }

  public void close() {
    if (executor != null && executor instanceof ExecutorService)
      ((ExecutorService) executor).shutdown();
  }

  public void init() throws IOException {
    URL s = getClass().getClassLoader().getResource(SERVICE_JAR_FILE);
    if (s == null)
      if (underTest) return;
      else throw new Error("No " + SERVICE_JAR_FILE + " resource in jar");
    service.getParentFile().mkdirs();
    IO.copy(s, service);
  }

  public Platform getPlatform() {
    return platform;
  }

  /**
   * Copy from the copy method in StructUtil. Did not want to drag that code in. maybe this actually
   * should go to struct.
   *
   * @param from
   * @param to
   * @param excludes
   * @return
   * @throws Exception
   */
  public static <T extends struct> T xcopy(struct from, T to, String... excludes) throws Exception {
    Arrays.sort(excludes);
    for (Field f : from.fields()) {
      if (Arrays.binarySearch(excludes, f.getName()) >= 0) continue;

      Object o = f.get(from);
      if (o == null) continue;

      Field tof = to.getField(f.getName());
      if (tof != null)
        try {
          tof.set(to, Converter.cnv(tof.getGenericType(), o));
        } catch (Exception e) {
          System.out.println(
              "Failed to convert "
                  + f.getName()
                  + " from "
                  + from.getClass()
                  + " to "
                  + to.getClass()
                  + " value "
                  + o
                  + " exception "
                  + e);
        }
    }

    return to;
  }

  public JpmRepo getLibrary() {
    return library;
  }

  public void setLocalInstall(boolean b) {
    localInstall = b;
  }

  public String what(String key, boolean oneliner) throws Exception {
    byte[] sha;

    Matcher m = SHA_P.matcher(key);
    if (m.matches()) {
      sha = Hex.toByteArray(key);
    } else {
      m = URL_P.matcher(key);
      if (m.matches()) {
        URL url = new URL(key);
        sha = SHA1.digest(url.openStream()).digest();
      } else {
        File jarfile = new File(key);
        if (!jarfile.exists()) {
          reporter.error("File does not exist: %s", jarfile.getCanonicalPath());
        }
        sha = SHA1.digest(jarfile).digest();
      }
    }
    reporter.trace("sha %s", Hex.toHexString(sha));
    Revision revision = library.getRevision(sha);
    if (revision == null) {
      return null;
    }

    StringBuilder sb = new StringBuilder();
    Formatter f = new Formatter(sb);
    Justif justif = new Justif(120, 20, 70, 20, 75);
    DateFormat dateFormat = DateFormat.getDateInstance();

    try {
      if (oneliner) {
        f.format("%20s %s%n", Hex.toHexString(revision._id), createCoord(revision));
      } else {
        f.format("Artifact: %s%n", revision.artifactId);
        if (revision.organization != null && revision.organization.name != null) {
          f.format(" (%s)", revision.organization.name);
        }
        f.format("%n");
        f.format("Coordinates\t0: %s%n", createCoord(revision));
        f.format("Created\t0: %s%n", dateFormat.format(new Date(revision.created)));
        f.format("Size\t0: %d%n", revision.size);
        f.format("Sha\t0: %s%n", Hex.toHexString(revision._id));
        f.format("URL\t0: %s%n", createJpmLink(revision));
        f.format("%n");
        f.format("%s%n", revision.description);
        f.format("%n");
        f.format("Dependencies\t0:%n");
        boolean flag = false;
        Iterable<RevisionRef> closure = library.getClosure(revision._id, true);
        for (RevisionRef dep : closure) {
          f.format(
              " - %s \t2- %s \t3- %s%n",
              dep.name, createCoord(dep), dateFormat.format(new Date(dep.created)));
          flag = true;
        }
        if (!flag) {
          f.format("     None%n");
        }
        f.format("%n");
      }
      f.flush();
      justif.wrap(sb);
      return sb.toString();
    } finally {
      f.close();
    }
  }

  private String createCoord(Revision rev) {
    return String.format("%s:%s@%s [%s]", rev.groupId, rev.artifactId, rev.version, rev.phase);
  }

  private String createCoord(RevisionRef rev) {
    return String.format("%s:%s@%s [%s]", rev.groupId, rev.artifactId, rev.version, rev.phase);
  }

  private String createJpmLink(Revision rev) {
    return String.format(
        "http://jpm4j.org/#!/p/sha/%s//%s", Hex.toHexString(rev._id), rev.baseline);
  }

  public class UpdateMemo {
    public CommandData current; // Works for commandData and
    // ServiceData, as ServiceData --|>
    // CommandData
    public RevisionRef best;
  }

  public void listUpdates(
      List<UpdateMemo> notFound,
      List<UpdateMemo> upToDate,
      List<UpdateMemo> toUpdate,
      CommandData data,
      boolean staged)
      throws Exception {

    // UpdateMemo memo = new UpdateMemo();
    // memo.current = data;
    //
    // Matcher m = data.coordinates == null ? null :
    // COORD_P.matcher(data.coordinates);
    //
    // if (data.version == null || m == null || !m.matches()) {
    // Revision revision = library.getRevision(data.sha);
    // if (revision == null) {
    // notFound.add(memo);
    // return;
    // }
    // data.version = new Version(revision.version);
    // data.coordinates = getCoordinates(revision);
    // if (data instanceof ServiceData) {
    // storeData(IO.getFile(new File(serviceDir, data.name), "data"), data);
    // } else {
    // storeData(IO.getFile(commandDir, data.name), data);
    // }
    // }
    //
    // Iterable< ? extends Program> programs =
    // library.getQueryPrograms(data.coordinates, 0, 0);
    // int count = 0;
    // RevisionRef best = null;
    // for (Program p : programs) {
    // // best = selectBest(p.revisions, staged, null);
    // count++;
    // }
    // if (count != 1 || best == null) {
    // notFound.add(memo);
    // return;
    // }
    // Version bestVersion = new Version(best.version);
    //
    // if (data.version.compareTo(bestVersion) < 0) { // Update available
    // memo.best = best;
    // toUpdate.add(memo);
    // } else { // up to date
    // upToDate.add(memo);
    // }
  }

  public void update(UpdateMemo memo) throws Exception {

    ArtifactData target = put(memo.best.urls.iterator().next());

    memo.current.version = new Version(memo.best.version);
    target.sync();
    memo.current.sha = target.sha;
    // memo.current.dependencies = target.dependencies;
    // memo.current.dependencies.add((new File(repoDir,
    // Hex.toHexString(target.sha))).getCanonicalPath());
    // memo.current.runbundles = target.runbundles;
    // memo.current.description = target.description;
    memo.current.time = target.time;

    if (memo.current instanceof ServiceData) {
      Service service = getService((ServiceData) memo.current);
      service.remove();
      createService((ServiceData) memo.current);
      IO.delete(new File(IO.getFile(serviceDir, memo.current.name), "data"));
      storeData(new File(IO.getFile(serviceDir, memo.current.name), "data"), memo.current);
    } else {
      platform.deleteCommand(memo.current);
      createCommand(memo.current, false);
      IO.delete(IO.getFile(commandDir, memo.current.name));
      storeData(IO.getFile(commandDir, memo.current.name), memo.current);
    }
  }

  private boolean isUrl(String coordinate) {
    return URL_P.matcher(coordinate).matches();
  }

  /**
   * Find programs
   *
   * @throws Exception
   */
  public List<Program> find(String query, int skip, int limit) throws Exception {
    return library.getQueryPrograms(query, skip, limit);
  }

  public boolean isWildcard(String coordinate) {
    return coordinate != null && coordinate.endsWith("@*");
  }

  public CommandData parseCommandData(ArtifactData artifact) throws Exception {
    File source = new File(artifact.file);
    if (!source.isFile()) throw new FileNotFoundException();

    CommandData data = new CommandData();
    data.sha = artifact.sha;
    data.jpmRepoDir = repoDir.getCanonicalPath();
    JarFile jar = new JarFile(source);
    try {
      reporter.trace("Parsing %s", source);
      Manifest m = jar.getManifest();
      Attributes main = m.getMainAttributes();
      data.name = data.bsn = main.getValue("Bundle-SymbolicName");
      String version = main.getValue("Bundle-Version");
      if (version == null) data.version = Version.LOWEST;
      else data.version = new Version(version);

      data.main = main.getValue("Main-Class");
      data.description = main.getValue("Bundle-Description");
      data.title = main.getValue("JPM-Name");

      reporter.trace("name " + data.name + " " + data.main + " " + data.title);
      DependencyCollector path = new DependencyCollector(this);
      path.add(artifact);
      DependencyCollector bundles = new DependencyCollector(this);
      if (main.getValue("JPM-Classpath") != null) {
        Parameters requires = OSGiHeader.parseHeader(main.getValue("JPM-Classpath"));

        for (Map.Entry<String, Attrs> e : requires.entrySet()) {
          path.add(e.getKey(), e.getValue().get("name")); // coordinate
        }
      } else if (!artifact.local) { // No JPM-Classpath, falling back to
        // server's revision
        // Iterable<RevisionRef> closure =
        // library.getClosure(artifact.sha,
        // false);
        // System.out.println("getting closure " + artifact.url + " " +
        // Strings.join("\n",closure));

        // if (closure != null) {
        // for (RevisionRef ref : closure) {
        // path.add(Hex.toHexString(ref.revision));
        // }
        // }
      }

      if (main.getValue("JPM-Runbundles") != null) {
        Parameters jpmrunbundles = OSGiHeader.parseHeader(main.getValue("JPM-Runbundles"));

        for (Map.Entry<String, Attrs> e : jpmrunbundles.entrySet()) {
          bundles.add(e.getKey(), e.getValue().get("name"));
        }
      }

      reporter.trace("collect digests runpath");
      data.dependencies.addAll(path.getDigests());
      reporter.trace("collect digests bundles");
      data.runbundles.addAll(bundles.getDigests());

      Parameters command = OSGiHeader.parseHeader(main.getValue("JPM-Command"));
      if (command.size() > 1) reporter.error("Only one command can be specified");

      for (Map.Entry<String, Attrs> e : command.entrySet()) {
        data.name = e.getKey();

        Attrs attrs = e.getValue();

        if (attrs.containsKey("jvmargs")) data.jvmArgs = attrs.get("jvmargs");

        if (attrs.containsKey("title")) data.title = attrs.get("title");

        if (data.title != null) data.title = data.name;
      }
      return data;
    } finally {
      jar.close();
    }
  }

  public void setUnderTest() {
    underTest = true;
  }

  /**
   * Turn the shas into a readable form
   *
   * @param dependencies
   * @return
   * @throws Exception
   */
  public List<?> toString(List<byte[]> dependencies) throws Exception {
    List<String> out = new ArrayList<String>();
    for (byte[] dependency : dependencies) {
      ArtifactData data = get(dependency);
      if (data == null) out.add(Hex.toHexString(dependency));
      else {
        out.add(Strings.display(data.name, Hex.toHexString(dependency)));
      }
    }
    return out;
  }

  /**
   * Get a list of candidates from a coordinate
   *
   * @param c
   * @throws Exception
   */
  public Iterable<Revision> getCandidates(Coordinate c) throws Exception {
    return library.getRevisionsByCoordinate(c);
  }

  /** Post install */
  public void doPostInstall() {
    getPlatform().doPostInstall();
  }
}
  public BankMessage makeMessage() {

    Scanner stdIn = new Scanner(System.in);
    int userInput;

    int[] opcodes = {CREATE, DEPOSIT, WITHDRAW, GETBALANCE, CLOSE};

    do {
      userInput = stdIn.nextInt();
    } while (!(userInput >= 1 && userInput <= 5));

    int opcode = opcodes[userInput - 1];
    int[] parameters = new int[2];

    // create a pattern only accepting positive numbers with at most 2 decimal places
    Pattern dollar =
        Pattern.compile(
            "^\\$?([1-9]{1}[0-9]{0,2}(\\,[0-9]{3})*(\\.[0-9]{0,2})?|[1-9]{1}[0-9]{0,}(\\.[0-9]{0,2})?|0(\\.[0-9]{0,2})?|(\\.[0-9]{1,2})?)$");
    String input = null;

    // populate parameters list
    switch (opcode) {
      case CREATE:
        do {
          System.out.println("How much for initial deposit?");
          input = stdIn.next();
        } while ((dollar.matcher(input).matches()) == false);
        parameters[0] = (int) (Double.parseDouble(input) * 100);

        break;

      case DEPOSIT:
        System.out.println("Which account?");
        parameters[0] = stdIn.nextInt();

        do {
          System.out.println("How much to deposit?");
          input = stdIn.next();
        } while ((dollar.matcher(input).matches()) == false);
        parameters[1] = (int) (Double.parseDouble(input) * 100);
        break;

      case WITHDRAW:
        System.out.println("Which account?");
        parameters[0] = stdIn.nextInt();

        do {
          System.out.println("How much to withdraw?");
          input = stdIn.next();
        } while ((dollar.matcher(input).matches()) == false);
        parameters[1] = (int) (Double.parseDouble(input) * 100);
        break;

      case GETBALANCE:
        System.out.println("Which account?");
        parameters[0] = stdIn.nextInt();
        break;

      case CLOSE:
        System.out.println("Which account?");
        parameters[0] = stdIn.nextInt();
        break;

      default:
        System.out.println("Invalid opcode. System failure.");
        System.exit(-1);
    }

    BankMessage output = buildMessage(opcode, parameters);
    return output;
  }
Ejemplo n.º 14
0
public class Main {

  private static Pattern lores =
      Pattern.compile(
          "post_photolores=\"https?://([-\\w\\.]+)+(:\\d+)?(/([\\w/_\\.]*(\\?\\S+)?)?)?\"");
  private static List<String> endings = new ArrayList<>();

  static {
    Main.endings.add("_1280");
    Main.endings.add("_700");
    Main.endings.add("_700");
    Main.endings.add("_500");
    Main.endings.add("_400");
    Main.endings.add("_250");
    Main.endings.add("_100");
  }

  private static Gui gui = null;
  private static HashMap<Post, Post> post_post_hash = new HashMap<>();
  protected static HashMap<Picture, Picture> pic_pic_hash = new HashMap<>();
  private static HashMap<Picture, Post> pic_post_hash = new HashMap<>();
  private static HashMap<Post, Post> dup_post_list = new HashMap<>();
  private static String blogname = "";
  private static File blogdir = null;

  public static Set<Picture> getPictures() {
    return Main.pic_pic_hash.keySet();
  }

  public static File getBlogDir() {
    return Main.blogdir;
  }

  public static void setBlogName(String blogname) {
    Main.blogname = blogname;
    Main.blogdir = new File("." + File.separator + Main.blogname + File.separator);
  }

  public static void main(String[] args) {
    if (args.length == 0) {
      gui = new Gui();
      gui.setVisible(true);
    } else if (args.length == 3 || args.length == 4) {
      Main.setBlogName(args[0]);
      Main.load();
      if (args.length == 3) {
        int start, limit;
        try {
          start = Integer.parseInt(args[1]);
          limit = Integer.parseInt(args[2]);
        } catch (Exception e) {
          Main.status("usage: Main <blogname> <start_page> <end_page> -hires");
          Main.status("start_page and end_page must be integers >= 1");
          return;
        }
        Main.run(start, limit);
        Main.save();
      } else if (args.length == 4 && args[3].equals("-hires")) {
        Main.downloadHiRes();
        Main.save();
      } else {
        Main.status("usage: Main <blogname> <start_page> <end_page> -hires");
      }
    } else {
      Main.status("usage: Main <blogname> <start_page> <end_page> -hires");
    }
  }

  protected static void reset() {
    Helper.removeDirectoryIfItExists(Helper.temp);
    Helper.removeDirectoryIfItExists(blogdir);
    Main.post_post_hash.clear();
    Main.pic_pic_hash.clear();
    Main.pic_post_hash.clear();
    Main.dup_post_list.clear();
  }

  public static void run(int start_page, int end_page) {
    if (start_page < 1 || end_page < 1) {
      Main.status("start_page and end_page must be integers >= 1");
      return;
    }
    int progress = 0;
    if (gui != null) {
      gui.setProgress(progress);
    }
    if (end_page >= start_page) {
      if (gui != null) {
        gui.setMaxProgress(end_page - start_page);
      }
      for (int i = start_page; i <= end_page; i++) {
        boolean exists =
            Main.handleURL(String.format("http://%s.tumblr.com/page/%s", Main.blogname, i));
        if (!exists) {
          Main.status(String.format("We ran out of posts to process at page %s.", i));
          break;
        }
        if (gui != null) {
          gui.setProgress(progress);
          progress++;
        }
      }
    } else {
      if (gui != null) {
        gui.setMaxProgress(start_page - end_page);
      }
      for (int i = start_page; i >= end_page; i--) {
        boolean exists =
            Main.handleURL(String.format("http://%s.tumblr.com/page/%s", Main.blogname, i));
        if (!exists) {
          Main.status(String.format("We ran out of posts to process at page %s.", i));
          break;
        }
        if (gui != null) {
          gui.setProgress(progress);
          progress++;
        }
      }
    }
    if (gui != null) {
      gui.setProgress(progress);
    }
    Main.writeDuplicates();
  }

  private static void writeDuplicates() {
    Main.status("Writing duplicates.");
    if (!dup_post_list.isEmpty()) {
      Main.status(String.format("%s\t%s", "older_post", "newer_post"));
      for (Post post : dup_post_list.keySet()) {
        Main.status(String.format("%s\t%s", post.post_id, dup_post_list.get(post).post_id));
      }
    } else {
      Main.status("There are no duplicates.");
    }
    Main.status("Writing duplicates done.");
  }

  public static void load() {
    Main.status("Loading databases.");
    File file = new File(blogdir, "picpic.db");
    List<Object> objects = Helper.loadObjectFromFile(file);
    if (objects == null || objects.size() != 1) {
      Main.error("Unable to load database files so creating new database.");
      reset();
    } else {
      Main.post_post_hash = (HashMap<Post, Post>) objects.get(0);
      Main.pic_pic_hash.clear();
      Main.pic_post_hash.clear();
      Main.dup_post_list.clear();
      Main.setupPosts();
    }
    Main.status("Done loading databases.");
  }

  public static void save() {
    Main.status("Saving databases.");
    File file = new File(blogdir, "picpic.db");
    List<Object> objects = new ArrayList<>();
    objects.add(Main.post_post_hash);
    Helper.saveObjectToFile(file, objects);
    Main.status("Done saving databases.");
  }

  private static boolean handleURL(String address) {
    Main.status(String.format("Processing page \"%s\".", address));
    try {
      NodeList posts = getPosts(address);
      if (posts.toNodeArray().length == 0) {
        return false;
      }
      for (Node post_node : posts.toNodeArray()) {
        if (post_node instanceof TagNode) {
          TagNode post = (TagNode) post_node;
          Post new_post = new Post(Long.parseLong(post.getAttribute("id").substring(5)));
          if (!Main.post_post_hash.containsKey(new_post)) {
            NodeList photo_posts = getPhotoPosts(post.getChildren());
            NodeList remarks = getRemarks(photo_posts);
            for (Node node : remarks.toNodeArray()) {
              Matcher matcher = lores.matcher(node.getText());
              String media_url = "";
              if (matcher.find()) {
                media_url = matcher.group();
                media_url = media_url.substring(17, media_url.length() - 1);
              }
              String thumb =
                  media_url.replace(
                      media_url.substring(media_url.lastIndexOf("_"), media_url.lastIndexOf(".")),
                      "_75sq");
              URL thumb_url = new URL(thumb);
              new_post.pictures.add(new Picture(new URL(media_url), thumb_url));
            }
            NodeList photoset_posts = getPhotosetPosts(post.getChildren());
            NodeList iframes = getIFrames(photoset_posts);
            for (Node node : iframes.toNodeArray()) {
              if (node instanceof TagNode) {
                String iframe_url = ((TagNode) node).getAttribute("src");
                Parser parser2 = new Parser(iframe_url);
                NodeList a_list = parser2.extractAllNodesThatMatch(new TagNameFilter("a"));
                Node[] a_array = a_list.toNodeArray();
                Node[] img_array =
                    a_list.extractAllNodesThatMatch(new TagNameFilter("img"), true).toNodeArray();
                String media_url;
                for (int i = 0; i < a_array.length; i++) {
                  media_url = ((TagNode) img_array[i]).getAttribute("src");
                  String thumb =
                      media_url.replace(
                          media_url.substring(
                              media_url.lastIndexOf("_"), media_url.lastIndexOf(".")),
                          "_75sq");
                  URL thumb_url = new URL(thumb);
                  new_post.pictures.add(new Picture(new URL(media_url), thumb_url));
                }
              }
            }
            Main.handlePost(new_post);
          } else {
            new_post = post_post_hash.get(new_post);
            handleNonDownloadPost(new_post);
          }
        }
      }
    } catch (Exception ex) {
      ex.printStackTrace();
      Main.status("Error handling post.");
    }
    return true;
  }

  private static NodeList getPosts(String url) throws ParserException {
    return new Parser(url)
        .extractAllNodesThatMatch(new HasAttributeFilter("class", "my_post load"));
  }

  private static NodeList getPhotoPosts(NodeList node_list) {
    return node_list.extractAllNodesThatMatch(
        new HasAttributeFilter("class", "my_photo_post"), true);
  }

  private static NodeList getPhotosetPosts(NodeList node_list) {
    return node_list.extractAllNodesThatMatch(
        new HasAttributeFilter("class", "my_photoset_post"), true);
  }

  private static NodeList getRemarks(NodeList node_list) {
    return node_list.extractAllNodesThatMatch(new NodeClassFilter(RemarkNode.class), true);
  }

  private static NodeList getIFrames(NodeList node_list) {
    return node_list.extractAllNodesThatMatch(new TagNameFilter("iframe"), true);
  }

  public static void downloadHiRes() {
    Main.status("Downloading hi res versions of photos in database.");
    if (gui != null) {
      gui.setProgress(0);
      gui.setMaxProgress(pic_pic_hash.keySet().size());
    }
    int progress = 0;
    for (Picture picture : pic_pic_hash.keySet()) {
      if (!picture.downloaded_hi) {
        tryResUrls(picture);
      }
      if (gui != null) {
        gui.setProgress(progress);
        progress++;
      }
    }
    if (gui != null) {
      gui.setProgress(progress);
      progress++;
    }
    Main.status("Done downloading hi res versions.");
  }

  private static void tryResUrls(Picture picture) {
    String hi_res = "";
    String url = picture.media_url.toString();
    for (String ending : Main.endings) {
      try {
        hi_res = url.replace(url.substring(url.lastIndexOf("_"), url.lastIndexOf(".")), ending);
        URL hi_url = new URL(hi_res);
        File hi_name = Helper.extractMediaFileNameFromURL(hi_url);
        if (hi_name.equals(picture.media_name)) {
          picture.hi_url = hi_url;
          picture.hi_name = hi_name;
          picture.downloaded_hi = true;
          break;
        } else {
          boolean success = Helper.downloadFileFromURLToFileInTemp(hi_url, hi_name);
          if (success) {
            picture.hi_url = hi_url;
            picture.hi_name = hi_name;
            picture.downloaded_hi = true;
            Helper.moveTempImageToStore(hi_name, new File(Main.blogdir, picture.md5_id));
            break;
          }
        }
      } catch (MalformedURLException ex) {
        Main.error(String.format("Attempted hi res url %s is a malformed URL.", hi_res));
      }
    }
  }

  private static void handlePost(Post post) {
    Main.post_post_hash.put(post, post);
    for (Picture picture : post.pictures) {
      Helper.downloadFileFromURLToFileInTemp(picture.thumb_url, picture.thumb_name);
      picture.md5_id = Helper.createMD5FromFileInTemp(picture.thumb_name);
      Helper.moveTempImageToStore(picture.thumb_name, new File(Main.blogdir, picture.md5_id));
      if (!Main.pic_pic_hash.containsKey(picture)) {
        Main.pic_pic_hash.put(picture, picture);
        Main.pic_post_hash.put(picture, post);
        Helper.downloadFileFromURLToFileInTemp(picture.media_url, picture.media_name);
        Helper.moveTempImageToStore(picture.media_name, new File(Main.blogdir, picture.md5_id));
      } else {
        if (!post.equals(Main.pic_post_hash.get(picture))) {
          dup_post_list.put(post, Main.pic_post_hash.get(picture));
        }
      }
    }
  }

  public static void setupPosts() {
    for (Post post : Main.post_post_hash.keySet()) {
      post = Main.post_post_hash.get(post);
      handleNonDownloadPost(post);
    }
  }

  private static void handleNonDownloadPost(Post post) {
    for (Picture picture : post.pictures) {
      if (!Main.pic_pic_hash.containsKey(picture)) {
        Main.pic_pic_hash.put(picture, picture);
        Main.pic_post_hash.put(picture, post);
      } else {
        if (!post.equals(Main.pic_post_hash.get(picture))) {
          dup_post_list.put(post, Main.pic_post_hash.get(picture));
        }
      }
    }
  }

  private static void status(String status) {
    if (gui == null) {
      System.out.println(status);
    } else {
      gui.setStatus(status);
    }
  }

  private static void error(String error) {
    if (gui == null) {
      System.err.println(error);
    } else {
      gui.setStatus(error);
    }
  }
}
Ejemplo n.º 15
0
/**
 * This class represents a single RESTXQ function.
 *
 * @author BaseX Team 2005-12, BSD License
 * @author Christian Gruen
 */
final class RestXqFunction implements Comparable<RestXqFunction> {
  /** Pattern for a single template. */
  private static final Pattern TEMPLATE = Pattern.compile("\\s*\\{\\s*\\$(.+?)\\s*\\}\\s*");

  /** Supported methods. */
  EnumSet<HTTPMethod> methods = EnumSet.allOf(HTTPMethod.class);
  /** Serialization parameters. */
  final SerializerProp output = new SerializerProp();
  /** Associated function. */
  final StaticUserFunc function;
  /** Associated module. */
  final RestXqModule module;
  /** Path. */
  RestXqPath path;

  /** Query parameters. */
  final ArrayList<RestXqParam> queryParams = new ArrayList<RestXqParam>();
  /** Form parameters. */
  final ArrayList<RestXqParam> formParams = new ArrayList<RestXqParam>();
  /** Header parameters. */
  final ArrayList<RestXqParam> headerParams = new ArrayList<RestXqParam>();
  /** Cookie parameters. */
  final ArrayList<RestXqParam> cookieParams = new ArrayList<RestXqParam>();

  /** Query context. */
  private final QueryContext context;
  /** Consumed media types. */
  private final StringList consumes = new StringList();
  /** Returned media types. */
  private final StringList produces = new StringList();
  /** Post/Put variable. */
  private QNm requestBody;

  /**
   * Constructor.
   *
   * @param uf associated user function
   * @param qc query context
   * @param m associated module
   */
  RestXqFunction(final StaticUserFunc uf, final QueryContext qc, final RestXqModule m) {
    function = uf;
    context = qc;
    module = m;
  }

  /**
   * Processes the HTTP request. Parses new modules and discards obsolete ones.
   *
   * @param http HTTP context
   * @throws Exception exception
   */
  void process(final HTTPContext http) throws Exception {
    try {
      module.process(http, this);
    } catch (final QueryException ex) {
      if (ex.file() == null) ex.info(function.info);
      throw ex;
    }
  }

  /**
   * Checks a function for RESTFful annotations.
   *
   * @return {@code true} if module contains relevant annotations
   * @throws QueryException query exception
   */
  boolean analyze() throws QueryException {
    // parse all annotations
    final EnumSet<HTTPMethod> mth = EnumSet.noneOf(HTTPMethod.class);
    final boolean[] declared = new boolean[function.args.length];
    boolean found = false;
    final int as = function.ann.size();
    for (int a = 0; a < as; a++) {
      final QNm name = function.ann.names[a];
      final Value value = function.ann.values[a];
      final byte[] local = name.local();
      final byte[] uri = name.uri();
      final boolean rexq = eq(uri, QueryText.RESTXQURI);
      if (rexq) {
        if (eq(PATH, local)) {
          // annotation "path"
          if (path != null) error(ANN_TWICE, "%", name.string());
          path = new RestXqPath(toString(value, name));
          for (final String s : path) {
            if (s.trim().startsWith("{")) checkVariable(s, AtomType.AAT, declared);
          }
        } else if (eq(CONSUMES, local)) {
          // annotation "consumes"
          strings(value, name, consumes);
        } else if (eq(PRODUCES, local)) {
          // annotation "produces"
          strings(value, name, produces);
        } else if (eq(QUERY_PARAM, local)) {
          // annotation "query-param"
          queryParams.add(param(value, name, declared));
        } else if (eq(FORM_PARAM, local)) {
          // annotation "form-param"
          formParams.add(param(value, name, declared));
        } else if (eq(HEADER_PARAM, local)) {
          // annotation "header-param"
          headerParams.add(param(value, name, declared));
        } else if (eq(COOKIE_PARAM, local)) {
          // annotation "cookie-param"
          cookieParams.add(param(value, name, declared));
        } else {
          // method annotations
          final HTTPMethod m = HTTPMethod.get(string(local));
          if (m == null) error(ANN_UNKNOWN, "%", name.string());
          if (!value.isEmpty()) {
            // remember post/put variable
            if (requestBody != null) error(ANN_TWICE, "%", name.string());
            if (m != POST && m != PUT) error(METHOD_VALUE, m);
            requestBody = checkVariable(toString(value, name), declared);
          }
          if (mth.contains(m)) error(ANN_TWICE, "%", name.string());
          mth.add(m);
        }
      } else if (eq(uri, QueryText.OUTPUTURI)) {
        // serialization parameters
        final String key = string(local);
        final String val = toString(value, name);
        if (output.get(key) == null) error(UNKNOWN_SER, key);
        output.set(key, val);
      }
      found |= rexq;
    }
    if (!mth.isEmpty()) methods = mth;

    if (found) {
      if (path == null) error(ANN_MISSING, PATH);
      for (int i = 0; i < declared.length; i++)
        if (!declared[i]) error(VAR_UNDEFINED, function.args[i].name.string());
    }
    return found;
  }

  /**
   * Checks if an HTTP request matches this function and its constraints.
   *
   * @param http http context
   * @return result of check
   */
  boolean matches(final HTTPContext http) {
    // check method, path, consumed and produced media type
    return methods.contains(http.method) && pathMatches(http) && consumes(http) && produces(http);
  }

  /**
   * Binds the annotated variables.
   *
   * @param http http context
   * @param arg argument array
   * @throws QueryException query exception
   * @throws IOException I/O exception
   */
  void bind(final HTTPContext http, final Expr[] arg) throws QueryException, IOException {
    // bind variables from segments
    for (int s = 0; s < path.size; s++) {
      final Matcher m = TEMPLATE.matcher(path.segment[s]);
      if (!m.find()) continue;
      final QNm qnm = new QNm(token(m.group(1)), context);
      bind(qnm, arg, new Atm(http.segment(s)));
    }

    // cache request body
    final String ct = http.contentType();
    IOContent body = null;

    if (requestBody != null) {
      body = cache(http, null);
      try {
        // bind request body in the correct format
        body.name(http.method + IO.XMLSUFFIX);
        bind(requestBody, arg, Parser.item(body, context.context.prop, ct));
      } catch (final IOException ex) {
        error(INPUT_CONV, ex);
      }
    }

    // bind query parameters
    final Map<String, String[]> params = http.params();
    for (final RestXqParam rxp : queryParams) bind(rxp, arg, params.get(rxp.key));

    // bind form parameters
    if (!formParams.isEmpty()) {
      if (MimeTypes.APP_FORM.equals(ct)) {
        // convert parameters encoded in a form
        body = cache(http, body);
        addParams(body.toString(), params);
      }
      for (final RestXqParam rxp : formParams) bind(rxp, arg, params.get(rxp.key));
    }

    // bind header parameters
    for (final RestXqParam rxp : headerParams) {
      final StringList sl = new StringList();
      final Enumeration<?> en = http.req.getHeaders(rxp.key);
      while (en.hasMoreElements()) {
        for (final String s : en.nextElement().toString().split(", *")) sl.add(s);
      }
      bind(rxp, arg, sl.toArray());
    }

    // bind cookie parameters
    final Cookie[] ck = http.req.getCookies();
    for (final RestXqParam rxp : cookieParams) {
      String v = null;
      if (ck != null) {
        for (final Cookie c : ck) {
          if (rxp.key.equals(c.getName())) v = c.getValue();
        }
      }
      if (v == null) bind(rxp, arg);
      else bind(rxp, arg, v);
    }
  }

  /**
   * Creates an exception with the specified message.
   *
   * @param msg message
   * @param ext error extension
   * @return exception
   * @throws QueryException query exception
   */
  QueryException error(final String msg, final Object... ext) throws QueryException {
    throw new QueryException(function.info, Err.BASX_RESTXQ, Util.info(msg, ext));
  }

  @Override
  public int compareTo(final RestXqFunction rxf) {
    return path.compareTo(rxf.path);
  }

  // PRIVATE METHODS ====================================================================

  /**
   * Checks the specified template and adds a variable.
   *
   * @param tmp template string
   * @param declared variable declaration flags
   * @return resulting variable
   * @throws QueryException query exception
   */
  private QNm checkVariable(final String tmp, final boolean[] declared) throws QueryException {
    return checkVariable(tmp, AtomType.ITEM, declared);
  }

  /**
   * Checks the specified template and adds a variable.
   *
   * @param tmp template string
   * @param type allowed type
   * @param declared variable declaration flags
   * @return resulting variable
   * @throws QueryException query exception
   */
  private QNm checkVariable(final String tmp, final Type type, final boolean[] declared)
      throws QueryException {
    final Var[] args = function.args;
    final Matcher m = TEMPLATE.matcher(tmp);
    if (!m.find()) error(INV_TEMPLATE, tmp);
    final byte[] vn = token(m.group(1));
    if (!XMLToken.isQName(vn)) error(INV_VARNAME, vn);
    final QNm qnm = new QNm(vn, context);
    int r = -1;
    while (++r < args.length && !args[r].name.eq(qnm)) ;
    if (r == args.length) error(UNKNOWN_VAR, vn);
    if (declared[r]) error(VAR_ASSIGNED, vn);
    final SeqType st = args[r].declaredType();
    if (args[r].checksType() && !st.type.instanceOf(type)) error(INV_VARTYPE, vn, type);
    declared[r] = true;
    return qnm;
  }

  /**
   * Checks if the path matches the HTTP request.
   *
   * @param http http context
   * @return result of check
   */
  private boolean pathMatches(final HTTPContext http) {
    return path.matches(http);
  }

  /**
   * Checks if the consumed content type matches.
   *
   * @param http http context
   * @return result of check
   */
  private boolean consumes(final HTTPContext http) {
    // return true if no type is given
    if (consumes.isEmpty()) return true;
    // return true if no content type is specified by the user
    final String ct = http.contentType();
    if (ct == null) return true;
    // check if any combination matches
    for (final String c : consumes) {
      if (MimeTypes.matches(c, ct)) return true;
    }
    return false;
  }

  /**
   * Checks if the produced content type matches.
   *
   * @param http http context
   * @return result of check
   */
  private boolean produces(final HTTPContext http) {
    // return true if no type is given
    if (produces.isEmpty()) return true;
    // check if any combination matches
    for (final String pr : http.produces()) {
      for (final String p : produces) {
        if (MimeTypes.matches(p, pr)) return true;
      }
    }
    return false;
  }

  /**
   * Binds the specified parameter to a variable.
   *
   * @param rxp parameter
   * @param args argument array
   * @param values values to be bound; the parameter's default value is assigned if the argument is
   *     {@code null} or empty
   * @throws QueryException query exception
   */
  private void bind(final RestXqParam rxp, final Expr[] args, final String... values)
      throws QueryException {
    final Value val;
    if (values == null || values.length == 0) {
      val = rxp.value;
    } else {
      final ValueBuilder vb = new ValueBuilder();
      for (final String s : values) vb.add(new Atm(s));
      val = vb.value();
    }
    bind(rxp.name, args, val);
  }

  /**
   * Binds the specified value to a variable.
   *
   * @param name variable name
   * @param args argument array
   * @param value value to be bound
   * @throws QueryException query exception
   */
  private void bind(final QNm name, final Expr[] args, final Value value) throws QueryException {
    // skip nulled values
    if (value == null) return;

    for (int i = 0; i < function.args.length; i++) {
      final Var var = function.args[i];
      if (!var.name.eq(name)) continue;
      // casts and binds the value
      args[i] = var.checkType(value, context, null);
      break;
    }
  }

  /**
   * Returns the specified value as an atomic string.
   *
   * @param value value
   * @param name name
   * @return string
   * @throws QueryException HTTP exception
   */
  private String toString(final Value value, final QNm name) throws QueryException {
    if (!(value instanceof Str)) error(ANN_STRING, "%", name.string(), value);
    return ((Str) value).toJava();
  }

  /**
   * Adds items to the specified list.
   *
   * @param value value
   * @param name name
   * @param list list to add values to
   * @throws QueryException HTTP exception
   */
  private void strings(final Value value, final QNm name, final StringList list)
      throws QueryException {

    final long vs = value.size();
    for (int v = 0; v < vs; v++) list.add(toString(value.itemAt(v), name));
  }

  /**
   * Returns a parameter.
   *
   * @param value value
   * @param name name
   * @param declared variable declaration flags
   * @return parameter
   * @throws QueryException HTTP exception
   */
  private RestXqParam param(final Value value, final QNm name, final boolean[] declared)
      throws QueryException {
    // [CG] RESTXQ: allow identical field names?
    final long vs = value.size();
    if (vs < 2) error(ANN_PARAMS, "%", name.string(), 2);
    // name of parameter
    final String key = toString(value.itemAt(0), name);
    // variable template
    final QNm qnm = checkVariable(toString(value.itemAt(1), name), declared);
    // default value
    final ValueBuilder vb = new ValueBuilder();
    for (int v = 2; v < vs; v++) vb.add(value.itemAt(v));
    return new RestXqParam(qnm, key, vb.value());
  }

  // PRIVATE STATIC METHODS =============================================================

  /**
   * Caches the request body, if not done yet.
   *
   * @param http http context
   * @param cache cache existing cache reference
   * @return cache
   * @throws IOException I/O exception
   */
  private static IOContent cache(final HTTPContext http, final IOContent cache) throws IOException {

    if (cache != null) return cache;
    final BufferInput bi = new BufferInput(http.req.getInputStream());
    final IOContent io = new IOContent(bi.content());
    io.name(http.method + IO.XMLSUFFIX);
    return io;
  }

  /**
   * Adds parameters from the passed on request body.
   *
   * @param body request body
   * @param params map parameters
   */
  private static void addParams(final String body, final Map<String, String[]> params) {
    for (final String nv : body.split("&")) {
      final String[] parts = nv.split("=", 2);
      if (parts.length < 2) continue;
      try {
        params.put(parts[0], new String[] {URLDecoder.decode(parts[1], Token.UTF8)});
      } catch (final Exception ex) {
        Util.notexpected(ex);
      }
    }
  }
}