private static final class MessageFormatter {
private static final Pattern variablePattern =
Pattern.compile("(?i)\\$(\\{[a-z0-9_]{1,}\\}|[a-z0-9_]{1,})");
private static final Pattern colorPattern = Pattern.compile("(?i)&[0-9A-FK-OR]");
private final Map<String, String> variables;
public MessageFormatter() {
variables = new HashMap<String, String>();
}
public synchronized void setVariable(String variable, String value) {
if (value == null) variables.remove(value);
else variables.put(variable, value);
}
public synchronized String format(String message) {
Matcher matcher = variablePattern.matcher(message);
while (matcher.find()) {
String variable = matcher.group();
variable = variable.substring(1);
if (variable.startsWith("{") && variable.endsWith("}"))
variable = variable.substring(1, variable.length() - 1);
String value = variables.get(variable);
if (value == null) value = "";
message =
message.replaceFirst(Pattern.quote(matcher.group()), Matcher.quoteReplacement(value));
}
matcher = colorPattern.matcher(message);
while (matcher.find())
message =
message.substring(0, matcher.start()) + "\247" + message.substring(matcher.end() - 1);
return message;
}
}
/**
* Get the noun and verb <roots> (i.e. 'the highest' synsets in WordNet) from the particular
* 'icfile' (Information Content) that you are applying. Store a noun <root>: a synset offset
* number of type Integer in nounroots: an ArrayList<Integer> defined in the constructor of this
* class. Store a verb <root>: a synset offset number of type Integer in verbroots: an
* ArrayList<Integer> defined in the constructor of this class.
*
* <p>An example line in an 'icfile', showing a noun <root>: 1740n 128767 ROOT
*/
private void getRoots() {
Pattern pn = Pattern.compile("[0-9]+n [0-9]+ ROOT"); // find noun <root>
Pattern pv = Pattern.compile("[0-9]+v [0-9]+ ROOT"); // find verb <root>
Matcher m = null;
String root = "";
try {
BufferedReader in = new BufferedReader(new FileReader(icfile));
String line;
while ((line = in.readLine()) != null) {
// nouns
m = pn.matcher(line);
if (m.matches()) {
root = (line.split("\\s")[0]).split("n")[0]; // !!! double split !!!
nounroots.add(Integer.parseInt(root));
}
// verbs
m = pv.matcher(line);
if (m.matches()) {
root = (line.split("\\s")[0]).split("v")[0]; // !!! double split !!!
verbroots.add(Integer.parseInt(root));
}
}
in.close();
} catch (IOException e) {
e.printStackTrace();
}
}
// Chamando Robot
public static void robot() throws Exception {
Database db = new Database();
db.connect();
ResultSet rs = Page.findAll(db);
Page p = null;
while ((p = Page.next(rs)) != null) {
String body = Robot.get(p.getUrl());
// procurar por urls dentro do body
// buscar por essas paginas
// String expr = "href=\"([^\"]*)";
String ereg = "href=\"https{0,1}:\\/\\/([^\"]*)\"";
Pattern pt = Pattern.compile(ereg);
Matcher m = pt.matcher(body);
while (m.find()) {
System.out.println(m.group());
String[] _url = m.group().split("\"");
Page.newUrl(_url[1]);
}
p.setBody(body);
p.update();
}
db.close();
}
private static String expand(String str) {
if (str == null) {
return null;
}
StringBuilder result = new StringBuilder();
Pattern re = Pattern.compile("^(.*?)\\$\\{([^}]*)\\}(.*)");
while (true) {
Matcher matcher = re.matcher(str);
if (matcher.matches()) {
result.append(matcher.group(1));
String property = matcher.group(2);
if (property.equals("/")) {
property = "file.separator";
}
String value = System.getProperty(property);
if (value != null) {
result.append(value);
}
str = matcher.group(3);
} else {
result.append(str);
break;
}
}
return result.toString();
}
String filter(String[] args, boolean include) {
verifyCommand(args, String.format(_filterHelp, args[0]), null, 3, 3);
Collection<String> list = new ArrayList<String>(Processor.split(args[1]));
Pattern pattern = Pattern.compile(args[2]);
for (Iterator<String> i = list.iterator(); i.hasNext(); ) {
if (pattern.matcher(i.next()).matches() == include) i.remove();
}
return Processor.join(list);
}
public static void loadPermissions(URL url) throws IOException, PermissionParseException {
BufferedReader in = new BufferedReader(new InputStreamReader(url.openStream()));
String line;
Pattern ignore = Pattern.compile("^\\s*(//.*)?$");
Pattern valid =
Pattern.compile("^\\s*permission\\s+(\\S+)" + "(\\s+\"([^\"]*)\"(,\\s+\"([^\"]*)\")?)?;$");
Set<Permission> perms = new HashSet<Permission>();
while ((line = in.readLine()) != null) {
if (ignore.matcher(line).matches()) {
continue;
}
Matcher matcher = valid.matcher(line);
if (!matcher.matches()) {
throw new PermissionParseException("invalid syntax: " + line);
}
int nGroups = matcher.groupCount();
String type = matcher.group(1);
String name = expand(nGroups >= 3 ? matcher.group(3) : null);
String actions = expand(nGroups >= 5 ? matcher.group(5) : null);
try {
Permission perm = getPermission(type, name, actions);
perms.add(perm);
} catch (Throwable e) {
String message =
String.format(
"could not instantiate permission: " + "type=%s name=%s actions=",
type, name, actions);
throw new PermissionParseException(message, e);
}
}
in.close();
permSet.addAll(perms);
}
private void processResponse(InputStream response) throws IOException {
Pattern p = Pattern.compile("<cdbId>\\s*(\\d+)");
BufferedReader in = new BufferedReader(new InputStreamReader(response));
String line = in.readLine();
while (line != null) {
System.out.println(line);
Matcher m = p.matcher(line);
if (m.find()) {
String idText = m.group(1);
if (idText != null) {
cdbId = Integer.parseInt(idText);
}
}
line = in.readLine();
}
}
@Override
@EventHandler
public void onChatReceived(ChatReceivedEvent event) {
super.onChatReceived(event);
String message = Util.stripColors(event.getMessage());
if (message.startsWith("Please register with \"/register")) {
String password = Util.generateRandomString(10 + random.nextInt(6));
bot.say("/register " + password + " " + password);
} else if (message.contains("You are not member of any faction.")
&& spamMessage != null
&& createFaction) {
String msg = "/f create " + Util.generateRandomString(7 + random.nextInt(4));
bot.say(msg);
}
for (String s : captchaList) {
Matcher captchaMatcher = Pattern.compile(s).matcher(message);
if (captchaMatcher.matches()) bot.say(captchaMatcher.group(1));
}
}
private static List<String> loadAccounts(String fileName) {
List<String> accounts = new ArrayList<String>();
try {
Pattern pattern = Pattern.compile("[\\w]{1,16}");
BufferedReader reader = new BufferedReader(new FileReader(new File(fileName)));
String line;
while ((line = reader.readLine()) != null) {
Matcher matcher = pattern.matcher(line);
if (!matcher.find()) continue;
String username = matcher.group();
if (!matcher.find()) continue;
String password = matcher.group();
accounts.add(username + ":" + password);
}
reader.close();
} catch (Exception exception) {
throw new RuntimeException(exception);
}
System.out.println("Loaded " + accounts.size() + " accounts.");
return accounts;
}
/**
* Subscribe to all channels whose name matches the regular expression. Note that to subscribe to
* all channels, you must specify ".*", not "*".
*/
public void subscribe(String regex, ZCMSubscriber sub) {
if (this.closed) throw new IllegalStateException();
SubscriptionRecord srec = new SubscriptionRecord();
srec.regex = regex;
srec.pat = Pattern.compile(regex);
srec.lcsub = sub;
synchronized (this) {
zcmjni.subscribe(regex, this);
}
synchronized (subscriptions) {
subscriptions.add(srec);
for (String channel : subscriptionsMap.keySet()) {
if (srec.pat.matcher(channel).matches()) {
ArrayList<SubscriptionRecord> subs = subscriptionsMap.get(channel);
subs.add(srec);
}
}
}
}
/**
* Provide a macro processor. This processor can replace variables in strings based on a properties
* and a domain. The domain can implement functions that start with a "_" and take args[], the names
* of these functions are available as functions in the macro processor (without the _). Macros can
* nest to any depth but may not contain loops. Add POSIX macros: ${#parameter} String length.
* ${parameter%word} Remove smallest suffix pattern. ${parameter%%word} Remove largest suffix
* pattern. ${parameter#word} Remove smallest prefix pattern. ${parameter##word} Remove largest
* prefix pattern.
*/
public class Macro {
Processor domain;
Object targets[];
boolean flattening;
public Macro(Processor domain, Object... targets) {
this.domain = domain;
this.targets = targets;
if (targets != null) {
for (Object o : targets) {
assert o != null;
}
}
}
public String process(String line, Processor source) {
return process(line, new Link(source, null, line));
}
String process(String line, Link link) {
StringBuilder sb = new StringBuilder();
process(line, 0, '\u0000', '\u0000', sb, link);
return sb.toString();
}
int process(CharSequence org, int index, char begin, char end, StringBuilder result, Link link) {
StringBuilder line = new StringBuilder(org);
int nesting = 1;
StringBuilder variable = new StringBuilder();
outer:
while (index < line.length()) {
char c1 = line.charAt(index++);
if (c1 == end) {
if (--nesting == 0) {
result.append(replace(variable.toString(), link));
return index;
}
} else if (c1 == begin) nesting++;
else if (c1 == '\\' && index < line.length() - 1 && line.charAt(index) == '$') {
// remove the escape backslash and interpret the dollar
// as a
// literal
index++;
variable.append('$');
continue outer;
} else if (c1 == '$' && index < line.length() - 2) {
char c2 = line.charAt(index);
char terminator = getTerminator(c2);
if (terminator != 0) {
index = process(line, index + 1, c2, terminator, variable, link);
continue outer;
}
} else if (c1 == '.' && index < line.length() && line.charAt(index) == '/') {
// Found the sequence ./
if (index == 1 || Character.isWhitespace(line.charAt(index - 2))) {
// make sure it is preceded by whitespace or starts at begin
index++;
variable.append(domain.getBase().getAbsolutePath());
variable.append('/');
continue outer;
}
}
variable.append(c1);
}
result.append(variable);
return index;
}
public static char getTerminator(char c) {
switch (c) {
case '(':
return ')';
case '[':
return ']';
case '{':
return '}';
case '<':
return '>';
case '\u00ab': // Guillemet double << >>
return '\u00bb';
case '\u2039': // Guillemet single
return '\u203a';
}
return 0;
}
protected String replace(String key, Link link) {
if (link != null && link.contains(key)) return "${infinite:" + link.toString() + "}";
if (key != null) {
key = key.trim();
if (key.length() > 0) {
Processor source = domain;
String value = null;
if (key.indexOf(';') < 0) {
Instruction ins = new Instruction(key);
if (!ins.isLiteral()) {
SortedList<String> sortedList = SortedList.fromIterator(domain.iterator());
StringBuilder sb = new StringBuilder();
String del = "";
for (String k : sortedList) {
if (ins.matches(k)) {
String v = replace(k, new Link(source, link, key));
if (v != null) {
sb.append(del);
del = ",";
sb.append(v);
}
}
}
return sb.toString();
}
}
while (value == null && source != null) {
value = source.getProperties().getProperty(key);
source = source.getParent();
}
if (value != null) return process(value, new Link(source, link, key));
value = doCommands(key, link);
if (value != null) return process(value, new Link(source, link, key));
if (key != null && key.trim().length() > 0) {
value = System.getProperty(key);
if (value != null) return value;
}
if (!flattening && !key.equals("@"))
domain.warning("No translation found for macro: " + key);
} else {
domain.warning("Found empty macro key");
}
} else {
domain.warning("Found null macro key");
}
return "${" + key + "}";
}
/**
* Parse the key as a command. A command consist of parameters separated by ':'.
*
* @param key
* @return
*/
static Pattern commands = Pattern.compile("(?<!\\\\);");
private String doCommands(String key, Link source) {
String[] args = commands.split(key);
if (args == null || args.length == 0) return null;
for (int i = 0; i < args.length; i++)
if (args[i].indexOf('\\') >= 0) args[i] = args[i].replaceAll("\\\\;", ";");
if (args[0].startsWith("^")) {
String varname = args[0].substring(1).trim();
Processor parent = source.start.getParent();
if (parent != null) return parent.getProperty(varname);
return null;
}
Processor rover = domain;
while (rover != null) {
String result = doCommand(rover, args[0], args);
if (result != null) return result;
rover = rover.getParent();
}
for (int i = 0; targets != null && i < targets.length; i++) {
String result = doCommand(targets[i], args[0], args);
if (result != null) return result;
}
return doCommand(this, args[0], args);
}
private String doCommand(Object target, String method, String[] args) {
if (target == null) ; // System.err.println("Huh? Target should never be null " +
// domain);
else {
String cname = "_" + method.replaceAll("-", "_");
try {
Method m = target.getClass().getMethod(cname, new Class[] {String[].class});
return (String) m.invoke(target, new Object[] {args});
} catch (NoSuchMethodException e) {
// Ignore
} catch (InvocationTargetException e) {
if (e.getCause() instanceof IllegalArgumentException) {
domain.error(
"%s, for cmd: %s, arguments; %s",
e.getCause().getMessage(), method, Arrays.toString(args));
} else {
domain.warning("Exception in replace: %s", e.getCause());
e.getCause().printStackTrace();
}
} catch (Exception e) {
domain.warning("Exception in replace: " + e + " method=" + method);
e.printStackTrace();
}
}
return null;
}
/**
* Return a unique list where the duplicates are removed.
*
* @param args
* @return
*/
static String _uniqHelp = "${uniq;<list> ...}";
public String _uniq(String args[]) {
verifyCommand(args, _uniqHelp, null, 1, Integer.MAX_VALUE);
Set<String> set = new LinkedHashSet<String>();
for (int i = 1; i < args.length; i++) {
Processor.split(args[i], set);
}
return Processor.join(set, ",");
}
public String _pathseparator(@SuppressWarnings("unused") String args[]) {
return File.pathSeparator;
}
public String _separator(@SuppressWarnings("unused") String args[]) {
return File.separator;
}
public String _filter(String args[]) {
return filter(args, false);
}
public String _filterout(String args[]) {
return filter(args, true);
}
static String _filterHelp = "${%s;<list>;<regex>}";
String filter(String[] args, boolean include) {
verifyCommand(args, String.format(_filterHelp, args[0]), null, 3, 3);
Collection<String> list = new ArrayList<String>(Processor.split(args[1]));
Pattern pattern = Pattern.compile(args[2]);
for (Iterator<String> i = list.iterator(); i.hasNext(); ) {
if (pattern.matcher(i.next()).matches() == include) i.remove();
}
return Processor.join(list);
}
static String _sortHelp = "${sort;<list>...}";
public String _sort(String args[]) {
verifyCommand(args, _sortHelp, null, 2, Integer.MAX_VALUE);
List<String> result = new ArrayList<String>();
for (int i = 1; i < args.length; i++) {
Processor.split(args[i], result);
}
Collections.sort(result);
return Processor.join(result);
}
static String _joinHelp = "${join;<list>...}";
public String _join(String args[]) {
verifyCommand(args, _joinHelp, null, 1, Integer.MAX_VALUE);
List<String> result = new ArrayList<String>();
for (int i = 1; i < args.length; i++) {
Processor.split(args[i], result);
}
return Processor.join(result);
}
static String _ifHelp = "${if;<condition>;<iftrue> [;<iffalse>] }";
public String _if(String args[]) {
verifyCommand(args, _ifHelp, null, 3, 4);
String condition = args[1].trim();
if (!condition.equalsIgnoreCase("false")) if (condition.length() != 0) return args[2];
if (args.length > 3) return args[3];
return "";
}
public String _now(@SuppressWarnings("unused") String args[]) {
return new Date().toString();
}
public static final String _fmodifiedHelp =
"${fmodified;<list of filenames>...}, return latest modification date";
public String _fmodified(String args[]) throws Exception {
verifyCommand(args, _fmodifiedHelp, null, 2, Integer.MAX_VALUE);
long time = 0;
Collection<String> names = new ArrayList<String>();
for (int i = 1; i < args.length; i++) {
Processor.split(args[i], names);
}
for (String name : names) {
File f = new File(name);
if (f.exists() && f.lastModified() > time) time = f.lastModified();
}
return "" + time;
}
public String _long2date(String args[]) {
try {
return new Date(Long.parseLong(args[1])).toString();
} catch (Exception e) {
e.printStackTrace();
}
return "not a valid long";
}
public String _literal(String args[]) {
if (args.length != 2)
throw new RuntimeException("Need a value for the ${literal;<value>} macro");
return "${" + args[1] + "}";
}
public String _def(String args[]) {
if (args.length != 2) throw new RuntimeException("Need a value for the ${def;<value>} macro");
return domain.getProperty(args[1], "");
}
/**
* replace ; <list> ; regex ; replace
*
* @param args
* @return
*/
public String _replace(String args[]) {
if (args.length != 4) {
domain.warning("Invalid nr of arguments to replace " + Arrays.asList(args));
return null;
}
String list[] = args[1].split("\\s*,\\s*");
StringBuilder sb = new StringBuilder();
String del = "";
for (int i = 0; i < list.length; i++) {
String element = list[i].trim();
if (!element.equals("")) {
sb.append(del);
sb.append(element.replaceAll(args[2], args[3]));
del = ", ";
}
}
return sb.toString();
}
public String _warning(String args[]) {
for (int i = 1; i < args.length; i++) {
domain.warning(process(args[i]));
}
return "";
}
public String _error(String args[]) {
for (int i = 1; i < args.length; i++) {
domain.error(process(args[i]));
}
return "";
}
/**
* toclassname ; <path>.class ( , <path>.class ) *
*
* @param args
* @return
*/
static String _toclassnameHelp =
"${classname;<list of class names>}, convert class paths to FQN class names ";
public String _toclassname(String args[]) {
verifyCommand(args, _toclassnameHelp, null, 2, 2);
Collection<String> paths = Processor.split(args[1]);
List<String> names = new ArrayList<String>(paths.size());
for (String path : paths) {
if (path.endsWith(".class")) {
String name = path.substring(0, path.length() - 6).replace('/', '.');
names.add(name);
} else if (path.endsWith(".java")) {
String name = path.substring(0, path.length() - 5).replace('/', '.');
names.add(name);
} else {
domain.warning(
"in toclassname, "
+ args[1]
+ " is not a class path because it does not end in .class");
}
}
return Processor.join(names, ",");
}
/**
* toclassname ; <path>.class ( , <path>.class ) *
*
* @param args
* @return
*/
static String _toclasspathHelp =
"${toclasspath;<list>[;boolean]}, convert a list of class names to paths";
public String _toclasspath(String args[]) {
verifyCommand(args, _toclasspathHelp, null, 2, 3);
boolean cl = true;
if (args.length > 2) cl = Boolean.valueOf(args[2]);
Collection<String> names = Processor.split(args[1]);
Collection<String> paths = new ArrayList<String>(names.size());
for (String name : names) {
String path = name.replace('.', '/') + (cl ? ".class" : "");
paths.add(path);
}
return Processor.join(paths, ",");
}
public String _dir(String args[]) {
if (args.length < 2) {
domain.warning("Need at least one file name for ${dir;...}");
return null;
}
String del = "";
StringBuilder sb = new StringBuilder();
for (int i = 1; i < args.length; i++) {
File f = domain.getFile(args[i]);
if (f.exists() && f.getParentFile().exists()) {
sb.append(del);
sb.append(f.getParentFile().getAbsolutePath());
del = ",";
}
}
return sb.toString();
}
public String _basename(String args[]) {
if (args.length < 2) {
domain.warning("Need at least one file name for ${basename;...}");
return null;
}
String del = "";
StringBuilder sb = new StringBuilder();
for (int i = 1; i < args.length; i++) {
File f = domain.getFile(args[i]);
if (f.exists() && f.getParentFile().exists()) {
sb.append(del);
sb.append(f.getName());
del = ",";
}
}
return sb.toString();
}
public String _isfile(String args[]) {
if (args.length < 2) {
domain.warning("Need at least one file name for ${isfile;...}");
return null;
}
boolean isfile = true;
for (int i = 1; i < args.length; i++) {
File f = new File(args[i]).getAbsoluteFile();
isfile &= f.isFile();
}
return isfile ? "true" : "false";
}
public String _isdir(String args[]) {
if (args.length < 2) {
domain.warning("Need at least one file name for ${isdir;...}");
return null;
}
boolean isdir = true;
for (int i = 1; i < args.length; i++) {
File f = new File(args[i]).getAbsoluteFile();
isdir &= f.isDirectory();
}
return isdir ? "true" : "false";
}
public String _tstamp(String args[]) {
String format = "yyyyMMddHHmm";
long now = System.currentTimeMillis();
TimeZone tz = TimeZone.getTimeZone("UTC");
if (args.length > 1) {
format = args[1];
}
if (args.length > 2) {
tz = TimeZone.getTimeZone(args[2]);
}
if (args.length > 3) {
now = Long.parseLong(args[3]);
}
if (args.length > 4) {
domain.warning("Too many arguments for tstamp: " + Arrays.toString(args));
}
SimpleDateFormat sdf = new SimpleDateFormat(format);
sdf.setTimeZone(tz);
return sdf.format(new Date(now));
}
/**
* Wildcard a directory. The lists can contain Instruction that are matched against the given
* directory ${lsr;<dir>;<list>(;<list>)*} ${lsa;<dir>;<list>(;<list>)*}
*
* @author aqute
*/
public String _lsr(String args[]) {
return ls(args, true);
}
public String _lsa(String args[]) {
return ls(args, false);
}
String ls(String args[], boolean relative) {
if (args.length < 2)
throw new IllegalArgumentException(
"the ${ls} macro must at least have a directory as parameter");
File dir = domain.getFile(args[1]);
if (!dir.isAbsolute())
throw new IllegalArgumentException(
"the ${ls} macro directory parameter is not absolute: " + dir);
if (!dir.exists())
throw new IllegalArgumentException(
"the ${ls} macro directory parameter does not exist: " + dir);
if (!dir.isDirectory())
throw new IllegalArgumentException(
"the ${ls} macro directory parameter points to a file instead of a directory: " + dir);
Collection<File> files = new ArrayList<File>(new SortedList<File>(dir.listFiles()));
for (int i = 2; i < args.length; i++) {
Instructions filters = new Instructions(args[i]);
files = filters.select(files, true);
}
List<String> result = new ArrayList<String>();
for (File file : files) result.add(relative ? file.getName() : file.getAbsolutePath());
return Processor.join(result, ",");
}
public String _currenttime(@SuppressWarnings("unused") String args[]) {
return Long.toString(System.currentTimeMillis());
}
/**
* Modify a version to set a version policy. Thed policy is a mask that is mapped to a version.
*
* <pre>
* + increment
* - decrement
* = maintain
* ˜ discard
*
* ==+ = maintain major, minor, increment micro, discard qualifier
* ˜˜˜= = just get the qualifier
* version="[${version;==;${@}},${version;=+;${@}})"
* </pre>
*
* @param args
* @return
*/
static final String MASK_STRING = "[\\-+=~0123456789]{0,3}[=~]?";
static final Pattern MASK = Pattern.compile(MASK_STRING);
static final String _versionHelp =
"${version;<mask>;<version>}, modify a version\n"
+ "<mask> ::= [ M [ M [ M [ MQ ]]]\n"
+ "M ::= '+' | '-' | MQ\n"
+ "MQ ::= '~' | '='";
static final Pattern _versionPattern[] = new Pattern[] {null, null, MASK, Verifier.VERSION};
public String _version(String args[]) {
verifyCommand(args, _versionHelp, null, 2, 3);
String mask = args[1];
Version version = null;
if (args.length >= 3) version = new Version(args[2]);
return version(version, mask);
}
String version(Version version, String mask) {
if (version == null) {
String v = domain.getProperty("@");
if (v == null) {
domain.error(
"No version specified for ${version} or ${range} and no implicit version ${@} either, mask=%s",
mask);
v = "0";
}
version = new Version(v);
}
StringBuilder sb = new StringBuilder();
String del = "";
for (int i = 0; i < mask.length(); i++) {
char c = mask.charAt(i);
String result = null;
if (c != '~') {
if (i == 3) {
result = version.getQualifier();
} else if (Character.isDigit(c)) {
// Handle masks like +00, =+0
result = String.valueOf(c);
} else {
int x = version.get(i);
switch (c) {
case '+':
x++;
break;
case '-':
x--;
break;
case '=':
break;
}
result = Integer.toString(x);
}
if (result != null) {
sb.append(del);
del = ".";
sb.append(result);
}
}
}
return sb.toString();
}
/**
* Schortcut for version policy
*
* <pre>
* -provide-policy : ${policy;[==,=+)}
* -consume-policy : ${policy;[==,+)}
* </pre>
*
* @param args
* @return
*/
static Pattern RANGE_MASK =
Pattern.compile("(\\[|\\()(" + MASK_STRING + "),(" + MASK_STRING + ")(\\]|\\))");
static String _rangeHelp =
"${range;<mask>[;<version>]}, range for version, if version not specified lookyp ${@}\n"
+ "<mask> ::= [ M [ M [ M [ MQ ]]]\n"
+ "M ::= '+' | '-' | MQ\n"
+ "MQ ::= '~' | '='";
static Pattern _rangePattern[] = new Pattern[] {null, RANGE_MASK};
public String _range(String args[]) {
verifyCommand(args, _rangeHelp, _rangePattern, 2, 3);
Version version = null;
if (args.length >= 3) version = new Version(args[2]);
else {
String v = domain.getProperty("@");
if (v == null) return null;
version = new Version(v);
}
String spec = args[1];
Matcher m = RANGE_MASK.matcher(spec);
m.matches();
String floor = m.group(1);
String floorMask = m.group(2);
String ceilingMask = m.group(3);
String ceiling = m.group(4);
String left = version(version, floorMask);
String right = version(version, ceilingMask);
StringBuilder sb = new StringBuilder();
sb.append(floor);
sb.append(left);
sb.append(",");
sb.append(right);
sb.append(ceiling);
String s = sb.toString();
VersionRange vr = new VersionRange(s);
if (!(vr.includes(vr.getHigh()) || vr.includes(vr.getLow()))) {
domain.error(
"${range} macro created an invalid range %s from %s and mask %s", s, version, spec);
}
return sb.toString();
}
/**
* System command. Execute a command and insert the result.
*
* @param args
* @param help
* @param patterns
* @param low
* @param high
*/
public String system_internal(boolean allowFail, String args[]) throws Exception {
verifyCommand(
args,
"${"
+ (allowFail ? "system-allow-fail" : "system")
+ ";<command>[;<in>]}, execute a system command",
null,
2,
3);
String command = args[1];
String input = null;
if (args.length > 2) {
input = args[2];
}
Process process = Runtime.getRuntime().exec(command, null, domain.getBase());
if (input != null) {
process.getOutputStream().write(input.getBytes("UTF-8"));
}
process.getOutputStream().close();
String s = IO.collect(process.getInputStream(), "UTF-8");
int exitValue = process.waitFor();
if (exitValue != 0) return exitValue + "";
if (!allowFail && (exitValue != 0)) {
domain.error("System command " + command + " failed with " + exitValue);
}
return s.trim();
}
public String _system(String args[]) throws Exception {
return system_internal(false, args);
}
public String _system_allow_fail(String args[]) throws Exception {
String result = "";
try {
result = system_internal(true, args);
} catch (Throwable t) {
/* ignore */
}
return result;
}
public String _env(String args[]) {
verifyCommand(args, "${env;<name>}, get the environmet variable", null, 2, 2);
try {
return System.getenv(args[1]);
} catch (Throwable t) {
return null;
}
}
/**
* Get the contents of a file.
*
* @param in
* @return
* @throws IOException
*/
public String _cat(String args[]) throws IOException {
verifyCommand(args, "${cat;<in>}, get the content of a file", null, 2, 2);
File f = domain.getFile(args[1]);
if (f.isFile()) {
return IO.collect(f);
} else if (f.isDirectory()) {
return Arrays.toString(f.list());
} else {
try {
URL url = new URL(args[1]);
return IO.collect(url, "UTF-8");
} catch (MalformedURLException mfue) {
// Ignore here
}
return null;
}
}
public static void verifyCommand(
String args[],
@SuppressWarnings("unused") String help,
Pattern[] patterns,
int low,
int high) {
String message = "";
if (args.length > high) {
message = "too many arguments";
} else if (args.length < low) {
message = "too few arguments";
} else {
for (int i = 0; patterns != null && i < patterns.length && i < args.length; i++) {
if (patterns[i] != null) {
Matcher m = patterns[i].matcher(args[i]);
if (!m.matches())
message +=
String.format(
"Argument %s (%s) does not match %s%n", i, args[i], patterns[i].pattern());
}
}
}
if (message.length() != 0) {
StringBuilder sb = new StringBuilder();
String del = "${";
for (String arg : args) {
sb.append(del);
sb.append(arg);
del = ";";
}
sb.append("}, is not understood. ");
sb.append(message);
throw new IllegalArgumentException(sb.toString());
}
}
// Helper class to track expansion of variables
// on the stack.
static class Link {
Link previous;
String key;
Processor start;
public Link(Processor start, Link previous, String key) {
this.previous = previous;
this.key = key;
this.start = start;
}
public boolean contains(String key) {
if (this.key.equals(key)) return true;
if (previous == null) return false;
return previous.contains(key);
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
String del = "[";
for (Link r = this; r != null; r = r.previous) {
sb.append(del);
sb.append(r.key);
del = ",";
}
sb.append("]");
return sb.toString();
}
}
/**
* Take all the properties and translate them to actual values. This method takes the set
* properties and traverse them over all entries, including the default properties for that
* properties. The values no longer contain macros.
*
* @return A new Properties with the flattened values
*/
public Properties getFlattenedProperties() {
// Some macros only work in a lower processor, so we
// do not report unknown macros while flattening
flattening = true;
try {
Properties flattened = new Properties();
Properties source = domain.getProperties();
for (Enumeration<?> e = source.propertyNames(); e.hasMoreElements(); ) {
String key = (String) e.nextElement();
if (!key.startsWith("_"))
if (key.startsWith("-")) flattened.put(key, source.getProperty(key));
else flattened.put(key, process(source.getProperty(key)));
}
return flattened;
} finally {
flattening = false;
}
}
public static final String _fileHelp =
"${file;<base>;<paths>...}, create correct OS dependent path";
public String _osfile(String args[]) {
verifyCommand(args, _fileHelp, null, 3, 3);
File base = new File(args[1]);
File f = Processor.getFile(base, args[2]);
return f.getAbsolutePath();
}
public String _path(String args[]) {
List<String> list = new ArrayList<String>();
for (int i = 1; i < args.length; i++) {
list.addAll(Processor.split(args[i]));
}
return Processor.join(list, File.pathSeparator);
}
public static Properties getParent(Properties p) {
try {
Field f = Properties.class.getDeclaredField("defaults");
f.setAccessible(true);
return (Properties) f.get(p);
} catch (Exception e) {
Field[] fields = Properties.class.getFields();
System.err.println(Arrays.toString(fields));
return null;
}
}
public String process(String line) {
return process(line, domain);
}
}
/**
* JPM is the Java package manager. It manages a local repository in the user global directory
* and/or a global directory. This class is the main entry point for the command line. This program
* maintains a repository, a list of installed commands, and a list of installed service. It
* provides the commands to changes these resources. All information is kept in a platform specific
* area. However, the layout of this area is standardized.
*
* <pre>
* platform/
* check check for write access
* repo/ repository
* <bsn>/ bsn directory
* <bsn>-<version>.jar jar file
* service/ All service
* <service>/ A service
* data Service data (JSON)
* wdir/ Working dir
* lock Lock file (if running, contains port)
* commands/
* <command> Command data (JSON)
* </pre>
*
* For each service, the platform must also have a user writable directory used for working dir,
* lock, and logging.
*
* <pre>
* platform var/
* wdir/ Working dir
* lock Lock file (exists only when running, contains UDP port)
* </pre>
*/
public class JustAnotherPackageManager {
private static final String SERVICE_JAR_FILE = "service.jar";
public static final String SERVICE = "service";
public static final String COMMANDS = "commands";
public static final String LOCK = "lock";
private static final String JPM_CACHE_LOCAL = "jpm.cache.local";
private static final String JPM_CACHE_GLOBAL = "jpm.cache.global";
static final String PERMISSION_ERROR =
"No write acces, might require administrator or root privileges (sudo in *nix)";
static JSONCodec codec = new JSONCodec();
static Pattern BSN_P =
Pattern.compile(
"([-a-z0-9_]+(?:\\.[-a-z0-9_]+)+)(?:@([0-9]+(?:\\.[0-9]+(?:\\.[0-9]+(?:\\.[-_a-z0-9]+)?)?)?))?",
Pattern.CASE_INSENSITIVE);
static Pattern COORD_P =
Pattern.compile(
"([-a-z0-9_.]+):([-a-z0-9_.]+)(?::([-a-z0-9_.]+))?(?:@([-a-z0-9._]+))?",
Pattern.CASE_INSENSITIVE);
static Pattern URL_P = Pattern.compile("([a-z]{3,6}:/.*)", Pattern.CASE_INSENSITIVE);
static Pattern CMD_P = Pattern.compile("([a-z_][a-z\\d_]*)", Pattern.CASE_INSENSITIVE);
static Pattern SHA_P = Pattern.compile("(?:sha:)?([a-fA-F0-9]{40,40})", Pattern.CASE_INSENSITIVE);
static Executor executor;
final File homeDir;
final File binDir;
final File repoDir;
final File commandDir;
final File serviceDir;
final File service;
final Platform platform;
final Reporter reporter;
JpmRepo library;
final List<Service> startedByDaemon = new ArrayList<Service>();
boolean localInstall = false;
private URLClient host;
private boolean underTest;
/**
* Constructor
*
* @throws IOException
*/
public JustAnotherPackageManager(Reporter reporter, Platform platform, File homeDir, File binDir)
throws IOException {
this.platform = platform;
this.reporter = reporter;
this.homeDir = homeDir;
if (!homeDir.exists() && !homeDir.mkdirs())
throw new IllegalArgumentException("Could not create directory " + homeDir);
repoDir = IO.getFile(homeDir, "repo");
if (!repoDir.exists() && !repoDir.mkdirs())
throw new IllegalArgumentException("Could not create directory " + repoDir);
commandDir = new File(homeDir, COMMANDS);
serviceDir = new File(homeDir, SERVICE);
commandDir.mkdir();
serviceDir.mkdir();
service = new File(repoDir, SERVICE_JAR_FILE);
if (!service.isFile()) init();
this.binDir = binDir;
if (!binDir.exists() && !binDir.mkdirs())
throw new IllegalArgumentException("Could not create bin directory " + binDir);
}
public String getArtifactIdFromCoord(String coord) {
Matcher m = COORD_P.matcher(coord);
if (m.matches()) {
return m.group(2);
} else {
return null;
}
}
public boolean hasAccess() {
assert (binDir != null);
assert (homeDir != null);
return binDir.canWrite() && homeDir.canWrite();
}
public File getHomeDir() {
return homeDir;
}
public File getRepoDir() {
return repoDir;
}
public File getBinDir() {
return binDir;
}
public List<ServiceData> getServices() throws Exception {
return getServices(serviceDir);
}
public List<ServiceData> getServices(File serviceDir) throws Exception {
List<ServiceData> result = new ArrayList<ServiceData>();
if (!serviceDir.exists()) {
return result;
}
for (File sdir : serviceDir.listFiles()) {
File dataFile = new File(sdir, "data");
ServiceData data = getData(ServiceData.class, dataFile);
result.add(data);
}
return result;
}
public List<CommandData> getCommands() throws Exception {
return getCommands(commandDir);
}
public List<CommandData> getCommands(File commandDir) throws Exception {
List<CommandData> result = new ArrayList<CommandData>();
if (!commandDir.exists()) {
return result;
}
for (File f : commandDir.listFiles()) {
CommandData data = getData(CommandData.class, f);
if (data != null) result.add(data);
}
return result;
}
public CommandData getCommand(String name) throws Exception {
File f = new File(commandDir, name);
if (!f.isFile()) return null;
return getData(CommandData.class, f);
}
/**
* Garbage collect repository
*
* @throws Exception
*/
public void gc() throws Exception {
HashSet<byte[]> deps = new HashSet<byte[]>();
// deps.add(SERVICE_JAR_FILE);
for (File cmd : commandDir.listFiles()) {
CommandData data = getData(CommandData.class, cmd);
addDependencies(deps, data);
}
for (File service : serviceDir.listFiles()) {
File dataFile = new File(service, "data");
ServiceData data = getData(ServiceData.class, dataFile);
addDependencies(deps, data);
}
int count = 0;
for (File f : repoDir.listFiles()) {
String name = f.getName();
if (!deps.contains(name)) {
if (!name.endsWith(".json")
|| !deps.contains(name.substring(0, name.length() - ".json".length()))) { // Remove
// json
// files
// only
// if
// the
// bin
// is
// going
// as
// well
f.delete();
count++;
} else {
}
}
}
System.out.format("Garbage collection done (%d file(s) removed)%n", count);
}
private void addDependencies(HashSet<byte[]> deps, CommandData data) {
for (byte[] dep : data.dependencies) {
deps.add(dep);
}
for (byte[] dep : data.runbundles) {
deps.add(dep);
}
}
public void deinit(Appendable out, boolean force) throws Exception {
Settings settings = new Settings(platform.getConfigFile());
if (!force) {
Justif justify = new Justif(80, 40);
StringBuilder sb = new StringBuilder();
Formatter f = new Formatter(sb);
try {
String list = listFiles(platform.getGlobal());
if (list != null) {
f.format("In global default environment:%n");
f.format(list);
}
list = listFiles(platform.getLocal());
if (list != null) {
f.format("In local default environment:%n");
f.format(list);
}
if (settings.containsKey(JPM_CACHE_GLOBAL)) {
list = listFiles(IO.getFile(settings.get(JPM_CACHE_GLOBAL)));
if (list != null) {
f.format("In global configured environment:%n");
f.format(list);
}
}
if (settings.containsKey(JPM_CACHE_LOCAL)) {
list = listFiles(IO.getFile(settings.get(JPM_CACHE_LOCAL)));
if (list != null) {
f.format("In local configured environment:%n");
f.format(list);
}
}
list = listSupportFiles();
if (list != null) {
f.format("jpm support files:%n");
f.format(list);
}
f.format("%n%n");
f.format(
"All files listed above will be deleted if deinit is run with the force flag set"
+ " (\"jpm deinit -f\" or \"jpm deinit --force\"%n%n");
f.flush();
justify.wrap(sb);
out.append(sb.toString());
} finally {
f.close();
}
} else { // i.e. if(force)
int count = 0;
File[] caches = {platform.getGlobal(), platform.getLocal(), null, null};
if (settings.containsKey(JPM_CACHE_LOCAL)) {
caches[2] = IO.getFile(settings.get(JPM_CACHE_LOCAL));
}
if (settings.containsKey(JPM_CACHE_GLOBAL)) {
caches[3] = IO.getFile(settings.get(JPM_CACHE_GLOBAL));
}
ArrayList<File> toDelete = new ArrayList<File>();
for (File cache : caches) {
if (cache == null || !cache.exists()) {
continue;
}
listFiles(cache, toDelete);
if (toDelete.size() > count) {
count = toDelete.size();
if (!cache.canWrite()) {
reporter.error(PERMISSION_ERROR + " (" + cache + ")");
return;
}
toDelete.add(cache);
}
}
listSupportFiles(toDelete);
for (File f : toDelete) {
if (f.exists() && !f.canWrite()) {
reporter.error(PERMISSION_ERROR + " (" + f + ")");
}
}
if (reporter.getErrors().size() > 0) {
return;
}
for (File f : toDelete) {
if (f.exists()) {
IO.deleteWithException(f);
}
}
}
}
// Adapter to list without planning to delete
private String listFiles(final File cache) throws Exception {
return listFiles(cache, null);
}
private String listFiles(final File cache, List<File> toDelete) throws Exception {
boolean stopServices = false;
if (toDelete == null) {
toDelete = new ArrayList<File>();
} else {
stopServices = true;
}
int count = 0;
Formatter f = new Formatter();
f.format(" - Cache:%n * %s%n", cache.getCanonicalPath());
f.format(" - Commands:%n");
for (CommandData cdata : getCommands(new File(cache, COMMANDS))) {
f.format(" * %s \t0 handle for \"%s\"%n", cdata.bin, cdata.name);
toDelete.add(new File(cdata.bin));
count++;
}
f.format(" - Services:%n");
for (ServiceData sdata : getServices(new File(cache, SERVICE))) {
if (sdata != null) {
f.format(" * %s \t0 service directory for \"%s\"%n", sdata.sdir, sdata.name);
toDelete.add(new File(sdata.sdir));
File initd = platform.getInitd(sdata);
if (initd != null && initd.exists()) {
f.format(" * %s \t0 init.d file for \"%s\"%n", initd.getCanonicalPath(), sdata.name);
toDelete.add(initd);
}
if (stopServices) {
Service s = getService(sdata);
try {
s.stop();
} catch (Exception e) {
}
}
count++;
}
}
f.format("%n");
String result = (count > 0) ? f.toString() : null;
f.close();
return result;
}
private String listSupportFiles() throws Exception { // Adapter to list
// without planning
// to delete
return listSupportFiles(null);
}
private String listSupportFiles(List<File> toDelete) throws Exception {
Formatter f = new Formatter();
try {
if (toDelete == null) {
toDelete = new ArrayList<File>();
}
int precount = toDelete.size();
File confFile = IO.getFile(platform.getConfigFile()).getCanonicalFile();
if (confFile.exists()) {
f.format(" * %s \t0 Config file%n", confFile);
toDelete.add(confFile);
}
String result = (toDelete.size() > precount) ? f.toString() : null;
return result;
} finally {
f.close();
}
}
/**
* @param data
* @param target
* @throws Exception
* @throws IOException
*/
public String createService(ServiceData data) throws Exception, IOException {
File sdir = new File(serviceDir, data.name);
if (!sdir.exists() && !sdir.mkdirs()) {
throw new IOException("Could not create directory " + data.sdir);
}
data.sdir = sdir.getAbsolutePath();
File lock = new File(data.sdir, LOCK);
data.lock = lock.getAbsolutePath();
if (data.work == null) data.work = new File(data.sdir, "work").getAbsolutePath();
if (data.user == null) data.user = platform.user();
if (data.user == null) data.user = "******";
new File(data.work).mkdir();
if (data.log == null) data.log = new File(data.sdir, "log").getAbsolutePath();
// TODO
// if (Data.validate(data) != null)
// return "Invalid service data: " + Data.validate(data);
if (service == null)
throw new RuntimeException(
"Missing biz.aQute.jpm.service in repo, should have been installed by init, try reiniting");
data.serviceLib = service.getAbsolutePath();
platform.chown(data.user, true, new File(data.sdir));
String s = platform.createService(data, null, false);
if (s == null) storeData(new File(data.sdir, "data"), data);
return s;
}
/**
* @param data
* @param target
* @throws Exception
* @throws IOException
*/
public String createCommand(CommandData data, boolean force) throws Exception, IOException {
// TODO
// if (Data.validate(data) != null)
// return "Invalid command data: " + Data.validate(data);
Map<String, String> map = null;
if (data.trace) {
map = new HashMap<String, String>();
map.put("java.security.manager", "aQute.jpm.service.TraceSecurityManager");
reporter.trace("tracing");
}
String s = platform.createCommand(data, map, force, service.getAbsolutePath());
if (s == null) storeData(new File(commandDir, data.name), data);
return s;
}
public void deleteCommand(String name) throws Exception {
CommandData cmd = getCommand(name);
if (cmd == null) throw new IllegalArgumentException("No such command " + name);
platform.deleteCommand(cmd);
File tobedel = new File(commandDir, name);
IO.deleteWithException(tobedel);
}
public Service getService(String serviceName) throws Exception {
File base = new File(serviceDir, serviceName);
return getService(base);
}
public Service getService(ServiceData sdata) throws Exception {
return getService(new File(sdata.sdir));
}
private Service getService(File base) throws Exception {
File dataFile = new File(base, "data");
if (!dataFile.isFile()) return null;
ServiceData data = getData(ServiceData.class, dataFile);
return new Service(this, data);
}
/**
* Verify that the jar file is correct. This also verifies ok when there are no checksums or.
*
* @throws IOException
*/
static Pattern MANIFEST_ENTRY = Pattern.compile("(META-INF/[^/]+)|(.*/)");
public String verify(JarFile jar, String... algorithms) throws IOException {
if (algorithms == null || algorithms.length == 0) algorithms = new String[] {"MD5", "SHA"};
else if (algorithms.length == 1 && algorithms[0].equals("-")) return null;
try {
Manifest m = jar.getManifest();
if (m.getEntries().isEmpty()) return "No name sections";
for (Enumeration<JarEntry> e = jar.entries(); e.hasMoreElements(); ) {
JarEntry je = e.nextElement();
if (MANIFEST_ENTRY.matcher(je.getName()).matches()) continue;
Attributes nameSection = m.getAttributes(je.getName());
if (nameSection == null) return "No name section for " + je.getName();
for (String algorithm : algorithms) {
try {
MessageDigest md = MessageDigest.getInstance(algorithm);
String expected = nameSection.getValue(algorithm + "-Digest");
if (expected != null) {
byte digest[] = Base64.decodeBase64(expected);
copy(jar.getInputStream(je), md);
if (!Arrays.equals(digest, md.digest()))
return "Invalid digest for "
+ je.getName()
+ ", "
+ expected
+ " != "
+ Base64.encodeBase64(md.digest());
} else reporter.error("could not find digest for " + algorithm + "-Digest");
} catch (NoSuchAlgorithmException nsae) {
return "Missing digest algorithm " + algorithm;
}
}
}
} catch (Exception e) {
return "Failed to verify due to exception: " + e.getMessage();
}
return null;
}
/**
* @param clazz
* @param dataFile
* @return
* @throws Exception
*/
private <T> T getData(Class<T> clazz, File dataFile) throws Exception {
try {
return codec.dec().from(dataFile).get(clazz);
} catch (Exception e) {
// e.printStackTrace();
// System.out.println("Cannot read data file "+dataFile+": " +
// IO.collect(dataFile));
return null;
}
}
private void storeData(File dataFile, Object o) throws Exception {
codec.enc().to(dataFile).put(o);
}
/**
* This is called when JPM runs in the background to start jobs
*
* @throws Exception
*/
public void daemon() throws Exception {
Runtime.getRuntime()
.addShutdownHook(
new Thread("Daemon shutdown") {
public void run() {
for (Service service : startedByDaemon) {
try {
reporter.error("Stopping " + service);
service.stop();
reporter.error("Stopped " + service);
} catch (Exception e) {
// Ignore
}
}
}
});
List<ServiceData> services = getServices();
Map<String, ServiceData> map = new HashMap<String, ServiceData>();
for (ServiceData d : services) {
map.put(d.name, d);
}
List<ServiceData> start = new ArrayList<ServiceData>();
Set<ServiceData> set = new HashSet<ServiceData>();
for (ServiceData sd : services) {
checkStartup(map, start, sd, set);
}
if (start.isEmpty()) reporter.warning("No services to start");
for (ServiceData sd : start) {
try {
Service service = getService(sd.name);
reporter.trace("Starting " + service);
String result = service.start();
if (result != null) reporter.error("Started error " + result);
else startedByDaemon.add(service);
reporter.trace("Started " + service);
} catch (Exception e) {
reporter.error("Cannot start daemon %s, due to %s", sd.name, e);
}
}
while (true) {
for (Service sd : startedByDaemon) {
try {
if (!sd.isRunning()) {
reporter.error("Starting due to failure " + sd);
String result = sd.start();
if (result != null) reporter.error("Started error " + result);
}
} catch (Exception e) {
reporter.error("Cannot start daemon %s, due to %s", sd, e);
}
}
Thread.sleep(10000);
}
}
private void checkStartup(
Map<String, ServiceData> map,
List<ServiceData> start,
ServiceData sd,
Set<ServiceData> cyclic) {
if (sd.after.isEmpty() || start.contains(sd)) return;
if (cyclic.contains(sd)) {
reporter.error("Cyclic dependency for " + sd.name);
return;
}
cyclic.add(sd);
for (String dependsOn : sd.after) {
if (dependsOn.equals("boot")) continue;
ServiceData deps = map.get(dependsOn);
if (deps == null) {
reporter.error("No such service " + dependsOn + " but " + sd.name + " depends on it");
} else {
checkStartup(map, start, deps, cyclic);
}
}
start.add(sd);
}
public void register(boolean user) throws Exception {
platform.installDaemon(user);
}
public ArtifactData putAsync(final URI uri) {
final ArtifactData data = new ArtifactData();
data.busy = true;
Runnable r =
new Runnable() {
public void run() {
try {
put(uri, data);
} catch (Throwable e) {
e.printStackTrace();
data.error = e.toString();
} finally {
reporter.trace("done downloading %s", uri);
data.done();
}
}
};
getExecutor().execute(r);
return data;
}
public ArtifactData put(final URI uri) throws Exception {
final ArtifactData data = new ArtifactData();
put(uri, data);
return data;
}
void put(final URI uri, ArtifactData data) throws Exception {
reporter.trace("put %s %s", uri, data);
File tmp = createTempFile(repoDir, "mtp", ".whatever");
tmp.deleteOnExit();
try {
copy(uri.toURL(), tmp);
byte[] sha = SHA1.digest(tmp).digest();
reporter.trace("SHA %s %s", uri, Hex.toHexString(sha));
ArtifactData existing = get(sha);
if (existing != null) {
reporter.trace("existing");
xcopy(existing, data);
return;
}
File meta = new File(repoDir, Hex.toHexString(sha) + ".json");
File file = new File(repoDir, Hex.toHexString(sha));
rename(tmp, file);
reporter.trace("file %s", file);
data.file = file.getAbsolutePath();
data.sha = sha;
data.busy = false;
CommandData cmddata = parseCommandData(data);
if (cmddata.bsn != null) {
data.name = cmddata.bsn + "-" + cmddata.version;
} else data.name = Strings.display(cmddata.title, cmddata.bsn, cmddata.name, uri);
codec.enc().to(meta).put(data);
reporter.trace("TD = " + data);
} finally {
tmp.delete();
reporter.trace("puted %s %s", uri, data);
}
}
public ArtifactData get(byte[] sha) throws Exception {
String name = Hex.toHexString(sha);
File data = IO.getFile(repoDir, name + ".json");
reporter.trace("artifact data file %s", data);
if (data.isFile()) { // Bin + metadata
ArtifactData artifact = codec.dec().from(data).get(ArtifactData.class);
artifact.file = IO.getFile(repoDir, name).getAbsolutePath();
return artifact;
}
File bin = IO.getFile(repoDir, name);
if (bin.exists()) { // Only bin
ArtifactData artifact = new ArtifactData();
artifact.file = bin.getAbsolutePath();
artifact.sha = sha;
return artifact;
}
return null;
}
public List<Revision> filter(Collection<Revision> list, EnumSet<Library.Phase> phases) {
List<Revision> filtered = new ArrayList<Library.Revision>();
for (Revision r : list) if (phases.contains(r.phase)) filtered.add(r);
return filtered;
}
public Map<String, Revision> latest(Collection<Revision> list) {
Map<String, Revision> programs = new HashMap<String, Library.Revision>();
for (Revision r : list) {
String coordinates = r.groupId + ":" + r.artifactId;
if (r.classifier != null) coordinates += ":" + r.classifier;
if (r.groupId.equals(Library.SHA_GROUP)) continue;
Revision current = programs.get(coordinates);
if (current == null) programs.put(coordinates, r);
else {
// who is better?
if (compare(r, current) >= 0) programs.put(coordinates, r);
}
}
return programs;
}
private int compare(Revision a, Revision b) {
if (Arrays.equals(a._id, b._id)) return 0;
Version va = getVersion(a);
Version vb = getVersion(b);
int n = va.compareTo(vb);
if (n != 0) return n;
if (a.created != b.created) return a.created > b.created ? 1 : -1;
for (int i = 0; i < a._id.length; i++)
if (a._id[i] != b._id[i]) return a._id[i] > b._id[i] ? 1 : -1;
return 0;
}
private Version getVersion(Revision a) {
if (a.qualifier != null) return new Version(a.baseline + "." + a.qualifier);
return new Version(a.baseline);
}
public String getCoordinates(Revision r) {
StringBuilder sb = new StringBuilder(r.groupId).append(":").append(r.artifactId);
if (r.classifier != null) sb.append(":").append(r.classifier);
sb.append("@").append(r.version);
return sb.toString();
}
public String getCoordinates(RevisionRef r) {
StringBuilder sb = new StringBuilder(r.groupId).append(":").append(r.artifactId).append(":");
if (r.classifier != null) sb.append(r.classifier).append("@");
sb.append(r.version);
return sb.toString();
}
public ArtifactData getCandidate(String key) throws Exception {
ArtifactData data = getCandidateAsync(key);
if (data != null) {
data.sync();
}
return data;
}
public ArtifactData getCandidateAsync(String arg) throws Exception {
reporter.trace("coordinate %s", arg);
if (isUrl(arg))
try {
ArtifactData data = putAsync(new URI(arg));
data.local = true;
return data;
} catch (Exception e) {
reporter.trace("hmm, not a valid url %s, will try the server", arg);
}
Coordinate c = new Coordinate(arg);
if (c.isSha()) {
ArtifactData r = get(c.getSha());
if (r != null) return r;
}
Revision revision = library.getRevisionByCoordinate(c);
if (revision == null) return null;
reporter.trace("revision %s", Hex.toHexString(revision._id));
ArtifactData ad = get(revision._id);
if (ad != null) {
reporter.trace("found in cache");
return ad;
}
URI url = revision.urls.iterator().next();
ArtifactData artifactData = putAsync(url);
artifactData.coordinate = c;
return artifactData;
}
public static Executor getExecutor() {
if (executor == null) executor = Executors.newFixedThreadPool(4);
return executor;
}
public static void setExecutor(Executor executor) {
JustAnotherPackageManager.executor = executor;
}
public void setLibrary(URI url) throws Exception {
if (url == null) url = new URI("http://repo.jpm4j.org/");
this.host = new URLClient(url.toString());
host.setReporter(reporter);
library = JSONRPCProxy.createRPC(JpmRepo.class, host, "jpm");
}
public void close() {
if (executor != null && executor instanceof ExecutorService)
((ExecutorService) executor).shutdown();
}
public void init() throws IOException {
URL s = getClass().getClassLoader().getResource(SERVICE_JAR_FILE);
if (s == null)
if (underTest) return;
else throw new Error("No " + SERVICE_JAR_FILE + " resource in jar");
service.getParentFile().mkdirs();
IO.copy(s, service);
}
public Platform getPlatform() {
return platform;
}
/**
* Copy from the copy method in StructUtil. Did not want to drag that code in. maybe this actually
* should go to struct.
*
* @param from
* @param to
* @param excludes
* @return
* @throws Exception
*/
public static <T extends struct> T xcopy(struct from, T to, String... excludes) throws Exception {
Arrays.sort(excludes);
for (Field f : from.fields()) {
if (Arrays.binarySearch(excludes, f.getName()) >= 0) continue;
Object o = f.get(from);
if (o == null) continue;
Field tof = to.getField(f.getName());
if (tof != null)
try {
tof.set(to, Converter.cnv(tof.getGenericType(), o));
} catch (Exception e) {
System.out.println(
"Failed to convert "
+ f.getName()
+ " from "
+ from.getClass()
+ " to "
+ to.getClass()
+ " value "
+ o
+ " exception "
+ e);
}
}
return to;
}
public JpmRepo getLibrary() {
return library;
}
public void setLocalInstall(boolean b) {
localInstall = b;
}
public String what(String key, boolean oneliner) throws Exception {
byte[] sha;
Matcher m = SHA_P.matcher(key);
if (m.matches()) {
sha = Hex.toByteArray(key);
} else {
m = URL_P.matcher(key);
if (m.matches()) {
URL url = new URL(key);
sha = SHA1.digest(url.openStream()).digest();
} else {
File jarfile = new File(key);
if (!jarfile.exists()) {
reporter.error("File does not exist: %s", jarfile.getCanonicalPath());
}
sha = SHA1.digest(jarfile).digest();
}
}
reporter.trace("sha %s", Hex.toHexString(sha));
Revision revision = library.getRevision(sha);
if (revision == null) {
return null;
}
StringBuilder sb = new StringBuilder();
Formatter f = new Formatter(sb);
Justif justif = new Justif(120, 20, 70, 20, 75);
DateFormat dateFormat = DateFormat.getDateInstance();
try {
if (oneliner) {
f.format("%20s %s%n", Hex.toHexString(revision._id), createCoord(revision));
} else {
f.format("Artifact: %s%n", revision.artifactId);
if (revision.organization != null && revision.organization.name != null) {
f.format(" (%s)", revision.organization.name);
}
f.format("%n");
f.format("Coordinates\t0: %s%n", createCoord(revision));
f.format("Created\t0: %s%n", dateFormat.format(new Date(revision.created)));
f.format("Size\t0: %d%n", revision.size);
f.format("Sha\t0: %s%n", Hex.toHexString(revision._id));
f.format("URL\t0: %s%n", createJpmLink(revision));
f.format("%n");
f.format("%s%n", revision.description);
f.format("%n");
f.format("Dependencies\t0:%n");
boolean flag = false;
Iterable<RevisionRef> closure = library.getClosure(revision._id, true);
for (RevisionRef dep : closure) {
f.format(
" - %s \t2- %s \t3- %s%n",
dep.name, createCoord(dep), dateFormat.format(new Date(dep.created)));
flag = true;
}
if (!flag) {
f.format(" None%n");
}
f.format("%n");
}
f.flush();
justif.wrap(sb);
return sb.toString();
} finally {
f.close();
}
}
private String createCoord(Revision rev) {
return String.format("%s:%s@%s [%s]", rev.groupId, rev.artifactId, rev.version, rev.phase);
}
private String createCoord(RevisionRef rev) {
return String.format("%s:%s@%s [%s]", rev.groupId, rev.artifactId, rev.version, rev.phase);
}
private String createJpmLink(Revision rev) {
return String.format(
"http://jpm4j.org/#!/p/sha/%s//%s", Hex.toHexString(rev._id), rev.baseline);
}
public class UpdateMemo {
public CommandData current; // Works for commandData and
// ServiceData, as ServiceData --|>
// CommandData
public RevisionRef best;
}
public void listUpdates(
List<UpdateMemo> notFound,
List<UpdateMemo> upToDate,
List<UpdateMemo> toUpdate,
CommandData data,
boolean staged)
throws Exception {
// UpdateMemo memo = new UpdateMemo();
// memo.current = data;
//
// Matcher m = data.coordinates == null ? null :
// COORD_P.matcher(data.coordinates);
//
// if (data.version == null || m == null || !m.matches()) {
// Revision revision = library.getRevision(data.sha);
// if (revision == null) {
// notFound.add(memo);
// return;
// }
// data.version = new Version(revision.version);
// data.coordinates = getCoordinates(revision);
// if (data instanceof ServiceData) {
// storeData(IO.getFile(new File(serviceDir, data.name), "data"), data);
// } else {
// storeData(IO.getFile(commandDir, data.name), data);
// }
// }
//
// Iterable< ? extends Program> programs =
// library.getQueryPrograms(data.coordinates, 0, 0);
// int count = 0;
// RevisionRef best = null;
// for (Program p : programs) {
// // best = selectBest(p.revisions, staged, null);
// count++;
// }
// if (count != 1 || best == null) {
// notFound.add(memo);
// return;
// }
// Version bestVersion = new Version(best.version);
//
// if (data.version.compareTo(bestVersion) < 0) { // Update available
// memo.best = best;
// toUpdate.add(memo);
// } else { // up to date
// upToDate.add(memo);
// }
}
public void update(UpdateMemo memo) throws Exception {
ArtifactData target = put(memo.best.urls.iterator().next());
memo.current.version = new Version(memo.best.version);
target.sync();
memo.current.sha = target.sha;
// memo.current.dependencies = target.dependencies;
// memo.current.dependencies.add((new File(repoDir,
// Hex.toHexString(target.sha))).getCanonicalPath());
// memo.current.runbundles = target.runbundles;
// memo.current.description = target.description;
memo.current.time = target.time;
if (memo.current instanceof ServiceData) {
Service service = getService((ServiceData) memo.current);
service.remove();
createService((ServiceData) memo.current);
IO.delete(new File(IO.getFile(serviceDir, memo.current.name), "data"));
storeData(new File(IO.getFile(serviceDir, memo.current.name), "data"), memo.current);
} else {
platform.deleteCommand(memo.current);
createCommand(memo.current, false);
IO.delete(IO.getFile(commandDir, memo.current.name));
storeData(IO.getFile(commandDir, memo.current.name), memo.current);
}
}
private boolean isUrl(String coordinate) {
return URL_P.matcher(coordinate).matches();
}
/**
* Find programs
*
* @throws Exception
*/
public List<Program> find(String query, int skip, int limit) throws Exception {
return library.getQueryPrograms(query, skip, limit);
}
public boolean isWildcard(String coordinate) {
return coordinate != null && coordinate.endsWith("@*");
}
public CommandData parseCommandData(ArtifactData artifact) throws Exception {
File source = new File(artifact.file);
if (!source.isFile()) throw new FileNotFoundException();
CommandData data = new CommandData();
data.sha = artifact.sha;
data.jpmRepoDir = repoDir.getCanonicalPath();
JarFile jar = new JarFile(source);
try {
reporter.trace("Parsing %s", source);
Manifest m = jar.getManifest();
Attributes main = m.getMainAttributes();
data.name = data.bsn = main.getValue("Bundle-SymbolicName");
String version = main.getValue("Bundle-Version");
if (version == null) data.version = Version.LOWEST;
else data.version = new Version(version);
data.main = main.getValue("Main-Class");
data.description = main.getValue("Bundle-Description");
data.title = main.getValue("JPM-Name");
reporter.trace("name " + data.name + " " + data.main + " " + data.title);
DependencyCollector path = new DependencyCollector(this);
path.add(artifact);
DependencyCollector bundles = new DependencyCollector(this);
if (main.getValue("JPM-Classpath") != null) {
Parameters requires = OSGiHeader.parseHeader(main.getValue("JPM-Classpath"));
for (Map.Entry<String, Attrs> e : requires.entrySet()) {
path.add(e.getKey(), e.getValue().get("name")); // coordinate
}
} else if (!artifact.local) { // No JPM-Classpath, falling back to
// server's revision
// Iterable<RevisionRef> closure =
// library.getClosure(artifact.sha,
// false);
// System.out.println("getting closure " + artifact.url + " " +
// Strings.join("\n",closure));
// if (closure != null) {
// for (RevisionRef ref : closure) {
// path.add(Hex.toHexString(ref.revision));
// }
// }
}
if (main.getValue("JPM-Runbundles") != null) {
Parameters jpmrunbundles = OSGiHeader.parseHeader(main.getValue("JPM-Runbundles"));
for (Map.Entry<String, Attrs> e : jpmrunbundles.entrySet()) {
bundles.add(e.getKey(), e.getValue().get("name"));
}
}
reporter.trace("collect digests runpath");
data.dependencies.addAll(path.getDigests());
reporter.trace("collect digests bundles");
data.runbundles.addAll(bundles.getDigests());
Parameters command = OSGiHeader.parseHeader(main.getValue("JPM-Command"));
if (command.size() > 1) reporter.error("Only one command can be specified");
for (Map.Entry<String, Attrs> e : command.entrySet()) {
data.name = e.getKey();
Attrs attrs = e.getValue();
if (attrs.containsKey("jvmargs")) data.jvmArgs = attrs.get("jvmargs");
if (attrs.containsKey("title")) data.title = attrs.get("title");
if (data.title != null) data.title = data.name;
}
return data;
} finally {
jar.close();
}
}
public void setUnderTest() {
underTest = true;
}
/**
* Turn the shas into a readable form
*
* @param dependencies
* @return
* @throws Exception
*/
public List<?> toString(List<byte[]> dependencies) throws Exception {
List<String> out = new ArrayList<String>();
for (byte[] dependency : dependencies) {
ArtifactData data = get(dependency);
if (data == null) out.add(Hex.toHexString(dependency));
else {
out.add(Strings.display(data.name, Hex.toHexString(dependency)));
}
}
return out;
}
/**
* Get a list of candidates from a coordinate
*
* @param c
* @throws Exception
*/
public Iterable<Revision> getCandidates(Coordinate c) throws Exception {
return library.getRevisionsByCoordinate(c);
}
/** Post install */
public void doPostInstall() {
getPlatform().doPostInstall();
}
}
public BankMessage makeMessage() {
Scanner stdIn = new Scanner(System.in);
int userInput;
int[] opcodes = {CREATE, DEPOSIT, WITHDRAW, GETBALANCE, CLOSE};
do {
userInput = stdIn.nextInt();
} while (!(userInput >= 1 && userInput <= 5));
int opcode = opcodes[userInput - 1];
int[] parameters = new int[2];
// create a pattern only accepting positive numbers with at most 2 decimal places
Pattern dollar =
Pattern.compile(
"^\\$?([1-9]{1}[0-9]{0,2}(\\,[0-9]{3})*(\\.[0-9]{0,2})?|[1-9]{1}[0-9]{0,}(\\.[0-9]{0,2})?|0(\\.[0-9]{0,2})?|(\\.[0-9]{1,2})?)$");
String input = null;
// populate parameters list
switch (opcode) {
case CREATE:
do {
System.out.println("How much for initial deposit?");
input = stdIn.next();
} while ((dollar.matcher(input).matches()) == false);
parameters[0] = (int) (Double.parseDouble(input) * 100);
break;
case DEPOSIT:
System.out.println("Which account?");
parameters[0] = stdIn.nextInt();
do {
System.out.println("How much to deposit?");
input = stdIn.next();
} while ((dollar.matcher(input).matches()) == false);
parameters[1] = (int) (Double.parseDouble(input) * 100);
break;
case WITHDRAW:
System.out.println("Which account?");
parameters[0] = stdIn.nextInt();
do {
System.out.println("How much to withdraw?");
input = stdIn.next();
} while ((dollar.matcher(input).matches()) == false);
parameters[1] = (int) (Double.parseDouble(input) * 100);
break;
case GETBALANCE:
System.out.println("Which account?");
parameters[0] = stdIn.nextInt();
break;
case CLOSE:
System.out.println("Which account?");
parameters[0] = stdIn.nextInt();
break;
default:
System.out.println("Invalid opcode. System failure.");
System.exit(-1);
}
BankMessage output = buildMessage(opcode, parameters);
return output;
}
public class Main {
private static Pattern lores =
Pattern.compile(
"post_photolores=\"https?://([-\\w\\.]+)+(:\\d+)?(/([\\w/_\\.]*(\\?\\S+)?)?)?\"");
private static List<String> endings = new ArrayList<>();
static {
Main.endings.add("_1280");
Main.endings.add("_700");
Main.endings.add("_700");
Main.endings.add("_500");
Main.endings.add("_400");
Main.endings.add("_250");
Main.endings.add("_100");
}
private static Gui gui = null;
private static HashMap<Post, Post> post_post_hash = new HashMap<>();
protected static HashMap<Picture, Picture> pic_pic_hash = new HashMap<>();
private static HashMap<Picture, Post> pic_post_hash = new HashMap<>();
private static HashMap<Post, Post> dup_post_list = new HashMap<>();
private static String blogname = "";
private static File blogdir = null;
public static Set<Picture> getPictures() {
return Main.pic_pic_hash.keySet();
}
public static File getBlogDir() {
return Main.blogdir;
}
public static void setBlogName(String blogname) {
Main.blogname = blogname;
Main.blogdir = new File("." + File.separator + Main.blogname + File.separator);
}
public static void main(String[] args) {
if (args.length == 0) {
gui = new Gui();
gui.setVisible(true);
} else if (args.length == 3 || args.length == 4) {
Main.setBlogName(args[0]);
Main.load();
if (args.length == 3) {
int start, limit;
try {
start = Integer.parseInt(args[1]);
limit = Integer.parseInt(args[2]);
} catch (Exception e) {
Main.status("usage: Main <blogname> <start_page> <end_page> -hires");
Main.status("start_page and end_page must be integers >= 1");
return;
}
Main.run(start, limit);
Main.save();
} else if (args.length == 4 && args[3].equals("-hires")) {
Main.downloadHiRes();
Main.save();
} else {
Main.status("usage: Main <blogname> <start_page> <end_page> -hires");
}
} else {
Main.status("usage: Main <blogname> <start_page> <end_page> -hires");
}
}
protected static void reset() {
Helper.removeDirectoryIfItExists(Helper.temp);
Helper.removeDirectoryIfItExists(blogdir);
Main.post_post_hash.clear();
Main.pic_pic_hash.clear();
Main.pic_post_hash.clear();
Main.dup_post_list.clear();
}
public static void run(int start_page, int end_page) {
if (start_page < 1 || end_page < 1) {
Main.status("start_page and end_page must be integers >= 1");
return;
}
int progress = 0;
if (gui != null) {
gui.setProgress(progress);
}
if (end_page >= start_page) {
if (gui != null) {
gui.setMaxProgress(end_page - start_page);
}
for (int i = start_page; i <= end_page; i++) {
boolean exists =
Main.handleURL(String.format("http://%s.tumblr.com/page/%s", Main.blogname, i));
if (!exists) {
Main.status(String.format("We ran out of posts to process at page %s.", i));
break;
}
if (gui != null) {
gui.setProgress(progress);
progress++;
}
}
} else {
if (gui != null) {
gui.setMaxProgress(start_page - end_page);
}
for (int i = start_page; i >= end_page; i--) {
boolean exists =
Main.handleURL(String.format("http://%s.tumblr.com/page/%s", Main.blogname, i));
if (!exists) {
Main.status(String.format("We ran out of posts to process at page %s.", i));
break;
}
if (gui != null) {
gui.setProgress(progress);
progress++;
}
}
}
if (gui != null) {
gui.setProgress(progress);
}
Main.writeDuplicates();
}
private static void writeDuplicates() {
Main.status("Writing duplicates.");
if (!dup_post_list.isEmpty()) {
Main.status(String.format("%s\t%s", "older_post", "newer_post"));
for (Post post : dup_post_list.keySet()) {
Main.status(String.format("%s\t%s", post.post_id, dup_post_list.get(post).post_id));
}
} else {
Main.status("There are no duplicates.");
}
Main.status("Writing duplicates done.");
}
public static void load() {
Main.status("Loading databases.");
File file = new File(blogdir, "picpic.db");
List<Object> objects = Helper.loadObjectFromFile(file);
if (objects == null || objects.size() != 1) {
Main.error("Unable to load database files so creating new database.");
reset();
} else {
Main.post_post_hash = (HashMap<Post, Post>) objects.get(0);
Main.pic_pic_hash.clear();
Main.pic_post_hash.clear();
Main.dup_post_list.clear();
Main.setupPosts();
}
Main.status("Done loading databases.");
}
public static void save() {
Main.status("Saving databases.");
File file = new File(blogdir, "picpic.db");
List<Object> objects = new ArrayList<>();
objects.add(Main.post_post_hash);
Helper.saveObjectToFile(file, objects);
Main.status("Done saving databases.");
}
private static boolean handleURL(String address) {
Main.status(String.format("Processing page \"%s\".", address));
try {
NodeList posts = getPosts(address);
if (posts.toNodeArray().length == 0) {
return false;
}
for (Node post_node : posts.toNodeArray()) {
if (post_node instanceof TagNode) {
TagNode post = (TagNode) post_node;
Post new_post = new Post(Long.parseLong(post.getAttribute("id").substring(5)));
if (!Main.post_post_hash.containsKey(new_post)) {
NodeList photo_posts = getPhotoPosts(post.getChildren());
NodeList remarks = getRemarks(photo_posts);
for (Node node : remarks.toNodeArray()) {
Matcher matcher = lores.matcher(node.getText());
String media_url = "";
if (matcher.find()) {
media_url = matcher.group();
media_url = media_url.substring(17, media_url.length() - 1);
}
String thumb =
media_url.replace(
media_url.substring(media_url.lastIndexOf("_"), media_url.lastIndexOf(".")),
"_75sq");
URL thumb_url = new URL(thumb);
new_post.pictures.add(new Picture(new URL(media_url), thumb_url));
}
NodeList photoset_posts = getPhotosetPosts(post.getChildren());
NodeList iframes = getIFrames(photoset_posts);
for (Node node : iframes.toNodeArray()) {
if (node instanceof TagNode) {
String iframe_url = ((TagNode) node).getAttribute("src");
Parser parser2 = new Parser(iframe_url);
NodeList a_list = parser2.extractAllNodesThatMatch(new TagNameFilter("a"));
Node[] a_array = a_list.toNodeArray();
Node[] img_array =
a_list.extractAllNodesThatMatch(new TagNameFilter("img"), true).toNodeArray();
String media_url;
for (int i = 0; i < a_array.length; i++) {
media_url = ((TagNode) img_array[i]).getAttribute("src");
String thumb =
media_url.replace(
media_url.substring(
media_url.lastIndexOf("_"), media_url.lastIndexOf(".")),
"_75sq");
URL thumb_url = new URL(thumb);
new_post.pictures.add(new Picture(new URL(media_url), thumb_url));
}
}
}
Main.handlePost(new_post);
} else {
new_post = post_post_hash.get(new_post);
handleNonDownloadPost(new_post);
}
}
}
} catch (Exception ex) {
ex.printStackTrace();
Main.status("Error handling post.");
}
return true;
}
private static NodeList getPosts(String url) throws ParserException {
return new Parser(url)
.extractAllNodesThatMatch(new HasAttributeFilter("class", "my_post load"));
}
private static NodeList getPhotoPosts(NodeList node_list) {
return node_list.extractAllNodesThatMatch(
new HasAttributeFilter("class", "my_photo_post"), true);
}
private static NodeList getPhotosetPosts(NodeList node_list) {
return node_list.extractAllNodesThatMatch(
new HasAttributeFilter("class", "my_photoset_post"), true);
}
private static NodeList getRemarks(NodeList node_list) {
return node_list.extractAllNodesThatMatch(new NodeClassFilter(RemarkNode.class), true);
}
private static NodeList getIFrames(NodeList node_list) {
return node_list.extractAllNodesThatMatch(new TagNameFilter("iframe"), true);
}
public static void downloadHiRes() {
Main.status("Downloading hi res versions of photos in database.");
if (gui != null) {
gui.setProgress(0);
gui.setMaxProgress(pic_pic_hash.keySet().size());
}
int progress = 0;
for (Picture picture : pic_pic_hash.keySet()) {
if (!picture.downloaded_hi) {
tryResUrls(picture);
}
if (gui != null) {
gui.setProgress(progress);
progress++;
}
}
if (gui != null) {
gui.setProgress(progress);
progress++;
}
Main.status("Done downloading hi res versions.");
}
private static void tryResUrls(Picture picture) {
String hi_res = "";
String url = picture.media_url.toString();
for (String ending : Main.endings) {
try {
hi_res = url.replace(url.substring(url.lastIndexOf("_"), url.lastIndexOf(".")), ending);
URL hi_url = new URL(hi_res);
File hi_name = Helper.extractMediaFileNameFromURL(hi_url);
if (hi_name.equals(picture.media_name)) {
picture.hi_url = hi_url;
picture.hi_name = hi_name;
picture.downloaded_hi = true;
break;
} else {
boolean success = Helper.downloadFileFromURLToFileInTemp(hi_url, hi_name);
if (success) {
picture.hi_url = hi_url;
picture.hi_name = hi_name;
picture.downloaded_hi = true;
Helper.moveTempImageToStore(hi_name, new File(Main.blogdir, picture.md5_id));
break;
}
}
} catch (MalformedURLException ex) {
Main.error(String.format("Attempted hi res url %s is a malformed URL.", hi_res));
}
}
}
private static void handlePost(Post post) {
Main.post_post_hash.put(post, post);
for (Picture picture : post.pictures) {
Helper.downloadFileFromURLToFileInTemp(picture.thumb_url, picture.thumb_name);
picture.md5_id = Helper.createMD5FromFileInTemp(picture.thumb_name);
Helper.moveTempImageToStore(picture.thumb_name, new File(Main.blogdir, picture.md5_id));
if (!Main.pic_pic_hash.containsKey(picture)) {
Main.pic_pic_hash.put(picture, picture);
Main.pic_post_hash.put(picture, post);
Helper.downloadFileFromURLToFileInTemp(picture.media_url, picture.media_name);
Helper.moveTempImageToStore(picture.media_name, new File(Main.blogdir, picture.md5_id));
} else {
if (!post.equals(Main.pic_post_hash.get(picture))) {
dup_post_list.put(post, Main.pic_post_hash.get(picture));
}
}
}
}
public static void setupPosts() {
for (Post post : Main.post_post_hash.keySet()) {
post = Main.post_post_hash.get(post);
handleNonDownloadPost(post);
}
}
private static void handleNonDownloadPost(Post post) {
for (Picture picture : post.pictures) {
if (!Main.pic_pic_hash.containsKey(picture)) {
Main.pic_pic_hash.put(picture, picture);
Main.pic_post_hash.put(picture, post);
} else {
if (!post.equals(Main.pic_post_hash.get(picture))) {
dup_post_list.put(post, Main.pic_post_hash.get(picture));
}
}
}
}
private static void status(String status) {
if (gui == null) {
System.out.println(status);
} else {
gui.setStatus(status);
}
}
private static void error(String error) {
if (gui == null) {
System.err.println(error);
} else {
gui.setStatus(error);
}
}
}
/**
* This class represents a single RESTXQ function.
*
* @author BaseX Team 2005-12, BSD License
* @author Christian Gruen
*/
final class RestXqFunction implements Comparable<RestXqFunction> {
/** Pattern for a single template. */
private static final Pattern TEMPLATE = Pattern.compile("\\s*\\{\\s*\\$(.+?)\\s*\\}\\s*");
/** Supported methods. */
EnumSet<HTTPMethod> methods = EnumSet.allOf(HTTPMethod.class);
/** Serialization parameters. */
final SerializerProp output = new SerializerProp();
/** Associated function. */
final StaticUserFunc function;
/** Associated module. */
final RestXqModule module;
/** Path. */
RestXqPath path;
/** Query parameters. */
final ArrayList<RestXqParam> queryParams = new ArrayList<RestXqParam>();
/** Form parameters. */
final ArrayList<RestXqParam> formParams = new ArrayList<RestXqParam>();
/** Header parameters. */
final ArrayList<RestXqParam> headerParams = new ArrayList<RestXqParam>();
/** Cookie parameters. */
final ArrayList<RestXqParam> cookieParams = new ArrayList<RestXqParam>();
/** Query context. */
private final QueryContext context;
/** Consumed media types. */
private final StringList consumes = new StringList();
/** Returned media types. */
private final StringList produces = new StringList();
/** Post/Put variable. */
private QNm requestBody;
/**
* Constructor.
*
* @param uf associated user function
* @param qc query context
* @param m associated module
*/
RestXqFunction(final StaticUserFunc uf, final QueryContext qc, final RestXqModule m) {
function = uf;
context = qc;
module = m;
}
/**
* Processes the HTTP request. Parses new modules and discards obsolete ones.
*
* @param http HTTP context
* @throws Exception exception
*/
void process(final HTTPContext http) throws Exception {
try {
module.process(http, this);
} catch (final QueryException ex) {
if (ex.file() == null) ex.info(function.info);
throw ex;
}
}
/**
* Checks a function for RESTFful annotations.
*
* @return {@code true} if module contains relevant annotations
* @throws QueryException query exception
*/
boolean analyze() throws QueryException {
// parse all annotations
final EnumSet<HTTPMethod> mth = EnumSet.noneOf(HTTPMethod.class);
final boolean[] declared = new boolean[function.args.length];
boolean found = false;
final int as = function.ann.size();
for (int a = 0; a < as; a++) {
final QNm name = function.ann.names[a];
final Value value = function.ann.values[a];
final byte[] local = name.local();
final byte[] uri = name.uri();
final boolean rexq = eq(uri, QueryText.RESTXQURI);
if (rexq) {
if (eq(PATH, local)) {
// annotation "path"
if (path != null) error(ANN_TWICE, "%", name.string());
path = new RestXqPath(toString(value, name));
for (final String s : path) {
if (s.trim().startsWith("{")) checkVariable(s, AtomType.AAT, declared);
}
} else if (eq(CONSUMES, local)) {
// annotation "consumes"
strings(value, name, consumes);
} else if (eq(PRODUCES, local)) {
// annotation "produces"
strings(value, name, produces);
} else if (eq(QUERY_PARAM, local)) {
// annotation "query-param"
queryParams.add(param(value, name, declared));
} else if (eq(FORM_PARAM, local)) {
// annotation "form-param"
formParams.add(param(value, name, declared));
} else if (eq(HEADER_PARAM, local)) {
// annotation "header-param"
headerParams.add(param(value, name, declared));
} else if (eq(COOKIE_PARAM, local)) {
// annotation "cookie-param"
cookieParams.add(param(value, name, declared));
} else {
// method annotations
final HTTPMethod m = HTTPMethod.get(string(local));
if (m == null) error(ANN_UNKNOWN, "%", name.string());
if (!value.isEmpty()) {
// remember post/put variable
if (requestBody != null) error(ANN_TWICE, "%", name.string());
if (m != POST && m != PUT) error(METHOD_VALUE, m);
requestBody = checkVariable(toString(value, name), declared);
}
if (mth.contains(m)) error(ANN_TWICE, "%", name.string());
mth.add(m);
}
} else if (eq(uri, QueryText.OUTPUTURI)) {
// serialization parameters
final String key = string(local);
final String val = toString(value, name);
if (output.get(key) == null) error(UNKNOWN_SER, key);
output.set(key, val);
}
found |= rexq;
}
if (!mth.isEmpty()) methods = mth;
if (found) {
if (path == null) error(ANN_MISSING, PATH);
for (int i = 0; i < declared.length; i++)
if (!declared[i]) error(VAR_UNDEFINED, function.args[i].name.string());
}
return found;
}
/**
* Checks if an HTTP request matches this function and its constraints.
*
* @param http http context
* @return result of check
*/
boolean matches(final HTTPContext http) {
// check method, path, consumed and produced media type
return methods.contains(http.method) && pathMatches(http) && consumes(http) && produces(http);
}
/**
* Binds the annotated variables.
*
* @param http http context
* @param arg argument array
* @throws QueryException query exception
* @throws IOException I/O exception
*/
void bind(final HTTPContext http, final Expr[] arg) throws QueryException, IOException {
// bind variables from segments
for (int s = 0; s < path.size; s++) {
final Matcher m = TEMPLATE.matcher(path.segment[s]);
if (!m.find()) continue;
final QNm qnm = new QNm(token(m.group(1)), context);
bind(qnm, arg, new Atm(http.segment(s)));
}
// cache request body
final String ct = http.contentType();
IOContent body = null;
if (requestBody != null) {
body = cache(http, null);
try {
// bind request body in the correct format
body.name(http.method + IO.XMLSUFFIX);
bind(requestBody, arg, Parser.item(body, context.context.prop, ct));
} catch (final IOException ex) {
error(INPUT_CONV, ex);
}
}
// bind query parameters
final Map<String, String[]> params = http.params();
for (final RestXqParam rxp : queryParams) bind(rxp, arg, params.get(rxp.key));
// bind form parameters
if (!formParams.isEmpty()) {
if (MimeTypes.APP_FORM.equals(ct)) {
// convert parameters encoded in a form
body = cache(http, body);
addParams(body.toString(), params);
}
for (final RestXqParam rxp : formParams) bind(rxp, arg, params.get(rxp.key));
}
// bind header parameters
for (final RestXqParam rxp : headerParams) {
final StringList sl = new StringList();
final Enumeration<?> en = http.req.getHeaders(rxp.key);
while (en.hasMoreElements()) {
for (final String s : en.nextElement().toString().split(", *")) sl.add(s);
}
bind(rxp, arg, sl.toArray());
}
// bind cookie parameters
final Cookie[] ck = http.req.getCookies();
for (final RestXqParam rxp : cookieParams) {
String v = null;
if (ck != null) {
for (final Cookie c : ck) {
if (rxp.key.equals(c.getName())) v = c.getValue();
}
}
if (v == null) bind(rxp, arg);
else bind(rxp, arg, v);
}
}
/**
* Creates an exception with the specified message.
*
* @param msg message
* @param ext error extension
* @return exception
* @throws QueryException query exception
*/
QueryException error(final String msg, final Object... ext) throws QueryException {
throw new QueryException(function.info, Err.BASX_RESTXQ, Util.info(msg, ext));
}
@Override
public int compareTo(final RestXqFunction rxf) {
return path.compareTo(rxf.path);
}
// PRIVATE METHODS ====================================================================
/**
* Checks the specified template and adds a variable.
*
* @param tmp template string
* @param declared variable declaration flags
* @return resulting variable
* @throws QueryException query exception
*/
private QNm checkVariable(final String tmp, final boolean[] declared) throws QueryException {
return checkVariable(tmp, AtomType.ITEM, declared);
}
/**
* Checks the specified template and adds a variable.
*
* @param tmp template string
* @param type allowed type
* @param declared variable declaration flags
* @return resulting variable
* @throws QueryException query exception
*/
private QNm checkVariable(final String tmp, final Type type, final boolean[] declared)
throws QueryException {
final Var[] args = function.args;
final Matcher m = TEMPLATE.matcher(tmp);
if (!m.find()) error(INV_TEMPLATE, tmp);
final byte[] vn = token(m.group(1));
if (!XMLToken.isQName(vn)) error(INV_VARNAME, vn);
final QNm qnm = new QNm(vn, context);
int r = -1;
while (++r < args.length && !args[r].name.eq(qnm)) ;
if (r == args.length) error(UNKNOWN_VAR, vn);
if (declared[r]) error(VAR_ASSIGNED, vn);
final SeqType st = args[r].declaredType();
if (args[r].checksType() && !st.type.instanceOf(type)) error(INV_VARTYPE, vn, type);
declared[r] = true;
return qnm;
}
/**
* Checks if the path matches the HTTP request.
*
* @param http http context
* @return result of check
*/
private boolean pathMatches(final HTTPContext http) {
return path.matches(http);
}
/**
* Checks if the consumed content type matches.
*
* @param http http context
* @return result of check
*/
private boolean consumes(final HTTPContext http) {
// return true if no type is given
if (consumes.isEmpty()) return true;
// return true if no content type is specified by the user
final String ct = http.contentType();
if (ct == null) return true;
// check if any combination matches
for (final String c : consumes) {
if (MimeTypes.matches(c, ct)) return true;
}
return false;
}
/**
* Checks if the produced content type matches.
*
* @param http http context
* @return result of check
*/
private boolean produces(final HTTPContext http) {
// return true if no type is given
if (produces.isEmpty()) return true;
// check if any combination matches
for (final String pr : http.produces()) {
for (final String p : produces) {
if (MimeTypes.matches(p, pr)) return true;
}
}
return false;
}
/**
* Binds the specified parameter to a variable.
*
* @param rxp parameter
* @param args argument array
* @param values values to be bound; the parameter's default value is assigned if the argument is
* {@code null} or empty
* @throws QueryException query exception
*/
private void bind(final RestXqParam rxp, final Expr[] args, final String... values)
throws QueryException {
final Value val;
if (values == null || values.length == 0) {
val = rxp.value;
} else {
final ValueBuilder vb = new ValueBuilder();
for (final String s : values) vb.add(new Atm(s));
val = vb.value();
}
bind(rxp.name, args, val);
}
/**
* Binds the specified value to a variable.
*
* @param name variable name
* @param args argument array
* @param value value to be bound
* @throws QueryException query exception
*/
private void bind(final QNm name, final Expr[] args, final Value value) throws QueryException {
// skip nulled values
if (value == null) return;
for (int i = 0; i < function.args.length; i++) {
final Var var = function.args[i];
if (!var.name.eq(name)) continue;
// casts and binds the value
args[i] = var.checkType(value, context, null);
break;
}
}
/**
* Returns the specified value as an atomic string.
*
* @param value value
* @param name name
* @return string
* @throws QueryException HTTP exception
*/
private String toString(final Value value, final QNm name) throws QueryException {
if (!(value instanceof Str)) error(ANN_STRING, "%", name.string(), value);
return ((Str) value).toJava();
}
/**
* Adds items to the specified list.
*
* @param value value
* @param name name
* @param list list to add values to
* @throws QueryException HTTP exception
*/
private void strings(final Value value, final QNm name, final StringList list)
throws QueryException {
final long vs = value.size();
for (int v = 0; v < vs; v++) list.add(toString(value.itemAt(v), name));
}
/**
* Returns a parameter.
*
* @param value value
* @param name name
* @param declared variable declaration flags
* @return parameter
* @throws QueryException HTTP exception
*/
private RestXqParam param(final Value value, final QNm name, final boolean[] declared)
throws QueryException {
// [CG] RESTXQ: allow identical field names?
final long vs = value.size();
if (vs < 2) error(ANN_PARAMS, "%", name.string(), 2);
// name of parameter
final String key = toString(value.itemAt(0), name);
// variable template
final QNm qnm = checkVariable(toString(value.itemAt(1), name), declared);
// default value
final ValueBuilder vb = new ValueBuilder();
for (int v = 2; v < vs; v++) vb.add(value.itemAt(v));
return new RestXqParam(qnm, key, vb.value());
}
// PRIVATE STATIC METHODS =============================================================
/**
* Caches the request body, if not done yet.
*
* @param http http context
* @param cache cache existing cache reference
* @return cache
* @throws IOException I/O exception
*/
private static IOContent cache(final HTTPContext http, final IOContent cache) throws IOException {
if (cache != null) return cache;
final BufferInput bi = new BufferInput(http.req.getInputStream());
final IOContent io = new IOContent(bi.content());
io.name(http.method + IO.XMLSUFFIX);
return io;
}
/**
* Adds parameters from the passed on request body.
*
* @param body request body
* @param params map parameters
*/
private static void addParams(final String body, final Map<String, String[]> params) {
for (final String nv : body.split("&")) {
final String[] parts = nv.split("=", 2);
if (parts.length < 2) continue;
try {
params.put(parts[0], new String[] {URLDecoder.decode(parts[1], Token.UTF8)});
} catch (final Exception ex) {
Util.notexpected(ex);
}
}
}
}