private static void finalize(
Configuration conf, JobConf jobconf, final Path destPath, String presevedAttributes)
throws IOException {
if (presevedAttributes == null) {
return;
}
EnumSet<FileAttribute> preseved = FileAttribute.parse(presevedAttributes);
if (!preseved.contains(FileAttribute.USER)
&& !preseved.contains(FileAttribute.GROUP)
&& !preseved.contains(FileAttribute.PERMISSION)) {
return;
}
FileSystem dstfs = destPath.getFileSystem(conf);
Path dstdirlist = new Path(jobconf.get(DST_DIR_LIST_LABEL));
SequenceFile.Reader in = null;
try {
in = new SequenceFile.Reader(dstdirlist.getFileSystem(jobconf), dstdirlist, jobconf);
Text dsttext = new Text();
FilePair pair = new FilePair();
for (; in.next(dsttext, pair); ) {
Path absdst = new Path(destPath, pair.output);
updatePermissions(pair.input, dstfs.getFileStatus(absdst), preseved, dstfs);
}
} finally {
checkAndClose(in);
}
}
private static void updatePermissions(
FileStatus src, FileStatus dst, EnumSet<FileAttribute> preseved, FileSystem destFileSys)
throws IOException {
String owner = null;
String group = null;
if (preseved.contains(FileAttribute.USER) && !src.getOwner().equals(dst.getOwner())) {
owner = src.getOwner();
}
if (preseved.contains(FileAttribute.GROUP) && !src.getGroup().equals(dst.getGroup())) {
group = src.getGroup();
}
if (owner != null || group != null) {
destFileSys.setOwner(dst.getPath(), owner, group);
}
if (preseved.contains(FileAttribute.PERMISSION)
&& !src.getPermission().equals(dst.getPermission())) {
destFileSys.setPermission(dst.getPath(), src.getPermission());
}
}
private FSDataOutputStream create(Path f, Reporter reporter, FileStatus srcstat)
throws IOException {
if (destFileSys.exists(f)) {
destFileSys.delete(f, false);
}
if (!preserve_status) {
return destFileSys.create(f, true, sizeBuf, reporter);
}
FsPermission permission =
preseved.contains(FileAttribute.PERMISSION) ? srcstat.getPermission() : null;
short replication =
preseved.contains(FileAttribute.REPLICATION)
? srcstat.getReplication()
: destFileSys.getDefaultReplication();
long blockSize =
preseved.contains(FileAttribute.BLOCK_SIZE)
? srcstat.getBlockSize()
: destFileSys.getDefaultBlockSize();
return destFileSys.create(f, permission, true, sizeBuf, replication, blockSize, reporter);
}
static EnumSet<FileAttribute> parse(String s) {
if (s == null || s.length() == 0) {
return EnumSet.allOf(FileAttribute.class);
}
EnumSet<FileAttribute> set = EnumSet.noneOf(FileAttribute.class);
FileAttribute[] attributes = values();
for (char c : s.toCharArray()) {
int i = 0;
for (; i < attributes.length && c != attributes[i].symbol; i++) ;
if (i < attributes.length) {
if (!set.contains(attributes[i])) {
set.add(attributes[i]);
} else {
throw new IllegalArgumentException(
"There are more than one '" + attributes[i].symbol + "' in " + s);
}
} else {
throw new IllegalArgumentException("'" + c + "' in " + s + " is undefined.");
}
}
return set;
}
static Arguments valueOf(String[] args, Configuration conf) throws IOException {
List<Path> srcs = new ArrayList<Path>();
Path dst = null;
Path log = null;
EnumSet<Options> flags = EnumSet.noneOf(Options.class);
String presevedAttributes = null;
String mapredSslConf = null;
long filelimit = Long.MAX_VALUE;
long sizelimit = Long.MAX_VALUE;
for (int idx = 0; idx < args.length; idx++) {
Options[] opt = Options.values();
int i = 0;
for (; i < opt.length && !args[idx].startsWith(opt[i].cmd); i++) ;
if (i < opt.length) {
flags.add(opt[i]);
if (opt[i] == Options.PRESERVE_STATUS) {
presevedAttributes = args[idx].substring(2);
FileAttribute.parse(presevedAttributes); // validation
} else if (opt[i] == Options.FILE_LIMIT) {
filelimit = Options.FILE_LIMIT.parseLong(args, ++idx);
} else if (opt[i] == Options.SIZE_LIMIT) {
sizelimit = Options.SIZE_LIMIT.parseLong(args, ++idx);
}
} else if ("-f".equals(args[idx])) {
if (++idx == args.length) {
throw new IllegalArgumentException("urilist_uri not specified in -f");
}
srcs.addAll(fetchFileList(conf, new Path(args[idx])));
} else if ("-log".equals(args[idx])) {
if (++idx == args.length) {
throw new IllegalArgumentException("logdir not specified in -log");
}
log = new Path(args[idx]);
} else if ("-mapredSslConf".equals(args[idx])) {
if (++idx == args.length) {
throw new IllegalArgumentException("ssl conf file not specified in -mapredSslConf");
}
mapredSslConf = args[idx];
} else if ("-m".equals(args[idx])) {
if (++idx == args.length) {
throw new IllegalArgumentException("num_maps not specified in -m");
}
try {
conf.setInt(MAX_MAPS_LABEL, Integer.valueOf(args[idx]));
} catch (NumberFormatException e) {
throw new IllegalArgumentException("Invalid argument to -m: " + args[idx]);
}
} else if ('-' == args[idx].codePointAt(0)) {
throw new IllegalArgumentException("Invalid switch " + args[idx]);
} else if (idx == args.length - 1) {
dst = new Path(args[idx]);
} else {
srcs.add(new Path(args[idx]));
}
}
// mandatory command-line parameters
if (srcs.isEmpty() || dst == null) {
throw new IllegalArgumentException("Missing " + (dst == null ? "dst path" : "src"));
}
// incompatible command-line flags
final boolean isOverwrite = flags.contains(Options.OVERWRITE);
final boolean isUpdate = flags.contains(Options.UPDATE);
final boolean isDelete = flags.contains(Options.DELETE);
if (isOverwrite && isUpdate) {
throw new IllegalArgumentException("Conflicting overwrite policies");
}
if (isDelete && !isOverwrite && !isUpdate) {
throw new IllegalArgumentException(
Options.DELETE.cmd
+ " must be specified with "
+ Options.OVERWRITE
+ " or "
+ Options.UPDATE
+ ".");
}
return new Arguments(
srcs, dst, log, flags, presevedAttributes, filelimit, sizelimit, mapredSslConf);
}
public final List<Sample> parse(
Reader reader, EnumSet<MissingPedField> missingFields, SampleDB sampleDB) {
final List<String> lines = new XReadLines(reader).readLines();
// What are the record offsets?
final int familyPos = missingFields.contains(MissingPedField.NO_FAMILY_ID) ? -1 : 0;
final int samplePos = familyPos + 1;
final int paternalPos = missingFields.contains(MissingPedField.NO_PARENTS) ? -1 : samplePos + 1;
final int maternalPos =
missingFields.contains(MissingPedField.NO_PARENTS) ? -1 : paternalPos + 1;
final int sexPos =
missingFields.contains(MissingPedField.NO_SEX) ? -1 : Math.max(maternalPos, samplePos) + 1;
final int phenotypePos =
missingFields.contains(MissingPedField.NO_PHENOTYPE)
? -1
: Math.max(sexPos, Math.max(maternalPos, samplePos)) + 1;
final int nExpectedFields =
MathUtils.arrayMaxInt(
Arrays.asList(samplePos, paternalPos, maternalPos, sexPos, phenotypePos))
+ 1;
// go through once and determine properties
int lineNo = 1;
boolean isQT = false;
final List<String[]> splits = new ArrayList<String[]>(lines.size());
for (final String line : lines) {
if (line.startsWith(commentMarker)) continue;
if (line.trim().equals("")) continue;
final String[] parts = line.split("\\s+");
if (parts.length != nExpectedFields)
throw new UserException.MalformedFile(
reader.toString(), "Bad PED line " + lineNo + ": wrong number of fields");
if (phenotypePos != -1) {
isQT = isQT || !CATAGORICAL_TRAIT_VALUES.contains(parts[phenotypePos]);
}
splits.add(parts);
lineNo++;
}
logger.info("Phenotype is other? " + isQT);
// now go through and parse each record
lineNo = 1;
final List<Sample> samples = new ArrayList<Sample>(splits.size());
for (final String[] parts : splits) {
String familyID = null, individualID, paternalID = null, maternalID = null;
Gender sex = Gender.UNKNOWN;
String quantitativePhenotype = Sample.UNSET_QT;
Affection affection = Affection.UNKNOWN;
if (familyPos != -1) familyID = maybeMissing(parts[familyPos]);
individualID = parts[samplePos];
if (paternalPos != -1) paternalID = maybeMissing(parts[paternalPos]);
if (maternalPos != -1) maternalID = maybeMissing(parts[maternalPos]);
if (sexPos != -1) {
if (parts[sexPos].equals(SEX_MALE)) sex = Gender.MALE;
else if (parts[sexPos].equals(SEX_FEMALE)) sex = Gender.FEMALE;
else sex = Gender.UNKNOWN;
}
if (phenotypePos != -1) {
if (isQT) {
if (parts[phenotypePos].equals(MISSING_VALUE1)) affection = Affection.UNKNOWN;
else {
affection = Affection.OTHER;
quantitativePhenotype = parts[phenotypePos];
}
} else {
if (parts[phenotypePos].equals(MISSING_VALUE1)) affection = Affection.UNKNOWN;
else if (parts[phenotypePos].equals(MISSING_VALUE2)) affection = Affection.UNKNOWN;
else if (parts[phenotypePos].equals(PHENOTYPE_UNAFFECTED))
affection = Affection.UNAFFECTED;
else if (parts[phenotypePos].equals(PHENOTYPE_AFFECTED)) affection = Affection.AFFECTED;
else
throw new ReviewedGATKException(
"Unexpected phenotype type " + parts[phenotypePos] + " at line " + lineNo);
}
}
final Sample s =
new Sample(
individualID,
sampleDB,
familyID,
paternalID,
maternalID,
sex,
affection,
quantitativePhenotype);
samples.add(s);
sampleDB.addSample(s);
lineNo++;
}
for (final Sample sample : new ArrayList<Sample>(samples)) {
Sample dad =
maybeAddImplicitSample(
sampleDB, sample.getPaternalID(), sample.getFamilyID(), Gender.MALE);
if (dad != null) samples.add(dad);
Sample mom =
maybeAddImplicitSample(
sampleDB, sample.getMaternalID(), sample.getFamilyID(), Gender.FEMALE);
if (mom != null) samples.add(mom);
}
return samples;
}
public List<Revision> filter(Collection<Revision> list, EnumSet<Library.Phase> phases) {
List<Revision> filtered = new ArrayList<Library.Revision>();
for (Revision r : list) if (phases.contains(r.phase)) filtered.add(r);
return filtered;
}
/**
* Checks if an HTTP request matches this function and its constraints.
*
* @param http http context
* @return result of check
*/
boolean matches(final HTTPContext http) {
// check method, path, consumed and produced media type
return methods.contains(http.method) && pathMatches(http) && consumes(http) && produces(http);
}
/**
* Checks a function for RESTFful annotations.
*
* @return {@code true} if module contains relevant annotations
* @throws QueryException query exception
*/
boolean analyze() throws QueryException {
// parse all annotations
final EnumSet<HTTPMethod> mth = EnumSet.noneOf(HTTPMethod.class);
final boolean[] declared = new boolean[function.args.length];
boolean found = false;
final int as = function.ann.size();
for (int a = 0; a < as; a++) {
final QNm name = function.ann.names[a];
final Value value = function.ann.values[a];
final byte[] local = name.local();
final byte[] uri = name.uri();
final boolean rexq = eq(uri, QueryText.RESTXQURI);
if (rexq) {
if (eq(PATH, local)) {
// annotation "path"
if (path != null) error(ANN_TWICE, "%", name.string());
path = new RestXqPath(toString(value, name));
for (final String s : path) {
if (s.trim().startsWith("{")) checkVariable(s, AtomType.AAT, declared);
}
} else if (eq(CONSUMES, local)) {
// annotation "consumes"
strings(value, name, consumes);
} else if (eq(PRODUCES, local)) {
// annotation "produces"
strings(value, name, produces);
} else if (eq(QUERY_PARAM, local)) {
// annotation "query-param"
queryParams.add(param(value, name, declared));
} else if (eq(FORM_PARAM, local)) {
// annotation "form-param"
formParams.add(param(value, name, declared));
} else if (eq(HEADER_PARAM, local)) {
// annotation "header-param"
headerParams.add(param(value, name, declared));
} else if (eq(COOKIE_PARAM, local)) {
// annotation "cookie-param"
cookieParams.add(param(value, name, declared));
} else {
// method annotations
final HTTPMethod m = HTTPMethod.get(string(local));
if (m == null) error(ANN_UNKNOWN, "%", name.string());
if (!value.isEmpty()) {
// remember post/put variable
if (requestBody != null) error(ANN_TWICE, "%", name.string());
if (m != POST && m != PUT) error(METHOD_VALUE, m);
requestBody = checkVariable(toString(value, name), declared);
}
if (mth.contains(m)) error(ANN_TWICE, "%", name.string());
mth.add(m);
}
} else if (eq(uri, QueryText.OUTPUTURI)) {
// serialization parameters
final String key = string(local);
final String val = toString(value, name);
if (output.get(key) == null) error(UNKNOWN_SER, key);
output.set(key, val);
}
found |= rexq;
}
if (!mth.isEmpty()) methods = mth;
if (found) {
if (path == null) error(ANN_MISSING, PATH);
for (int i = 0; i < declared.length; i++)
if (!declared[i]) error(VAR_UNDEFINED, function.args[i].name.string());
}
return found;
}