Esempio n. 1
0
  private static void finalize(
      Configuration conf, JobConf jobconf, final Path destPath, String presevedAttributes)
      throws IOException {
    if (presevedAttributes == null) {
      return;
    }
    EnumSet<FileAttribute> preseved = FileAttribute.parse(presevedAttributes);
    if (!preseved.contains(FileAttribute.USER)
        && !preseved.contains(FileAttribute.GROUP)
        && !preseved.contains(FileAttribute.PERMISSION)) {
      return;
    }

    FileSystem dstfs = destPath.getFileSystem(conf);
    Path dstdirlist = new Path(jobconf.get(DST_DIR_LIST_LABEL));
    SequenceFile.Reader in = null;
    try {
      in = new SequenceFile.Reader(dstdirlist.getFileSystem(jobconf), dstdirlist, jobconf);
      Text dsttext = new Text();
      FilePair pair = new FilePair();
      for (; in.next(dsttext, pair); ) {
        Path absdst = new Path(destPath, pair.output);
        updatePermissions(pair.input, dstfs.getFileStatus(absdst), preseved, dstfs);
      }
    } finally {
      checkAndClose(in);
    }
  }
Esempio n. 2
0
 private static void updatePermissions(
     FileStatus src, FileStatus dst, EnumSet<FileAttribute> preseved, FileSystem destFileSys)
     throws IOException {
   String owner = null;
   String group = null;
   if (preseved.contains(FileAttribute.USER) && !src.getOwner().equals(dst.getOwner())) {
     owner = src.getOwner();
   }
   if (preseved.contains(FileAttribute.GROUP) && !src.getGroup().equals(dst.getGroup())) {
     group = src.getGroup();
   }
   if (owner != null || group != null) {
     destFileSys.setOwner(dst.getPath(), owner, group);
   }
   if (preseved.contains(FileAttribute.PERMISSION)
       && !src.getPermission().equals(dst.getPermission())) {
     destFileSys.setPermission(dst.getPath(), src.getPermission());
   }
 }
Esempio n. 3
0
    private FSDataOutputStream create(Path f, Reporter reporter, FileStatus srcstat)
        throws IOException {
      if (destFileSys.exists(f)) {
        destFileSys.delete(f, false);
      }
      if (!preserve_status) {
        return destFileSys.create(f, true, sizeBuf, reporter);
      }

      FsPermission permission =
          preseved.contains(FileAttribute.PERMISSION) ? srcstat.getPermission() : null;
      short replication =
          preseved.contains(FileAttribute.REPLICATION)
              ? srcstat.getReplication()
              : destFileSys.getDefaultReplication();
      long blockSize =
          preseved.contains(FileAttribute.BLOCK_SIZE)
              ? srcstat.getBlockSize()
              : destFileSys.getDefaultBlockSize();
      return destFileSys.create(f, permission, true, sizeBuf, replication, blockSize, reporter);
    }
Esempio n. 4
0
    static EnumSet<FileAttribute> parse(String s) {
      if (s == null || s.length() == 0) {
        return EnumSet.allOf(FileAttribute.class);
      }

      EnumSet<FileAttribute> set = EnumSet.noneOf(FileAttribute.class);
      FileAttribute[] attributes = values();
      for (char c : s.toCharArray()) {
        int i = 0;
        for (; i < attributes.length && c != attributes[i].symbol; i++) ;
        if (i < attributes.length) {
          if (!set.contains(attributes[i])) {
            set.add(attributes[i]);
          } else {
            throw new IllegalArgumentException(
                "There are more than one '" + attributes[i].symbol + "' in " + s);
          }
        } else {
          throw new IllegalArgumentException("'" + c + "' in " + s + " is undefined.");
        }
      }
      return set;
    }
Esempio n. 5
0
    static Arguments valueOf(String[] args, Configuration conf) throws IOException {
      List<Path> srcs = new ArrayList<Path>();
      Path dst = null;
      Path log = null;
      EnumSet<Options> flags = EnumSet.noneOf(Options.class);
      String presevedAttributes = null;
      String mapredSslConf = null;
      long filelimit = Long.MAX_VALUE;
      long sizelimit = Long.MAX_VALUE;

      for (int idx = 0; idx < args.length; idx++) {
        Options[] opt = Options.values();
        int i = 0;
        for (; i < opt.length && !args[idx].startsWith(opt[i].cmd); i++) ;

        if (i < opt.length) {
          flags.add(opt[i]);
          if (opt[i] == Options.PRESERVE_STATUS) {
            presevedAttributes = args[idx].substring(2);
            FileAttribute.parse(presevedAttributes); // validation
          } else if (opt[i] == Options.FILE_LIMIT) {
            filelimit = Options.FILE_LIMIT.parseLong(args, ++idx);
          } else if (opt[i] == Options.SIZE_LIMIT) {
            sizelimit = Options.SIZE_LIMIT.parseLong(args, ++idx);
          }
        } else if ("-f".equals(args[idx])) {
          if (++idx == args.length) {
            throw new IllegalArgumentException("urilist_uri not specified in -f");
          }
          srcs.addAll(fetchFileList(conf, new Path(args[idx])));
        } else if ("-log".equals(args[idx])) {
          if (++idx == args.length) {
            throw new IllegalArgumentException("logdir not specified in -log");
          }
          log = new Path(args[idx]);
        } else if ("-mapredSslConf".equals(args[idx])) {
          if (++idx == args.length) {
            throw new IllegalArgumentException("ssl conf file not specified in -mapredSslConf");
          }
          mapredSslConf = args[idx];
        } else if ("-m".equals(args[idx])) {
          if (++idx == args.length) {
            throw new IllegalArgumentException("num_maps not specified in -m");
          }
          try {
            conf.setInt(MAX_MAPS_LABEL, Integer.valueOf(args[idx]));
          } catch (NumberFormatException e) {
            throw new IllegalArgumentException("Invalid argument to -m: " + args[idx]);
          }
        } else if ('-' == args[idx].codePointAt(0)) {
          throw new IllegalArgumentException("Invalid switch " + args[idx]);
        } else if (idx == args.length - 1) {
          dst = new Path(args[idx]);
        } else {
          srcs.add(new Path(args[idx]));
        }
      }
      // mandatory command-line parameters
      if (srcs.isEmpty() || dst == null) {
        throw new IllegalArgumentException("Missing " + (dst == null ? "dst path" : "src"));
      }
      // incompatible command-line flags
      final boolean isOverwrite = flags.contains(Options.OVERWRITE);
      final boolean isUpdate = flags.contains(Options.UPDATE);
      final boolean isDelete = flags.contains(Options.DELETE);
      if (isOverwrite && isUpdate) {
        throw new IllegalArgumentException("Conflicting overwrite policies");
      }
      if (isDelete && !isOverwrite && !isUpdate) {
        throw new IllegalArgumentException(
            Options.DELETE.cmd
                + " must be specified with "
                + Options.OVERWRITE
                + " or "
                + Options.UPDATE
                + ".");
      }
      return new Arguments(
          srcs, dst, log, flags, presevedAttributes, filelimit, sizelimit, mapredSslConf);
    }
Esempio n. 6
0
  public final List<Sample> parse(
      Reader reader, EnumSet<MissingPedField> missingFields, SampleDB sampleDB) {
    final List<String> lines = new XReadLines(reader).readLines();

    // What are the record offsets?
    final int familyPos = missingFields.contains(MissingPedField.NO_FAMILY_ID) ? -1 : 0;
    final int samplePos = familyPos + 1;
    final int paternalPos = missingFields.contains(MissingPedField.NO_PARENTS) ? -1 : samplePos + 1;
    final int maternalPos =
        missingFields.contains(MissingPedField.NO_PARENTS) ? -1 : paternalPos + 1;
    final int sexPos =
        missingFields.contains(MissingPedField.NO_SEX) ? -1 : Math.max(maternalPos, samplePos) + 1;
    final int phenotypePos =
        missingFields.contains(MissingPedField.NO_PHENOTYPE)
            ? -1
            : Math.max(sexPos, Math.max(maternalPos, samplePos)) + 1;
    final int nExpectedFields =
        MathUtils.arrayMaxInt(
                Arrays.asList(samplePos, paternalPos, maternalPos, sexPos, phenotypePos))
            + 1;

    // go through once and determine properties
    int lineNo = 1;
    boolean isQT = false;
    final List<String[]> splits = new ArrayList<String[]>(lines.size());
    for (final String line : lines) {
      if (line.startsWith(commentMarker)) continue;
      if (line.trim().equals("")) continue;

      final String[] parts = line.split("\\s+");

      if (parts.length != nExpectedFields)
        throw new UserException.MalformedFile(
            reader.toString(), "Bad PED line " + lineNo + ": wrong number of fields");

      if (phenotypePos != -1) {
        isQT = isQT || !CATAGORICAL_TRAIT_VALUES.contains(parts[phenotypePos]);
      }

      splits.add(parts);
      lineNo++;
    }
    logger.info("Phenotype is other? " + isQT);

    // now go through and parse each record
    lineNo = 1;
    final List<Sample> samples = new ArrayList<Sample>(splits.size());
    for (final String[] parts : splits) {
      String familyID = null, individualID, paternalID = null, maternalID = null;
      Gender sex = Gender.UNKNOWN;
      String quantitativePhenotype = Sample.UNSET_QT;
      Affection affection = Affection.UNKNOWN;

      if (familyPos != -1) familyID = maybeMissing(parts[familyPos]);
      individualID = parts[samplePos];
      if (paternalPos != -1) paternalID = maybeMissing(parts[paternalPos]);
      if (maternalPos != -1) maternalID = maybeMissing(parts[maternalPos]);

      if (sexPos != -1) {
        if (parts[sexPos].equals(SEX_MALE)) sex = Gender.MALE;
        else if (parts[sexPos].equals(SEX_FEMALE)) sex = Gender.FEMALE;
        else sex = Gender.UNKNOWN;
      }

      if (phenotypePos != -1) {
        if (isQT) {
          if (parts[phenotypePos].equals(MISSING_VALUE1)) affection = Affection.UNKNOWN;
          else {
            affection = Affection.OTHER;
            quantitativePhenotype = parts[phenotypePos];
          }
        } else {
          if (parts[phenotypePos].equals(MISSING_VALUE1)) affection = Affection.UNKNOWN;
          else if (parts[phenotypePos].equals(MISSING_VALUE2)) affection = Affection.UNKNOWN;
          else if (parts[phenotypePos].equals(PHENOTYPE_UNAFFECTED))
            affection = Affection.UNAFFECTED;
          else if (parts[phenotypePos].equals(PHENOTYPE_AFFECTED)) affection = Affection.AFFECTED;
          else
            throw new ReviewedGATKException(
                "Unexpected phenotype type " + parts[phenotypePos] + " at line " + lineNo);
        }
      }

      final Sample s =
          new Sample(
              individualID,
              sampleDB,
              familyID,
              paternalID,
              maternalID,
              sex,
              affection,
              quantitativePhenotype);
      samples.add(s);
      sampleDB.addSample(s);
      lineNo++;
    }

    for (final Sample sample : new ArrayList<Sample>(samples)) {
      Sample dad =
          maybeAddImplicitSample(
              sampleDB, sample.getPaternalID(), sample.getFamilyID(), Gender.MALE);
      if (dad != null) samples.add(dad);

      Sample mom =
          maybeAddImplicitSample(
              sampleDB, sample.getMaternalID(), sample.getFamilyID(), Gender.FEMALE);
      if (mom != null) samples.add(mom);
    }

    return samples;
  }
Esempio n. 7
0
  public List<Revision> filter(Collection<Revision> list, EnumSet<Library.Phase> phases) {
    List<Revision> filtered = new ArrayList<Library.Revision>();
    for (Revision r : list) if (phases.contains(r.phase)) filtered.add(r);

    return filtered;
  }
Esempio n. 8
0
 /**
  * Checks if an HTTP request matches this function and its constraints.
  *
  * @param http http context
  * @return result of check
  */
 boolean matches(final HTTPContext http) {
   // check method, path, consumed and produced media type
   return methods.contains(http.method) && pathMatches(http) && consumes(http) && produces(http);
 }
Esempio n. 9
0
  /**
   * Checks a function for RESTFful annotations.
   *
   * @return {@code true} if module contains relevant annotations
   * @throws QueryException query exception
   */
  boolean analyze() throws QueryException {
    // parse all annotations
    final EnumSet<HTTPMethod> mth = EnumSet.noneOf(HTTPMethod.class);
    final boolean[] declared = new boolean[function.args.length];
    boolean found = false;
    final int as = function.ann.size();
    for (int a = 0; a < as; a++) {
      final QNm name = function.ann.names[a];
      final Value value = function.ann.values[a];
      final byte[] local = name.local();
      final byte[] uri = name.uri();
      final boolean rexq = eq(uri, QueryText.RESTXQURI);
      if (rexq) {
        if (eq(PATH, local)) {
          // annotation "path"
          if (path != null) error(ANN_TWICE, "%", name.string());
          path = new RestXqPath(toString(value, name));
          for (final String s : path) {
            if (s.trim().startsWith("{")) checkVariable(s, AtomType.AAT, declared);
          }
        } else if (eq(CONSUMES, local)) {
          // annotation "consumes"
          strings(value, name, consumes);
        } else if (eq(PRODUCES, local)) {
          // annotation "produces"
          strings(value, name, produces);
        } else if (eq(QUERY_PARAM, local)) {
          // annotation "query-param"
          queryParams.add(param(value, name, declared));
        } else if (eq(FORM_PARAM, local)) {
          // annotation "form-param"
          formParams.add(param(value, name, declared));
        } else if (eq(HEADER_PARAM, local)) {
          // annotation "header-param"
          headerParams.add(param(value, name, declared));
        } else if (eq(COOKIE_PARAM, local)) {
          // annotation "cookie-param"
          cookieParams.add(param(value, name, declared));
        } else {
          // method annotations
          final HTTPMethod m = HTTPMethod.get(string(local));
          if (m == null) error(ANN_UNKNOWN, "%", name.string());
          if (!value.isEmpty()) {
            // remember post/put variable
            if (requestBody != null) error(ANN_TWICE, "%", name.string());
            if (m != POST && m != PUT) error(METHOD_VALUE, m);
            requestBody = checkVariable(toString(value, name), declared);
          }
          if (mth.contains(m)) error(ANN_TWICE, "%", name.string());
          mth.add(m);
        }
      } else if (eq(uri, QueryText.OUTPUTURI)) {
        // serialization parameters
        final String key = string(local);
        final String val = toString(value, name);
        if (output.get(key) == null) error(UNKNOWN_SER, key);
        output.set(key, val);
      }
      found |= rexq;
    }
    if (!mth.isEmpty()) methods = mth;

    if (found) {
      if (path == null) error(ANN_MISSING, PATH);
      for (int i = 0; i < declared.length; i++)
        if (!declared[i]) error(VAR_UNDEFINED, function.args[i].name.string());
    }
    return found;
  }