Ejemplo n.º 1
1
    private File getHfdsFileToTmpFile(String hdfsPath, HdfsConfiguration configuration) {
      try {
        String fname = hdfsPath.substring(hdfsPath.lastIndexOf('/'));

        File outputDest = File.createTempFile(fname, ".hdfs");
        if (outputDest.exists()) {
          outputDest.delete();
        }

        HdfsInfo hdfsInfo = HdfsInfoFactory.newHdfsInfo(hdfsPath);
        FileSystem fileSystem = hdfsInfo.getFileSystem();
        FileUtil.copy(fileSystem, new Path(hdfsPath), outputDest, false, fileSystem.getConf());
        try {
          FileUtil.copyMerge(
              fileSystem, // src
              new Path(hdfsPath),
              FileSystem.getLocal(new Configuration()), // dest
              new Path(outputDest.toURI()),
              false,
              fileSystem.getConf(),
              null);
        } catch (IOException e) {
          return outputDest;
        }

        return new File(outputDest, fname);
      } catch (IOException ex) {
        throw new RuntimeCamelException(ex);
      }
    }
Ejemplo n.º 2
0
  @BeforeClass
  public static void clusterSetupAtBegining()
      throws IOException, LoginException, URISyntaxException {
    SupportsBlocks = true;
    CONF.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);

    cluster =
        new MiniDFSCluster.Builder(CONF)
            .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(2))
            .numDataNodes(2)
            .build();
    cluster.waitClusterUp();

    fHdfs = cluster.getFileSystem(0);
    fHdfs2 = cluster.getFileSystem(1);
    fHdfs
        .getConf()
        .set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, FsConstants.VIEWFS_URI.toString());
    fHdfs2
        .getConf()
        .set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, FsConstants.VIEWFS_URI.toString());

    defaultWorkingDirectory =
        fHdfs.makeQualified(
            new Path("/user/" + UserGroupInformation.getCurrentUser().getShortUserName()));
    defaultWorkingDirectory2 =
        fHdfs2.makeQualified(
            new Path("/user/" + UserGroupInformation.getCurrentUser().getShortUserName()));

    fHdfs.mkdirs(defaultWorkingDirectory);
    fHdfs2.mkdirs(defaultWorkingDirectory2);
  }
Ejemplo n.º 3
0
 @Test
 public void testNonDefaultFS() throws IOException {
   FileSystem fs = cluster.getFileSystem();
   Configuration conf = fs.getConf();
   conf.set(DFSConfigKeys.FS_DEFAULT_NAME_KEY, fs.getUri().toString());
   TestTrash.trashNonDefaultFS(conf);
 }
Ejemplo n.º 4
0
  @SuppressWarnings("unchecked")
  public static <V extends Writable> List<V> readValues(Path path, FileSystem fs, int max) {
    List<V> list = new ArrayList<V>();

    try {
      int k = 0;
      SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, fs.getConf());

      Writable key = (Writable) reader.getKeyClass().newInstance();
      V value = (V) reader.getValueClass().newInstance();

      while (reader.next(key, value)) {
        k++;
        list.add(value);
        if (k >= max) {
          break;
        }

        value = (V) reader.getValueClass().newInstance();
      }
      reader.close();
    } catch (Exception e) {
      throw new RuntimeException("Error reading SequenceFile " + path);
    }

    return list;
  }
Ejemplo n.º 5
0
    private void writeBuffer(
        String destPath, ArrayList<Pair<LogFileKey, LogFileValue>> buffer, int part)
        throws IOException {
      Path path = new Path(destPath, String.format("part-r-%05d", part++));
      FileSystem ns = fs.getFileSystemByPath(path);

      @SuppressWarnings("deprecation")
      MapFile.Writer output =
          new MapFile.Writer(
              ns.getConf(), ns, path.toString(), LogFileKey.class, LogFileValue.class);
      try {
        Collections.sort(
            buffer,
            new Comparator<Pair<LogFileKey, LogFileValue>>() {
              @Override
              public int compare(
                  Pair<LogFileKey, LogFileValue> o1, Pair<LogFileKey, LogFileValue> o2) {
                return o1.getFirst().compareTo(o2.getFirst());
              }
            });
        for (Pair<LogFileKey, LogFileValue> entry : buffer) {
          output.append(entry.getFirst(), entry.getSecond());
        }
      } finally {
        output.close();
      }
    }
Ejemplo n.º 6
0
  /**
   * Reads key-value pairs from a SequenceFile, up to a maximum number.
   *
   * @param path path to file
   * @param max maximum of key-value pairs to read
   * @return list of key-value pairs
   */
  @SuppressWarnings("unchecked")
  public static <K extends Writable, V extends Writable> List<PairOfWritables<K, V>> readFile(
      Path path, FileSystem fs, int max) throws IOException {
    List<PairOfWritables<K, V>> list = new ArrayList<PairOfWritables<K, V>>();

    try {
      int k = 0;
      SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, fs.getConf());

      K key;
      V value;

      key = (K) reader.getKeyClass().newInstance();
      value = (V) reader.getValueClass().newInstance();

      while (reader.next(key, value)) {
        k++;
        list.add(new PairOfWritables<K, V>(key, value));
        if (k >= max) {
          break;
        }

        // Create new objects, because the key, value gets reused
        key = (K) reader.getKeyClass().newInstance();
        value = (V) reader.getValueClass().newInstance();
      }
      reader.close();
    } catch (IllegalAccessException e) {
      throw new RuntimeException("Error reading SequenceFile: " + e);
    } catch (InstantiationException e) {
      throw new RuntimeException("Error reading SequenceFile: " + e);
    }

    return list;
  }
Ejemplo n.º 7
0
 /** Constructor */
 public DFSAdminCommand(FileSystem fs) {
   super(fs.getConf());
   if (!(fs instanceof DistributedFileSystem)) {
     throw new IllegalArgumentException(
         "FileSystem " + fs.getUri() + " is not a distributed file system");
   }
   this.dfs = (DistributedFileSystem) fs;
 }
Ejemplo n.º 8
0
 // creates a file but does not close it
 static FSDataOutputStream createFile(FileSystem fileSys, Path name, int repl) throws IOException {
   System.out.println("createFile: Created " + name + " with " + repl + " replica.");
   FSDataOutputStream stm =
       fileSys.create(
           name,
           true,
           fileSys.getConf().getInt("io.file.buffer.size", 4096),
           (short) repl,
           (long) blockSize);
   return stm;
 }
 // creates a file and returns a descriptor for writing to it
 private static FSDataOutputStream writeFile(
     FileSystem fileSys, Path name, short repl, long blockSize) throws IOException {
   FSDataOutputStream stm =
       fileSys.create(
           name,
           true,
           fileSys.getConf().getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096),
           repl,
           blockSize);
   return stm;
 }
Ejemplo n.º 10
0
 private void writeFile(FileSystem fileSys, Path name, int repl) throws IOException {
   FSDataOutputStream stm =
       fileSys.create(
           name,
           true,
           fileSys.getConf().getInt("io.file.buffer.size", 4096),
           (short) repl,
           (long) blockSize);
   byte[] buffer = new byte[fileSize];
   Random rand = new Random(seed);
   rand.nextBytes(buffer);
   stm.write(buffer);
   stm.close();
 }
 private FSDataOutputStream writeFile(FileSystem fileSys, Path name, int repl) throws IOException {
   FSDataOutputStream stm =
       fileSys.create(
           name,
           true,
           fileSys.getConf().getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096),
           (short) repl,
           blockSize);
   byte[] buffer = new byte[fileSize];
   Random rand = new Random(seed);
   rand.nextBytes(buffer);
   stm.write(buffer);
   return stm;
 }
Ejemplo n.º 12
0
  private FileSystemWriter(FileSystem fs, Path path, DatasetDescriptor descriptor) {
    Preconditions.checkNotNull(fs, "File system is not defined");
    Preconditions.checkNotNull(path, "Destination directory is not defined");
    Preconditions.checkNotNull(descriptor, "Descriptor is not defined");
    this.fs = fs;
    this.directory = path;
    this.descriptor = descriptor;
    this.conf = new Configuration(fs.getConf());
    this.state = ReaderWriterState.NEW;

    // copy file format settings from custom properties to the Configuration
    for (String prop : descriptor.listProperties()) {
      conf.set(prop, descriptor.getProperty(prop));
    }
  }
Ejemplo n.º 13
0
 private Collection<TestDataModel> collectContent(FileSystem fs, FileStatus status)
     throws IOException {
   Collection<TestDataModel> results = new ArrayList<TestDataModel>();
   ModelInput<TestDataModel> input =
       TemporaryStorage.openInput(fs.getConf(), TestDataModel.class, status.getPath());
   try {
     TestDataModel model = new TestDataModel();
     while (input.readTo(model)) {
       results.add(model.copy());
     }
   } finally {
     input.close();
   }
   return results;
 }
Ejemplo n.º 14
0
  // creates a file using DistributedFileSystem.createNonRecursive()
  static FSDataOutputStream createNonRecursive(
      FileSystem fs, Path name, int repl, boolean overwrite) throws IOException {
    System.out.println("createNonRecursive: Created " + name + " with " + repl + " replica.");
    FSDataOutputStream stm =
        ((DistributedFileSystem) fs)
            .createNonRecursive(
                name,
                FsPermission.getDefault(),
                overwrite,
                fs.getConf().getInt("io.file.buffer.size", 4096),
                (short) repl,
                (long) blockSize,
                null);

    return stm;
  }
 //
 // creates a file and populate it with data.
 //
 private void createOldFile(FileSystem fileSys, Path name, int repl, int numBlocks, long blocksize)
     throws IOException {
   FSDataOutputStream stm =
       fileSys.create(
           name,
           true,
           fileSys.getConf().getInt("io.file.buffer.size", 4096),
           (short) repl,
           blocksize);
   // fill data into file
   final byte[] b = new byte[(int) blocksize];
   for (int i = 0; i < numBlocks; i++) {
     stm.write(b);
   }
   stm.close();
 }
  @Test
  public void testDirectoryFilterUnfixableFiles() throws Exception {
    conf = new Configuration();
    dfsCluster = new MiniDFSCluster(conf, NUM_DATANODES, true, null);
    dfsCluster.waitActive();
    FileSystem fs = dfsCluster.getFileSystem();

    Utils.loadTestCodecs(conf, 3, 5, 1, 3, "/destraid", "/destraidrs", false, true);
    try {
      Configuration testConf = fs.getConf();
      BlockIntegrityMonitor blockFixer = new LocalBlockIntegrityMonitor(testConf, false);

      String p1 = "/user/foo/f1";
      String p2 = "/user/foo/f2";
      String p3 = "/user1/foo/bar/f1";
      String p4 = "/a/b";
      String p5 = "/c";
      String p6 = "/destraidrs/user";
      String p7 = "/destraid/user1/foo";

      fs.mkdirs(new Path(p6));

      List<String> fileList = new ArrayList<String>();
      fileList.add(p1);
      fileList.add(p2);
      fileList.add(p3);
      fileList.add(p4);
      fileList.add(p5);

      blockFixer.filterUnreconstructableSourceFiles(fs, fileList.iterator());
      // p3 and p5 should be filtered out.
      assertEquals(3, fileList.size());

      Set<String> filtered = new HashSet<String>();
      for (String p : fileList) filtered.add(p);
      assertFalse("File not filtered", filtered.contains(p3));
      assertFalse("File not filtered", filtered.contains(p5));

      fileList.add(p3);
      fs.mkdirs(new Path(p7));
      blockFixer.filterUnreconstructableSourceFiles(fs, fileList.iterator());
      // Nothing is filtered.
      assertEquals(4, fileList.size());
    } finally {
      dfsCluster.shutdown();
    }
  }
Ejemplo n.º 17
0
  private static int readSequenceFile(Path path, FileSystem fs, int max) throws IOException {
    SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, fs.getConf());

    System.out.println("Reading " + path + "...\n");
    try {
      System.out.println("Key type: " + reader.getKeyClass().toString());
      System.out.println("Value type: " + reader.getValueClass().toString() + "\n");
    } catch (Exception e) {
      throw new RuntimeException("Error: loading key/value class");
    }

    Writable key = null, value;
    int n = 0;
    try {
      if (Tuple.class.isAssignableFrom(reader.getKeyClass())) {
        key = TUPLE_FACTORY.newTuple();
      } else {
        key = (Writable) reader.getKeyClass().newInstance();
      }

      if (Tuple.class.isAssignableFrom(reader.getValueClass())) {
        value = TUPLE_FACTORY.newTuple();
      } else {
        value = (Writable) reader.getValueClass().newInstance();
      }

      while (reader.next(key, value)) {
        System.out.println("Record " + n);
        System.out.println("Key: " + key + "\nValue: " + value);
        System.out.println("----------------------------------------");
        n++;

        if (n >= max) break;
      }
      reader.close();
      System.out.println(n + " records read.\n");
    } catch (Exception e) {
      e.printStackTrace();
    }

    return n;
  }
Ejemplo n.º 18
0
  private void cloneOutput() throws IOException {

    List<FileStatus> listStatus = getOutputMappings();

    /*
     * Initialize to empty list, in which case swap() will be a no-op. The reference is then replaced with a real list, which is
     * used in the subsequent iterations.
     */
    List<Path> crushInput = emptyList();

    Text srcFile = new Text();
    Text crushOut = new Text();
    Text prevCrushOut = new Text();

    for (FileStatus partFile : listStatus) {
      Path path = partFile.getPath();

      Reader reader = new Reader(fs, path, fs.getConf());

      try {
        while (reader.next(srcFile, crushOut)) {
          if (!crushOut.equals(prevCrushOut)) {
            swap(crushInput, prevCrushOut.toString());

            prevCrushOut.set(crushOut);
            crushInput = new LinkedList<Path>();
          }

          crushInput.add(new Path(srcFile.toString()));
        }
      } finally {
        try {
          reader.close();
        } catch (IOException e) {
          LOG.warn("Trapped exception when closing " + path, e);
        }
      }

      swap(crushInput, prevCrushOut.toString());
    }
  }
  public static RawSequenceFileWriter createWriter(
      FileSystem fs,
      Configuration conf,
      Path name,
      Class keyClass,
      Class valClass,
      CompressionType compressionType)
      throws IOException {

    return new RawSequenceFileWriter(
        fs,
        conf,
        name,
        keyClass,
        valClass,
        fs.getConf().getInt("io.file.buffer.size", 65536),
        fs.getDefaultReplication(),
        fs.getDefaultBlockSize(),
        null,
        new Metadata());
  }
 /**
  * Get token from the token sequence file.
  *
  * @param authPath
  * @param proxyUserName
  * @return Token for proxyUserName if it exists.
  * @throws IOException
  */
 private Optional<Token> getTokenFromSeqFile(String authPath, String proxyUserName)
     throws IOException {
   Closer closer = Closer.create();
   try {
     FileSystem localFs = FileSystem.getLocal(new Configuration());
     SequenceFile.Reader tokenReader =
         closer.register(new SequenceFile.Reader(localFs, new Path(authPath), localFs.getConf()));
     Text key = new Text();
     Token value = new Token();
     while (tokenReader.next(key, value)) {
       LOG.info("Found token for " + key);
       if (key.toString().equals(proxyUserName)) {
         return Optional.of(value);
       }
     }
   } catch (Throwable t) {
     throw closer.rethrow(t);
   } finally {
     closer.close();
   }
   return Optional.absent();
 }
Ejemplo n.º 21
0
  public static HiveServer create(
      Map<String, String> properties,
      File baseDir,
      File confDir,
      File logDir,
      FileSystem fileSystem)
      throws Exception {

    if (!properties.containsKey(WAREHOUSE_DIR)) {
      LOGGER.info("fileSystem " + fileSystem.getClass().getSimpleName());
      if (fileSystem instanceof DistributedFileSystem) {
        String dfsUri = FileSystem.getDefaultUri(fileSystem.getConf()).toString();
        LOGGER.info("dfsUri " + dfsUri);
        properties.put(WAREHOUSE_DIR, dfsUri + "/data");
        fileSystem.mkdirs(new Path("/data/"), new FsPermission((short) 0777));
      } else {
        properties.put(WAREHOUSE_DIR, new File(baseDir, "warehouse").getPath());
        fileSystem.mkdirs(new Path("/", "warehouse"), new FsPermission((short) 0777));
      }
    }
    LOGGER.info("Setting an readable path to hive.exec.scratchdir");
    properties.put("hive.exec.scratchdir", new File(baseDir, "scratchdir").getPath());

    if (!properties.containsKey(METASTORE_CONNECTION_URL)) {
      properties.put(
          METASTORE_CONNECTION_URL,
          String.format(
              "jdbc:derby:;databaseName=%s;create=true", new File(baseDir, "metastore").getPath()));
    }
    if (!properties.containsKey(HS2_PORT)) {
      properties.put(HS2_PORT, String.valueOf(findPort()));
    }
    if (!properties.containsKey(SUPPORT_CONCURRENCY)) {
      properties.put(SUPPORT_CONCURRENCY, "false");
    }
    if (!properties.containsKey(HADOOPBIN)) {
      properties.put(HADOOPBIN, "./target/test-classes/hadoop");
    }

    // Modify the test resource to have executable permission
    java.nio.file.Path hadoopPath =
        FileSystems.getDefault().getPath("target/test-classes", "hadoop");
    if (hadoopPath != null) {
      hadoopPath.toFile().setExecutable(true);
    }

    properties.put(METASTORE_SETUGI, "true");
    properties.put(METASTORE_CLIENT_TIMEOUT, "100");
    properties.put(ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS.varname, "true");

    properties.put(ConfVars.HIVESTATSAUTOGATHER.varname, "false");
    properties.put(ConfVars.HIVE_STATS_COLLECT_SCANCOLS.varname, "true");
    String hadoopBinPath = properties.get(HADOOPBIN);
    Assert.assertNotNull(hadoopBinPath, "Hadoop Bin");
    File hadoopBin = new File(hadoopBinPath);
    if (!hadoopBin.isFile()) {
      Assert.fail(
          "Path to hadoop bin "
              + hadoopBin.getPath()
              + " is invalid. "
              + "Perhaps you missed the download-hadoop profile.");
    }

    /*
     * This hack, setting the hiveSiteURL field removes a previous hack involving
     * setting of system properties for each property. Although both are hacks,
     * I prefer this hack because once the system properties are set they can
     * affect later tests unless those tests clear them. This hack allows for
     * a clean switch to a new set of defaults when a new HiveConf object is created.
     */
    Reflection.staticField("hiveSiteURL").ofType(URL.class).in(HiveConf.class).set(null);
    HiveConf hiveConf = new HiveConf();
    for (Map.Entry<String, String> entry : properties.entrySet()) {
      LOGGER.info(entry.getKey() + " => " + entry.getValue());
      hiveConf.set(entry.getKey(), entry.getValue());
    }
    File hiveSite = new File(confDir, "hive-site.xml");

    hiveConf.set(HIVESERVER2_IMPERSONATION, "false");
    OutputStream out = new FileOutputStream(hiveSite);
    hiveConf.writeXml(out);
    out.close();

    Reflection.staticField("hiveSiteURL")
        .ofType(URL.class)
        .in(HiveConf.class)
        .set(hiveSite.toURI().toURL());

    LOGGER.info("Creating InternalHiveServer");
    return new InternalHiveServer(hiveConf);
  }
Ejemplo n.º 22
0
  @Override
  public void visit(State state, Properties props) throws Exception {
    boolean userExists = SecurityHelper.getTabUserExists(state);
    Connector conn;
    try {
      conn =
          state
              .getInstance()
              .getConnector(
                  SecurityHelper.getTabUserName(state), SecurityHelper.getTabUserPass(state));
    } catch (AccumuloSecurityException ae) {
      if (ae.getErrorCode().equals(SecurityErrorCode.BAD_CREDENTIALS)) {
        if (userExists)
          throw new AccumuloException(
              "User didn't exist when they should (or worse- password mismatch)", ae);
        else return;
      }
      throw new AccumuloException("Unexpected exception!", ae);
    }
    String action = props.getProperty("action", "_random");
    TablePermission tp;
    if ("_random".equalsIgnoreCase(action)) {
      Random r = new Random();
      tp = TablePermission.values()[r.nextInt(TablePermission.values().length)];
    } else {
      tp = TablePermission.valueOf(action);
    }

    boolean tableExists = SecurityHelper.getTableExists(state);
    boolean hasPerm = SecurityHelper.getTabPerm(state, SecurityHelper.getTabUserName(state), tp);

    String tableName = state.getString("secTableName");

    switch (tp) {
      case READ:
        Authorizations auths =
            SecurityHelper.getUserAuths(state, SecurityHelper.getTabUserName(state));
        boolean canRead =
            SecurityHelper.getTabPerm(
                state, SecurityHelper.getTabUserName(state), TablePermission.READ);
        try {
          Scanner scan =
              conn.createScanner(
                  tableName,
                  conn.securityOperations()
                      .getUserAuthorizations(SecurityHelper.getTabUserName(state)));
          int seen = 0;
          Iterator<Entry<Key, Value>> iter = scan.iterator();
          while (iter.hasNext()) {
            Entry<Key, Value> entry = iter.next();
            Key k = entry.getKey();
            seen++;
            if (!auths.contains(k.getColumnVisibilityData()))
              throw new AccumuloException(
                  "Got data I should not be capable of seeing: " + k + " table " + tableName);
          }
          if (!canRead)
            throw new AccumuloException(
                "Was able to read when I shouldn't have had the perm with connection user "
                    + conn.whoami()
                    + " table "
                    + tableName);
          for (Entry<String, Integer> entry : SecurityHelper.getAuthsMap(state).entrySet()) {
            if (auths.contains(entry.getKey().getBytes())) seen = seen - entry.getValue();
          }
          if (seen != 0) throw new AccumuloException("Got mismatched amounts of data");
        } catch (TableNotFoundException tnfe) {
          if (tableExists)
            throw new AccumuloException(
                "Accumulo and test suite out of sync: table " + tableName, tnfe);
          return;
        } catch (AccumuloSecurityException ae) {
          if (ae.getErrorCode().equals(SecurityErrorCode.PERMISSION_DENIED)) {
            if (canRead)
              throw new AccumuloException(
                  "Table read permission out of sync with Accumulo: table " + tableName, ae);
            else return;
          }
          throw new AccumuloException("Unexpected exception!", ae);
        } catch (RuntimeException re) {
          if (re.getCause() instanceof AccumuloSecurityException
              && ((AccumuloSecurityException) re.getCause())
                  .getErrorCode()
                  .equals(SecurityErrorCode.PERMISSION_DENIED)) {
            if (canRead)
              throw new AccumuloException(
                  "Table read permission out of sync with Accumulo: table " + tableName,
                  re.getCause());
            else return;
          }
          throw new AccumuloException("Unexpected exception!", re);
        }

        break;
      case WRITE:
        String key = SecurityHelper.getLastKey(state) + "1";
        Mutation m = new Mutation(new Text(key));
        for (String s : SecurityHelper.getAuthsArray()) {
          m.put(new Text(), new Text(), new ColumnVisibility(s), new Value("value".getBytes()));
        }
        BatchWriter writer;
        try {
          writer = conn.createBatchWriter(tableName, 9000l, 0l, 1);
        } catch (TableNotFoundException tnfe) {
          if (tableExists)
            throw new AccumuloException("Table didn't exist when it should have: " + tableName);
          return;
        }
        boolean works = true;
        try {
          writer.addMutation(m);
        } catch (MutationsRejectedException mre) {
          throw new AccumuloException("Mutation exception!", mre);
        }
        if (works)
          for (String s : SecurityHelper.getAuthsArray())
            SecurityHelper.increaseAuthMap(state, s, 1);
        break;
      case BULK_IMPORT:
        key = SecurityHelper.getLastKey(state) + "1";
        SortedSet<Key> keys = new TreeSet<Key>();
        for (String s : SecurityHelper.getAuthsArray()) {
          Key k = new Key(key, "", "", s);
          keys.add(k);
        }
        Path dir = new Path("/tmp", "bulk_" + UUID.randomUUID().toString());
        Path fail = new Path(dir.toString() + "_fail");
        FileSystem fs = SecurityHelper.getFs(state);
        FileSKVWriter f =
            FileOperations.getInstance()
                .openWriter(
                    dir + "/securityBulk." + RFile.EXTENSION,
                    fs,
                    fs.getConf(),
                    AccumuloConfiguration.getDefaultConfiguration());
        f.startDefaultLocalityGroup();
        fs.mkdirs(fail);
        for (Key k : keys) f.append(k, new Value("Value".getBytes()));
        f.close();
        try {
          conn.tableOperations().importDirectory(tableName, dir.toString(), fail.toString(), true);
        } catch (TableNotFoundException tnfe) {
          if (tableExists)
            throw new AccumuloException("Table didn't exist when it should have: " + tableName);
          return;
        } catch (AccumuloSecurityException ae) {
          if (ae.getErrorCode().equals(SecurityErrorCode.PERMISSION_DENIED)) {
            if (hasPerm)
              throw new AccumuloException(
                  "Bulk Import failed when it should have worked: " + tableName);
            return;
          }
          throw new AccumuloException("Unexpected exception!", ae);
        }
        for (String s : SecurityHelper.getAuthsArray()) SecurityHelper.increaseAuthMap(state, s, 1);
        fs.delete(dir, true);
        fs.delete(fail, true);

        if (!hasPerm)
          throw new AccumuloException(
              "Bulk Import succeeded when it should have failed: " + dir + " table " + tableName);
        break;
      case ALTER_TABLE:
        AlterTable.renameTable(conn, state, tableName, tableName + "plus", hasPerm, tableExists);
        break;

      case GRANT:
        props.setProperty("task", "grant");
        props.setProperty("perm", "random");
        props.setProperty("source", "table");
        props.setProperty("target", "system");
        AlterTablePerm.alter(state, props);
        break;

      case DROP_TABLE:
        props.setProperty("source", "table");
        DropTable.dropTable(state, props);
        break;
    }
  }
Ejemplo n.º 23
0
  /**
   * Log job submitted event to history. Creates a new file in history for the job. if history file
   * creation fails, it disables history for all other events.
   *
   * @param jobConfPath path to job conf xml file in HDFS.
   * @param submitTime time when job tracker received the job
   * @throws IOException
   */
  public void logSubmitted(String jobConfPath, long submitTime, String jobTrackerId)
      throws IOException {

    if (disableHistory) {
      return;
    }

    // create output stream for logging in hadoop.job.history.location
    int defaultBufferSize = logDirFs.getConf().getInt("io.file.buffer.size", 4096);

    try {
      FSDataOutputStream out = null;
      PrintWriter writer = null;

      // In case the old JT is still running, but we can't connect to it, we
      // should ensure that it won't write to our (new JT's) job history file.
      if (logDirFs.exists(logFile)) {
        LOG.info("Remove the old history file " + logFile);
        logDirFs.delete(logFile, true);
      }

      out =
          logDirFs.create(
              logFile,
              new FsPermission(HISTORY_FILE_PERMISSION),
              true,
              defaultBufferSize,
              logDirFs.getDefaultReplication(),
              jobHistoryBlockSize,
              null);

      writer = new PrintWriter(out);

      fileManager.addWriter(jobId, writer);

      // cache it ...
      fileManager.setHistoryFile(jobId, logFile);

      writers = fileManager.getWriters(jobId);
      if (null != writers) {
        log(
            writers,
            RecordTypes.Meta,
            new Keys[] {Keys.VERSION},
            new String[] {String.valueOf(JobHistory.VERSION)});
      }

      String jobName = getJobName();
      String user = getUserName();

      // add to writer as well
      log(
          writers,
          RecordTypes.Job,
          new Keys[] {
            Keys.JOBID, Keys.JOBNAME, Keys.USER, Keys.SUBMIT_TIME, Keys.JOBCONF, Keys.JOBTRACKERID
          },
          new String[] {
            jobId.toString(), jobName, user,
            String.valueOf(submitTime), jobConfPath, jobTrackerId
          });

    } catch (IOException e) {
      // Disable history if we have errors other than in the user log.
      disableHistory = true;
    }

    /* Storing the job conf on the log dir */
    Path jobFilePath = new Path(logDir, CoronaJobHistoryFilesManager.getConfFilename(jobId));
    fileManager.setConfFile(jobId, jobFilePath);
    FSDataOutputStream jobFileOut = null;
    try {
      if (!logDirFs.exists(jobFilePath)) {
        jobFileOut =
            logDirFs.create(
                jobFilePath,
                new FsPermission(HISTORY_FILE_PERMISSION),
                true,
                defaultBufferSize,
                logDirFs.getDefaultReplication(),
                logDirFs.getDefaultBlockSize(),
                null);
        conf.writeXml(jobFileOut);
        jobFileOut.close();
      }
    } catch (IOException ioe) {
      LOG.error("Failed to store job conf in the log dir", ioe);
    } finally {
      if (jobFileOut != null) {
        try {
          jobFileOut.close();
        } catch (IOException ie) {
          LOG.info(
              "Failed to close the job configuration file " + StringUtils.stringifyException(ie));
        }
      }
    }
  }
Ejemplo n.º 24
0
  /**
   * Moves the skipped files to the output directory. Called when operation in normal (non-clone)
   * mode.
   */
  private void moveOutput() throws IOException {

    List<FileStatus> listStatus = getOutputMappings();

    Text srcFile = new Text();
    Text crushOut = new Text();

    Set<String> crushOutputFiles = new HashSet<String>(nBuckets);

    for (FileStatus partFile : listStatus) {
      Path path = partFile.getPath();

      Reader reader = new Reader(fs, path, fs.getConf());

      try {
        while (reader.next(srcFile, crushOut)) {
          crushOutputFiles.add(new Path(crushOut.toString()).toUri().getPath());
        }
      } finally {
        try {
          reader.close();
        } catch (IOException e) {
          LOG.warn("Trapped exception when closing " + path, e);
        }
      }
    }

    assert crushOutputFiles.size() == nBuckets;

    /*
     * The crushoutput files will appear in a subdirectory of the output directory. The subdirectory will be the full path of the
     * input directory that was crushed. E.g.
     *
     * Crush input:
     * /user/me/input/dir1/file1
     * /user/me/input/dir1/file2
     * /user/me/input/dir2/file3
     * /user/me/input/dir2/file4
     * /user/me/input/dir3/dir4/file5
     * /user/me/input/dir3/dir4/file6
     *
     * Crush output:
     * /user/me/output/user/me/input/dir1/crushed_file ...
     * /user/me/output/user/me/input/dir2/crushed_file ...
     * /user/me/output/user/me/input/dir2/dir3/dir4/crushed_file ...
     *
     * We need to collapse this down to:
     * /user/me/output/dir1/crushed_file ...
     * /user/me/output/dir2/crushed_file ...
     * /user/me/output/dir2/dir3/dir4/crushed_file ...
     */
    String srcDirName = fs.makeQualified(srcDir).toUri().getPath();

    String destName = fs.makeQualified(dest).toUri().getPath();
    String partToReplace = fs.makeQualified(outDir).toUri().getPath() + "/crush" + srcDirName;

    print(Verbosity.INFO, "\n\nCopying crush files to " + destName);

    for (String crushOutputFile : crushOutputFiles) {
      Path srcPath = new Path(crushOutputFile);
      Path destPath =
          new Path(destName + crushOutputFile.substring(partToReplace.length())).getParent();

      rename(srcPath, destPath, null);
    }

    print(Verbosity.INFO, "\n\nMoving skipped files to " + destName);

    /*
     * Don't forget to move the files that were not crushed to the output dir so that the output dir has all the data that was in
     * the input dir, the difference being there are fewer files in the output dir.
     */
    for (String name : skippedFiles) {
      Path srcPath = new Path(name);
      Path destPath = new Path(destName + name.substring(srcDirName.length())).getParent();

      rename(srcPath, destPath, null);
    }
  }
Ejemplo n.º 25
0
 public FileLinkInputStream(final FileSystem fs, final FileLink fileLink) throws IOException {
   this(fs, fileLink, fs.getConf().getInt("io.file.buffer.size", 4096));
 }
Ejemplo n.º 26
0
 @Override
 public Object getConf() {
   return mFs.getConf();
 }
Ejemplo n.º 27
0
  static void bulkLoadLots(Logger log, State state, Environment env, Value value) throws Exception {
    final Path dir = new Path("/tmp", "bulk_" + UUID.randomUUID().toString());
    final Path fail = new Path(dir.toString() + "_fail");
    final DefaultConfiguration defaultConfiguration =
        AccumuloConfiguration.getDefaultConfiguration();
    final Random rand = (Random) state.get("rand");
    final FileSystem fs = (FileSystem) state.get("fs");
    fs.mkdirs(fail);
    final int parts = rand.nextInt(10) + 1;

    TreeSet<Integer> startRows = new TreeSet<Integer>();
    startRows.add(0);
    while (startRows.size() < parts) startRows.add(rand.nextInt(LOTS));

    List<String> printRows = new ArrayList<String>(startRows.size());
    for (Integer row : startRows) printRows.add(String.format(FMT, row));

    String markerColumnQualifier = String.format("%07d", counter.incrementAndGet());
    log.debug(
        "preparing bulk files with start rows "
            + printRows
            + " last row "
            + String.format(FMT, LOTS - 1)
            + " marker "
            + markerColumnQualifier);

    List<Integer> rows = new ArrayList<Integer>(startRows);
    rows.add(LOTS);

    for (int i = 0; i < parts; i++) {
      String fileName = dir + "/" + String.format("part_%d.", i) + RFile.EXTENSION;
      FileSKVWriter f =
          FileOperations.getInstance().openWriter(fileName, fs, fs.getConf(), defaultConfiguration);
      f.startDefaultLocalityGroup();
      int start = rows.get(i);
      int end = rows.get(i + 1);
      for (int j = start; j < end; j++) {
        Text row = new Text(String.format(FMT, j));
        for (Column col : COLNAMES) {
          f.append(new Key(row, col.getColumnFamily(), col.getColumnQualifier()), value);
        }
        f.append(new Key(row, MARKER_CF, new Text(markerColumnQualifier)), ONE);
      }
      f.close();
    }
    env.getConnector()
        .tableOperations()
        .importDirectory(Setup.getTableName(), dir.toString(), fail.toString(), true);
    fs.delete(dir, true);
    FileStatus[] failures = fs.listStatus(fail);
    if (failures != null && failures.length > 0) {
      state.set("bulkImportSuccess", "false");
      throw new Exception(failures.length + " failure files found importing files from " + dir);
    }
    fs.delete(fail, true);
    log.debug(
        "Finished bulk import, start rows "
            + printRows
            + " last row "
            + String.format(FMT, LOTS - 1)
            + " marker "
            + markerColumnQualifier);
  }
Ejemplo n.º 28
0
 @SuppressWarnings("static-access")
 public URI getDFSUri() throws IOException {
   return fileSystem.getDefaultUri(fileSystem.getConf());
 }
Ejemplo n.º 29
0
  /**
   * Test case that stops a writer after finalizing a block but before calling completeFile,
   * recovers a file from another writer, starts writing from that writer, and then has the old
   * lease holder call completeFile
   */
  @Test(timeout = 60000)
  public void testCompleteOtherLeaseHoldersFile() throws Throwable {
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();

    try {
      cluster.waitActive();
      NameNode preSpyNN = cluster.getNameNode();
      NameNode spyNN = spy(preSpyNN);

      // Delay completeFile
      DelayAnswer delayer = new DelayAnswer();
      doAnswer(delayer).when(spyNN).complete(anyString(), anyString(), (Block) anyObject());

      DFSClient client = new DFSClient(null, spyNN, conf, null);
      file1 = new Path("/testCompleteOtherLease");
      final OutputStream stm = client.create("/testCompleteOtherLease", true);

      // write 1/2 block
      AppendTestUtil.write(stm, 0, 4096);
      final AtomicReference<Throwable> err = new AtomicReference<Throwable>();
      Thread t =
          new Thread() {
            public void run() {
              try {
                stm.close();
              } catch (Throwable t) {
                err.set(t);
              }
            }
          };
      t.start();
      LOG.info("Waiting for close to get to latch...");
      delayer.waitForCall();

      // At this point, the block is finalized on the DNs, but the file
      // has not been completed in the NN.
      // Lose the leases
      LOG.info("Killing lease checker");
      client.leasechecker.interruptAndJoin();

      FileSystem fs1 = cluster.getFileSystem();
      FileSystem fs2 = AppendTestUtil.createHdfsWithDifferentUsername(fs1.getConf());

      LOG.info("Recovering file");
      recoverFile(fs2);

      LOG.info("Opening file for append from new fs");
      FSDataOutputStream appenderStream = fs2.append(file1);

      LOG.info("Writing some data from new appender");
      AppendTestUtil.write(appenderStream, 0, 4096);

      LOG.info("Telling old close to proceed.");
      delayer.proceed();
      LOG.info("Waiting for close to finish.");
      t.join();
      LOG.info("Close finished.");

      // We expect that close will get a "Lease mismatch"
      // error.
      Throwable thrownByClose = err.get();
      assertNotNull(thrownByClose);
      assertTrue(thrownByClose instanceof IOException);
      if (!thrownByClose.getMessage().contains("Lease mismatch")) throw thrownByClose;

      // The appender should be able to close properly
      appenderStream.close();
    } finally {
      cluster.shutdown();
    }
  }
  @BeforeClass
  public static void setupTestStaticConfiguration() throws Exception {
    LOGGER.info("AbstractTestWithStaticConfiguration setupTestStaticConfiguration");
    properties = Maps.newHashMap();
    if (!policyOnHdfs) {
      policyOnHdfs = new Boolean(System.getProperty("sentry.e2etest.policyonhdfs", "false"));
    }
    if (testServerType != null) {
      properties.put("sentry.e2etest.hiveServer2Type", testServerType);
    }
    baseDir = Files.createTempDir();
    LOGGER.info("BaseDir = " + baseDir);
    logDir = assertCreateDir(new File(baseDir, "log"));
    confDir = assertCreateDir(new File(baseDir, "etc"));
    dataDir = assertCreateDir(new File(baseDir, "data"));
    policyFileLocation = new File(confDir, HiveServerFactory.AUTHZ_PROVIDER_FILENAME);

    String dfsType = System.getProperty(DFSFactory.FS_TYPE);
    dfs = DFSFactory.create(dfsType, baseDir, testServerType);
    fileSystem = dfs.getFileSystem();

    PolicyFile policyFile =
        PolicyFile.setAdminOnServer1(ADMIN1)
            .setUserGroupMapping(StaticUserGroup.getStaticMapping());
    policyFile.write(policyFileLocation);

    String policyURI;
    if (policyOnHdfs) {
      String dfsUri = FileSystem.getDefaultUri(fileSystem.getConf()).toString();
      LOGGER.error("dfsUri " + dfsUri);
      policyURI =
          dfsUri + System.getProperty("sentry.e2etest.hive.policy.location", "/user/hive/sentry");
      policyURI += "/" + HiveServerFactory.AUTHZ_PROVIDER_FILENAME;
    } else {
      policyURI = policyFileLocation.getPath();
    }

    boolean startSentry = new Boolean(System.getProperty(EXTERNAL_SENTRY_SERVICE, "false"));
    if ("true".equalsIgnoreCase(System.getProperty(ENABLE_SENTRY_HA, "false"))) {
      enableSentryHA = true;
    }
    if (useSentryService && (!startSentry)) {
      setupSentryService();
    }

    if (enableHiveConcurrency) {
      properties.put(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "true");
      properties.put(
          HiveConf.ConfVars.HIVE_TXN_MANAGER.varname,
          "org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager");
      properties.put(
          HiveConf.ConfVars.HIVE_LOCK_MANAGER.varname,
          "org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager");
    }

    hiveServer = create(properties, baseDir, confDir, logDir, policyURI, fileSystem);
    hiveServer.start();
    createContext();

    // Create tmp as scratch dir if it doesn't exist
    Path tmpPath = new Path("/tmp");
    if (!fileSystem.exists(tmpPath)) {
      fileSystem.mkdirs(tmpPath, new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL));
    }
  }