private File getHfdsFileToTmpFile(String hdfsPath, HdfsConfiguration configuration) {
try {
String fname = hdfsPath.substring(hdfsPath.lastIndexOf('/'));
File outputDest = File.createTempFile(fname, ".hdfs");
if (outputDest.exists()) {
outputDest.delete();
}
HdfsInfo hdfsInfo = HdfsInfoFactory.newHdfsInfo(hdfsPath);
FileSystem fileSystem = hdfsInfo.getFileSystem();
FileUtil.copy(fileSystem, new Path(hdfsPath), outputDest, false, fileSystem.getConf());
try {
FileUtil.copyMerge(
fileSystem, // src
new Path(hdfsPath),
FileSystem.getLocal(new Configuration()), // dest
new Path(outputDest.toURI()),
false,
fileSystem.getConf(),
null);
} catch (IOException e) {
return outputDest;
}
return new File(outputDest, fname);
} catch (IOException ex) {
throw new RuntimeCamelException(ex);
}
}
@BeforeClass
public static void clusterSetupAtBegining()
throws IOException, LoginException, URISyntaxException {
SupportsBlocks = true;
CONF.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
cluster =
new MiniDFSCluster.Builder(CONF)
.nnTopology(MiniDFSNNTopology.simpleFederatedTopology(2))
.numDataNodes(2)
.build();
cluster.waitClusterUp();
fHdfs = cluster.getFileSystem(0);
fHdfs2 = cluster.getFileSystem(1);
fHdfs
.getConf()
.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, FsConstants.VIEWFS_URI.toString());
fHdfs2
.getConf()
.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, FsConstants.VIEWFS_URI.toString());
defaultWorkingDirectory =
fHdfs.makeQualified(
new Path("/user/" + UserGroupInformation.getCurrentUser().getShortUserName()));
defaultWorkingDirectory2 =
fHdfs2.makeQualified(
new Path("/user/" + UserGroupInformation.getCurrentUser().getShortUserName()));
fHdfs.mkdirs(defaultWorkingDirectory);
fHdfs2.mkdirs(defaultWorkingDirectory2);
}
@Test
public void testNonDefaultFS() throws IOException {
FileSystem fs = cluster.getFileSystem();
Configuration conf = fs.getConf();
conf.set(DFSConfigKeys.FS_DEFAULT_NAME_KEY, fs.getUri().toString());
TestTrash.trashNonDefaultFS(conf);
}
@SuppressWarnings("unchecked")
public static <V extends Writable> List<V> readValues(Path path, FileSystem fs, int max) {
List<V> list = new ArrayList<V>();
try {
int k = 0;
SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, fs.getConf());
Writable key = (Writable) reader.getKeyClass().newInstance();
V value = (V) reader.getValueClass().newInstance();
while (reader.next(key, value)) {
k++;
list.add(value);
if (k >= max) {
break;
}
value = (V) reader.getValueClass().newInstance();
}
reader.close();
} catch (Exception e) {
throw new RuntimeException("Error reading SequenceFile " + path);
}
return list;
}
private void writeBuffer(
String destPath, ArrayList<Pair<LogFileKey, LogFileValue>> buffer, int part)
throws IOException {
Path path = new Path(destPath, String.format("part-r-%05d", part++));
FileSystem ns = fs.getFileSystemByPath(path);
@SuppressWarnings("deprecation")
MapFile.Writer output =
new MapFile.Writer(
ns.getConf(), ns, path.toString(), LogFileKey.class, LogFileValue.class);
try {
Collections.sort(
buffer,
new Comparator<Pair<LogFileKey, LogFileValue>>() {
@Override
public int compare(
Pair<LogFileKey, LogFileValue> o1, Pair<LogFileKey, LogFileValue> o2) {
return o1.getFirst().compareTo(o2.getFirst());
}
});
for (Pair<LogFileKey, LogFileValue> entry : buffer) {
output.append(entry.getFirst(), entry.getSecond());
}
} finally {
output.close();
}
}
/**
* Reads key-value pairs from a SequenceFile, up to a maximum number.
*
* @param path path to file
* @param max maximum of key-value pairs to read
* @return list of key-value pairs
*/
@SuppressWarnings("unchecked")
public static <K extends Writable, V extends Writable> List<PairOfWritables<K, V>> readFile(
Path path, FileSystem fs, int max) throws IOException {
List<PairOfWritables<K, V>> list = new ArrayList<PairOfWritables<K, V>>();
try {
int k = 0;
SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, fs.getConf());
K key;
V value;
key = (K) reader.getKeyClass().newInstance();
value = (V) reader.getValueClass().newInstance();
while (reader.next(key, value)) {
k++;
list.add(new PairOfWritables<K, V>(key, value));
if (k >= max) {
break;
}
// Create new objects, because the key, value gets reused
key = (K) reader.getKeyClass().newInstance();
value = (V) reader.getValueClass().newInstance();
}
reader.close();
} catch (IllegalAccessException e) {
throw new RuntimeException("Error reading SequenceFile: " + e);
} catch (InstantiationException e) {
throw new RuntimeException("Error reading SequenceFile: " + e);
}
return list;
}
/** Constructor */
public DFSAdminCommand(FileSystem fs) {
super(fs.getConf());
if (!(fs instanceof DistributedFileSystem)) {
throw new IllegalArgumentException(
"FileSystem " + fs.getUri() + " is not a distributed file system");
}
this.dfs = (DistributedFileSystem) fs;
}
// creates a file but does not close it
static FSDataOutputStream createFile(FileSystem fileSys, Path name, int repl) throws IOException {
System.out.println("createFile: Created " + name + " with " + repl + " replica.");
FSDataOutputStream stm =
fileSys.create(
name,
true,
fileSys.getConf().getInt("io.file.buffer.size", 4096),
(short) repl,
(long) blockSize);
return stm;
}
// creates a file and returns a descriptor for writing to it
private static FSDataOutputStream writeFile(
FileSystem fileSys, Path name, short repl, long blockSize) throws IOException {
FSDataOutputStream stm =
fileSys.create(
name,
true,
fileSys.getConf().getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096),
repl,
blockSize);
return stm;
}
private void writeFile(FileSystem fileSys, Path name, int repl) throws IOException {
FSDataOutputStream stm =
fileSys.create(
name,
true,
fileSys.getConf().getInt("io.file.buffer.size", 4096),
(short) repl,
(long) blockSize);
byte[] buffer = new byte[fileSize];
Random rand = new Random(seed);
rand.nextBytes(buffer);
stm.write(buffer);
stm.close();
}
private FSDataOutputStream writeFile(FileSystem fileSys, Path name, int repl) throws IOException {
FSDataOutputStream stm =
fileSys.create(
name,
true,
fileSys.getConf().getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096),
(short) repl,
blockSize);
byte[] buffer = new byte[fileSize];
Random rand = new Random(seed);
rand.nextBytes(buffer);
stm.write(buffer);
return stm;
}
private FileSystemWriter(FileSystem fs, Path path, DatasetDescriptor descriptor) {
Preconditions.checkNotNull(fs, "File system is not defined");
Preconditions.checkNotNull(path, "Destination directory is not defined");
Preconditions.checkNotNull(descriptor, "Descriptor is not defined");
this.fs = fs;
this.directory = path;
this.descriptor = descriptor;
this.conf = new Configuration(fs.getConf());
this.state = ReaderWriterState.NEW;
// copy file format settings from custom properties to the Configuration
for (String prop : descriptor.listProperties()) {
conf.set(prop, descriptor.getProperty(prop));
}
}
private Collection<TestDataModel> collectContent(FileSystem fs, FileStatus status)
throws IOException {
Collection<TestDataModel> results = new ArrayList<TestDataModel>();
ModelInput<TestDataModel> input =
TemporaryStorage.openInput(fs.getConf(), TestDataModel.class, status.getPath());
try {
TestDataModel model = new TestDataModel();
while (input.readTo(model)) {
results.add(model.copy());
}
} finally {
input.close();
}
return results;
}
// creates a file using DistributedFileSystem.createNonRecursive()
static FSDataOutputStream createNonRecursive(
FileSystem fs, Path name, int repl, boolean overwrite) throws IOException {
System.out.println("createNonRecursive: Created " + name + " with " + repl + " replica.");
FSDataOutputStream stm =
((DistributedFileSystem) fs)
.createNonRecursive(
name,
FsPermission.getDefault(),
overwrite,
fs.getConf().getInt("io.file.buffer.size", 4096),
(short) repl,
(long) blockSize,
null);
return stm;
}
//
// creates a file and populate it with data.
//
private void createOldFile(FileSystem fileSys, Path name, int repl, int numBlocks, long blocksize)
throws IOException {
FSDataOutputStream stm =
fileSys.create(
name,
true,
fileSys.getConf().getInt("io.file.buffer.size", 4096),
(short) repl,
blocksize);
// fill data into file
final byte[] b = new byte[(int) blocksize];
for (int i = 0; i < numBlocks; i++) {
stm.write(b);
}
stm.close();
}
@Test
public void testDirectoryFilterUnfixableFiles() throws Exception {
conf = new Configuration();
dfsCluster = new MiniDFSCluster(conf, NUM_DATANODES, true, null);
dfsCluster.waitActive();
FileSystem fs = dfsCluster.getFileSystem();
Utils.loadTestCodecs(conf, 3, 5, 1, 3, "/destraid", "/destraidrs", false, true);
try {
Configuration testConf = fs.getConf();
BlockIntegrityMonitor blockFixer = new LocalBlockIntegrityMonitor(testConf, false);
String p1 = "/user/foo/f1";
String p2 = "/user/foo/f2";
String p3 = "/user1/foo/bar/f1";
String p4 = "/a/b";
String p5 = "/c";
String p6 = "/destraidrs/user";
String p7 = "/destraid/user1/foo";
fs.mkdirs(new Path(p6));
List<String> fileList = new ArrayList<String>();
fileList.add(p1);
fileList.add(p2);
fileList.add(p3);
fileList.add(p4);
fileList.add(p5);
blockFixer.filterUnreconstructableSourceFiles(fs, fileList.iterator());
// p3 and p5 should be filtered out.
assertEquals(3, fileList.size());
Set<String> filtered = new HashSet<String>();
for (String p : fileList) filtered.add(p);
assertFalse("File not filtered", filtered.contains(p3));
assertFalse("File not filtered", filtered.contains(p5));
fileList.add(p3);
fs.mkdirs(new Path(p7));
blockFixer.filterUnreconstructableSourceFiles(fs, fileList.iterator());
// Nothing is filtered.
assertEquals(4, fileList.size());
} finally {
dfsCluster.shutdown();
}
}
private static int readSequenceFile(Path path, FileSystem fs, int max) throws IOException {
SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, fs.getConf());
System.out.println("Reading " + path + "...\n");
try {
System.out.println("Key type: " + reader.getKeyClass().toString());
System.out.println("Value type: " + reader.getValueClass().toString() + "\n");
} catch (Exception e) {
throw new RuntimeException("Error: loading key/value class");
}
Writable key = null, value;
int n = 0;
try {
if (Tuple.class.isAssignableFrom(reader.getKeyClass())) {
key = TUPLE_FACTORY.newTuple();
} else {
key = (Writable) reader.getKeyClass().newInstance();
}
if (Tuple.class.isAssignableFrom(reader.getValueClass())) {
value = TUPLE_FACTORY.newTuple();
} else {
value = (Writable) reader.getValueClass().newInstance();
}
while (reader.next(key, value)) {
System.out.println("Record " + n);
System.out.println("Key: " + key + "\nValue: " + value);
System.out.println("----------------------------------------");
n++;
if (n >= max) break;
}
reader.close();
System.out.println(n + " records read.\n");
} catch (Exception e) {
e.printStackTrace();
}
return n;
}
private void cloneOutput() throws IOException {
List<FileStatus> listStatus = getOutputMappings();
/*
* Initialize to empty list, in which case swap() will be a no-op. The reference is then replaced with a real list, which is
* used in the subsequent iterations.
*/
List<Path> crushInput = emptyList();
Text srcFile = new Text();
Text crushOut = new Text();
Text prevCrushOut = new Text();
for (FileStatus partFile : listStatus) {
Path path = partFile.getPath();
Reader reader = new Reader(fs, path, fs.getConf());
try {
while (reader.next(srcFile, crushOut)) {
if (!crushOut.equals(prevCrushOut)) {
swap(crushInput, prevCrushOut.toString());
prevCrushOut.set(crushOut);
crushInput = new LinkedList<Path>();
}
crushInput.add(new Path(srcFile.toString()));
}
} finally {
try {
reader.close();
} catch (IOException e) {
LOG.warn("Trapped exception when closing " + path, e);
}
}
swap(crushInput, prevCrushOut.toString());
}
}
public static RawSequenceFileWriter createWriter(
FileSystem fs,
Configuration conf,
Path name,
Class keyClass,
Class valClass,
CompressionType compressionType)
throws IOException {
return new RawSequenceFileWriter(
fs,
conf,
name,
keyClass,
valClass,
fs.getConf().getInt("io.file.buffer.size", 65536),
fs.getDefaultReplication(),
fs.getDefaultBlockSize(),
null,
new Metadata());
}
/**
* Get token from the token sequence file.
*
* @param authPath
* @param proxyUserName
* @return Token for proxyUserName if it exists.
* @throws IOException
*/
private Optional<Token> getTokenFromSeqFile(String authPath, String proxyUserName)
throws IOException {
Closer closer = Closer.create();
try {
FileSystem localFs = FileSystem.getLocal(new Configuration());
SequenceFile.Reader tokenReader =
closer.register(new SequenceFile.Reader(localFs, new Path(authPath), localFs.getConf()));
Text key = new Text();
Token value = new Token();
while (tokenReader.next(key, value)) {
LOG.info("Found token for " + key);
if (key.toString().equals(proxyUserName)) {
return Optional.of(value);
}
}
} catch (Throwable t) {
throw closer.rethrow(t);
} finally {
closer.close();
}
return Optional.absent();
}
public static HiveServer create(
Map<String, String> properties,
File baseDir,
File confDir,
File logDir,
FileSystem fileSystem)
throws Exception {
if (!properties.containsKey(WAREHOUSE_DIR)) {
LOGGER.info("fileSystem " + fileSystem.getClass().getSimpleName());
if (fileSystem instanceof DistributedFileSystem) {
String dfsUri = FileSystem.getDefaultUri(fileSystem.getConf()).toString();
LOGGER.info("dfsUri " + dfsUri);
properties.put(WAREHOUSE_DIR, dfsUri + "/data");
fileSystem.mkdirs(new Path("/data/"), new FsPermission((short) 0777));
} else {
properties.put(WAREHOUSE_DIR, new File(baseDir, "warehouse").getPath());
fileSystem.mkdirs(new Path("/", "warehouse"), new FsPermission((short) 0777));
}
}
LOGGER.info("Setting an readable path to hive.exec.scratchdir");
properties.put("hive.exec.scratchdir", new File(baseDir, "scratchdir").getPath());
if (!properties.containsKey(METASTORE_CONNECTION_URL)) {
properties.put(
METASTORE_CONNECTION_URL,
String.format(
"jdbc:derby:;databaseName=%s;create=true", new File(baseDir, "metastore").getPath()));
}
if (!properties.containsKey(HS2_PORT)) {
properties.put(HS2_PORT, String.valueOf(findPort()));
}
if (!properties.containsKey(SUPPORT_CONCURRENCY)) {
properties.put(SUPPORT_CONCURRENCY, "false");
}
if (!properties.containsKey(HADOOPBIN)) {
properties.put(HADOOPBIN, "./target/test-classes/hadoop");
}
// Modify the test resource to have executable permission
java.nio.file.Path hadoopPath =
FileSystems.getDefault().getPath("target/test-classes", "hadoop");
if (hadoopPath != null) {
hadoopPath.toFile().setExecutable(true);
}
properties.put(METASTORE_SETUGI, "true");
properties.put(METASTORE_CLIENT_TIMEOUT, "100");
properties.put(ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS.varname, "true");
properties.put(ConfVars.HIVESTATSAUTOGATHER.varname, "false");
properties.put(ConfVars.HIVE_STATS_COLLECT_SCANCOLS.varname, "true");
String hadoopBinPath = properties.get(HADOOPBIN);
Assert.assertNotNull(hadoopBinPath, "Hadoop Bin");
File hadoopBin = new File(hadoopBinPath);
if (!hadoopBin.isFile()) {
Assert.fail(
"Path to hadoop bin "
+ hadoopBin.getPath()
+ " is invalid. "
+ "Perhaps you missed the download-hadoop profile.");
}
/*
* This hack, setting the hiveSiteURL field removes a previous hack involving
* setting of system properties for each property. Although both are hacks,
* I prefer this hack because once the system properties are set they can
* affect later tests unless those tests clear them. This hack allows for
* a clean switch to a new set of defaults when a new HiveConf object is created.
*/
Reflection.staticField("hiveSiteURL").ofType(URL.class).in(HiveConf.class).set(null);
HiveConf hiveConf = new HiveConf();
for (Map.Entry<String, String> entry : properties.entrySet()) {
LOGGER.info(entry.getKey() + " => " + entry.getValue());
hiveConf.set(entry.getKey(), entry.getValue());
}
File hiveSite = new File(confDir, "hive-site.xml");
hiveConf.set(HIVESERVER2_IMPERSONATION, "false");
OutputStream out = new FileOutputStream(hiveSite);
hiveConf.writeXml(out);
out.close();
Reflection.staticField("hiveSiteURL")
.ofType(URL.class)
.in(HiveConf.class)
.set(hiveSite.toURI().toURL());
LOGGER.info("Creating InternalHiveServer");
return new InternalHiveServer(hiveConf);
}
@Override
public void visit(State state, Properties props) throws Exception {
boolean userExists = SecurityHelper.getTabUserExists(state);
Connector conn;
try {
conn =
state
.getInstance()
.getConnector(
SecurityHelper.getTabUserName(state), SecurityHelper.getTabUserPass(state));
} catch (AccumuloSecurityException ae) {
if (ae.getErrorCode().equals(SecurityErrorCode.BAD_CREDENTIALS)) {
if (userExists)
throw new AccumuloException(
"User didn't exist when they should (or worse- password mismatch)", ae);
else return;
}
throw new AccumuloException("Unexpected exception!", ae);
}
String action = props.getProperty("action", "_random");
TablePermission tp;
if ("_random".equalsIgnoreCase(action)) {
Random r = new Random();
tp = TablePermission.values()[r.nextInt(TablePermission.values().length)];
} else {
tp = TablePermission.valueOf(action);
}
boolean tableExists = SecurityHelper.getTableExists(state);
boolean hasPerm = SecurityHelper.getTabPerm(state, SecurityHelper.getTabUserName(state), tp);
String tableName = state.getString("secTableName");
switch (tp) {
case READ:
Authorizations auths =
SecurityHelper.getUserAuths(state, SecurityHelper.getTabUserName(state));
boolean canRead =
SecurityHelper.getTabPerm(
state, SecurityHelper.getTabUserName(state), TablePermission.READ);
try {
Scanner scan =
conn.createScanner(
tableName,
conn.securityOperations()
.getUserAuthorizations(SecurityHelper.getTabUserName(state)));
int seen = 0;
Iterator<Entry<Key, Value>> iter = scan.iterator();
while (iter.hasNext()) {
Entry<Key, Value> entry = iter.next();
Key k = entry.getKey();
seen++;
if (!auths.contains(k.getColumnVisibilityData()))
throw new AccumuloException(
"Got data I should not be capable of seeing: " + k + " table " + tableName);
}
if (!canRead)
throw new AccumuloException(
"Was able to read when I shouldn't have had the perm with connection user "
+ conn.whoami()
+ " table "
+ tableName);
for (Entry<String, Integer> entry : SecurityHelper.getAuthsMap(state).entrySet()) {
if (auths.contains(entry.getKey().getBytes())) seen = seen - entry.getValue();
}
if (seen != 0) throw new AccumuloException("Got mismatched amounts of data");
} catch (TableNotFoundException tnfe) {
if (tableExists)
throw new AccumuloException(
"Accumulo and test suite out of sync: table " + tableName, tnfe);
return;
} catch (AccumuloSecurityException ae) {
if (ae.getErrorCode().equals(SecurityErrorCode.PERMISSION_DENIED)) {
if (canRead)
throw new AccumuloException(
"Table read permission out of sync with Accumulo: table " + tableName, ae);
else return;
}
throw new AccumuloException("Unexpected exception!", ae);
} catch (RuntimeException re) {
if (re.getCause() instanceof AccumuloSecurityException
&& ((AccumuloSecurityException) re.getCause())
.getErrorCode()
.equals(SecurityErrorCode.PERMISSION_DENIED)) {
if (canRead)
throw new AccumuloException(
"Table read permission out of sync with Accumulo: table " + tableName,
re.getCause());
else return;
}
throw new AccumuloException("Unexpected exception!", re);
}
break;
case WRITE:
String key = SecurityHelper.getLastKey(state) + "1";
Mutation m = new Mutation(new Text(key));
for (String s : SecurityHelper.getAuthsArray()) {
m.put(new Text(), new Text(), new ColumnVisibility(s), new Value("value".getBytes()));
}
BatchWriter writer;
try {
writer = conn.createBatchWriter(tableName, 9000l, 0l, 1);
} catch (TableNotFoundException tnfe) {
if (tableExists)
throw new AccumuloException("Table didn't exist when it should have: " + tableName);
return;
}
boolean works = true;
try {
writer.addMutation(m);
} catch (MutationsRejectedException mre) {
throw new AccumuloException("Mutation exception!", mre);
}
if (works)
for (String s : SecurityHelper.getAuthsArray())
SecurityHelper.increaseAuthMap(state, s, 1);
break;
case BULK_IMPORT:
key = SecurityHelper.getLastKey(state) + "1";
SortedSet<Key> keys = new TreeSet<Key>();
for (String s : SecurityHelper.getAuthsArray()) {
Key k = new Key(key, "", "", s);
keys.add(k);
}
Path dir = new Path("/tmp", "bulk_" + UUID.randomUUID().toString());
Path fail = new Path(dir.toString() + "_fail");
FileSystem fs = SecurityHelper.getFs(state);
FileSKVWriter f =
FileOperations.getInstance()
.openWriter(
dir + "/securityBulk." + RFile.EXTENSION,
fs,
fs.getConf(),
AccumuloConfiguration.getDefaultConfiguration());
f.startDefaultLocalityGroup();
fs.mkdirs(fail);
for (Key k : keys) f.append(k, new Value("Value".getBytes()));
f.close();
try {
conn.tableOperations().importDirectory(tableName, dir.toString(), fail.toString(), true);
} catch (TableNotFoundException tnfe) {
if (tableExists)
throw new AccumuloException("Table didn't exist when it should have: " + tableName);
return;
} catch (AccumuloSecurityException ae) {
if (ae.getErrorCode().equals(SecurityErrorCode.PERMISSION_DENIED)) {
if (hasPerm)
throw new AccumuloException(
"Bulk Import failed when it should have worked: " + tableName);
return;
}
throw new AccumuloException("Unexpected exception!", ae);
}
for (String s : SecurityHelper.getAuthsArray()) SecurityHelper.increaseAuthMap(state, s, 1);
fs.delete(dir, true);
fs.delete(fail, true);
if (!hasPerm)
throw new AccumuloException(
"Bulk Import succeeded when it should have failed: " + dir + " table " + tableName);
break;
case ALTER_TABLE:
AlterTable.renameTable(conn, state, tableName, tableName + "plus", hasPerm, tableExists);
break;
case GRANT:
props.setProperty("task", "grant");
props.setProperty("perm", "random");
props.setProperty("source", "table");
props.setProperty("target", "system");
AlterTablePerm.alter(state, props);
break;
case DROP_TABLE:
props.setProperty("source", "table");
DropTable.dropTable(state, props);
break;
}
}
/**
* Log job submitted event to history. Creates a new file in history for the job. if history file
* creation fails, it disables history for all other events.
*
* @param jobConfPath path to job conf xml file in HDFS.
* @param submitTime time when job tracker received the job
* @throws IOException
*/
public void logSubmitted(String jobConfPath, long submitTime, String jobTrackerId)
throws IOException {
if (disableHistory) {
return;
}
// create output stream for logging in hadoop.job.history.location
int defaultBufferSize = logDirFs.getConf().getInt("io.file.buffer.size", 4096);
try {
FSDataOutputStream out = null;
PrintWriter writer = null;
// In case the old JT is still running, but we can't connect to it, we
// should ensure that it won't write to our (new JT's) job history file.
if (logDirFs.exists(logFile)) {
LOG.info("Remove the old history file " + logFile);
logDirFs.delete(logFile, true);
}
out =
logDirFs.create(
logFile,
new FsPermission(HISTORY_FILE_PERMISSION),
true,
defaultBufferSize,
logDirFs.getDefaultReplication(),
jobHistoryBlockSize,
null);
writer = new PrintWriter(out);
fileManager.addWriter(jobId, writer);
// cache it ...
fileManager.setHistoryFile(jobId, logFile);
writers = fileManager.getWriters(jobId);
if (null != writers) {
log(
writers,
RecordTypes.Meta,
new Keys[] {Keys.VERSION},
new String[] {String.valueOf(JobHistory.VERSION)});
}
String jobName = getJobName();
String user = getUserName();
// add to writer as well
log(
writers,
RecordTypes.Job,
new Keys[] {
Keys.JOBID, Keys.JOBNAME, Keys.USER, Keys.SUBMIT_TIME, Keys.JOBCONF, Keys.JOBTRACKERID
},
new String[] {
jobId.toString(), jobName, user,
String.valueOf(submitTime), jobConfPath, jobTrackerId
});
} catch (IOException e) {
// Disable history if we have errors other than in the user log.
disableHistory = true;
}
/* Storing the job conf on the log dir */
Path jobFilePath = new Path(logDir, CoronaJobHistoryFilesManager.getConfFilename(jobId));
fileManager.setConfFile(jobId, jobFilePath);
FSDataOutputStream jobFileOut = null;
try {
if (!logDirFs.exists(jobFilePath)) {
jobFileOut =
logDirFs.create(
jobFilePath,
new FsPermission(HISTORY_FILE_PERMISSION),
true,
defaultBufferSize,
logDirFs.getDefaultReplication(),
logDirFs.getDefaultBlockSize(),
null);
conf.writeXml(jobFileOut);
jobFileOut.close();
}
} catch (IOException ioe) {
LOG.error("Failed to store job conf in the log dir", ioe);
} finally {
if (jobFileOut != null) {
try {
jobFileOut.close();
} catch (IOException ie) {
LOG.info(
"Failed to close the job configuration file " + StringUtils.stringifyException(ie));
}
}
}
}
/**
* Moves the skipped files to the output directory. Called when operation in normal (non-clone)
* mode.
*/
private void moveOutput() throws IOException {
List<FileStatus> listStatus = getOutputMappings();
Text srcFile = new Text();
Text crushOut = new Text();
Set<String> crushOutputFiles = new HashSet<String>(nBuckets);
for (FileStatus partFile : listStatus) {
Path path = partFile.getPath();
Reader reader = new Reader(fs, path, fs.getConf());
try {
while (reader.next(srcFile, crushOut)) {
crushOutputFiles.add(new Path(crushOut.toString()).toUri().getPath());
}
} finally {
try {
reader.close();
} catch (IOException e) {
LOG.warn("Trapped exception when closing " + path, e);
}
}
}
assert crushOutputFiles.size() == nBuckets;
/*
* The crushoutput files will appear in a subdirectory of the output directory. The subdirectory will be the full path of the
* input directory that was crushed. E.g.
*
* Crush input:
* /user/me/input/dir1/file1
* /user/me/input/dir1/file2
* /user/me/input/dir2/file3
* /user/me/input/dir2/file4
* /user/me/input/dir3/dir4/file5
* /user/me/input/dir3/dir4/file6
*
* Crush output:
* /user/me/output/user/me/input/dir1/crushed_file ...
* /user/me/output/user/me/input/dir2/crushed_file ...
* /user/me/output/user/me/input/dir2/dir3/dir4/crushed_file ...
*
* We need to collapse this down to:
* /user/me/output/dir1/crushed_file ...
* /user/me/output/dir2/crushed_file ...
* /user/me/output/dir2/dir3/dir4/crushed_file ...
*/
String srcDirName = fs.makeQualified(srcDir).toUri().getPath();
String destName = fs.makeQualified(dest).toUri().getPath();
String partToReplace = fs.makeQualified(outDir).toUri().getPath() + "/crush" + srcDirName;
print(Verbosity.INFO, "\n\nCopying crush files to " + destName);
for (String crushOutputFile : crushOutputFiles) {
Path srcPath = new Path(crushOutputFile);
Path destPath =
new Path(destName + crushOutputFile.substring(partToReplace.length())).getParent();
rename(srcPath, destPath, null);
}
print(Verbosity.INFO, "\n\nMoving skipped files to " + destName);
/*
* Don't forget to move the files that were not crushed to the output dir so that the output dir has all the data that was in
* the input dir, the difference being there are fewer files in the output dir.
*/
for (String name : skippedFiles) {
Path srcPath = new Path(name);
Path destPath = new Path(destName + name.substring(srcDirName.length())).getParent();
rename(srcPath, destPath, null);
}
}
public FileLinkInputStream(final FileSystem fs, final FileLink fileLink) throws IOException {
this(fs, fileLink, fs.getConf().getInt("io.file.buffer.size", 4096));
}
@Override
public Object getConf() {
return mFs.getConf();
}
static void bulkLoadLots(Logger log, State state, Environment env, Value value) throws Exception {
final Path dir = new Path("/tmp", "bulk_" + UUID.randomUUID().toString());
final Path fail = new Path(dir.toString() + "_fail");
final DefaultConfiguration defaultConfiguration =
AccumuloConfiguration.getDefaultConfiguration();
final Random rand = (Random) state.get("rand");
final FileSystem fs = (FileSystem) state.get("fs");
fs.mkdirs(fail);
final int parts = rand.nextInt(10) + 1;
TreeSet<Integer> startRows = new TreeSet<Integer>();
startRows.add(0);
while (startRows.size() < parts) startRows.add(rand.nextInt(LOTS));
List<String> printRows = new ArrayList<String>(startRows.size());
for (Integer row : startRows) printRows.add(String.format(FMT, row));
String markerColumnQualifier = String.format("%07d", counter.incrementAndGet());
log.debug(
"preparing bulk files with start rows "
+ printRows
+ " last row "
+ String.format(FMT, LOTS - 1)
+ " marker "
+ markerColumnQualifier);
List<Integer> rows = new ArrayList<Integer>(startRows);
rows.add(LOTS);
for (int i = 0; i < parts; i++) {
String fileName = dir + "/" + String.format("part_%d.", i) + RFile.EXTENSION;
FileSKVWriter f =
FileOperations.getInstance().openWriter(fileName, fs, fs.getConf(), defaultConfiguration);
f.startDefaultLocalityGroup();
int start = rows.get(i);
int end = rows.get(i + 1);
for (int j = start; j < end; j++) {
Text row = new Text(String.format(FMT, j));
for (Column col : COLNAMES) {
f.append(new Key(row, col.getColumnFamily(), col.getColumnQualifier()), value);
}
f.append(new Key(row, MARKER_CF, new Text(markerColumnQualifier)), ONE);
}
f.close();
}
env.getConnector()
.tableOperations()
.importDirectory(Setup.getTableName(), dir.toString(), fail.toString(), true);
fs.delete(dir, true);
FileStatus[] failures = fs.listStatus(fail);
if (failures != null && failures.length > 0) {
state.set("bulkImportSuccess", "false");
throw new Exception(failures.length + " failure files found importing files from " + dir);
}
fs.delete(fail, true);
log.debug(
"Finished bulk import, start rows "
+ printRows
+ " last row "
+ String.format(FMT, LOTS - 1)
+ " marker "
+ markerColumnQualifier);
}
@SuppressWarnings("static-access")
public URI getDFSUri() throws IOException {
return fileSystem.getDefaultUri(fileSystem.getConf());
}
/**
* Test case that stops a writer after finalizing a block but before calling completeFile,
* recovers a file from another writer, starts writing from that writer, and then has the old
* lease holder call completeFile
*/
@Test(timeout = 60000)
public void testCompleteOtherLeaseHoldersFile() throws Throwable {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
try {
cluster.waitActive();
NameNode preSpyNN = cluster.getNameNode();
NameNode spyNN = spy(preSpyNN);
// Delay completeFile
DelayAnswer delayer = new DelayAnswer();
doAnswer(delayer).when(spyNN).complete(anyString(), anyString(), (Block) anyObject());
DFSClient client = new DFSClient(null, spyNN, conf, null);
file1 = new Path("/testCompleteOtherLease");
final OutputStream stm = client.create("/testCompleteOtherLease", true);
// write 1/2 block
AppendTestUtil.write(stm, 0, 4096);
final AtomicReference<Throwable> err = new AtomicReference<Throwable>();
Thread t =
new Thread() {
public void run() {
try {
stm.close();
} catch (Throwable t) {
err.set(t);
}
}
};
t.start();
LOG.info("Waiting for close to get to latch...");
delayer.waitForCall();
// At this point, the block is finalized on the DNs, but the file
// has not been completed in the NN.
// Lose the leases
LOG.info("Killing lease checker");
client.leasechecker.interruptAndJoin();
FileSystem fs1 = cluster.getFileSystem();
FileSystem fs2 = AppendTestUtil.createHdfsWithDifferentUsername(fs1.getConf());
LOG.info("Recovering file");
recoverFile(fs2);
LOG.info("Opening file for append from new fs");
FSDataOutputStream appenderStream = fs2.append(file1);
LOG.info("Writing some data from new appender");
AppendTestUtil.write(appenderStream, 0, 4096);
LOG.info("Telling old close to proceed.");
delayer.proceed();
LOG.info("Waiting for close to finish.");
t.join();
LOG.info("Close finished.");
// We expect that close will get a "Lease mismatch"
// error.
Throwable thrownByClose = err.get();
assertNotNull(thrownByClose);
assertTrue(thrownByClose instanceof IOException);
if (!thrownByClose.getMessage().contains("Lease mismatch")) throw thrownByClose;
// The appender should be able to close properly
appenderStream.close();
} finally {
cluster.shutdown();
}
}
@BeforeClass
public static void setupTestStaticConfiguration() throws Exception {
LOGGER.info("AbstractTestWithStaticConfiguration setupTestStaticConfiguration");
properties = Maps.newHashMap();
if (!policyOnHdfs) {
policyOnHdfs = new Boolean(System.getProperty("sentry.e2etest.policyonhdfs", "false"));
}
if (testServerType != null) {
properties.put("sentry.e2etest.hiveServer2Type", testServerType);
}
baseDir = Files.createTempDir();
LOGGER.info("BaseDir = " + baseDir);
logDir = assertCreateDir(new File(baseDir, "log"));
confDir = assertCreateDir(new File(baseDir, "etc"));
dataDir = assertCreateDir(new File(baseDir, "data"));
policyFileLocation = new File(confDir, HiveServerFactory.AUTHZ_PROVIDER_FILENAME);
String dfsType = System.getProperty(DFSFactory.FS_TYPE);
dfs = DFSFactory.create(dfsType, baseDir, testServerType);
fileSystem = dfs.getFileSystem();
PolicyFile policyFile =
PolicyFile.setAdminOnServer1(ADMIN1)
.setUserGroupMapping(StaticUserGroup.getStaticMapping());
policyFile.write(policyFileLocation);
String policyURI;
if (policyOnHdfs) {
String dfsUri = FileSystem.getDefaultUri(fileSystem.getConf()).toString();
LOGGER.error("dfsUri " + dfsUri);
policyURI =
dfsUri + System.getProperty("sentry.e2etest.hive.policy.location", "/user/hive/sentry");
policyURI += "/" + HiveServerFactory.AUTHZ_PROVIDER_FILENAME;
} else {
policyURI = policyFileLocation.getPath();
}
boolean startSentry = new Boolean(System.getProperty(EXTERNAL_SENTRY_SERVICE, "false"));
if ("true".equalsIgnoreCase(System.getProperty(ENABLE_SENTRY_HA, "false"))) {
enableSentryHA = true;
}
if (useSentryService && (!startSentry)) {
setupSentryService();
}
if (enableHiveConcurrency) {
properties.put(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "true");
properties.put(
HiveConf.ConfVars.HIVE_TXN_MANAGER.varname,
"org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager");
properties.put(
HiveConf.ConfVars.HIVE_LOCK_MANAGER.varname,
"org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager");
}
hiveServer = create(properties, baseDir, confDir, logDir, policyURI, fileSystem);
hiveServer.start();
createContext();
// Create tmp as scratch dir if it doesn't exist
Path tmpPath = new Path("/tmp");
if (!fileSystem.exists(tmpPath)) {
fileSystem.mkdirs(tmpPath, new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL));
}
}
