@Test
public void testGetReplicaCount() throws IOException {
Path testPath = new Path(TestUtils.createTempDir().getAbsolutePath());
FileSystem fs = testPath.getFileSystem(new Configuration());
fs.mkdirs(testPath);
assertEquals(HadoopStoreBuilderUtils.getDataChunkFiles(fs, testPath, 0, 0).length, 0);
fs.create(new Path(testPath, "0_0_1.data"));
fs.create(new Path(testPath, "0_0_1data"));
fs.create(new Path(testPath, "0_0_2.index"));
fs.create(new Path(testPath, "0_0.data"));
assertEquals(HadoopStoreBuilderUtils.getDataChunkFiles(fs, testPath, 0, 0).length, 1);
assertEquals(HadoopStoreBuilderUtils.getDataChunkFiles(fs, testPath, 0, 0, 1).length, 1);
assertEquals(HadoopStoreBuilderUtils.getDataChunkFiles(fs, testPath, 1, 0, 1).length, 0);
assertEquals(HadoopStoreBuilderUtils.getDataChunkFiles(fs, testPath, 0, 1).length, 0);
fs.create(new Path(testPath, "1_0_0.data"));
fs.create(new Path(testPath, "1_0"));
assertEquals(HadoopStoreBuilderUtils.getDataChunkFiles(fs, testPath, 1, 0).length, 1);
assertEquals(HadoopStoreBuilderUtils.getDataChunkFiles(fs, testPath, 1, 0, 0).length, 1);
fs.create(new Path(testPath, "1_0_1.data"));
fs.create(new Path(testPath, "1_0_1data"));
fs.create(new Path(testPath, "1_0_1.index"));
assertEquals(HadoopStoreBuilderUtils.getDataChunkFiles(fs, testPath, 1, 0).length, 2);
assertEquals(HadoopStoreBuilderUtils.getDataChunkFiles(fs, testPath, 1, 0, 1).length, 1);
fs.create(new Path(testPath, "1_0_2.data"));
assertEquals(HadoopStoreBuilderUtils.getDataChunkFiles(fs, testPath, 1, 0).length, 3);
}
public void testLease() throws Exception {
Configuration conf = new Configuration();
MiniDFSCluster cluster = new MiniDFSCluster(conf, 2, true, null);
try {
FileSystem fs = cluster.getFileSystem();
assertTrue(fs.mkdirs(dir));
Path a = new Path(dir, "a");
Path b = new Path(dir, "b");
DataOutputStream a_out = fs.create(a);
a_out.writeBytes("something");
assertTrue(hasLease(cluster, a));
assertTrue(!hasLease(cluster, b));
DataOutputStream b_out = fs.create(b);
b_out.writeBytes("something");
assertTrue(hasLease(cluster, a));
assertTrue(hasLease(cluster, b));
a_out.close();
b_out.close();
assertTrue(!hasLease(cluster, a));
assertTrue(!hasLease(cluster, b));
fs.delete(dir, true);
} finally {
if (cluster != null) {
cluster.shutdown();
}
}
}
/**
* Creates an output segment file and sets up the output streams to point at it. If the file
* already exists, retries with a different filename. This is a bit nasty -- after all, {@link
* FileOutputFormat}'s work directory concept is supposed to prevent filename clashes -- but it
* looks like Amazon Elastic MapReduce prevents use of per-task work directories if the output of
* a job is on S3.
*
* <p>TODO: Investigate this and find a better solution.
*/
private void createSegment() throws IOException {
segmentsAttempted = 0;
bytesWritten = 0;
boolean success = false;
while (!success) {
Path path =
workOutputPath.suffix(String.format(extensionFormat, segmentsCreated, segmentsAttempted));
FileSystem fs = path.getFileSystem(conf);
try {
// The o.a.h.mapred OutputFormats overwrite existing files, whereas
// the o.a.h.mapreduce OutputFormats don't overwrite. Bizarre...
// Here, overwrite if progress != null, i.e. if using mapred API.
FSDataOutputStream fsStream =
(progress == null) ? fs.create(path, false) : fs.create(path, progress);
byteStream = new CountingOutputStream(new BufferedOutputStream(fsStream));
dataStream =
new DataOutputStream(codec == null ? byteStream : codec.createOutputStream(byteStream));
segmentsCreated++;
logger.info("Writing to output file: {}", path);
success = true;
} catch (IOException e) {
if (e.getMessage().startsWith("File already exists")) {
logger.warn("Tried to create file {} but it already exists; retrying.", path);
segmentsAttempted++; // retry
} else {
throw e;
}
}
}
}
@Test
public void testReadFileContents() throws Exception {
Path testPath = new Path(TestUtils.createTempDir().getAbsolutePath(), "tempFile");
FileSystem fs = testPath.getFileSystem(new Configuration());
fs.create(testPath);
// 1) Read back empty file
String emptyString = HadoopStoreBuilderUtils.readFileContents(fs, testPath, 1024);
Assert.assertEquals(emptyString.length(), 0);
// 2) Read back random bytes
byte[] randomBytes = writeRandomData(testPath, 10);
// Read back data
Assert.assertEquals(
HadoopStoreBuilderUtils.readFileContents(fs, testPath, 1024), new String(randomBytes));
// 3) Write a json string
fs.delete(testPath, true);
fs.create(testPath);
ReadOnlyStorageMetadata metadata = new ReadOnlyStorageMetadata();
metadata.add(ReadOnlyStorageMetadata.FORMAT, ReadOnlyStorageFormat.READONLY_V2.getCode());
// Write file contents
new FileOutputStream(testPath.toString()).write(metadata.toJsonString().getBytes());
ReadOnlyStorageMetadata readMetadata =
new ReadOnlyStorageMetadata(HadoopStoreBuilderUtils.readFileContents(fs, testPath, 1024));
Assert.assertEquals(
readMetadata.get(ReadOnlyStorageMetadata.FORMAT),
ReadOnlyStorageFormat.READONLY_V2.getCode());
}
@SuppressWarnings("rawtypes")
static Path createConfigurationFileInFs(
FileSystem fs, String appHome, Map stormConf, YarnConfiguration yarnConf) throws IOException {
// dump stringwriter's content into FS conf/storm.yaml
Path confDst =
new Path(fs.getHomeDirectory(), appHome + Path.SEPARATOR + STORM_CONF_PATH_STRING);
Path dirDst = confDst.getParent();
fs.mkdirs(dirDst);
// storm.yaml
FSDataOutputStream out = fs.create(confDst);
Yaml yaml = new Yaml();
OutputStreamWriter writer = new OutputStreamWriter(out);
rmNulls(stormConf);
yaml.dump(stormConf, writer);
writer.close();
out.close();
// yarn-site.xml
Path yarn_site_xml = new Path(dirDst, "yarn-site.xml");
out = fs.create(yarn_site_xml);
writer = new OutputStreamWriter(out);
yarnConf.writeXml(writer);
writer.close();
out.close();
// logback.xml
Path logback_xml = new Path(dirDst, "logback.xml");
out = fs.create(logback_xml);
CreateLogbackXML(out);
out.close();
return dirDst;
}
@Override
public RecordWriter<IEtlKey, CamusWrapper> getDataRecordWriter(
TaskAttemptContext context, String fileName, CamusWrapper data, FileOutputCommitter committer)
throws IOException, InterruptedException {
// If recordDelimiter hasn't been initialized, do so now
if (recordDelimiter == null) {
recordDelimiter =
context.getConfiguration().get(ETL_OUTPUT_RECORD_DELIMITER, DEFAULT_RECORD_DELIMITER);
}
// Get the filename for this RecordWriter.
Path path =
new Path(
committer.getWorkPath(),
EtlMultiOutputFormat.getUniqueFile(context, fileName, getFilenameExtension()));
// final FSDataOutputStream writer =
// path.getFileSystem(context.getConfiguration()).create(path);
FileSystem fs = path.getFileSystem(context.getConfiguration());
DataOutputStream writer; // = fs.create(path, false);
if (isCompressed) {
return new ByteRecordWriter(
new DataOutputStream(codec.createOutputStream(fs.create(path, false))), recordDelimiter);
} else {
return new ByteRecordWriter(fs.create(path, false), recordDelimiter);
}
}
@Override
public void close() {
System.err.println(
"Target: " + vocE.size() + " types. Writing to " + job_.get("root", null) + "/vocab.E");
System.err.println(
"Source: " + vocF.size() + " types .Writing to " + job_.get("root", null) + "/vocab.F");
// write out vocabulary to file
try {
FileSystem fs = FileSystem.get(job_);
DataOutputStream dos =
new DataOutputStream(
new BufferedOutputStream(fs.create(new Path(job_.get("root", null) + "/vocab.E"))));
((VocabularyWritable) vocE).write(dos);
dos.close();
DataOutputStream dos2 =
new DataOutputStream(
new BufferedOutputStream(fs.create(new Path(job_.get("root", null) + "/vocab.F"))));
((VocabularyWritable) vocF).write(dos2);
dos2.close();
} catch (IOException e) {
throw new RuntimeException("Vocab couldn't be written to disk.\n" + e.toString());
}
}
public void testSubworkflowLib() throws Exception {
XConfiguration protoConf = getBaseProtoConf();
WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W");
FileSystem fs = getFileSystem();
Path parentLibJar = new Path(getFsTestCaseDir(), "lib/parentLibrary.jar");
fs.create(parentLibJar);
assertTrue(fs.exists(parentLibJar));
String defaultConf = workflow.getConf();
XConfiguration newConf = new XConfiguration(new StringReader(defaultConf));
newConf.set(OozieClient.LIBPATH, parentLibJar.getParent().toString());
workflow.setConf(newConf.toXmlString());
Path subWorkflowAppPath = new Path(getFsTestCaseDir().toString(), "subwf");
Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml")));
writer.write(APP1);
writer.close();
Path subwfLibJar = new Path(subWorkflowAppPath, "lib/subwfLibrary.jar");
fs.create(subwfLibJar);
assertTrue(fs.exists(subwfLibJar));
final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0);
action.setConf(
"<sub-workflow xmlns='uri:oozie:workflow:0.1' name='subwf'>"
+ " <app-path>"
+ subWorkflowAppPath
+ File.separator
+ "workflow.xml"
+ "</app-path>"
+ "</sub-workflow>");
SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor();
subWorkflow.start(new Context(workflow, action), action);
final OozieClient oozieClient =
subWorkflow.getWorkflowClient(
new Context(workflow, action), SubWorkflowActionExecutor.LOCAL);
waitFor(
JOB_TIMEOUT,
new Predicate() {
public boolean evaluate() throws Exception {
return oozieClient.getJobInfo(action.getExternalId()).getStatus()
== WorkflowJob.Status.SUCCEEDED;
}
});
assertEquals(
WorkflowJob.Status.SUCCEEDED, oozieClient.getJobInfo(action.getExternalId()).getStatus());
subWorkflow.check(new Context(workflow, action), action);
assertEquals(WorkflowAction.Status.DONE, action.getStatus());
subWorkflow.end(new Context(workflow, action), action);
assertEquals(WorkflowAction.Status.OK, action.getStatus());
WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
WorkflowJob wf = oozieClient.getJobInfo(action.getExternalId());
Configuration childConf = new XConfiguration(new StringReader(wf.getConf()));
childConf = wps.createProtoActionConf(childConf, "authToken", true);
assertEquals(childConf.get(WorkflowAppService.APP_LIB_PATH_LIST), subwfLibJar.toString());
}
private static void writeFile(FileSystem fs, Path name, CompressionCodec codec, String contents)
throws IOException {
OutputStream stm;
if (codec == null) {
stm = fs.create(name);
} else {
stm = codec.createOutputStream(fs.create(name));
}
stm.write(contents.getBytes());
stm.close();
}
protected Entity storeEntity(EntityType type, String name, String resource, String writeEndpoint)
throws Exception {
Unmarshaller unmarshaller = type.getUnmarshaller();
ConfigurationStore store = ConfigurationStore.get();
switch (type) {
case CLUSTER:
Cluster cluster = (Cluster) unmarshaller.unmarshal(this.getClass().getResource(resource));
if (name != null) {
store.remove(type, name);
cluster.setName(name);
}
store.publish(type, cluster);
if (writeEndpoint != null) {
ClusterHelper.getInterface(cluster, Interfacetype.WRITE).setEndpoint(writeEndpoint);
FileSystem fs = new Path(writeEndpoint).getFileSystem(EmbeddedCluster.newConfiguration());
fs.create(
new Path(
ClusterHelper.getLocation(cluster, ClusterLocationType.WORKING).getPath(),
"libext/FEED/retention/ext.jar"))
.close();
fs.create(
new Path(
ClusterHelper.getLocation(cluster, ClusterLocationType.WORKING).getPath(),
"libext/FEED/replication/ext.jar"))
.close();
}
return cluster;
case FEED:
Feed feed = (Feed) unmarshaller.unmarshal(this.getClass().getResource(resource));
if (name != null) {
store.remove(type, name);
feed.setName(name);
}
store.publish(type, feed);
return feed;
case PROCESS:
Process process = (Process) unmarshaller.unmarshal(this.getClass().getResource(resource));
if (name != null) {
store.remove(type, name);
process.setName(name);
}
store.publish(type, process);
return process;
default:
}
throw new IllegalArgumentException("Unhandled type: " + type);
}
private List<Path> createTestData() throws Exception {
List<Path> list = new ArrayList<Path>();
Configuration conf = new Configuration();
conf.set("fs.default.name", "hdfs://localhost:8020");
FileSystem fs = FileSystem.get(conf);
fs.mkdirs(new Path("/user/guest"));
fs.setOwner(new Path("/user/guest"), REMOTE_USER, "users");
DateFormat formatter = new SimpleDateFormat("yyyy/MM/dd/HH/mm");
formatter.setTimeZone(TimeZone.getTimeZone("UTC"));
Date date = new Date(System.currentTimeMillis() + 3 * 3600000);
Path path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file");
fs.create(path).close();
date = new Date(date.getTime() - 3600000);
path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file");
fs.create(path).close();
date = new Date(date.getTime() - 3600000);
path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file");
fs.create(path).close();
date = new Date(date.getTime() - 3600000);
path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file");
list.add(path);
fs.create(path).close();
date = new Date(date.getTime() - 3600000);
path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file");
list.add(path);
fs.create(path).close();
date = new Date(date.getTime() - 3600000);
path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file");
list.add(path);
fs.create(path).close();
date = new Date(date.getTime() - 3600000);
path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file");
list.add(path);
fs.create(path).close();
date = new Date(date.getTime() - 3600000);
path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file");
list.add(path);
fs.create(path).close();
date = new Date(date.getTime() - 3600000);
path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file");
list.add(path);
fs.create(path).close();
date = new Date(date.getTime() - 3600000);
path = new Path("/examples/input-data/rawLogs/" + formatter.format(date) + "/file");
list.add(path);
fs.create(path).close();
new FsShell(conf)
.run(new String[] {"-chown", "-R", "guest:users", "/examples/input-data/rawLogs"});
return list;
}
/** See MAPREDUCE-5902 for context on why this test is critical for ecosystem interoperability. */
@org.junit.Test
public void testEncodedPaths() throws Exception {
// FileSystem fs2 = FileSystem.getLocal(new Configuration());
FileSystem fs2 = fs;
Path encodedFiles = new Path("/tmp/encodedTest" + System.currentTimeMillis());
fs2.mkdirs(encodedFiles);
fs2.create(new Path(encodedFiles, "a"));
fs2.create(new Path(encodedFiles, "a%2"));
fs2.create(new Path(encodedFiles, "a%2a"));
fs2.create(new Path(encodedFiles, "a%3a"));
fs2.create(new Path(encodedFiles, "a%4a"));
Assert.assertEquals(5, fs2.listStatus(encodedFiles).length);
fs2.delete(encodedFiles);
}
/** Tests getPos() functionality. */
@Test
public void testGetPos() throws IOException {
final Path testFile = new Path("/testfile+1");
// Write a test file.
FSDataOutputStream out = hdfs.create(testFile, true);
out.writeBytes("0123456789");
out.close();
FSDataInputStream in = hftpFs.open(testFile);
// Test read().
for (int i = 0; i < 5; ++i) {
assertEquals(i, in.getPos());
in.read();
}
// Test read(b, off, len).
assertEquals(5, in.getPos());
byte[] buffer = new byte[10];
assertEquals(2, in.read(buffer, 0, 2));
assertEquals(7, in.getPos());
// Test read(b).
int bytesRead = in.read(buffer);
assertEquals(7 + bytesRead, in.getPos());
// Test EOF.
for (int i = 0; i < 100; ++i) {
in.read();
}
assertEquals(10, in.getPos());
in.close();
}
public Path write(Message... messages) throws Exception {
synchronized (WriteUsingMR.class) {
outputPath = TestUtils.someTemporaryFilePath();
Path inputPath = TestUtils.someTemporaryFilePath();
FileSystem fileSystem = inputPath.getFileSystem(conf);
fileSystem.create(inputPath);
inputMessages = Collections.unmodifiableList(Arrays.asList(messages));
final Job job = new Job(conf, "write");
// input not really used
TextInputFormat.addInputPath(job, inputPath);
job.setInputFormatClass(TextInputFormat.class);
job.setMapperClass(WritingMapper.class);
job.setNumReduceTasks(0);
job.setOutputFormatClass(ProtoParquetOutputFormat.class);
ProtoParquetOutputFormat.setOutputPath(job, outputPath);
ProtoParquetOutputFormat.setProtobufClass(job, TestUtils.inferRecordsClass(messages));
waitForJob(job);
inputMessages = null;
return outputPath;
}
}
private void writeIndexDescriptors(ETwinIndexDescriptor ETwinIndexDescriptor) throws IOException {
Configuration conf = getConf();
FileSystem fs = (new Path(IndexConfig.index.get()).getFileSystem(conf));
FileStatus[] fileStats = fs.globStatus(new Path(IndexConfig.index.get(), "*"));
// We write one indexDescriptor per generated index segment.
// Something to consider: right now it's a straight-up serialized Thrift object.
// Would it be better to do the LzoBase64Line thing, so that we can apply our tools?
// or extend the tools?
for (int i = 0; i < fileStats.length; i++) {
ETwinIndexDescriptor.setIndexPart(i);
FileStatus stat = fileStats[i];
Path idxPath =
new Path(stat.getPath().getParent(), "_" + stat.getPath().getName() + ".indexmeta");
FSDataOutputStream os = fs.create(idxPath, true);
@SuppressWarnings("unchecked")
ThriftWritable<ETwinIndexDescriptor> writable =
(ThriftWritable<ETwinIndexDescriptor>)
ThriftWritable.newInstance(ETwinIndexDescriptor.getClass());
writable.set(ETwinIndexDescriptor);
writable.write(os);
os.close();
}
}
/** {@inheritDoc} */
@Override
public int create(String path) throws IOException {
long startTime = System.currentTimeMillis();
FSDataOutputStream out = fileSystem.create(new Path(path), true);
out.close();
return (int) (System.currentTimeMillis() - startTime);
}
@Override
public void compute() {
long step = this.getSuperstep();
if (step == 2) {
int[] maxScanValue =
(int[])
((ArrayPrimitiveWritable) this.getAggregatedValue(Scan1Computation.MAX_AGG)).get();
logger.info("max scan: {}", step, Arrays.toString(maxScanValue));
String dir =
getConf()
.get(
"mapred.output.dir",
getConf().get("mapreduce.output.fileoutputformat.outputdir"));
String path = dir + "/_vertex_" + maxScanValue[0] + "_value_" + maxScanValue[1];
Path pt = new Path(path);
try {
FileSystem fs = FileSystem.get(new Configuration());
BufferedWriter br = new BufferedWriter(new OutputStreamWriter(fs.create(pt, true)));
br.close();
} catch (IOException e) {
e.printStackTrace();
throw new IllegalStateException("Could not write to file: " + path);
} finally {
this.haltComputation();
}
}
}
@Override
public void setup(Reducer<IntWritable, Text, NullWritable, NullWritable>.Context context) {
Configuration conf = context.getConfiguration();
FileSystem fs;
try {
fs = FileSystem.get(conf);
} catch (Exception e) {
throw new RuntimeException("Error opening the FileSystem!");
}
RetrievalEnvironment env = null;
try {
env = new RetrievalEnvironment(conf.get(Constants.IndexPath), fs);
} catch (IOException e) {
throw new RuntimeException("Unable to create RetrievalEnvironment!");
}
collectionDocumentCount = env.readCollectionDocumentCount();
try {
out = fs.create(new Path(env.getTermDocVectorsForwardIndex()), true);
out.writeInt(env.readDocnoOffset());
out.writeInt(collectionDocumentCount);
} catch (Exception e) {
throw new RuntimeException("Error in creating files!");
}
}
public static void fileTreeCreation() throws IOException {
Configuration conf = new Configuration();
FileSystem fs = null;
FSDataOutputStream out = null;
String s = null;
BufferedReader keyboardin = new BufferedReader((new InputStreamReader(System.in)));
String q = " ";
String wuri;
try {
while (!q.equals("q")) {
System.out.println("Enter the uri");
wuri = keyboardin.readLine();
try {
System.out.println("Write a story");
fs = FileSystem.get(URI.create(wuri), conf);
out = fs.create(new Path(wuri));
while (!(s = keyboardin.readLine()).equals("-1")) {
out.write(s.getBytes("UTF-8"));
}
} finally {
IOUtils.closeStream(out);
System.out.format("Story written to %s%n", wuri);
}
System.out.println("Do you want to exit: q");
q = keyboardin.readLine();
}
} finally {
keyboardin.close();
}
}
private void executePostProcessing(DistCpOptions options) throws IOException {
Path targetPath = options.getTargetPath();
FileSystem fs = targetPath.getFileSystem(getConf());
List<Path> inPaths = options.getSourcePaths();
assert inPaths.size() == 1 : "Source paths more than 1 can't be handled";
Path sourcePath = inPaths.get(0);
Path includePath = new Path(getConf().get("falcon.include.path"));
assert includePath
.toString()
.substring(0, sourcePath.toString().length())
.equals(sourcePath.toString())
: "Source path is not a subset of include path";
String relativePath = includePath.toString().substring(sourcePath.toString().length());
String fixedPath = getFixedPath(relativePath);
FileStatus[] files = fs.globStatus(new Path(targetPath.toString() + "/" + fixedPath));
if (files != null) {
for (FileStatus file : files) {
fs.create(new Path(file.getPath(), EntityUtil.SUCCEEDED_FILE_NAME)).close();
LOG.info("Created " + new Path(file.getPath(), EntityUtil.SUCCEEDED_FILE_NAME));
}
} else {
LOG.info(
"No files present in path: "
+ new Path(targetPath.toString() + "/" + fixedPath).toString());
}
}
private SrcFileInfo createFile(
Configuration conf,
FileSystem fs,
Path path,
int numPartitions,
int numKeysPerPartition,
int startKey)
throws IOException {
FSDataOutputStream outStream = fs.create(path);
int currentKey = startKey;
SrcFileInfo srcFileInfo = new SrcFileInfo();
srcFileInfo.indexedRecords = new TezIndexRecord[numPartitions];
srcFileInfo.path = path;
for (int i = 0; i < numPartitions; i++) {
long pos = outStream.getPos();
IFile.Writer writer =
new IFile.Writer(conf, outStream, IntWritable.class, IntWritable.class, null, null, null);
for (int j = 0; j < numKeysPerPartition; j++) {
writer.append(new IntWritable(currentKey), new IntWritable(currentKey));
currentKey++;
}
writer.close();
srcFileInfo.indexedRecords[i] =
new TezIndexRecord(pos, writer.getRawLength(), writer.getCompressedLength());
}
outStream.close();
return srcFileInfo;
}
/**
* upload file.
*
* @param directory hadoop directory will be saved uploaded file.
* @param filename file name.
* @param is file stream
* @param overwrite if true if the file is exist, it'll be overwrited.
* @return
*/
public boolean uploadFile(Path directory, String filename, InputStream is, boolean overwrite) {
boolean isSuccess = false;
FSDataOutputStream fos = null;
try {
log.debug("try to upload " + filename + ", to=" + directory.getName());
Path p = new Path(directory, new Path(filename));
FileSystem fs = getDFS();
if (fs.getFileStatus(directory).isDir() == false) {
throw new IOException(directory + " isn't directory.");
} else if (fs.exists(p)) {
if (overwrite) {
delete(p, true);
} else {
throw new IOException(p + " already exist.");
}
}
fos = fs.create(p);
BufferedInputStream bis = new BufferedInputStream(is);
// IOUtils.copyBytes(bis,fos,8192,true);
copyBytes(bis, fos, 8192, true);
isSuccess = true;
} catch (IOException e) {
log.error("error", e);
} finally {
close(fos);
}
return isSuccess;
}
private void touch(FileSystem fs, String path, boolean generateFiles) throws Exception {
if (generateFiles) {
fs.create(new Path(path)).close();
} else {
fs.mkdirs(new Path(path));
}
}
@org.junit.Test
public void testPermissions() throws Exception {
Path myFile = new Path("filePerm.txt");
fs.create(myFile);
short perm = 0777;
fs.setPermission(myFile, new FsPermission(perm));
assertEquals(fs.getFileStatus(myFile).getPermission().toShort(), perm);
perm = 0700;
fs.setPermission(myFile, new FsPermission(perm));
assertEquals(fs.getFileStatus(myFile).getPermission().toShort(), perm);
fs.delete(myFile);
assertFalse(fs.exists(myFile));
/* directory permissions */
Path directory = new Path("aa/bb/cc");
perm = 0700;
fs.mkdirs(directory, new FsPermission(perm));
assertEquals(fs.getFileStatus(directory).getPermission().toShort(), perm);
fs.delete(new Path("aa"), true);
assertFalse(fs.exists(directory));
perm = 0777;
fs.mkdirs(directory, new FsPermission(perm));
assertEquals(fs.getFileStatus(directory).getPermission().toShort(), perm);
fs.delete(new Path("aa"), true);
assertFalse(fs.exists(directory));
}
private Path[] createFiles() throws IOException {
int numberOfStreams = Math.max(2, rnd.nextInt(10));
mergeFactor = Math.max(mergeFactor, numberOfStreams);
LOG.info("No of streams : " + numberOfStreams);
Path[] paths = new Path[numberOfStreams];
for (int i = 0; i < numberOfStreams; i++) {
paths[i] = new Path(baseDir, "ifile_" + i + ".out");
FSDataOutputStream out = fs.create(paths[i]);
// write data with RLE
IFile.Writer writer = new IFile.Writer(conf, out, keyClass, valClass, null, null, null, true);
Map<Writable, Writable> data = createData();
for (Map.Entry<Writable, Writable> entry : data.entrySet()) {
writer.append(entry.getKey(), entry.getValue());
originalData.put(entry.getKey(), entry.getValue());
if (rnd.nextInt() % 2 == 0) {
for (int j = 0; j < rnd.nextInt(100); j++) {
// add some duplicate keys
writer.append(entry.getKey(), entry.getValue());
originalData.put(entry.getKey(), entry.getValue());
}
}
}
LOG.info("Wrote " + data.size() + " in " + paths[i]);
data.clear();
writer.close();
out.close();
}
return paths;
}
/**
* Regression test for HDFS-2742. The issue in this bug was: - DN does a block report while file
* is open. This BR contains the block in RBW state. - Standby queues the RBW state in
* PendingDatanodeMessages - Standby processes edit logs during failover. Before fixing this bug,
* it was mistakenly applying the RBW reported state after the block had been completed, causing
* the block to get marked corrupt. Instead, we should now be applying the RBW message on OP_ADD,
* and then the FINALIZED message on OP_CLOSE.
*/
@Test
public void testBlockReportsWhileFileBeingWritten() throws Exception {
FSDataOutputStream out = fs.create(TEST_FILE_PATH);
try {
AppendTestUtil.write(out, 0, 10);
out.hflush();
// Block report will include the RBW replica, but will be
// queued on the StandbyNode.
cluster.triggerBlockReports();
} finally {
IOUtils.closeStream(out);
}
cluster.transitionToStandby(0);
cluster.transitionToActive(1);
// Verify that no replicas are marked corrupt, and that the
// file is readable from the failed-over standby.
BlockManagerTestUtil.updateState(nn1.getNamesystem().getBlockManager());
BlockManagerTestUtil.updateState(nn2.getNamesystem().getBlockManager());
assertEquals(0, nn1.getNamesystem().getCorruptReplicaBlocks());
assertEquals(0, nn2.getNamesystem().getCorruptReplicaBlocks());
DFSTestUtil.readFile(fs, TEST_FILE_PATH);
}
@Test
public void test() throws Exception {
Connector c = getConnector();
// make a table
String tableName = getUniqueNames(1)[0];
c.tableOperations().create(tableName);
// write to it
BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
Mutation m = new Mutation("row");
m.put("cf", "cq", "value");
bw.addMutation(m);
bw.close();
// create a fake _tmp file in its directory
String id = c.tableOperations().tableIdMap().get(tableName);
FileSystem fs = getCluster().getFileSystem();
Path tmp = new Path("/accumulo/tables/" + id + "/default_tablet/junk.rf_tmp");
fs.create(tmp).close();
for (ProcessReference tserver : getCluster().getProcesses().get(ServerType.TABLET_SERVER)) {
getCluster().killProcess(ServerType.TABLET_SERVER, tserver);
}
getCluster().start();
Scanner scanner = c.createScanner(tableName, Authorizations.EMPTY);
FunctionalTestUtils.count(scanner);
assertFalse(fs.exists(tmp));
}
public void store(Path output) throws IOException {
FileSystem fs = output.getFileSystem(HadoopUtils.createConfiguration());
fs.delete(output, true);
OutputStream os = fs.create(output, true);
store(os);
os.close();
}
protected Path marshal(
Cluster cluster, JAXBElement<?> jaxbElement, JAXBContext jaxbContext, Path outPath)
throws FalconException {
try {
Marshaller marshaller = jaxbContext.createMarshaller();
marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE);
if (LOG.isDebugEnabled()) {
StringWriter writer = new StringWriter();
marshaller.marshal(jaxbElement, writer);
LOG.debug("Writing definition to {} on cluster {}", outPath, cluster.getName());
LOG.debug(writer.getBuffer().toString());
}
FileSystem fs =
HadoopClientFactory.get()
.createProxiedFileSystem(outPath.toUri(), ClusterHelper.getConfiguration(cluster));
OutputStream out = fs.create(outPath);
try {
marshaller.marshal(jaxbElement, out);
} finally {
out.close();
}
LOG.info("Marshalled {} to {}", jaxbElement.getDeclaredType(), outPath);
return outPath;
} catch (Exception e) {
throw new FalconException("Unable to marshall app object", e);
}
}
public void testConfigNotPropagation() throws Exception {
Path subWorkflowAppPath = getFsTestCaseDir();
FileSystem fs = getFileSystem();
Writer writer = new OutputStreamWriter(fs.create(new Path(subWorkflowAppPath, "workflow.xml")));
writer.write(APP1);
writer.close();
XConfiguration protoConf = getBaseProtoConf();
WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W");
String defaultConf = workflow.getConf();
XConfiguration newConf = new XConfiguration(new StringReader(defaultConf));
newConf.set("abc", "xyz");
workflow.setConf(newConf.toXmlString());
final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0);
action.setConf(
"<sub-workflow xmlns='uri:oozie:workflow:0.1' name='subwf'>"
+ " <app-path>"
+ subWorkflowAppPath
+ File.separator
+ "workflow.xml"
+ "</app-path>"
+ " <configuration>"
+ " <property>"
+ " <name>a</name>"
+ " <value>A</value>"
+ " </property>"
+ " </configuration>"
+ "</sub-workflow>");
SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor();
subWorkflow.start(new Context(workflow, action), action);
final OozieClient oozieClient =
subWorkflow.getWorkflowClient(
new Context(workflow, action), SubWorkflowActionExecutor.LOCAL);
waitFor(
JOB_TIMEOUT,
new Predicate() {
public boolean evaluate() throws Exception {
return oozieClient.getJobInfo(action.getExternalId()).getStatus()
== WorkflowJob.Status.SUCCEEDED;
}
});
assertEquals(
WorkflowJob.Status.SUCCEEDED, oozieClient.getJobInfo(action.getExternalId()).getStatus());
subWorkflow.check(new Context(workflow, action), action);
assertEquals(WorkflowAction.Status.DONE, action.getStatus());
subWorkflow.end(new Context(workflow, action), action);
assertEquals(WorkflowAction.Status.OK, action.getStatus());
WorkflowJob wf = oozieClient.getJobInfo(action.getExternalId());
Configuration childConf = new XConfiguration(new StringReader(wf.getConf()));
assertNull(childConf.get("abc"));
}