/** @param args */
@SuppressWarnings("deprecation")
public static void main(String[] args) throws IOException, URISyntaxException {
String name = "/home/naga/dept";
@SuppressWarnings("resource")
BufferedReader br = new BufferedReader(new FileReader(name));
String line = br.readLine();
String uri = "/nyse/";
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(new URI("hdfs://hadoop:9000"), conf);
Path path = new Path(uri);
Text key = new Text();
Text value = new Text();
MapFile.Writer writer = null;
try {
writer = new MapFile.Writer(conf, fs, uri, key.getClass(), value.getClass());
while (line != null) {
String parts[] = line.split("\\t");
key.set(parts[0]);
value.set(parts[1]);
writer.append(key, value);
line = br.readLine();
}
} finally {
IOUtils.closeStream(writer);
}
}
public static void main(String[] args) throws IOException {
// TODO Auto-generated method stub
String uri = args[0];
Configuration conf = new Configuration();
Path path = new Path(uri);
FileSystem fs = FileSystem.get(URI.create(uri), conf);
IntWritable key = new IntWritable();
Text value = new Text();
SequenceFile.Writer writer = null;
try {
writer = SequenceFile.createWriter(fs, conf, path, key.getClass(), value.getClass());
for (int i = 0; i < 100; i++) {
key.set(100 - i);
value.set(DATA[i % DATA.length]);
writer.append(key, value);
}
} finally {
IOUtils.closeStream(writer);
}
}
@Before
public void setUp() throws Exception {
// create local Pig server
pigServer = UnitTestUtil.makePigServer();
// create temp SequenceFile
File tempFile = File.createTempFile("test", ".txt");
tempFilename = tempFile.getAbsolutePath();
Path path = new Path("file:///" + tempFilename);
Configuration conf = new Configuration();
FileSystem fs = path.getFileSystem(conf);
IntWritable key = new IntWritable();
Text value = new Text();
SequenceFile.Writer writer = null;
try {
writer = SequenceFile.createWriter(fs, conf, path, key.getClass(), value.getClass());
for (int i = 0; i < DATA.length; ++i) {
key.set(i);
value.set(DATA[i]);
writer.append(key, value);
}
} finally {
IOUtils.closeStream(writer);
}
}
public long generate(boolean recurse) throws IOException {
long result = 0;
// read from input path
// create new Content object and add it to the SequenceFile
Text key = new Text();
BehemothDocument value = new BehemothDocument();
SequenceFile.Writer writer = null;
try {
Configuration conf = getConf();
FileSystem fs = output.getFileSystem(conf);
writer = SequenceFile.createWriter(fs, conf, output, key.getClass(), value.getClass());
PerformanceFileFilter pff = new PerformanceFileFilter(writer, key, value, conf, reporter);
// iterate on the files in the source dir
result = processFiles(conf, input, recurse, pff);
} finally {
IOUtils.closeStream(writer);
}
return result;
}