Exemplo n.º 1
0
 /** @param args */
 @SuppressWarnings("deprecation")
 public static void main(String[] args) throws IOException, URISyntaxException {
   String name = "/home/naga/dept";
   @SuppressWarnings("resource")
   BufferedReader br = new BufferedReader(new FileReader(name));
   String line = br.readLine();
   String uri = "/nyse/";
   Configuration conf = new Configuration();
   FileSystem fs = FileSystem.get(new URI("hdfs://hadoop:9000"), conf);
   Path path = new Path(uri);
   Text key = new Text();
   Text value = new Text();
   MapFile.Writer writer = null;
   try {
     writer = new MapFile.Writer(conf, fs, uri, key.getClass(), value.getClass());
     while (line != null) {
       String parts[] = line.split("\\t");
       key.set(parts[0]);
       value.set(parts[1]);
       writer.append(key, value);
       line = br.readLine();
     }
   } finally {
     IOUtils.closeStream(writer);
   }
 }
Exemplo n.º 2
0
  public static void main(String[] args) throws IOException {
    // TODO Auto-generated method stub
    String uri = args[0];

    Configuration conf = new Configuration();

    Path path = new Path(uri);

    FileSystem fs = FileSystem.get(URI.create(uri), conf);

    IntWritable key = new IntWritable();
    Text value = new Text();
    SequenceFile.Writer writer = null;
    try {
      writer = SequenceFile.createWriter(fs, conf, path, key.getClass(), value.getClass());

      for (int i = 0; i < 100; i++) {
        key.set(100 - i);
        value.set(DATA[i % DATA.length]);

        writer.append(key, value);
      }
    } finally {
      IOUtils.closeStream(writer);
    }
  }
  @Before
  public void setUp() throws Exception {
    // create local Pig server
    pigServer = UnitTestUtil.makePigServer();

    // create temp SequenceFile
    File tempFile = File.createTempFile("test", ".txt");
    tempFilename = tempFile.getAbsolutePath();
    Path path = new Path("file:///" + tempFilename);
    Configuration conf = new Configuration();
    FileSystem fs = path.getFileSystem(conf);
    IntWritable key = new IntWritable();
    Text value = new Text();
    SequenceFile.Writer writer = null;
    try {
      writer = SequenceFile.createWriter(fs, conf, path, key.getClass(), value.getClass());
      for (int i = 0; i < DATA.length; ++i) {
        key.set(i);
        value.set(DATA[i]);
        writer.append(key, value);
      }
    } finally {
      IOUtils.closeStream(writer);
    }
  }
Exemplo n.º 4
0
  public long generate(boolean recurse) throws IOException {
    long result = 0;
    // read from input path
    // create new Content object and add it to the SequenceFile
    Text key = new Text();
    BehemothDocument value = new BehemothDocument();
    SequenceFile.Writer writer = null;
    try {
      Configuration conf = getConf();
      FileSystem fs = output.getFileSystem(conf);
      writer = SequenceFile.createWriter(fs, conf, output, key.getClass(), value.getClass());
      PerformanceFileFilter pff = new PerformanceFileFilter(writer, key, value, conf, reporter);
      // iterate on the files in the source dir
      result = processFiles(conf, input, recurse, pff);

    } finally {
      IOUtils.closeStream(writer);
    }
    return result;
  }