// Writes cars into a text file before closing @Override protected void finalize() throws FileNotFoundException, IOException { File file = new File(this.fileName); FileWriter fileWriter = new FileWriter(file); BufferedWriter bWriter = new BufferedWriter(fileWriter); try { for (Car car : cars) { if (car != null) { bWriter.write(car.getMake() + System.getProperty("line.separator")); bWriter.write(car.getYear() + System.getProperty("line.separator")); bWriter.write(car.getPrice() + System.getProperty("line.separator")); bWriter.write(car.getVin() + System.getProperty("line.separator")); } } } catch (Exception e) { } finally { bWriter.close(); fileWriter.close(); } }
@Test public void testProjection() throws IOException { Path path = writeCarsToParquetFile(1, CompressionCodecName.UNCOMPRESSED, false); Configuration conf = new Configuration(); Schema schema = Car.getClassSchema(); List<Schema.Field> fields = schema.getFields(); // Schema.Parser parser = new Schema.Parser(); List<Schema.Field> projectedFields = new ArrayList<Schema.Field>(); for (Schema.Field field : fields) { String name = field.name(); if ("optionalExtra".equals(name) || "serviceHistory".equals(name)) { continue; } // Schema schemaClone = parser.parse(field.schema().toString(false)); Schema.Field fieldClone = new Schema.Field(name, field.schema(), field.doc(), field.defaultValue()); projectedFields.add(fieldClone); } Schema projectedSchema = Schema.createRecord( schema.getName(), schema.getDoc(), schema.getNamespace(), schema.isError()); projectedSchema.setFields(projectedFields); AvroReadSupport.setRequestedProjection(conf, projectedSchema); ParquetReader<Car> reader = new AvroParquetReader<Car>(conf, path); for (Car car = reader.read(); car != null; car = reader.read()) { assertEquals(car.getDoors() != null, true); assertEquals(car.getEngine() != null, true); assertEquals(car.getMake() != null, true); assertEquals(car.getModel() != null, true); assertEquals(car.getYear() != null, true); assertEquals(car.getVin() != null, true); assertNull(car.getOptionalExtra()); assertNull(car.getServiceHistory()); } }