private Schema namespacelessSchemaFor(Class<?> type) { return schemaCache.computeIfAbsent( type, clazz -> { Schema schema = ReflectData.get().getSchema(clazz); // kind of a hack to set an empty namespace :) return new Schema.Parser().parse(schema.toString().replace(schema.getNamespace(), "")); }); }
@Test public void testProjection() throws IOException { Path path = writeCarsToParquetFile(1, CompressionCodecName.UNCOMPRESSED, false); Configuration conf = new Configuration(); Schema schema = Car.getClassSchema(); List<Schema.Field> fields = schema.getFields(); // Schema.Parser parser = new Schema.Parser(); List<Schema.Field> projectedFields = new ArrayList<Schema.Field>(); for (Schema.Field field : fields) { String name = field.name(); if ("optionalExtra".equals(name) || "serviceHistory".equals(name)) { continue; } // Schema schemaClone = parser.parse(field.schema().toString(false)); Schema.Field fieldClone = new Schema.Field(name, field.schema(), field.doc(), field.defaultValue()); projectedFields.add(fieldClone); } Schema projectedSchema = Schema.createRecord( schema.getName(), schema.getDoc(), schema.getNamespace(), schema.isError()); projectedSchema.setFields(projectedFields); AvroReadSupport.setRequestedProjection(conf, projectedSchema); ParquetReader<Car> reader = new AvroParquetReader<Car>(conf, path); for (Car car = reader.read(); car != null; car = reader.read()) { assertEquals(car.getDoors() != null, true); assertEquals(car.getEngine() != null, true); assertEquals(car.getMake() != null, true); assertEquals(car.getModel() != null, true); assertEquals(car.getYear() != null, true); assertEquals(car.getVin() != null, true); assertNull(car.getOptionalExtra()); assertNull(car.getServiceHistory()); } }
/** Initialize this sensor. */ @Override public final void init() { schema = Schema.parse(getScheme()); uri = EntityUriBuilder.nativeUri(schema.getNamespace(), schema.getName()); LOG.debug("Sensor storing to URI: {}", uri); }