Example #1
0
 /**
  * Load the information in the SnapshotManifest. Called by SnapshotManifest.open()
  *
  * <p>If the format is v2 and there is no data-manifest, means that we are loading an in-progress
  * snapshot. Since we support rolling-upgrades, we loook for v1 and v2 regions format.
  */
 private void load() throws IOException {
   switch (getSnapshotFormat(desc)) {
     case SnapshotManifestV1.DESCRIPTOR_VERSION:
       {
         this.htd =
             FSTableDescriptors.getTableDescriptorFromFs(fs, workingDir).getHTableDescriptor();
         ThreadPoolExecutor tpool = createExecutor("SnapshotManifestLoader");
         try {
           this.regionManifests =
               SnapshotManifestV1.loadRegionManifests(conf, tpool, fs, workingDir, desc);
         } finally {
           tpool.shutdown();
         }
         break;
       }
     case SnapshotManifestV2.DESCRIPTOR_VERSION:
       {
         SnapshotDataManifest dataManifest = readDataManifest();
         if (dataManifest != null) {
           htd = ProtobufUtil.convertToHTableDesc(dataManifest.getTableSchema());
           regionManifests = dataManifest.getRegionManifestsList();
         } else {
           // Compatibility, load the v1 regions
           // This happens only when the snapshot is in-progress and the cache wants to refresh.
           List<SnapshotRegionManifest> v1Regions, v2Regions;
           ThreadPoolExecutor tpool = createExecutor("SnapshotManifestLoader");
           try {
             v1Regions = SnapshotManifestV1.loadRegionManifests(conf, tpool, fs, workingDir, desc);
             v2Regions = SnapshotManifestV2.loadRegionManifests(conf, tpool, fs, workingDir, desc);
           } catch (InvalidProtocolBufferException e) {
             throw new CorruptedSnapshotException(
                 "unable to parse region manifest " + e.getMessage(), e);
           } finally {
             tpool.shutdown();
           }
           if (v1Regions != null && v2Regions != null) {
             regionManifests =
                 new ArrayList<SnapshotRegionManifest>(v1Regions.size() + v2Regions.size());
             regionManifests.addAll(v1Regions);
             regionManifests.addAll(v2Regions);
           } else if (v1Regions != null) {
             regionManifests = v1Regions;
           } else /* if (v2Regions != null) */ {
             regionManifests = v2Regions;
           }
         }
         break;
       }
     default:
       throw new CorruptedSnapshotException(
           "Invalid Snapshot version: " + desc.getVersion(),
           ProtobufUtil.createSnapshotDesc(desc));
   }
 }
Example #2
0
 /*
  * Write the SnapshotDataManifest file
  */
 private void writeDataManifest(final SnapshotDataManifest manifest) throws IOException {
   FSDataOutputStream stream = fs.create(new Path(workingDir, DATA_MANIFEST_NAME));
   try {
     manifest.writeTo(stream);
   } finally {
     stream.close();
   }
 }
Example #3
0
  /*
   * In case of rolling-upgrade, we try to read all the formats and build
   * the snapshot with the latest format.
   */
  private void convertToV2SingleManifest() throws IOException {
    // Try to load v1 and v2 regions
    List<SnapshotRegionManifest> v1Regions, v2Regions;
    ThreadPoolExecutor tpool = createExecutor("SnapshotManifestLoader");
    try {
      v1Regions = SnapshotManifestV1.loadRegionManifests(conf, tpool, fs, workingDir, desc);
      v2Regions = SnapshotManifestV2.loadRegionManifests(conf, tpool, fs, workingDir, desc);
    } finally {
      tpool.shutdown();
    }

    SnapshotDataManifest.Builder dataManifestBuilder = SnapshotDataManifest.newBuilder();
    dataManifestBuilder.setTableSchema(ProtobufUtil.convertToTableSchema(htd));

    if (v1Regions != null && v1Regions.size() > 0) {
      dataManifestBuilder.addAllRegionManifests(v1Regions);
    }
    if (v2Regions != null && v2Regions.size() > 0) {
      dataManifestBuilder.addAllRegionManifests(v2Regions);
    }

    // Write the v2 Data Manifest.
    // Once the data-manifest is written, the snapshot can be considered complete.
    // Currently snapshots are written in a "temporary" directory and later
    // moved to the "complated" snapshot directory.
    SnapshotDataManifest dataManifest = dataManifestBuilder.build();
    writeDataManifest(dataManifest);
    this.regionManifests = dataManifest.getRegionManifestsList();

    // Remove the region manifests. Everything is now in the data-manifest.
    // The delete operation is "relaxed", unless we get an exception we keep going.
    // The extra files in the snapshot directory will not give any problem,
    // since they have the same content as the data manifest, and even by re-reading
    // them we will get the same information.
    if (v1Regions != null && v1Regions.size() > 0) {
      for (SnapshotRegionManifest regionManifest : v1Regions) {
        SnapshotManifestV1.deleteRegionManifest(fs, workingDir, regionManifest);
      }
    }
    if (v2Regions != null && v2Regions.size() > 0) {
      for (SnapshotRegionManifest regionManifest : v2Regions) {
        SnapshotManifestV2.deleteRegionManifest(fs, workingDir, regionManifest);
      }
    }
  }
Example #4
0
 /*
  * Read the SnapshotDataManifest file
  */
 private SnapshotDataManifest readDataManifest() throws IOException {
   FSDataInputStream in = null;
   try {
     in = fs.open(new Path(workingDir, DATA_MANIFEST_NAME));
     CodedInputStream cin = CodedInputStream.newInstance(in);
     cin.setSizeLimit(manifestSizeLimit);
     return SnapshotDataManifest.parseFrom(cin);
   } catch (FileNotFoundException e) {
     return null;
   } catch (InvalidProtocolBufferException e) {
     throw new CorruptedSnapshotException("unable to parse data manifest " + e.getMessage(), e);
   } finally {
     if (in != null) in.close();
   }
 }