private void writeIndex(
      String reason, IndexMetaData indexMetaData, @Nullable IndexMetaData previousIndexMetaData)
      throws Exception {
    logger.trace("[{}] writing state, reason [{}]", indexMetaData.index(), reason);
    XContentBuilder builder = XContentFactory.contentBuilder(format, new BytesStreamOutput());
    builder.startObject();
    IndexMetaData.Builder.toXContent(indexMetaData, builder, formatParams);
    builder.endObject();
    builder.flush();

    String stateFileName = "state-" + indexMetaData.version();
    Exception lastFailure = null;
    boolean wroteAtLeastOnce = false;
    for (File indexLocation : nodeEnv.indexLocations(new Index(indexMetaData.index()))) {
      File stateLocation = new File(indexLocation, "_state");
      FileSystemUtils.mkdirs(stateLocation);
      File stateFile = new File(stateLocation, stateFileName);

      FileOutputStream fos = null;
      try {
        fos = new FileOutputStream(stateFile);
        BytesReference bytes = builder.bytes();
        fos.write(bytes.array(), bytes.arrayOffset(), bytes.length());
        fos.getChannel().force(true);
        fos.close();
        wroteAtLeastOnce = true;
      } catch (Exception e) {
        lastFailure = e;
      } finally {
        IOUtils.closeWhileHandlingException(fos);
      }
    }

    if (!wroteAtLeastOnce) {
      logger.warn("[{}]: failed to state", lastFailure, indexMetaData.index());
      throw new IOException(
          "failed to write state for [" + indexMetaData.index() + "]", lastFailure);
    }

    // delete the old files
    if (previousIndexMetaData != null
        && previousIndexMetaData.version() != indexMetaData.version()) {
      for (File indexLocation : nodeEnv.indexLocations(new Index(indexMetaData.index()))) {
        File[] files = new File(indexLocation, "_state").listFiles();
        if (files == null) {
          continue;
        }
        for (File file : files) {
          if (!file.getName().startsWith("state-")) {
            continue;
          }
          if (file.getName().equals(stateFileName)) {
            continue;
          }
          file.delete();
        }
      }
    }
  }
Exemplo n.º 2
0
    public static void toXContent(
        IndexMetaData indexMetaData, XContentBuilder builder, ToXContent.Params params)
        throws IOException {
      builder.startObject(indexMetaData.index(), XContentBuilder.FieldCaseConversion.NONE);

      builder.field("version", indexMetaData.version());
      builder.field("state", indexMetaData.state().toString().toLowerCase(Locale.ENGLISH));

      boolean binary = params.paramAsBoolean("binary", false);

      builder.startObject("settings");
      for (Map.Entry<String, String> entry : indexMetaData.settings().getAsMap().entrySet()) {
        builder.field(entry.getKey(), entry.getValue());
      }
      builder.endObject();

      builder.startArray("mappings");
      for (Map.Entry<String, MappingMetaData> entry : indexMetaData.mappings().entrySet()) {
        if (binary) {
          builder.value(entry.getValue().source().compressed());
        } else {
          byte[] data = entry.getValue().source().uncompressed();
          XContentParser parser = XContentFactory.xContent(data).createParser(data);
          Map<String, Object> mapping = parser.mapOrdered();
          parser.close();
          builder.map(mapping);
        }
      }
      builder.endArray();

      for (Map.Entry<String, Custom> entry : indexMetaData.customs().entrySet()) {
        builder.startObject(entry.getKey(), XContentBuilder.FieldCaseConversion.NONE);
        lookupFactorySafe(entry.getKey()).toXContent(entry.getValue(), builder, params);
        builder.endObject();
      }

      builder.startObject("aliases");
      for (AliasMetaData alias : indexMetaData.aliases().values()) {
        AliasMetaData.Builder.toXContent(alias, builder, params);
      }
      builder.endObject();

      builder.endObject();
    }
Exemplo n.º 3
0
 public static void writeTo(IndexMetaData indexMetaData, StreamOutput out) throws IOException {
   out.writeUTF(indexMetaData.index());
   out.writeLong(indexMetaData.version());
   out.writeByte(indexMetaData.state().id());
   writeSettingsToStream(indexMetaData.settings(), out);
   out.writeVInt(indexMetaData.mappings().size());
   for (MappingMetaData mappingMd : indexMetaData.mappings().values()) {
     MappingMetaData.writeTo(mappingMd, out);
   }
   out.writeVInt(indexMetaData.aliases().size());
   for (AliasMetaData aliasMd : indexMetaData.aliases().values()) {
     AliasMetaData.Builder.writeTo(aliasMd, out);
   }
   out.writeVInt(indexMetaData.customs().size());
   for (Map.Entry<String, Custom> entry : indexMetaData.customs().entrySet()) {
     out.writeUTF(entry.getKey());
     lookupFactorySafe(entry.getKey()).writeTo(entry.getValue(), out);
   }
 }
  @Override
  public void clusterChanged(ClusterChangedEvent event) {
    if (event.state().blocks().disableStatePersistence()) {
      // reset the current metadata, we need to start fresh...
      this.currentMetaData = null;
      return;
    }

    MetaData newMetaData = event.state().metaData();
    // we don't check if metaData changed, since we might be called several times and we need to
    // check dangling...

    boolean success = true;
    // only applied to master node, writing the global and index level states
    if (event.state().nodes().localNode().masterNode()) {
      // check if the global state changed?
      if (currentMetaData == null || !MetaData.isGlobalStateEquals(currentMetaData, newMetaData)) {
        try {
          writeGlobalState("changed", newMetaData, currentMetaData);
        } catch (Exception e) {
          success = false;
        }
      }

      // check and write changes in indices
      for (IndexMetaData indexMetaData : newMetaData) {
        String writeReason = null;
        IndexMetaData currentIndexMetaData;
        if (currentMetaData == null) {
          // a new event..., check from the state stored
          currentIndexMetaData = loadIndex(indexMetaData.index());
        } else {
          currentIndexMetaData = currentMetaData.index(indexMetaData.index());
        }
        if (currentIndexMetaData == null) {
          writeReason = "freshly created";
        } else if (currentIndexMetaData.version() != indexMetaData.version()) {
          writeReason =
              "version changed from ["
                  + currentIndexMetaData.version()
                  + "] to ["
                  + indexMetaData.version()
                  + "]";
        }

        // we update the writeReason only if we really need to write it
        if (writeReason == null) {
          continue;
        }

        try {
          writeIndex(writeReason, indexMetaData, currentIndexMetaData);
        } catch (Exception e) {
          success = false;
        }
      }
    }

    // delete indices that were there before, but are deleted now
    // we need to do it so they won't be detected as dangling
    if (nodeEnv.hasNodeFile()) {
      if (currentMetaData != null) {
        // only delete indices when we already received a state (currentMetaData != null)
        // and we had a go at processing dangling indices at least once
        // this will also delete the _state of the index itself
        for (IndexMetaData current : currentMetaData) {
          if (danglingIndices.containsKey(current.index())) {
            continue;
          }
          if (!newMetaData.hasIndex(current.index())) {
            logger.debug(
                "[{}] deleting index that is no longer part of the metadata (indices: [{}])",
                current.index(),
                newMetaData.indices().keySet());
            FileSystemUtils.deleteRecursively(nodeEnv.indexLocations(new Index(current.index())));
          }
        }
      }
    }

    // handle dangling indices, we handle those for all nodes that have a node file (data or master)
    if (nodeEnv.hasNodeFile()) {
      if (danglingTimeout.millis() >= 0) {
        synchronized (danglingMutex) {
          for (String danglingIndex : danglingIndices.keySet()) {
            if (newMetaData.hasIndex(danglingIndex)) {
              logger.debug("[{}] no longer dangling (created), removing", danglingIndex);
              DanglingIndex removed = danglingIndices.remove(danglingIndex);
              removed.future.cancel(false);
            }
          }
          // delete indices that are no longer part of the metadata
          try {
            for (String indexName : nodeEnv.findAllIndices()) {
              // if we have the index on the metadata, don't delete it
              if (newMetaData.hasIndex(indexName)) {
                continue;
              }
              if (danglingIndices.containsKey(indexName)) {
                // already dangling, continue
                continue;
              }
              IndexMetaData indexMetaData = loadIndex(indexName);
              if (indexMetaData != null) {
                if (danglingTimeout.millis() == 0) {
                  logger.info(
                      "[{}] dangling index, exists on local file system, but not in cluster metadata, timeout set to 0, deleting now",
                      indexName);
                  FileSystemUtils.deleteRecursively(nodeEnv.indexLocations(new Index(indexName)));
                } else {
                  logger.info(
                      "[{}] dangling index, exists on local file system, but not in cluster metadata, scheduling to delete in [{}], auto import to cluster state [{}]",
                      indexName,
                      danglingTimeout,
                      autoImportDangled);
                  danglingIndices.put(
                      indexName,
                      new DanglingIndex(
                          indexName,
                          threadPool.schedule(
                              danglingTimeout,
                              ThreadPool.Names.SAME,
                              new RemoveDanglingIndex(indexName))));
                }
              }
            }
          } catch (Exception e) {
            logger.warn("failed to find dangling indices", e);
          }
        }
      }
      if (autoImportDangled.shouldImport() && !danglingIndices.isEmpty()) {
        final List<IndexMetaData> dangled = Lists.newArrayList();
        for (String indexName : danglingIndices.keySet()) {
          IndexMetaData indexMetaData = loadIndex(indexName);
          if (indexMetaData == null) {
            logger.debug("failed to find state for dangling index [{}]", indexName);
            continue;
          }
          // we might have someone copying over an index, renaming the directory, handle that
          if (!indexMetaData.index().equals(indexName)) {
            logger.info(
                "dangled index directory name is [{}], state name is [{}], renaming to directory name",
                indexName,
                indexMetaData.index());
            indexMetaData =
                IndexMetaData.newIndexMetaDataBuilder(indexMetaData).index(indexName).build();
          }
          if (autoImportDangled == AutoImportDangledState.CLOSED) {
            indexMetaData =
                IndexMetaData.newIndexMetaDataBuilder(indexMetaData)
                    .state(IndexMetaData.State.CLOSE)
                    .build();
          }
          if (indexMetaData != null) {
            dangled.add(indexMetaData);
          }
        }
        IndexMetaData[] dangledIndices = dangled.toArray(new IndexMetaData[dangled.size()]);
        try {
          allocateDangledIndices.allocateDangled(
              dangledIndices,
              new LocalAllocateDangledIndices.Listener() {
                @Override
                public void onResponse(
                    LocalAllocateDangledIndices.AllocateDangledResponse response) {
                  logger.trace("allocated dangled");
                }

                @Override
                public void onFailure(Throwable e) {
                  logger.info("failed to send allocated dangled", e);
                }
              });
        } catch (Exception e) {
          logger.warn("failed to send allocate dangled", e);
        }
      }
    }

    if (success) {
      currentMetaData = newMetaData;
    }
  }