private void writeIndex( String reason, IndexMetaData indexMetaData, @Nullable IndexMetaData previousIndexMetaData) throws Exception { logger.trace("[{}] writing state, reason [{}]", indexMetaData.index(), reason); XContentBuilder builder = XContentFactory.contentBuilder(format, new BytesStreamOutput()); builder.startObject(); IndexMetaData.Builder.toXContent(indexMetaData, builder, formatParams); builder.endObject(); builder.flush(); String stateFileName = "state-" + indexMetaData.version(); Exception lastFailure = null; boolean wroteAtLeastOnce = false; for (File indexLocation : nodeEnv.indexLocations(new Index(indexMetaData.index()))) { File stateLocation = new File(indexLocation, "_state"); FileSystemUtils.mkdirs(stateLocation); File stateFile = new File(stateLocation, stateFileName); FileOutputStream fos = null; try { fos = new FileOutputStream(stateFile); BytesReference bytes = builder.bytes(); fos.write(bytes.array(), bytes.arrayOffset(), bytes.length()); fos.getChannel().force(true); fos.close(); wroteAtLeastOnce = true; } catch (Exception e) { lastFailure = e; } finally { IOUtils.closeWhileHandlingException(fos); } } if (!wroteAtLeastOnce) { logger.warn("[{}]: failed to state", lastFailure, indexMetaData.index()); throw new IOException( "failed to write state for [" + indexMetaData.index() + "]", lastFailure); } // delete the old files if (previousIndexMetaData != null && previousIndexMetaData.version() != indexMetaData.version()) { for (File indexLocation : nodeEnv.indexLocations(new Index(indexMetaData.index()))) { File[] files = new File(indexLocation, "_state").listFiles(); if (files == null) { continue; } for (File file : files) { if (!file.getName().startsWith("state-")) { continue; } if (file.getName().equals(stateFileName)) { continue; } file.delete(); } } } }
public static void toXContent( IndexMetaData indexMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject(indexMetaData.index(), XContentBuilder.FieldCaseConversion.NONE); builder.field("version", indexMetaData.version()); builder.field("state", indexMetaData.state().toString().toLowerCase(Locale.ENGLISH)); boolean binary = params.paramAsBoolean("binary", false); builder.startObject("settings"); for (Map.Entry<String, String> entry : indexMetaData.settings().getAsMap().entrySet()) { builder.field(entry.getKey(), entry.getValue()); } builder.endObject(); builder.startArray("mappings"); for (Map.Entry<String, MappingMetaData> entry : indexMetaData.mappings().entrySet()) { if (binary) { builder.value(entry.getValue().source().compressed()); } else { byte[] data = entry.getValue().source().uncompressed(); XContentParser parser = XContentFactory.xContent(data).createParser(data); Map<String, Object> mapping = parser.mapOrdered(); parser.close(); builder.map(mapping); } } builder.endArray(); for (Map.Entry<String, Custom> entry : indexMetaData.customs().entrySet()) { builder.startObject(entry.getKey(), XContentBuilder.FieldCaseConversion.NONE); lookupFactorySafe(entry.getKey()).toXContent(entry.getValue(), builder, params); builder.endObject(); } builder.startObject("aliases"); for (AliasMetaData alias : indexMetaData.aliases().values()) { AliasMetaData.Builder.toXContent(alias, builder, params); } builder.endObject(); builder.endObject(); }
public static void writeTo(IndexMetaData indexMetaData, StreamOutput out) throws IOException { out.writeUTF(indexMetaData.index()); out.writeLong(indexMetaData.version()); out.writeByte(indexMetaData.state().id()); writeSettingsToStream(indexMetaData.settings(), out); out.writeVInt(indexMetaData.mappings().size()); for (MappingMetaData mappingMd : indexMetaData.mappings().values()) { MappingMetaData.writeTo(mappingMd, out); } out.writeVInt(indexMetaData.aliases().size()); for (AliasMetaData aliasMd : indexMetaData.aliases().values()) { AliasMetaData.Builder.writeTo(aliasMd, out); } out.writeVInt(indexMetaData.customs().size()); for (Map.Entry<String, Custom> entry : indexMetaData.customs().entrySet()) { out.writeUTF(entry.getKey()); lookupFactorySafe(entry.getKey()).writeTo(entry.getValue(), out); } }
@Override public void clusterChanged(ClusterChangedEvent event) { if (event.state().blocks().disableStatePersistence()) { // reset the current metadata, we need to start fresh... this.currentMetaData = null; return; } MetaData newMetaData = event.state().metaData(); // we don't check if metaData changed, since we might be called several times and we need to // check dangling... boolean success = true; // only applied to master node, writing the global and index level states if (event.state().nodes().localNode().masterNode()) { // check if the global state changed? if (currentMetaData == null || !MetaData.isGlobalStateEquals(currentMetaData, newMetaData)) { try { writeGlobalState("changed", newMetaData, currentMetaData); } catch (Exception e) { success = false; } } // check and write changes in indices for (IndexMetaData indexMetaData : newMetaData) { String writeReason = null; IndexMetaData currentIndexMetaData; if (currentMetaData == null) { // a new event..., check from the state stored currentIndexMetaData = loadIndex(indexMetaData.index()); } else { currentIndexMetaData = currentMetaData.index(indexMetaData.index()); } if (currentIndexMetaData == null) { writeReason = "freshly created"; } else if (currentIndexMetaData.version() != indexMetaData.version()) { writeReason = "version changed from [" + currentIndexMetaData.version() + "] to [" + indexMetaData.version() + "]"; } // we update the writeReason only if we really need to write it if (writeReason == null) { continue; } try { writeIndex(writeReason, indexMetaData, currentIndexMetaData); } catch (Exception e) { success = false; } } } // delete indices that were there before, but are deleted now // we need to do it so they won't be detected as dangling if (nodeEnv.hasNodeFile()) { if (currentMetaData != null) { // only delete indices when we already received a state (currentMetaData != null) // and we had a go at processing dangling indices at least once // this will also delete the _state of the index itself for (IndexMetaData current : currentMetaData) { if (danglingIndices.containsKey(current.index())) { continue; } if (!newMetaData.hasIndex(current.index())) { logger.debug( "[{}] deleting index that is no longer part of the metadata (indices: [{}])", current.index(), newMetaData.indices().keySet()); FileSystemUtils.deleteRecursively(nodeEnv.indexLocations(new Index(current.index()))); } } } } // handle dangling indices, we handle those for all nodes that have a node file (data or master) if (nodeEnv.hasNodeFile()) { if (danglingTimeout.millis() >= 0) { synchronized (danglingMutex) { for (String danglingIndex : danglingIndices.keySet()) { if (newMetaData.hasIndex(danglingIndex)) { logger.debug("[{}] no longer dangling (created), removing", danglingIndex); DanglingIndex removed = danglingIndices.remove(danglingIndex); removed.future.cancel(false); } } // delete indices that are no longer part of the metadata try { for (String indexName : nodeEnv.findAllIndices()) { // if we have the index on the metadata, don't delete it if (newMetaData.hasIndex(indexName)) { continue; } if (danglingIndices.containsKey(indexName)) { // already dangling, continue continue; } IndexMetaData indexMetaData = loadIndex(indexName); if (indexMetaData != null) { if (danglingTimeout.millis() == 0) { logger.info( "[{}] dangling index, exists on local file system, but not in cluster metadata, timeout set to 0, deleting now", indexName); FileSystemUtils.deleteRecursively(nodeEnv.indexLocations(new Index(indexName))); } else { logger.info( "[{}] dangling index, exists on local file system, but not in cluster metadata, scheduling to delete in [{}], auto import to cluster state [{}]", indexName, danglingTimeout, autoImportDangled); danglingIndices.put( indexName, new DanglingIndex( indexName, threadPool.schedule( danglingTimeout, ThreadPool.Names.SAME, new RemoveDanglingIndex(indexName)))); } } } } catch (Exception e) { logger.warn("failed to find dangling indices", e); } } } if (autoImportDangled.shouldImport() && !danglingIndices.isEmpty()) { final List<IndexMetaData> dangled = Lists.newArrayList(); for (String indexName : danglingIndices.keySet()) { IndexMetaData indexMetaData = loadIndex(indexName); if (indexMetaData == null) { logger.debug("failed to find state for dangling index [{}]", indexName); continue; } // we might have someone copying over an index, renaming the directory, handle that if (!indexMetaData.index().equals(indexName)) { logger.info( "dangled index directory name is [{}], state name is [{}], renaming to directory name", indexName, indexMetaData.index()); indexMetaData = IndexMetaData.newIndexMetaDataBuilder(indexMetaData).index(indexName).build(); } if (autoImportDangled == AutoImportDangledState.CLOSED) { indexMetaData = IndexMetaData.newIndexMetaDataBuilder(indexMetaData) .state(IndexMetaData.State.CLOSE) .build(); } if (indexMetaData != null) { dangled.add(indexMetaData); } } IndexMetaData[] dangledIndices = dangled.toArray(new IndexMetaData[dangled.size()]); try { allocateDangledIndices.allocateDangled( dangledIndices, new LocalAllocateDangledIndices.Listener() { @Override public void onResponse( LocalAllocateDangledIndices.AllocateDangledResponse response) { logger.trace("allocated dangled"); } @Override public void onFailure(Throwable e) { logger.info("failed to send allocated dangled", e); } }); } catch (Exception e) { logger.warn("failed to send allocate dangled", e); } } } if (success) { currentMetaData = newMetaData; } }