protected void addViewParameters( FacesContext ctx, String viewId, Map<String, List<String>> existingParameters) { UIViewRoot currentRoot = ctx.getViewRoot(); String currentViewId = currentRoot.getViewId(); Collection<UIViewParameter> toViewParams; Collection<UIViewParameter> currentViewParams; boolean currentIsSameAsNew = false; currentViewParams = ViewMetadata.getViewParameters(currentRoot); if (currentViewId.equals(viewId)) { currentIsSameAsNew = true; toViewParams = currentViewParams; } else { ViewDeclarationLanguage pdl = getViewDeclarationLanguage(ctx, viewId); ViewMetadata viewMetadata = pdl.getViewMetadata(ctx, viewId); UIViewRoot root = viewMetadata.createMetadataView(ctx); toViewParams = ViewMetadata.getViewParameters(root); } if (toViewParams.isEmpty()) { return; } for (UIViewParameter viewParam : toViewParams) { String value; // don't bother looking at view parameter if it's been overridden if (existingParameters.containsKey(viewParam.getName())) { continue; } else if (paramHasValueExpression(viewParam)) { value = viewParam.getStringValueFromModel(ctx); } else { // Anonymous view parameter: // Get string value from UIViewParameter instance stored in current view if (currentIsSameAsNew) { value = viewParam.getStringValue(ctx); } // ...or transfer string value from matching UIViewParameter instance stored in current view else { value = getStringValueToTransfer(ctx, viewParam, currentViewParams); } } if (value != null) { List<String> existing = existingParameters.get(viewParam.getName()); if (existing == null) { existing = new ArrayList<String>(4); existingParameters.put(viewParam.getName(), existing); } existing.add(value); } } }
/** * Build an {@link IndexSegment} from the compacting merge of an index partition. * * @return The {@link BuildResult}. */ protected BuildResult doTask() throws Exception { final Event e = new Event( resourceManager.getFederation(), new EventResource(vmd.indexMetadata), OverflowActionEnum.Merge, vmd.getParams()) .start(); BuildResult buildResult = null; try { try { if (resourceManager.isOverflowAllowed()) throw new IllegalStateException(); /* * Build the index segment. * * Note: Since this is a compacting merge the view on the old * journal as of the last commit time will be fully captured by * the generated index segment. However, writes buffered by the * live journal WILL NOT be present in that index segment and * the post-condition view will include those writes. */ // build the index segment. buildResult = resourceManager.buildIndexSegment( vmd.name, vmd.getView(), true /* compactingMerge */, vmd.commitTime, null /* fromKey */, null /* toKey */, e); } finally { /* * Release our hold on the source view - we only needed it when * we did the index segment build. */ clearRefs(); } if (buildResult.builder.getCheckpoint().length >= resourceManager.nominalShardSize) { /* * If sumSegBytes exceeds the threshold, then do a split here. */ // FIXME reconcile return type and enable post-merge split. // return new SplitCompactViewTask(vmd.name, buildResult); } /* * @todo error handling should be inside of the atomic update task * since it has more visibility into the state changes and when we * can no longer delete the new index segment. */ try { // scale-out index UUID. final UUID indexUUID = vmd.indexMetadata.getIndexUUID(); // submit task and wait for it to complete concurrencyManager .submit( new AtomicUpdateCompactingMergeTask( resourceManager, concurrencyManager, vmd.name, indexUUID, buildResult, e.newSubEvent(OverflowSubtaskEnum.AtomicUpdate, vmd.getParams()))) .get(); // /* // * Verify that the view was updated. If the atomic update task // * runs correctly then it will replace the IndexMetadata object // * on the mutable BTree with a new view containing only the live // * journal and the new index segment (for a compacting merge). // * We verify that right now to make sure that the state change // * to the BTree was noticed and resulted in a commit before // * returning control to us here. // * // * @todo comment this out or replicate for the index build task // * also? // */ // concurrencyManager // .submit( // new VerifyAtomicUpdateTask(resourceManager, // concurrencyManager, vmd.name, // indexUUID, result)).get(); } catch (Throwable t) { // make it releasable. resourceManager.retentionSetRemove(buildResult.segmentMetadata.getUUID()); // delete the generated index segment. resourceManager.deleteResource( buildResult.segmentMetadata.getUUID(), false /* isJournal */); // re-throw the exception throw new Exception(t); } if (resourceManager.compactingMergeWithAfterAction) { /* * Consider possible after-actions now that the view is compact. * If any is selected, then it will be executed in the current * thread. */ final AbstractTask<?> afterActionTask = chooseAfterActionTask(); if (afterActionTask != null) { afterActionTask.call(); } } return buildResult; } finally { if (buildResult != null) { /* * At this point the index segment was either incorporated into * the new view in a restart safe manner or there was an error. * Either way, we now remove the index segment store's UUID from * the retentionSet so it will be subject to the release policy * of the StoreManager. */ resourceManager.retentionSetRemove(buildResult.segmentMetadata.getUUID()); } e.end(); } }
@Override protected void clearRefs() { vmd.clearRef(); }
/** * Now that the index partition is compact, decide if we will take any after action, such as * {move, join, split, tailSplit, scatterSplit, etc). All of these operations are much cheaper * while the index is compact which is why we do them here. * * <p>Note: asynchronous overflow processing WILL NOT complete until the CompactingMergeTask is * done. This means that we will still be reading from the same journal. As long as we are reading * from the same ordered set of resources the lastCommitTime chosen here is somewhat arbitrary. * * <p>The updated view metadata as of the last commit time on the live journal. * * <p>FIXME Concurrent operations can replace the view definition. However, what would not be good * is if they changed the set of resources in the view. The AtomicUpdate of the after action task * MUST check for this precondition (same set of resources in the view) and abort (and clean up * any intermediate files) if the precondition has been violated (no harm is done if we abort, * just some lost work). * * @todo split + move and friends seem unnecessarily complicated. We can just move anything that * is compact. [Clean up the tasks to remove this stuff.] * @todo We might be better off running {@link #chooseAfterActionTask()} from inside of the atomic * update and then doing any work there while we have the lock on the shard. This will prevent * any new data from building up and can help ensure that the preconditions for the operation * remain valid. This might also help simplify the HA design. * @todo Once we have flow control on writes we can save the DS a lot of work by not accepting new * writes for an index partition when we are going to compact it, move it, split it, etc. */ private AbstractTask<?> chooseAfterActionTask() { final ViewMetadata vmd = new ViewMetadata( resourceManager, resourceManager.getLiveJournal().getLastCommitTime(), this.vmd.name, resourceManager.getIndexCounters(this.vmd.name)); /* * Scatter split? * * Note: Scatter splits are considered before tail splits and normal * splits since they can only be taken when there is a single index * partition for a scale-out index. The other kinds of splits are used * once the index has already been distributed onto the cluster by a * scatter split. */ { final ScatterSplitConfiguration ssc = vmd.indexMetadata.getScatterSplitConfiguration(); if ( // only a single index partitions? (vmd.getIndexPartitionCount() == 1L) // // scatter splits enabled for service && resourceManager.scatterSplitEnabled // // scatter splits enabled for index && ssc.isEnabled() // // The view is compact (only one segment). && vmd.compactView // // trigger scatter split before too much data builds up in one place. && vmd.getPercentOfSplit() >= ssc.getPercentOfSplitThreshold()) { // Target data services for the new index partitions. final UUID[] moveTargets = getScatterSplitTargets(ssc); if (moveTargets != null) { // #of splits. final int nsplits = ssc.getIndexPartitionCount() == 0 // ? (2 * moveTargets.length) // two per data service. : ssc.getIndexPartitionCount() // ; if (log.isInfoEnabled()) log.info("will scatter: " + vmd); // scatter split task. return new ScatterSplitTask(vmd, nsplits, moveTargets); } } } /* * Tail split? * * Note: We can do a tail split as long as we are "close" to a full * index partition. We have an expectation that the head of the split * will be over the minimum capacity. While the tail of the split MIGHT * be under the minimum capacity, if there are continued heavy writes on * the tail then it will should reach the minimum capacity for an index * partition by the time the live journal overflows again. */ if (vmd.isTailSplit() && false) { /* * FIXME The current tailSplit implementation operations against the * BTree, NOT the FusedView and NOT the IndexSegment. It needs to be * refactored before it can be an after action for a compacting * merge. * * It is written to identify the separator key based on an * examination of the mutable BTree. Once it has the separator key * it then does a normal build for each key-range. [@todo It * probably should use a compacting merge in order to avoid sharing * index segments across shards.] */ if (log.isInfoEnabled()) log.info("Will tailSpl" + vmd.name); return new SplitTailTask(vmd, null /* moveTarget */); } /* * Should split? * * Note: Split is NOT allowed if the index is currently being moved * onto this data service. Split, join, and move are all disallowed * until the index partition move is complete since each of them * would cause the index partition to become invalidated. */ if (vmd.getPercentOfSplit() > 1.0) { if (log.isInfoEnabled()) log.info("will split : " + vmd); return new SplitIndexPartitionTask(vmd, (UUID) null /* moveTarget */); } /* * Join undercapacity shard (either with local rightSibling or move to * join with remote rightSibling). * * If the rightSibling of an undercapacity index partition is also local * then a {@link JoinIndexPartitionTask} is used to join those index * partitions. * * If the rightSibling of an undercapacity index partition is remote, * then a {@link MoveTask} is created to move the undercapacity index * partition to the remove data service. * * Note: joins are only considered when the rightSibling of an index * partition exists. The last index partition has [rightSeparatorKey == * null] and there is no rightSibling for that index partition. * * @todo What kinds of guarantees do we have that a local rightSibling * will be around by the time the JoinIndexPartitionTask runs? * * @todo This has even more assumptions about [lastCommitTime] than the * other tasks. All these tasks need to be reviewed to make sure that * there are no gaps created by this refactor. Running these after * action tasks while we hold the write lock on the source shard could * probably help us to reduce the possibility of any such problems but * might require a revisit / refactor / simplification of the tasks. * * FIXME Make sure that we are not running compacting merges as part of * the split, scatter split and other tasks. Some tasks used to do this * in order to have a compact view. */ if (resourceManager.joinsEnabled && vmd.pmd.getRightSeparatorKey() != null && vmd.getPercentOfSplit() < resourceManager.percentOfJoinThreshold) { final String scaleOutIndexName = vmd.indexMetadata.getName(); final PartitionLocator rightSiblingLocator = getRightSiblingLocator(scaleOutIndexName, vmd.commitTime); if (rightSiblingLocator != null) { final UUID targetDataServiceUUID = rightSiblingLocator.getDataServiceUUID(); final String[] resources = new String[2]; // the underutilized index partition. resources[0] = DataService.getIndexPartitionName(scaleOutIndexName, vmd.pmd.getPartitionId()); // its right sibling (may be local or remote). resources[1] = DataService.getIndexPartitionName( scaleOutIndexName, rightSiblingLocator.getPartitionId()); if (resourceManager.getDataServiceUUID().equals(targetDataServiceUUID)) { /* * JOIN underutilized index partition with its local * rightSibling. * * Note: This is only joining two index partitions at a * time. It's possible to do more than that if it happens * that N > 2 underutilized sibling index partitions are on * the same data service, but that is a relatively unlikely * combination of events. */ if (log.isInfoEnabled()) log.info("Will JOIN: " + Arrays.toString(resources)); final String rightSiblingName = DataService.getIndexPartitionName( scaleOutIndexName, rightSiblingLocator.getPartitionId()); final ViewMetadata vmd2 = new ViewMetadata( resourceManager, vmd.commitTime, rightSiblingName, resourceManager.getIndexCounters(rightSiblingName)); return new JoinIndexPartitionTask( resourceManager, vmd.commitTime, resources, new ViewMetadata[] {vmd, vmd2}); } else { /* * MOVE underutilized index partition to data service * hosting the right sibling. * * @todo The decision to join shards is asymmetric (an * undercapacity shard is moved to its rightSibling). * However, it is possible that its rightSibling was also * undercapacity and was either moved to or locally joined * with its rightSibling (in which case its partition * identifier would have been changed). To avoid these edge * cases there could be a global synchronous agreement for * move/join decisions */ if (log.isInfoEnabled()) { // get the target service name. String targetDataServiceName; try { targetDataServiceName = resourceManager .getFederation() .getDataService(targetDataServiceUUID) .getServiceName(); } catch (Throwable t) { targetDataServiceName = targetDataServiceUUID.toString(); } log.info( "willMoveToJoinWithRightSibling" + "( " + vmd.name + " -> " + targetDataServiceName // + ", leftSibling=" + resources[0] // + ", rightSibling=" + resources[1] // + ")"); } return new MoveTask(vmd, targetDataServiceUUID); } } // rightSibling != null } // if(join) /* * Move (to shed or redistribute load). * * @todo We should prefer to move smaller shards (faster to move) or * "hotter" shards (sheds more workload). There should be a way to * estimate how much workload will be transferred so we know when we are * done. * * FIXME We should limit the #of shards that we move in a given period * of time to allow both this host and the target host an opportunity to * adapt to their new load. [An exception would be if this host was * critically overloaded, but that should probably be handled by * different logic.] */ ILoadBalancerService loadBalancerService = null; if (vmd.getPercentOfSplit() < resourceManager.maximumMovePercentOfSplit && resourceManager.maximumMovesPerTarget != 0 && resourceManager.getLiveJournal().getName2Addr().rangeCount() > resourceManager.minimumActiveIndexPartitions && (loadBalancerService = getLoadBalancerService()) != null && shouldMove(loadBalancerService)) { // the UUID of this data service. final UUID sourceServiceUUID = resourceManager.getDataServiceUUID(); // Obtain UUID of a relatively underutilized data service. final UUID targetDataServiceUUID = getMoveTarget(sourceServiceUUID, loadBalancerService); if (targetDataServiceUUID != null) { if (log.isInfoEnabled()) { // get the target service name. String targetDataServiceName; try { targetDataServiceName = resourceManager .getFederation() .getDataService(targetDataServiceUUID) .getServiceName(); } catch (Throwable t) { targetDataServiceName = targetDataServiceUUID.toString(); } log.info("willMove" + "( " + vmd.name + " -> " + targetDataServiceName + ")"); } // Move the shard to the target host. return new MoveTask(vmd, targetDataServiceUUID); } } // No after action was chosen. return null; }