/** * Main method for testing this class. * * @param args should contain arguments to the filter: use -h for help */ public static void main(String[] args) { try { if (Utils.getFlag('b', args)) { Filter.batchFilterFile(new AddExpression(), args); } else { Filter.filterFile(new AddExpression(), args); } } catch (Exception ex) { System.out.println(ex.getMessage()); } }
public Document processRecord(Document record, int position) throws Exception { for (HashMap<String, String> map : check_fields) { String fieldXpath = map.get("Field Name"); String regexp = map.get("Field Value"); Filter.apply(record, new OpDelete(), fieldXpath, new TextMatches(regexp)); } return record; }
public void report(JCDiagnostic diag) { if (!diag.isFlagSet(JCDiagnostic.DiagnosticFlag.NON_DEFERRABLE) && (filter == null || filter.accepts(diag))) { deferred.add(diag); } else { prev.report(diag); } }
public boolean include(Object object) { if (object instanceof IResource) { IResource resource = (IResource) object; IPath path = IResource.FILE == resource.getType() ? resource.getFullPath() : resource.getFullPath().addTrailingSeparator(); object = path.toPortableString(); } for (Iterator iter = filters.iterator(); iter.hasNext(); ) { Filter filter = (Filter) iter.next(); if (filter.matches(object)) { return filter.inclusive(); } } return default_; }
/** * Sets the format of the input instances. * * @param instanceInfo an Instances object containing the input instance structure (any instances * contained in the object are ignored - only the structure is required). * @return true if the outputFormat may be collected immediately * @exception Exception if the format couldn't be set successfully */ public boolean setInputFormat(Instances instanceInfo) throws Exception { convertInfixToPostfix(new String(m_infixExpression)); super.setInputFormat(instanceInfo); Instances outputFormat = new Instances(instanceInfo, 0); Attribute newAttribute; if (m_Debug) { newAttribute = new Attribute(m_postFixExpVector.toString()); } else if (m_attributeName.compareTo("expression") != 0) { newAttribute = new Attribute(m_attributeName); } else { newAttribute = new Attribute(m_infixExpression); } outputFormat.insertAttributeAt(newAttribute, instanceInfo.numAttributes()); setOutputFormat(outputFormat); return true; }
protected void process() { int i, j, len, ch, chunkLength; long progOff, progLen; float f1; // io AudioFile inF = null; AudioFile outF = null; AudioFileDescr inStream; AudioFileDescr outStream; FloatFile[] floatF = null; File tempFile[] = null; // buffers float[][] inBuf, outBuf; float[] win; float[] convBuf1, convBuf2; float[] tempFlt; int inChanNum, inLength, inputStep, outputStep, winSize; int transLen, skip, inputLen, outputLen, fltLen; int framesRead, framesWritten; float warp, a1, b0, b1, x0, x1, y0, y1, b0init; Param ampRef = new Param(1.0, Param.ABS_AMP); // transform-Referenz Param peakGain; float gain = 1.0f; // gain abs amp float maxAmp = 0.0f; PathField ggOutput; topLevel: try { // ---- open input, output ---- // input inF = AudioFile.openAsRead(new File(pr.text[PR_INPUTFILE])); inStream = inF.getDescr(); inChanNum = inStream.channels; inLength = (int) inStream.length; // this helps to prevent errors from empty files! if ((inLength * inChanNum) < 1) throw new EOFException(ERR_EMPTY); // .... check running .... if (!threadRunning) break topLevel; // output ggOutput = (PathField) gui.getItemObj(GG_OUTPUTFILE); if (ggOutput == null) throw new IOException(ERR_MISSINGPROP); outStream = new AudioFileDescr(inStream); ggOutput.fillStream(outStream); outF = AudioFile.openAsWrite(outStream); // .... check running .... if (!threadRunning) break topLevel; // ---- parameter inits ---- warp = Math.max(-0.98f, Math.min(0.98f, (float) (pr.para[PR_WARP].val / 100))); // DAFx2000 'b' f1 = (1.0f - warp) / (1.0f + warp); // DAFx2000 (25) winSize = 32 << pr.intg[PR_FRAMESIZE]; // DAFx2000 'N' j = winSize >> 1; transLen = (int) (f1 * winSize + 0.5f); // DAFx2000 'P' (26) i = pr.intg[PR_OVERLAP] + 1; while (((float) transLen / (float) i) > j) i++; inputStep = (int) (((float) transLen / (float) i) + 0.5f); // DAFx2000 'L' fltLen = Math.max(winSize, transLen); // System.out.println( "inputStep "+inputStep+"; winSize "+winSize+"; transLen "+transLen+"; // fltLen "+fltLen+"; warp "+warp+"; � "+f1 ); win = Filter.createFullWindow(winSize, Filter.WIN_HANNING); // DAFx2000 (27) outputStep = inputStep; b0init = (float) Math.sqrt(1.0f - warp * warp); progOff = 0; progLen = (long) inLength * (2 + inChanNum); // + winSize; tempFlt = new float[fltLen]; inputLen = winSize + inputStep; inBuf = new float[inChanNum][inputLen]; outputLen = transLen + outputStep; outBuf = new float[inChanNum][outputLen]; // normalization requires temp files if (pr.intg[PR_GAINTYPE] == GAIN_UNITY) { tempFile = new File[inChanNum]; floatF = new FloatFile[inChanNum]; for (ch = 0; ch < inChanNum; ch++) { // first zero them because an exception might be thrown tempFile[ch] = null; floatF[ch] = null; } for (ch = 0; ch < inChanNum; ch++) { tempFile[ch] = IOUtil.createTempFile(); floatF[ch] = new FloatFile(tempFile[ch], GenericFile.MODE_OUTPUT); } progLen += (long) inLength; } else { gain = (float) ((Param.transform(pr.para[PR_GAIN], Param.ABS_AMP, ampRef, null)).val); } // .... check running .... if (!threadRunning) break topLevel; // ----==================== the real stuff ====================---- framesRead = 0; framesWritten = 0; skip = 0; while (threadRunning && (framesWritten < inLength)) { chunkLength = Math.min(inputLen, inLength - framesRead + skip); // ---- read input chunk ---- len = Math.max(0, chunkLength - skip); inF.readFrames(inBuf, skip, len); framesRead += len; progOff += len; // off += len; // .... progress .... setProgression((float) progOff / (float) progLen); // .... check running .... if (!threadRunning) break topLevel; // zero padding if (chunkLength < inputLen) { for (ch = 0; ch < inChanNum; ch++) { convBuf1 = inBuf[ch]; for (i = chunkLength; i < convBuf1.length; i++) { convBuf1[i] = 0.0f; } } } for (ch = 0; threadRunning && (ch < inChanNum); ch++) { convBuf1 = inBuf[ch]; convBuf2 = outBuf[ch]; for (i = 0, j = fltLen; i < winSize; i++) { tempFlt[--j] = convBuf1[i] * win[i]; } while (j > 0) { tempFlt[--j] = 0.0f; } a1 = -warp; // inital allpass b0 = b0init; b1 = 0.0f; for (j = 0; j < transLen; j++) { x1 = 0.0f; y1 = 0.0f; // for( i = 0; i < transLen; i++ ) { // DAFx2000 (2 resp. 3) for (i = 0; i < fltLen; i++) { // DAFx2000 (2 resp. 3) x0 = tempFlt[i]; y0 = b0 * x0 + b1 * x1 - a1 * y1; tempFlt[i] = y0; // (work with double precision while computing cascades) y1 = y0; x1 = x0; } a1 = -warp; // cascaded allpasses b0 = -warp; b1 = 1.0f; convBuf2[j] += (float) y1; } // .... progress .... progOff += chunkLength - skip; setProgression((float) progOff / (float) progLen); } // for channels // .... check running .... if (!threadRunning) break topLevel; chunkLength = Math.min(outputStep, inLength - framesWritten); // ---- write output chunk ---- if (floatF != null) { for (ch = 0; ch < inChanNum; ch++) { floatF[ch].writeFloats(outBuf[ch], 0, chunkLength); } progOff += chunkLength; // off += len; framesWritten += chunkLength; // .... progress .... setProgression((float) progOff / (float) progLen); } else { for (ch = 0; ch < inChanNum; ch++) { Util.mult(outBuf[ch], 0, chunkLength, gain); } outF.writeFrames(outBuf, 0, chunkLength); progOff += chunkLength; // off += len; framesWritten += chunkLength; // .... progress .... setProgression((float) progOff / (float) progLen); } // .... check running .... if (!threadRunning) break topLevel; // check max amp for (ch = 0; ch < inChanNum; ch++) { convBuf1 = outBuf[ch]; for (i = 0; i < chunkLength; i++) { f1 = Math.abs(convBuf1[i]); if (f1 > maxAmp) { maxAmp = f1; } } } // overlaps skip = winSize; for (ch = 0; ch < inChanNum; ch++) { System.arraycopy(inBuf[ch], inputStep, inBuf[ch], 0, winSize); convBuf1 = outBuf[ch]; System.arraycopy(convBuf1, outputStep, convBuf1, 0, transLen); for (i = transLen; i < outputLen; ) { convBuf1[i++] = 0.0f; } } } // until framesWritten == outLength // .... check running .... if (!threadRunning) break topLevel; // ----==================== normalize output ====================---- if (pr.intg[PR_GAINTYPE] == GAIN_UNITY) { peakGain = new Param((double) maxAmp, Param.ABS_AMP); gain = (float) (Param.transform( pr.para[PR_GAIN], Param.ABS_AMP, new Param(1.0 / peakGain.val, peakGain.unit), null)) .val; normalizeAudioFile(floatF, outF, inBuf, gain, 1.0f); maxAmp *= gain; for (ch = 0; ch < inChanNum; ch++) { floatF[ch].cleanUp(); floatF[ch] = null; tempFile[ch].delete(); tempFile[ch] = null; } } // .... check running .... if (!threadRunning) break topLevel; // ---- Finish ---- outF.close(); outF = null; outStream = null; inF.close(); inF = null; inStream = null; inBuf = null; // inform about clipping/ low level handleClipping(maxAmp); } catch (IOException e1) { setError(e1); } catch (OutOfMemoryError e2) { inStream = null; outStream = null; inBuf = null; convBuf1 = null; convBuf2 = null; System.gc(); setError(new Exception(ERR_MEMORY)); ; } // ---- cleanup (topLevel) ---- if (inF != null) { inF.cleanUp(); inF = null; } if (outF != null) { outF.cleanUp(); outF = null; } if (floatF != null) { for (ch = 0; ch < floatF.length; ch++) { if (floatF[ch] != null) { floatF[ch].cleanUp(); floatF[ch] = null; } if (tempFile[ch] != null) { tempFile[ch].delete(); tempFile[ch] = null; } } } } // process()
/** * Sets the format of the input instances. * * @param instanceInfo an Instances object containing the input instance structure (any instances * contained in the object are ignored - only the structure is required). * @return true if the outputFormat may be collected immediately */ public boolean setInputFormat(Instances instanceInfo) throws Exception { super.setInputFormat(instanceInfo); setOutputFormat(instanceInfo); return true; }
/** * Updates the config file during the upgrade process. * * @param configPath The original path to the file. * @param filter The filter to select entries. Only useful for modify change type. * @param changeType The change type which must be applied to ldif lines. * @param ldifLines The change record ldif lines. For ADD change type, the first line must be the * dn. For DELETE change type, the first and only line must be the dn. * @throws IOException If an Exception occurs during the input output methods. * @return The changes number that have occurred. */ static int updateConfigFile( final String configPath, final Filter filter, final ChangeOperationType changeType, final String... ldifLines) throws IOException { final File original = new File(configPath); final File copyConfig = File.createTempFile("copyConfig", ".tmp", original.getParentFile()); int changeCount = 0; LDIFEntryReader entryReader = null; LDIFEntryWriter writer = null; try { final Schema schema = getUpgradeSchema(); entryReader = new LDIFEntryReader(new FileInputStream(configPath)).setSchema(schema); writer = new LDIFEntryWriter(new FileOutputStream(copyConfig)); writer.setWrapColumn(80); // Writes the header on the new file. writer.writeComment(INFO_CONFIG_FILE_HEADER.get()); writer.setWrapColumn(0); boolean entryAlreadyExist = false; DN ldifDN = null; if (filter == null && (changeType == ADD || changeType == DELETE)) { // The first line should start with dn: ldifDN = DN.valueOf(ldifLines[0].replaceFirst("dn: ", "")); } final Filter f = filter != null ? filter : Filter.alwaysFalse(); final Matcher matcher = f.matcher(schema); while (entryReader.hasNext()) { Entry entry = entryReader.readEntry(); final DN entryDN = entry.getName(); // Searching for the related entries if (changeType == MODIFY && matcher.matches(entry) == ConditionResult.TRUE) { try { final ModifyRequest mr = Requests.newModifyRequest(readLDIFLines(entryDN, changeType, ldifLines)); entry = Entries.modifyEntryPermissive(entry, mr.getModifications()); changeCount++; logger.debug( LocalizableMessage.raw("The following entry has been modified : %s", entryDN)); } catch (Exception ex) { logger.error(LocalizableMessage.raw(ex.getMessage())); } } if (entryDN.equals(ldifDN)) { logger.debug(LocalizableMessage.raw("Entry %s found", entryDN)); entryAlreadyExist = true; if (changeType == DELETE) { entry = null; changeCount++; logger.debug( LocalizableMessage.raw("The following entry has been deleted : %s", entryDN)); } } if (entry != null) { writer.writeEntry(entry); } } if (changeType == ADD && !entryAlreadyExist) { final AddRequest ar = Requests.newAddRequest(ldifLines); writer.writeEntry(ar); logger.debug( LocalizableMessage.raw( "Entry successfully added %s in %s", ldifDN, original.getAbsolutePath())); changeCount++; } } catch (Exception ex) { throw new IOException(ex.getMessage()); } finally { // The reader and writer must be close before renaming files. // Otherwise it causes exceptions under windows OS. StaticUtils.close(entryReader, writer); } try { // Renaming the file, overwriting previous one. rename(copyConfig, new File(configPath)); } catch (IOException e) { logger.error(LocalizableMessage.raw(e.getMessage())); deleteRecursively(original); throw e; } return changeCount; }
public void report(JCDiagnostic diag) { if (filter == null || filter.accepts(diag)) deferred.add(diag); else prev.report(diag); }
/** @throws Exception */ private void loadWorkload() throws Exception { final boolean debug = LOG.isDebugEnabled(); // Workload Trace if (this.params.containsKey(PARAM_WORKLOAD)) { assert (this.catalog_db != null) : "Missing catalog!"; String path = new File(this.params.get(PARAM_WORKLOAD)).getAbsolutePath(); boolean weightedTxns = this.getBooleanParam(PARAM_WORKLOAD_XACT_WEIGHTS, false); if (debug) LOG.debug("Use Transaction Weights in Limits: " + weightedTxns); // This will prune out duplicate trace records... if (params.containsKey(PARAM_WORKLOAD_REMOVE_DUPES)) { DuplicateTraceFilter filter = new DuplicateTraceFilter(); this.workload_filter = (this.workload_filter != null ? filter.attach(this.workload_filter) : filter); if (debug) LOG.debug("Attached " + filter.debugImpl()); } // TRANSACTION OFFSET if (params.containsKey(PARAM_WORKLOAD_XACT_OFFSET)) { this.workload_xact_offset = Long.parseLong(params.get(PARAM_WORKLOAD_XACT_OFFSET)); ProcedureLimitFilter filter = new ProcedureLimitFilter(-1l, this.workload_xact_offset, weightedTxns); // Important! The offset should go in the front! this.workload_filter = (this.workload_filter != null ? filter.attach(this.workload_filter) : filter); if (debug) LOG.debug("Attached " + filter.debugImpl()); } // BASE PARTITIONS if (params.containsKey(PARAM_WORKLOAD_RANDOM_PARTITIONS) || params.containsKey(PARAM_WORKLOAD_BASE_PARTITIONS)) { BasePartitionTxnFilter filter = new BasePartitionTxnFilter(new PartitionEstimator(catalog_db)); // FIXED LIST if (params.containsKey(PARAM_WORKLOAD_BASE_PARTITIONS)) { for (String p_str : this.getParam(PARAM_WORKLOAD_BASE_PARTITIONS).split(",")) { workload_base_partitions.add(Integer.valueOf(p_str)); } // FOR // RANDOM } else { double factor = this.getDoubleParam(PARAM_WORKLOAD_RANDOM_PARTITIONS); List<Integer> all_partitions = new ArrayList<Integer>(CatalogUtil.getAllPartitionIds(catalog_db)); Collections.shuffle(all_partitions, new Random()); workload_base_partitions.addAll( all_partitions.subList(0, (int) (all_partitions.size() * factor))); } filter.addPartitions(workload_base_partitions); this.workload_filter = (this.workload_filter != null ? this.workload_filter.attach(filter) : filter); if (debug) LOG.debug("Attached " + filter.debugImpl()); } // Txn Limit this.workload_xact_limit = this.getLongParam(PARAM_WORKLOAD_XACT_LIMIT); Histogram<String> proc_histogram = null; // Include/exclude procedures from the traces if (params.containsKey(PARAM_WORKLOAD_PROC_INCLUDE) || params.containsKey(PARAM_WORKLOAD_PROC_EXCLUDE)) { Filter filter = new ProcedureNameFilter(weightedTxns); // INCLUDE String temp = params.get(PARAM_WORKLOAD_PROC_INCLUDE); if (temp != null && !temp.equals(ProcedureNameFilter.INCLUDE_ALL)) { // We can take the counts for PROC_INCLUDE and scale them // with the multiplier double multiplier = 1.0d; if (this.hasDoubleParam(PARAM_WORKLOAD_PROC_INCLUDE_MULTIPLIER)) { multiplier = this.getDoubleParam(PARAM_WORKLOAD_PROC_INCLUDE_MULTIPLIER); if (debug) LOG.debug("Workload Procedure Multiplier: " + multiplier); } // Default Txn Frequencies String procinclude = params.get(PARAM_WORKLOAD_PROC_INCLUDE); if (procinclude.equalsIgnoreCase("default")) { procinclude = AbstractProjectBuilder.getProjectBuilder(catalog_type) .getTransactionFrequencyString(); } Map<String, Integer> limits = new HashMap<String, Integer>(); int total_unlimited = 0; int total = 0; for (String proc_name : procinclude.split(",")) { int limit = -1; // Check if there is a limit for this procedure if (proc_name.contains(":")) { String pieces[] = proc_name.split(":"); proc_name = pieces[0]; limit = (int) Math.round(Integer.parseInt(pieces[1]) * multiplier); } if (limit < 0) { if (proc_histogram == null) { if (debug) LOG.debug("Generating procedure histogram from workload file"); proc_histogram = WorkloadUtil.getProcedureHistogram(new File(path)); } limit = (int) proc_histogram.get(proc_name, 0); total_unlimited += limit; } else { total += limit; } limits.put(proc_name, limit); } // FOR // If we have a workload limit and some txns that we want // to get unlimited // records from, then we want to modify the other txns so // that we fill in the "gap" if (this.workload_xact_limit != null && total_unlimited > 0) { int remaining = this.workload_xact_limit.intValue() - total - total_unlimited; if (remaining > 0) { for (Entry<String, Integer> e : limits.entrySet()) { double ratio = e.getValue() / (double) total; e.setValue((int) Math.ceil(e.getValue() + (ratio * remaining))); } // FOR } } Histogram<String> proc_multiplier_histogram = null; if (debug) { if (proc_histogram != null) LOG.debug("Full Workload Histogram:\n" + proc_histogram); proc_multiplier_histogram = new Histogram<String>(); } total = 0; for (Entry<String, Integer> e : limits.entrySet()) { if (debug) proc_multiplier_histogram.put(e.getKey(), e.getValue()); ((ProcedureNameFilter) filter).include(e.getKey(), e.getValue()); total += e.getValue(); } // FOR if (debug) LOG.debug("Multiplier Histogram [total=" + total + "]:\n" + proc_multiplier_histogram); } // EXCLUDE temp = params.get(PARAM_WORKLOAD_PROC_EXCLUDE); if (temp != null) { for (String proc_name : params.get(PARAM_WORKLOAD_PROC_EXCLUDE).split(",")) { ((ProcedureNameFilter) filter).exclude(proc_name); } // FOR } // Sampling!! if (this.getBooleanParam(PARAM_WORKLOAD_PROC_SAMPLE, false)) { if (debug) LOG.debug("Attaching sampling filter"); if (proc_histogram == null) proc_histogram = WorkloadUtil.getProcedureHistogram(new File(path)); Map<String, Integer> proc_includes = ((ProcedureNameFilter) filter).getProcIncludes(); SamplingFilter sampling_filter = new SamplingFilter(proc_includes, proc_histogram); filter = sampling_filter; if (debug) LOG.debug("Workload Procedure Histogram:\n" + proc_histogram); } // Attach our new filter to the chain (or make it the head if // it's the first one) this.workload_filter = (this.workload_filter != null ? this.workload_filter.attach(filter) : filter); if (debug) LOG.debug("Attached " + filter.debugImpl()); } // TRANSACTION LIMIT if (this.workload_xact_limit != null) { ProcedureLimitFilter filter = new ProcedureLimitFilter(this.workload_xact_limit, weightedTxns); this.workload_filter = (this.workload_filter != null ? this.workload_filter.attach(filter) : filter); if (debug) LOG.debug("Attached " + filter.debugImpl()); } // QUERY LIMIT if (params.containsKey(PARAM_WORKLOAD_QUERY_LIMIT)) { this.workload_query_limit = Long.parseLong(params.get(PARAM_WORKLOAD_QUERY_LIMIT)); QueryLimitFilter filter = new QueryLimitFilter(this.workload_query_limit); this.workload_filter = (this.workload_filter != null ? this.workload_filter.attach(filter) : filter); } if (this.workload_filter != null && debug) LOG.debug("Workload Filters: " + this.workload_filter.toString()); this.workload = new Workload(this.catalog); this.workload.load(path, this.catalog_db, this.workload_filter); this.workload_path = new File(path).getAbsolutePath(); if (this.workload_filter != null) this.workload_filter.reset(); } // Workload Statistics if (this.catalog_db != null) { this.stats = new WorkloadStatistics(this.catalog_db); if (this.params.containsKey(PARAM_STATS)) { String path = this.params.get(PARAM_STATS); if (debug) LOG.debug("Loading in workload statistics from '" + path + "'"); this.stats_path = new File(path).getAbsolutePath(); try { this.stats.load(path, this.catalog_db); } catch (Throwable ex) { throw new RuntimeException("Failed to load stats file '" + this.stats_path + "'", ex); } } // Scaling if (this.params.containsKey(PARAM_STATS_SCALE_FACTOR)) { double scale_factor = this.getDoubleParam(PARAM_STATS_SCALE_FACTOR); LOG.info("Scaling TableStatistics: " + scale_factor); AbstractTableStatisticsGenerator generator = AbstractTableStatisticsGenerator.factory( this.catalog_db, this.catalog_type, scale_factor); generator.apply(this.stats); } } }