@Override public boolean waitMsecs(int msecs) { if (msecs < 0) { throw new RuntimeException("waitMsecs: msecs cannot be negative!"); } long maxMsecs = time.getMilliseconds() + msecs; int curMsecTimeout = 0; lock.lock(); try { while (!eventOccurred) { curMsecTimeout = Math.min(msecs, msecPeriod); if (LOG.isDebugEnabled()) { LOG.debug("waitMsecs: Wait for " + curMsecTimeout); } try { boolean signaled = cond.await(curMsecTimeout, TimeUnit.MILLISECONDS); if (LOG.isDebugEnabled()) { LOG.debug("waitMsecs: Got timed signaled of " + signaled); } } catch (InterruptedException e) { throw new IllegalStateException( "waitMsecs: Caught interrupted " + "exception on cond.await() " + curMsecTimeout, e); } if (time.getMilliseconds() > maxMsecs) { return false; } msecs = Math.max(0, msecs - curMsecTimeout); progressable.progress(); // go around again } } finally { lock.unlock(); } return true; }
/** * If the key is to be associated with a valid value, a mutation is created for it with the given * table and columns. In the event the value in the column is missing (i.e., null), then it is * marked for {@link Deletion}. Similarly, if the entire value for a key is missing (i.e., null), * then the entire key is marked for {@link Deletion}. * * @param keyColumns the key to write. * @param values the values to write. * @throws IOException */ @Override public void write(Map<String, ByteBuffer> keyColumns, List<ByteBuffer> values) throws IOException { TokenRange range = ringCache.getRange(getPartitionKey(keyColumns)); // get the client for the given range, or create a new one final InetAddress address = ringCache.getEndpoints(range).get(0); RangeClient client = clients.get(address); if (client == null) { // haven't seen keys for this range: create new client client = new RangeClient(ringCache.getEndpoints(range)); client.start(); clients.put(address, client); } // add primary key columns to the bind variables List<ByteBuffer> allValues = new ArrayList<ByteBuffer>(values); for (ColumnMetadata column : partitionKeyColumns) allValues.add(keyColumns.get(column.getName())); for (ColumnMetadata column : clusterColumns) allValues.add(keyColumns.get(column.getName())); client.put(allValues); if (progressable != null) progressable.progress(); if (context != null) HadoopCompat.progress(context); }
@Override public synchronized void write(BytesWritable key, BytesWritable value) throws IOException { try { mWriter.put(key.getBytes(), value.getBytes()); // Send a progress to the job manager to inform it that the task is still running. mProgress.progress(); } catch (TachyonException e) { throw new IOException(e); } }
public static <K extends Object, V extends Object> void writeFile( RawKeyValueIterator records, Writer<K, V> writer, Progressable progressable) throws IOException { long recordCtr = 0; while (records.next()) { writer.append(records.getKey(), records.getValue()); if ((++recordCtr % PROGRESS_BAR) == 0) { progressable.progress(); } } }
public static long copyFileToZipStream( File file, ZipOutputStream zipOutputStream, Progressable progressable) throws IOException { createNewZipEntry(zipOutputStream, file); long numRead = 0; try (FileInputStream inputStream = new FileInputStream(file)) { byte[] buf = new byte[0x10000]; for (int bytesRead = inputStream.read(buf); bytesRead >= 0; bytesRead = inputStream.read(buf)) { progressable.progress(); if (bytesRead == 0) { continue; } zipOutputStream.write(buf, 0, bytesRead); progressable.progress(); numRead += bytesRead; } } zipOutputStream.closeEntry(); progressable.progress(); return numRead; }
public static <K extends Object, V extends Object> void writeFile( RawKeyValueIterator records, Writer<K, V> writer, Progressable progressable, Configuration conf) throws IOException { long progressBar = conf.getLong("mapred.merge.recordsBeforeProgress", 10000); long recordCtr = 0; while (records.next()) { writer.append(records.getKey(), records.getValue()); if (((recordCtr++) % progressBar) == 0) { progressable.progress(); } } }
private void testLuceneIndexRecordReader( ArrayList<String> queryStrings, ArrayList<Path> indexPaths, ArrayList<ArrayList<ArrayList<Integer>>> indexesQueriesDocIds) throws Exception { LuceneIndexInputSplit split = createStrictMock(LuceneIndexInputSplit.class); expect(split.getIndexDirs()).andReturn(indexPaths); replay(split); Configuration conf = new Configuration(); TaskAttemptContext context = createStrictMock(TaskAttemptContext.class); expect(HadoopCompat.getConfiguration(context)).andStubReturn(conf); ((Progressable) context).progress(); // casting to avoid Hadoop 2 incompatibility expectLastCall().atLeastOnce(); replay(context); LuceneIndexInputFormat.setQueries(queryStrings, conf); LuceneIndexRecordReader<IntWritable> rr = createMockBuilder(MockRecordReader.class) .addMockedMethod("openIndex") .addMockedMethod("createSearcher") .createMock(); Query[] queries = new Query[queryStrings.size()]; for (int i = 0; i < queries.length; i++) { Query query = createStrictMock(Query.class); replay(query); queries[i] = query; expect(rr.deserializeQuery(queryStrings.get(i))).andReturn(query); } for (int index = 0; index < indexPaths.size(); index++) { IndexReader reader = createStrictMock(IndexReader.class); expect(reader.maxDoc()).andStubReturn(4); replay(reader); expect(rr.openIndex(indexPaths.get(index), conf)).andReturn(reader); IndexSearcher searcher = createStrictMock(IndexSearcher.class); expect(rr.createSearcher(reader)).andReturn(searcher); for (int query = 0; query < queries.length; query++) { final ArrayList<Integer> ids = indexesQueriesDocIds.get(index).get(query); final Capture<Collector> collectorCapture = new Capture<Collector>(); expect(searcher.getIndexReader()).andReturn(reader); searcher.search(eq(queries[query]), capture(collectorCapture)); expectLastCall() .andAnswer( new IAnswer<Void>() { @Override public Void answer() throws Throwable { for (int id : ids) { collectorCapture.getValue().collect(id); } return null; } }); for (int docId : ids) { expect(searcher.doc(docId)).andReturn(docs[docId]); } } replay(searcher); } replay(rr); rr.initialize(split, context); float prevProgress = -1; for (int index = 0; index < indexesQueriesDocIds.size(); index++) { for (int query = 0; query < indexesQueriesDocIds.get(index).size(); query++) { for (int docId : indexesQueriesDocIds.get(index).get(query)) { assertTrue(rr.nextKeyValue()); assertEquals(query, rr.getCurrentKey().get()); assertEquals(docsAndValues.get(docs[docId]), (Integer) rr.getCurrentValue().get()); float newProgress = rr.getProgress(); assertTrue(newProgress > prevProgress); assertTrue(newProgress <= 1.0); } } } assertFalse(rr.nextKeyValue()); assertFalse(rr.nextKeyValue()); verifyAll(); }
@Override public void waitForever() { while (!waitMsecs(msecPeriod)) { progressable.progress(); } }
private void progress() { if (progressable != null) progressable.progress(); }