@Override
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler)
throws Exception {
Object usrLogin = request.getSession().getAttribute(Constants.SESS_USER_KEY);
String sevPath = request.getServletPath();
logger.info("ServletPath: " + sevPath);
String rootPath = "";
if (!StringUtil.isBlankOrNull(sevPath)
&& !sevPath.equals("/" + showLogin)
&& !sevPath.equals("/" + login)) {
int len = sevPath.split("/").length;
for (int i = 2; i < len; i++) {
rootPath += "../";
}
if (usrLogin == null) {
// response.sendRedirect(rootPath+showLogin);
StringBuffer toLoginScript = new StringBuffer();
toLoginScript.append("<script type=\"text/javascript\">");
toLoginScript.append("top.window.location=\"" + rootPath + showLogin + "\"");
toLoginScript.append("</script>");
logger.info(toLoginScript);
PrintWriter writer = response.getWriter();
writer.write(toLoginScript.toString());
writer.flush();
}
}
return true;
}
void processPage(String pageUri) {
String search = "href=\"";
try {
log.info("Calling to Google: " + pageUri);
String inputString = UrlUtils.getURL(pageUri);
log.info(inputString);
int next = 0;
Pattern cite = Pattern.compile("<cite>(.*?)</cite>");
Matcher matcher = cite.matcher(inputString);
while (matcher.find()) {
String newURI =
"http://"
+ matcher
.group(1)
.replaceAll("\"", "")
.replaceAll("<b>|</b>", "")
.replaceAll("[ \\t\\n\\r]+", "")
.trim();
log.info(newURI);
profiles.addDeviceIfNotAlreadyKnown(newURI);
}
} catch (Exception e) {
log.error(e.toString(), e);
System.exit(0);
}
}
// method of AbstractService
@Override
public synchronized void stop() {
LOG.info("stop of UdaShuffleHandler");
rdmaChannel.close();
super.stop();
LOG.info("stop of UdaShuffleHandler is done");
}
@Override
public void deleteResource() throws Exception {
log.info("delete resource: " + path + " ...");
Operation op = new Remove(address);
ComplexResult res = connection.executeComplex(op);
if (!res.isSuccess())
throw new IllegalArgumentException(
"Delete for [" + path + "] failed: " + res.getFailureDescription());
if (path.contains("server-group")) {
// This was a server group level deployment - TODO do we also need to remove the entry in
// /deployments ?
/*
for (PROPERTY_VALUE val : address) {
if (val.getKey().equals("deployment")) {
ComplexResult res2 = connection.executeComplex(new Operation("remove",val.getKey(),val.getValue()));
if (!res2.isSuccess())
throw new IllegalArgumentException("Removal of [" + path + "] falied : " + res2.getFailureDescription());
}
}
*/
}
log.info(" ... done");
}
/**
* Get a proxy connection to a remote server
*
* @param protocol protocol class
* @param addr remote address
* @param timeout time in milliseconds before giving up
* @return the proxy
* @throws IOException if the far end through a RemoteException
*/
static <V extends VersionedProtocol> V waitForProxy(
Class<V> protocol, InetSocketAddress addr, long timeout) throws IOException {
long startTime = System.currentTimeMillis();
IOException ioe;
while (true) {
try {
return getProxy(protocol, addr);
} catch (ConnectException se) { // namenode has not been started
LOG.info("Server at " + addr + " not available yet, Zzzzz...");
ioe = se;
} catch (SocketTimeoutException te) { // namenode is busy
LOG.info("Problem connecting to server: " + addr);
ioe = te;
}
// check if timed out
if (System.currentTimeMillis() - timeout >= startTime) {
throw ioe;
}
// wait for retry
try {
Thread.sleep(1000);
} catch (InterruptedException ie) {
// IGNORE
}
}
}
/**
* Optionally accepts 1 argument. If the argument evaluates to true ({@link
* Boolean#parseBoolean(String)}, the main method will NOT run the SQL statements in the
* "destroyDdl" Resource.
*
* @param args
*/
public static void main(String[] args) throws Exception {
LOG.info("loading applicationContext: " + CONFIG);
ApplicationContext context = new ClassPathXmlApplicationContext(CONFIG);
boolean firstRun = false;
if (args.length == 1) {
firstRun = Boolean.parseBoolean(args[0]);
}
InitializeSchedulingAssistantDatabase init = new InitializeSchedulingAssistantDatabase();
init.setDataSource((DataSource) context.getBean("dataSource"));
if (!firstRun) {
Resource destroyDdl = (Resource) context.getBean("destroyDdl");
if (null != destroyDdl) {
String destroySql = IOUtils.toString(destroyDdl.getInputStream());
init.executeDdl(destroySql);
LOG.warn("existing tables removed");
}
}
Resource createDdl = (Resource) context.getBean("createDdl");
String createSql = IOUtils.toString(createDdl.getInputStream());
init.executeDdl(createSql);
LOG.info("database initialization complete");
}
public void doRun() {
try {
log.info("Export started");
openDocument(writer);
// -----------------------------------------------------------------
// XML
// -----------------------------------------------------------------
beforeXML(writer);
for (XMLHrConverter converter : xmlConverters) {
converter.write(writer, params);
}
afterXML(writer);
closeDocument(writer);
StreamUtils.closeZipEntry(zipOutputStream);
log.info("Export done");
} catch (Exception ex) {
ex.printStackTrace();
throw new RuntimeException("Writing failed" + ex);
} finally {
writer.closeWriter();
StreamUtils.closeOutputStream(zipOutputStream);
NameMappingUtil.clearMapping();
}
}
/**
* Refresh the includes/excludes information.
*
* @throws IOException
*/
public synchronized void refreshNodes() throws IOException {
hostsReader.refresh();
LOG.info(
"After refresh Included hosts: "
+ hostsReader.getHostNames().size()
+ " Excluded hosts: "
+ hostsReader.getExcludedHosts().size());
Set<String> newHosts = hostsReader.getHostNames();
Set<String> newExcludes = hostsReader.getExcludedHosts();
Set<ClusterNode> hostsToExclude = new HashSet<ClusterNode>();
for (ClusterNode tmpNode : nameToNode.values()) {
String host = tmpNode.getHost();
// Check if not included or explicitly excluded.
if (!newHosts.contains(host) || newExcludes.contains(host)) {
hostsToExclude.add(tmpNode);
}
}
for (ClusterNode node : hostsToExclude) {
synchronized (node) {
for (Map.Entry<ResourceType, RunnableIndices> entry : typeToIndices.entrySet()) {
ResourceType type = entry.getKey();
RunnableIndices r = entry.getValue();
if (r.hasRunnable(node)) {
LOG.info(
"Node "
+ node.getName()
+ " is no longer "
+ type
+ " runnable because it is excluded");
r.deleteRunnable(node);
}
}
}
}
}
@Override
@KuneTransactional
public void execute(final JobExecutionContext context) throws JobExecutionException {
LOG.info(String.format("Immediate notifications cron job start, %s", pendingManager));
pendingManager.sendImmediateNotifications();
LOG.info(String.format("Immediate notifications cron job end, %s", pendingManager));
}
public static void shutdown() throws InterruptedException {
excutor.shutdownNow();
while (!excutor.awaitTermination(5, TimeUnit.SECONDS)) {
log.info("await for keyManager shutdown!");
}
log.info("keyManager has shutdown!");
}
/**
* Remove the @Ignore to try out timeout and retry asettings
*
* @throws IOException
*/
@Ignore
@Test
public void testTimeoutAndRetries() throws IOException {
Configuration localConfig = HBaseConfiguration.create(this.conf);
// This override mocks up our exists/get call to throw a RegionServerStoppedException.
localConfig.set("hbase.client.connection.impl", RpcTimeoutConnection.class.getName());
HTable table = new HTable(localConfig, TableName.META_TABLE_NAME);
Throwable t = null;
LOG.info("Start");
try {
// An exists call turns into a get w/ a flag.
table.exists(new Get(Bytes.toBytes("abc")));
} catch (SocketTimeoutException e) {
// I expect this exception.
LOG.info("Got expected exception", e);
t = e;
} catch (RetriesExhaustedException e) {
// This is the old, unwanted behavior. If we get here FAIL!!!
fail();
} finally {
table.close();
}
LOG.info("Stop");
assertTrue(t != null);
}
/** This test provides a demonstration of testing membership in a collection. */
@Test
public void testMemberOf() {
log.info("*** testMemberOf() ***");
CriteriaBuilder cb = em.getCriteriaBuilder();
CriteriaQuery<Clerk> qdef = cb.createQuery(Clerk.class);
// select c from Clerk c where c.firstName = 'Manny'
Root<Clerk> c = qdef.from(Clerk.class);
qdef.select(c).where(cb.equal(c.get("firstName"), "Manny"));
Clerk clerk = em.createQuery(qdef).getSingleResult();
// find all sales that involve this clerk
CriteriaQuery<Sale> qdef2 = cb.createQuery(Sale.class);
// select s from Sale s
// where :clerk MEMBER OF s.clerks",
Root<Sale> s = qdef2.from(Sale.class);
qdef2.select(s).where(cb.isMember(clerk, s.<List<Clerk>>get("clerks")));
List<Sale> sales = em.createQuery(qdef2).getResultList();
for (Sale result : sales) {
log.info("found=" + result);
}
assertEquals("unexpected number of rows", 2, sales.size());
}
/** This test provides a demonstration of using a math formual within the where clause. */
@Test
public void testFormulas() {
log.info("*** testFormulas() ***");
CriteriaBuilder cb = em.getCriteriaBuilder();
CriteriaQuery<Number> qdef = cb.createQuery(Number.class);
// select count(s) from Sale s
// where (s.amount * :tax) > :amount"
Root<Sale> s = qdef.from(Sale.class);
qdef.select(cb.count(s))
.where(
cb.greaterThan(
cb.prod(s.<BigDecimal>get("amount"), cb.parameter(BigDecimal.class, "tax")),
new BigDecimal(10.0)));
TypedQuery<Number> query = em.createQuery(qdef);
// keep raising taxes until somebody pays $10.00 in tax
double tax = 0.05;
for (;
query.setParameter("tax", new BigDecimal(tax)).getSingleResult().intValue() == 0;
tax += 0.01) {
log.debug("tax=" + NumberFormat.getPercentInstance().format(tax));
}
log.info("raise taxes to: " + NumberFormat.getPercentInstance().format(tax));
assertEquals("unexpected level for tax:" + tax, 0.07, tax, .01);
}
/** This test provides an example collection path using an LEFT OUTER JOIN */
@Test
public void testOuterJoin() {
log.info("*** testOuterJoin() ***");
CriteriaBuilder cb = em.getCriteriaBuilder();
CriteriaQuery<Object[]> qdef = cb.createQuery(Object[].class);
// select c.id, c.firstName, sale.amount
// from Clerk c
// LEFT JOIN c.sales sale
Root<Clerk> c = qdef.from(Clerk.class);
Join<Clerk, Sale> sale = c.join("sales", JoinType.LEFT);
qdef.select(cb.array(c.get("id"), c.get("firstName"), sale.get("amount")));
TypedQuery<Object[]> query = em.createQuery(qdef);
List<Object[]> results = query.getResultList();
assertTrue("no results", results.size() > 0);
for (Object[] result : results) {
assertEquals("unexpected result length", 3, result.length);
Long id = (Long) result[0];
String name = (String) result[1];
BigDecimal amount = (BigDecimal) result[2];
log.info("clerk.id=" + id + ", clerk.firstName=" + name + ", amount=" + amount);
}
}
@VisibleForTesting
void init(LinuxContainerExecutor lce, ResourceCalculatorPlugin plugin) throws IOException {
initConfig();
// mount cgroups if requested
if (cgroupMount && cgroupMountPath != null) {
ArrayList<String> cgroupKVs = new ArrayList<String>();
cgroupKVs.add(CONTROLLER_CPU + "=" + cgroupMountPath + "/" + CONTROLLER_CPU);
lce.mountCgroups(cgroupKVs, cgroupPrefix);
}
initializeControllerPaths();
// cap overall usage to the number of cores allocated to YARN
yarnProcessors = NodeManagerHardwareUtils.getContainersCores(plugin, conf);
int systemProcessors = plugin.getNumProcessors();
if (systemProcessors != (int) yarnProcessors) {
LOG.info("YARN containers restricted to " + yarnProcessors + " cores");
int[] limits = getOverallLimits(yarnProcessors);
updateCgroup(CONTROLLER_CPU, "", CPU_PERIOD_US, String.valueOf(limits[0]));
updateCgroup(CONTROLLER_CPU, "", CPU_QUOTA_US, String.valueOf(limits[1]));
} else if (cpuLimitsExist()) {
LOG.info("Removing CPU constraints for YARN containers.");
updateCgroup(CONTROLLER_CPU, "", CPU_QUOTA_US, String.valueOf(-1));
}
}
@Override
protected void serviceStop() throws Exception {
if (drainEventsOnStop) {
blockNewEvents = true;
LOG.info("AsyncDispatcher is draining to stop, igonring any new events.");
synchronized (waitForDrained) {
while (!drained && eventHandlingThread.isAlive()) {
waitForDrained.wait(1000);
LOG.info(
"Waiting for AsyncDispatcher to drain. Thread state is :"
+ eventHandlingThread.getState());
}
}
}
stopped = true;
if (eventHandlingThread != null) {
eventHandlingThread.interrupt();
try {
eventHandlingThread.join();
} catch (InterruptedException ie) {
LOG.warn("Interrupted Exception while stopping", ie);
}
}
// stop all the components
super.serviceStop();
}
private void storeOrUpdateRMDelegationTokenState(
RMDelegationTokenIdentifier identifier, Long renewDate, boolean isUpdate) throws Exception {
Path nodeCreatePath =
getNodePath(
rmDTSecretManagerRoot, DELEGATION_TOKEN_PREFIX + identifier.getSequenceNumber());
RMDelegationTokenIdentifierData identifierData =
new RMDelegationTokenIdentifierData(identifier, renewDate);
if (isUpdate) {
LOG.info("Updating RMDelegationToken_" + identifier.getSequenceNumber());
updateFile(nodeCreatePath, identifierData.toByteArray(), true);
} else {
LOG.info("Storing RMDelegationToken_" + identifier.getSequenceNumber());
writeFileWithRetries(nodeCreatePath, identifierData.toByteArray(), true);
// store sequence number
Path latestSequenceNumberPath =
getNodePath(
rmDTSecretManagerRoot,
DELEGATION_TOKEN_SEQUENCE_NUMBER_PREFIX + identifier.getSequenceNumber());
LOG.info(
"Storing " + DELEGATION_TOKEN_SEQUENCE_NUMBER_PREFIX + identifier.getSequenceNumber());
if (dtSequenceNumberPath == null) {
if (!createFileWithRetries(latestSequenceNumberPath)) {
throw new Exception("Failed to create " + latestSequenceNumberPath);
}
} else {
if (!renameFileWithRetries(dtSequenceNumberPath, latestSequenceNumberPath)) {
throw new Exception("Failed to rename " + dtSequenceNumberPath);
}
}
dtSequenceNumberPath = latestSequenceNumberPath;
}
}
/**
* Initialize the PersistenceManagerFactory for the given location.
*
* @throws IllegalArgumentException in case of illegal property values
* @throws IOException if the properties could not be loaded from the given location
* @throws JDOException in case of JDO initialization errors
*/
public void afterPropertiesSet() throws IllegalArgumentException, IOException, JDOException {
if (this.persistenceManagerFactoryName != null) {
if (this.configLocation != null || !this.jdoPropertyMap.isEmpty()) {
throw new IllegalStateException(
"'configLocation'/'jdoProperties' not supported in "
+ "combination with 'persistenceManagerFactoryName' - specify one or the other, not both");
}
if (logger.isInfoEnabled()) {
logger.info(
"Building new JDO PersistenceManagerFactory for name '"
+ this.persistenceManagerFactoryName
+ "'");
}
this.persistenceManagerFactory =
newPersistenceManagerFactory(this.persistenceManagerFactoryName);
} else {
Map<String, Object> mergedProps = new HashMap<String, Object>();
if (this.configLocation != null) {
if (logger.isInfoEnabled()) {
logger.info("Loading JDO config from [" + this.configLocation + "]");
}
CollectionUtils.mergePropertiesIntoMap(
PropertiesLoaderUtils.loadProperties(this.configLocation), mergedProps);
}
mergedProps.putAll(this.jdoPropertyMap);
logger.info("Building new JDO PersistenceManagerFactory");
this.persistenceManagerFactory = newPersistenceManagerFactory(mergedProps);
}
// Build default JdoDialect if none explicitly specified.
if (this.jdoDialect == null) {
this.jdoDialect =
new DefaultJdoDialect(this.persistenceManagerFactory.getConnectionFactory());
}
}
/**
* 使用Authorization code方式鉴权时，用授权码换取Access Token
*
* @param oAuth
* @return
* @throws Exception
*/
public static boolean accessToken(OAuthV2 oAuth) throws Exception {
if (null == Q_HTTP_CLIENT) {
throw new OAuthClientException("1001");
}
log.info(
"AuthorizeCode = "
+ oAuth.getAuthorizeCode()
+ "\nOpenid = "
+ oAuth.getOpenid()
+ "\nOpenkey ="
+ oAuth.getOpenkey());
String url = OAuthConstants.OAUTH_V2_GET_ACCESS_TOKEN_URL;
String queryString = QStrOperate.getQueryString(oAuth.getAccessTokenByCodeParamsList());
log.info("authorization queryString = " + queryString);
String responseData = Q_HTTP_CLIENT.simpleHttpGet(url, queryString);
log.info("authorization responseData = " + responseData);
if (!parseAccessToken(responseData, oAuth)) { // Access Token 授权不通过
oAuth.setStatus(3);
return false;
} else {
return true;
}
}
private static void createControlFile(
FileSystem fs,
int fileSize, // in MB
int nrFiles)
throws IOException {
LOG.info("creating control file: " + fileSize + " mega bytes, " + nrFiles + " files");
fs.delete(CONTROL_DIR, true);
for (int i = 0; i < nrFiles; i++) {
String name = getFileName(i);
Path controlFile = new Path(CONTROL_DIR, "in_file_" + name);
SequenceFile.Writer writer = null;
try {
writer =
SequenceFile.createWriter(
fs, fsConfig, controlFile, Text.class, LongWritable.class, CompressionType.NONE);
writer.append(new Text(name), new LongWritable(fileSize));
} catch (Exception e) {
throw new IOException(e.getLocalizedMessage());
} finally {
if (writer != null) writer.close();
writer = null;
}
}
LOG.info("created control files for: " + nrFiles + " files");
}
public void initCapability() {
try {
log.info("INFO: Before test, getting queue...");
Assert.assertNotNull(queueManagerFactory);
queueCapability = queueManagerFactory.create(mockResource);
queueCapability.initialize();
protocolManager.getProtocolSessionManagerWithContext(
mockResource.getResourceId(), newSessionContextNetconf());
// Test elements not null
log.info("Checking chassis factory");
Assert.assertNotNull(chassisFactory);
log.info("Checking capability descriptor");
Assert.assertNotNull(mockResource.getResourceDescriptor().getCapabilityDescriptor("chassis"));
log.info("Creating chassis capability");
chassisCapability = chassisFactory.create(mockResource);
Assert.assertNotNull(chassisCapability);
chassisCapability.initialize();
mockResource.addCapability(chassisCapability);
mockResource.addCapability(queueCapability);
} catch (Exception e) {
e.printStackTrace();
log.error(e.getMessage());
if (ExceptionUtils.getRootCause(e) != null)
log.error(ExceptionUtils.getRootCause(e).getMessage());
Assert.fail(e.getMessage());
}
}
@Test
public void testTimelinePutErrors() throws Exception {
TimelinePutResponse TimelinePutErrors = new TimelinePutResponse();
TimelinePutError error1 = new TimelinePutError();
error1.setEntityId("entity id 1");
error1.setEntityId("entity type 1");
error1.setErrorCode(TimelinePutError.NO_START_TIME);
TimelinePutErrors.addError(error1);
List<TimelinePutError> response = new ArrayList<TimelinePutError>();
response.add(error1);
TimelinePutError error2 = new TimelinePutError();
error2.setEntityId("entity id 2");
error2.setEntityId("entity type 2");
error2.setErrorCode(TimelinePutError.IO_EXCEPTION);
response.add(error2);
TimelinePutErrors.addErrors(response);
LOG.info("Errors in JSON:");
LOG.info(TimelineUtils.dumpTimelineRecordtoJSON(TimelinePutErrors, true));
Assert.assertEquals(3, TimelinePutErrors.getErrors().size());
TimelinePutError e = TimelinePutErrors.getErrors().get(0);
Assert.assertEquals(error1.getEntityId(), e.getEntityId());
Assert.assertEquals(error1.getEntityType(), e.getEntityType());
Assert.assertEquals(error1.getErrorCode(), e.getErrorCode());
e = TimelinePutErrors.getErrors().get(1);
Assert.assertEquals(error1.getEntityId(), e.getEntityId());
Assert.assertEquals(error1.getEntityType(), e.getEntityType());
Assert.assertEquals(error1.getErrorCode(), e.getErrorCode());
e = TimelinePutErrors.getErrors().get(2);
Assert.assertEquals(error2.getEntityId(), e.getEntityId());
Assert.assertEquals(error2.getEntityType(), e.getEntityType());
Assert.assertEquals(error2.getErrorCode(), e.getErrorCode());
}
/** check that the requested path is listed in the user permissions file */
private boolean checkPath(String userID, X509Certificate cert, String pathInfo) {
if (!checkUser(userID, cert)) {
return false;
}
Set<Path> pathSet = permsMap.get(userID);
if (pathSet == null) {
LOG.info("User " + userID + " is not listed in the user permissions file");
return false;
}
if (pathInfo == null || pathInfo.length() == 0) {
LOG.info("Can't get file path from HTTPS request; user is " + userID);
return false;
}
Path userPath = new Path(pathInfo);
while (userPath != null) {
if (LOG.isDebugEnabled()) {
LOG.debug("\n Checking file path " + userPath);
}
if (pathSet.contains(userPath)) return true;
userPath = userPath.getParent();
}
LOG.info("User " + userID + " is not authorized to access " + pathInfo);
return false;
}
@Override
public void closeOp(boolean abort) throws HiveException {
// ricardoj checking mapper hashmap usage
memoryMXBean = ManagementFactory.getMemoryMXBean();
System.gc();
System.gc();
System.gc();
System.gc();
System.gc();
System.gc();
long usedMemory = memoryMXBean.getHeapMemoryUsage().getUsed();
LOG.info("ricardoj memory usage after deleting tables: " + usedMemory / (1024 * 1024) + "MB");
if (mapJoinTables != null) {
for (HashMapWrapper<?, ?> hashTable : mapJoinTables.values()) {
hashTable.close();
}
}
mapJoinTables = null;
// ricardoj
System.gc();
System.gc();
System.gc();
System.gc();
System.gc();
System.gc();
usedMemory = memoryMXBean.getHeapMemoryUsage().getUsed();
LOG.info("ricardoj memory usage after deleting tables: " + usedMemory / (1024 * 1024) + "MB");
super.closeOp(abort);
}
// TODO check if minimum required fields are set?
@Override
public int sendMail() {
int sendResult = -999;
if (mail != null) {
LOGGER.trace(
"Sending Mail: \nfrom = '"
+ mail.getFrom()
+ "'\n"
+ "replyTo='"
+ mail.getTo()
+ "'\n"
+ "\n"
+ mail);
MailSenderPluginApi mailPlugin =
(MailSenderPluginApi) getContext().getWiki().getPluginApi("mailsender", getContext());
sendResult = mailPlugin.sendMail(mail, getMailConfiguration());
LOGGER.info(
"Sent Mail from '"
+ mail.getFrom()
+ "' to '"
+ mail.getTo()
+ "'. Result was '"
+ sendResult
+ "'. Time: "
+ Calendar.getInstance().getTimeInMillis());
} else {
LOGGER.info(
"Mail Object is null. Send result was '"
+ sendResult
+ "'. Time: "
+ Calendar.getInstance().getTimeInMillis());
}
return sendResult;
}
/** Test of execute method, of class ToolProcessor. */
public void testExecuteFileIdentify() throws Exception {
Tool tool = repo.getTool("file");
String tmpInputFile = this.getClass().getClassLoader().getResource("ps2pdf-input.ps").getFile();
LOG.debug("tmpInputFile = " + tmpInputFile);
LOG.info("TEST file-identify");
ToolProcessor processor = new ToolProcessor(tool);
Operation operation = processor.findOperation("identify");
processor.setOperation(operation);
Map<String, String> mapInput = new HashMap<String, String>();
mapInput.put("input", tmpInputFile);
processor.setInputFileParameters(mapInput);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
processor.next(new StreamProcessor(baos));
try {
processor.execute();
} catch (IOException ex) {
LOG.error("Exception during execution (maybe unresolved system dependency?): " + ex);
}
LOG.info("output: " + new String(baos.toByteArray()));
}
/*
* Recover file.
* Try and open file in append mode.
* Doing this, we get a hold of the file that crashed writer
* was writing to. Once we have it, close it. This will
* allow subsequent reader to see up to last sync.
* NOTE: This is the same algorithm that HBase uses for file recovery
* @param fs
* @throws Exception
*/
private void recoverFile(final FileSystem fs) throws Exception {
LOG.info("Recovering File Lease");
// set the soft limit to be 1 second so that the
// namenode triggers lease recovery upon append request
cluster.setLeasePeriod(1000, FSConstants.LEASE_HARDLIMIT_PERIOD);
// Trying recovery
int tries = 60;
boolean recovered = false;
FSDataOutputStream out = null;
while (!recovered && tries-- > 0) {
try {
out = fs.append(file1);
LOG.info("Successfully opened for appends");
recovered = true;
} catch (IOException e) {
LOG.info("Failed open for append, waiting on lease recovery");
try {
Thread.sleep(1000);
} catch (InterruptedException ex) {
// ignore it and try again
}
}
}
if (out != null) {
out.close();
}
if (!recovered) {
fail("Recovery should take < 1 min");
}
LOG.info("Past out lease recovery");
}
public void testExecuteFileIdentifyStdin() throws Exception {
LOG.info("TEST file-identify-stdin");
// This test may throw an "Broken pipe" IOException
// because file needs not to read the whole file data from
// stdin and will terminate while the other thread is reading streams.
Tool tool = repo.getTool("file");
String tmpInputFile = this.getClass().getClassLoader().getResource("ps2pdf-input.ps").getFile();
ToolProcessor processor = new ToolProcessor(tool);
Operation operation = processor.findOperation("identify-stdin");
processor.setOperation(operation);
FileInputStream fin = new FileInputStream(new File(tmpInputFile));
StreamProcessor in = new StreamProcessor(fin);
in.next(processor);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
baos = new ByteArrayOutputStream();
processor.next(new StreamProcessor(baos));
try {
in.execute();
} catch (IOException ex) {
LOG.error("Exception during execution (maybe unresolved system dependency?): " + ex);
}
LOG.info("output: " + new String(baos.toByteArray()));
}
/*
* check to see if hit the limit for max # completed apps kept
*/
protected synchronized void checkAppNumCompletedLimit() {
// check apps kept in state store.
while (completedAppsInStateStore > this.maxCompletedAppsInStateStore) {
ApplicationId removeId = completedApps.get(completedApps.size() - completedAppsInStateStore);
RMApp removeApp = rmContext.getRMApps().get(removeId);
LOG.info(
"Max number of completed apps kept in state store met:"
+ " maxCompletedAppsInStateStore = "
+ maxCompletedAppsInStateStore
+ ", removing app "
+ removeApp.getApplicationId()
+ " from state store.");
rmContext.getStateStore().removeApplication(removeApp);
completedAppsInStateStore--;
}
// check apps kept in memorty.
while (completedApps.size() > this.maxCompletedAppsInMemory) {
ApplicationId removeId = completedApps.remove();
LOG.info(
"Application should be expired, max number of completed apps"
+ " kept in memory met: maxCompletedAppsInMemory = "
+ this.maxCompletedAppsInMemory
+ ", removing app "
+ removeId
+ " from memory: ");
rmContext.getRMApps().remove(removeId);
this.applicationACLsManager.removeApplication(removeId);
}
}
// method of AbstractService
@Override
public synchronized void start() {
LOG.info("start of UdaShuffleHandler");
rdmaChannel = new UdaPluginSH(config);
super.start();
LOG.info("start of UdaShuffleHandler is done");
}