Exemplo n.º 1
0
 /**
  * Sample rows from a frame. Can be unlucky for small sampling fractions - will continue calling
  * itself until at least 1 row is returned.
  *
  * @param fr Input frame
  * @param rows Approximate number of rows to sample (across all chunks)
  * @param seed Seed for RNG
  * @return Sampled frame
  */
 public static Frame sampleFrame(Frame fr, final long rows, final long seed) {
   if (fr == null) return null;
   final float fraction = rows > 0 ? (float) rows / fr.numRows() : 1.f;
   if (fraction >= 1.f) return fr;
   Frame r =
       new MRTask2() {
         @Override
         public void map(Chunk[] cs, NewChunk[] ncs) {
           final Random rng = getDeterRNG(seed + cs[0].cidx());
           int count = 0;
           for (int r = 0; r < cs[0]._len; r++)
             if (rng.nextFloat() < fraction || (count == 0 && r == cs[0]._len - 1)) {
               count++;
               for (int i = 0; i < ncs.length; i++) {
                 ncs[i].addNum(cs[i].at0(r));
               }
             }
         }
       }.doAll(fr.numCols(), fr).outputFrame(fr.names(), fr.domains());
   if (r.numRows() == 0) {
     Log.warn(
         "You asked for "
             + rows
             + " rows (out of "
             + fr.numRows()
             + "), but you got none (seed="
             + seed
             + ").");
     Log.warn("Let's try again. You've gotta ask yourself a question: \"Do I feel lucky?\"");
     return sampleFrame(fr, rows, seed + 1);
   }
   return r;
 }
Exemplo n.º 2
0
    // Handle the case where some centers go dry.  Rescue only 1 cluster
    // per iteration ('cause we only tracked the 1 worst row)
    boolean cleanupBadClusters(
        Lloyds task,
        final Vec[] vecs,
        final double[][] centers,
        final double[] means,
        final double[] mults,
        final int[] modes) {
      // Find any bad clusters
      int clu;
      for (clu = 0; clu < _parms._k; clu++) if (task._size[clu] == 0) break;
      if (clu == _parms._k) return false; // No bad clusters

      long row = task._worst_row;
      Log.warn("KMeans: Re-initializing cluster " + clu + " to row " + row);
      data(centers[clu] = task._cMeans[clu], vecs, row, means, mults, modes);
      task._size[clu] =
          1; // FIXME: PUBDEV-871 Some other cluster had their membership count reduced by one!
      // (which one?)

      // Find any MORE bad clusters; we only fixed the first one
      for (clu = 0; clu < _parms._k; clu++) if (task._size[clu] == 0) break;
      if (clu == _parms._k) return false; // No MORE bad clusters

      // If we see 2 or more bad rows, just re-run Lloyds to get the
      // next-worst row.  We don't count this as an iteration, because
      // we're not really adjusting the centers, we're trying to get
      // some centers *at-all*.
      Log.warn("KMeans: Re-running Lloyds to re-init another cluster");
      if (_reinit_attempts++ < _parms._k) {
        return true; // Rerun Lloyds, and assign points to centroids
      } else {
        _reinit_attempts = 0;
        return false;
      }
    }
Exemplo n.º 3
0
  /** Initialize SecondaryNameNode. */
  private void initialize(Configuration conf) throws IOException {
    // initiate Java VM metrics
    JvmMetrics.init("SecondaryNameNode", conf.get("session.id"));

    // Create connection to the namenode.
    shouldRun = true;
    nameNodeAddr = NameNode.getAddress(conf);

    this.conf = conf;
    this.namenode =
        (NamenodeProtocol)
            RPC.waitForProxy(
                NamenodeProtocol.class, NamenodeProtocol.versionID, nameNodeAddr, conf);

    // initialize checkpoint directories
    fsName = getInfoServer();
    checkpointDirs = FSImage.getCheckpointDirs(conf, "/tmp/hadoop/dfs/namesecondary");
    checkpointEditsDirs = FSImage.getCheckpointEditsDirs(conf, "/tmp/hadoop/dfs/namesecondary");
    checkpointImage = new CheckpointStorage(conf);
    checkpointImage.recoverCreate(checkpointDirs, checkpointEditsDirs);

    // Initialize other scheduling parameters from the configuration
    checkpointPeriod = conf.getLong("fs.checkpoint.period", 3600);
    checkpointSize = conf.getLong("fs.checkpoint.size", 4194304);

    // initialize the webserver for uploading files.
    String infoAddr =
        NetUtils.getServerAddress(
            conf,
            "dfs.secondary.info.bindAddress",
            "dfs.secondary.info.port",
            "dfs.secondary.http.address");
    InetSocketAddress infoSocAddr = NetUtils.createSocketAddr(infoAddr);
    infoBindAddress = infoSocAddr.getHostName();
    int tmpInfoPort = infoSocAddr.getPort();
    infoServer = new HttpServer("secondary", infoBindAddress, tmpInfoPort, tmpInfoPort == 0, conf);
    infoServer.setAttribute("name.system.image", checkpointImage);
    this.infoServer.setAttribute("name.conf", conf);
    infoServer.addInternalServlet("getimage", "/getimage", GetImageServlet.class);
    infoServer.start();

    // The web-server port can be ephemeral... ensure we have the correct info
    infoPort = infoServer.getPort();
    conf.set("dfs.secondary.http.address", infoBindAddress + ":" + infoPort);
    LOG.info("Secondary Web-server up at: " + infoBindAddress + ":" + infoPort);
    LOG.warn(
        "Checkpoint Period   :"
            + checkpointPeriod
            + " secs "
            + "("
            + checkpointPeriod / 60
            + " min)");
    LOG.warn(
        "Log Size Trigger    :"
            + checkpointSize
            + " bytes "
            + "("
            + checkpointSize / 1024
            + " KB)");
  }
Exemplo n.º 4
0
  /** Servlet GET request: handles event requests. */
  public void doGet(HttpServletRequest request, HttpServletResponse response)
      throws ServletException, IOException {
    Event event = null;

    try {
      // Event parm identifies event type from the client
      String eventType = Servlets.getParameter(request, P_EVENT);

      // Always must have an event type
      if (eventType == null) {
        Log.warn("Pushlet.doGet(): bad request, no event specified");
        response.sendError(HttpServletResponse.SC_BAD_REQUEST, "No eventType specified");
        return;
      }
      // Create Event and set attributes from parameters
      event = new Event(eventType);
      for (Enumeration e = request.getParameterNames(); e.hasMoreElements(); ) {
        String nextAttribute = (String) e.nextElement();
        event.setField(nextAttribute, request.getParameter(nextAttribute));
      }
    } catch (Throwable t) {
      // Error creating event
      Log.warn("Pushlet: Error creating event in doGet(): ", t);
      response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
      return;
    }
    // Handle parsed request
    doRequest(event, request, response);
  }
Exemplo n.º 5
0
  /**
   * 'handler' can be of any type that implements 'exportedInterface', but only methods declared by
   * the interface (and its superinterfaces) will be invocable.
   */
  public <T> InAppServer(
      String name,
      String portFilename,
      InetAddress inetAddress,
      Class<T> exportedInterface,
      T handler) {
    this.fullName = name + "Server";
    this.exportedInterface = exportedInterface;
    this.handler = handler;

    // In the absence of authentication, we shouldn't risk starting a server as root.
    if (System.getProperty("user.name").equals("root")) {
      Log.warn(
          "InAppServer: refusing to start unauthenticated server \"" + fullName + "\" as root!");
      return;
    }

    try {
      File portFile = FileUtilities.fileFromString(portFilename);
      secretFile = new File(portFile.getPath() + ".secret");
      Thread serverThread = new Thread(new ConnectionAccepter(portFile, inetAddress), fullName);
      // If there are no other threads left, the InApp server shouldn't keep us alive.
      serverThread.setDaemon(true);
      serverThread.start();
    } catch (Throwable th) {
      Log.warn("InAppServer: couldn't start \"" + fullName + "\".", th);
    }
    writeNewSecret();
  }
Exemplo n.º 6
0
  public void testLoggingWithNullParameters() {
    Log log = this.getLogObject();

    assertNotNull(log);

    log.debug(null);

    log.debug(null, null);

    log.debug(log.getClass().getName() + ": debug statement");

    log.debug(
        log.getClass().getName() + ": debug statement w/ null exception", new RuntimeException());

    log.error(null);

    log.error(null, null);

    log.error(log.getClass().getName() + ": error statement");

    log.error(
        log.getClass().getName() + ": error statement w/ null exception", new RuntimeException());

    log.fatal(null);

    log.fatal(null, null);

    log.fatal(log.getClass().getName() + ": fatal statement");

    log.fatal(
        log.getClass().getName() + ": fatal statement w/ null exception", new RuntimeException());

    log.info(null);

    log.info(null, null);

    log.info(log.getClass().getName() + ": info statement");

    log.info(
        log.getClass().getName() + ": info statement w/ null exception", new RuntimeException());

    log.trace(null);

    log.trace(null, null);

    log.trace(log.getClass().getName() + ": trace statement");

    log.trace(
        log.getClass().getName() + ": trace statement w/ null exception", new RuntimeException());

    log.warn(null);

    log.warn(null, null);

    log.warn(log.getClass().getName() + ": warn statement");

    log.warn(
        log.getClass().getName() + ": warn statement w/ null exception", new RuntimeException());
  }
Exemplo n.º 7
0
 /** Shut down this instance of the datanode. Returns only after shutdown is complete. */
 public void shutdown() {
   shouldRun = false;
   try {
     if (infoServer != null) infoServer.stop();
   } catch (Exception e) {
     LOG.warn("Exception shutting down SecondaryNameNode", e);
   }
   try {
     if (checkpointImage != null) checkpointImage.close();
   } catch (IOException e) {
     LOG.warn(StringUtils.stringifyException(e));
   }
 }
Exemplo n.º 8
0
 /** Generic request handler (GET+POST). */
 protected void doRequest(
     Event anEvent, HttpServletRequest request, HttpServletResponse response) {
   // Must have valid event type.
   String eventType = anEvent.getEventType();
   try {
     // Get Session: either by creating (on Join eventType)
     // or by id (any other eventType, since client is supposed to have
     // joined).
     Session session = null;
     if (eventType.startsWith(Protocol.E_JOIN)) {
       // Join request: create new subscriber
       session = SessionManager.getInstance().createSession(anEvent);
       String userAgent = request.getHeader("User-Agent");
       if (userAgent != null) {
         userAgent = userAgent.toLowerCase();
       } else {
         userAgent = "unknown";
       }
       session.setUserAgent(userAgent);
     } else {
       // Must be a request for existing Session
       // Get id
       String id = anEvent.getField(P_ID);
       // We must have an id value
       if (id == null) {
         response.sendError(HttpServletResponse.SC_BAD_REQUEST, "No id specified");
         Log.warn("Pushlet: bad request, no id specified event=" + eventType);
         return;
       }
       // We have an id: get the session object
       session = SessionManager.getInstance().getSession(id);
       // Check for invalid id
       if (session == null) {
         response.sendError(HttpServletResponse.SC_BAD_REQUEST, "Invalid or expired id: " + id);
         Log.warn("Pushlet:  bad request, no session found id=" + id + " event=" + eventType);
         return;
       }
     }
     // ASSERTION: we have a valid Session
     // Let Controller handle request further
     // including exceptions
     Command command = Command.create(session, anEvent, request, response);
     session.getController().doCommand(command);
   } catch (Throwable t) {
     // Hmm we should never ever get here
     Log.warn("Pushlet:  Exception in doRequest() event=" + eventType, t);
     t.printStackTrace();
     response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
   }
 }
Exemplo n.º 9
0
  @Test(dataProvider = "composeAllPermutationsOfSamInputResource")
  public void queryInputResourcePermutation(final SamInputResource resource) throws IOException {
    final SamReader reader = SamReaderFactory.makeDefault().open(resource);
    LOG.info(String.format("Query from %s ...", resource));
    if (reader.hasIndex()) {
      final StopWatch stopWatch = new StopWatch();
      stopWatch.start();
      final SAMRecordIterator q1 = reader.query("chr1", 500000, 100000000, true);
      observedRecordOrdering1.add(Iterables.slurp(q1));
      q1.close();
      final SAMRecordIterator q20 = reader.query("chr20", 1, 1000000, true);
      observedRecordOrdering20.add(Iterables.slurp(q20));
      q20.close();
      final SAMRecordIterator q3 = reader.query("chr3", 1, 10000000, true);
      observedRecordOrdering3.add(Iterables.slurp(q3));
      q3.close();
      stopWatch.stop();
      LOG.info(String.format("Finished queries in %sms", stopWatch.getElapsedTime()));

      Assert.assertEquals(
          observedRecordOrdering1.size(), 1, "read different records for chromosome 1");
      Assert.assertEquals(
          observedRecordOrdering20.size(), 1, "read different records for chromosome 20");
      Assert.assertEquals(
          observedRecordOrdering3.size(), 1, "read different records for chromosome 3");
    } else if (resource.indexMaybe() != null) {
      LOG.warn("Resource has an index source, but is not indexed: " + resource);
    } else {
      LOG.info("Skipping query operation: no index.");
    }
    reader.close();
  }
Exemplo n.º 10
0
  @Test
  @SuppressWarnings("deprecation")
  public void testOurLogLogging() {
    final Log logger = new Log();

    logger.trace("Foobar TRACE");
    AppenderForTests.hasNoLastEvent("at Trace level");
    assertFalse(logger.isTraceEnabled());

    logger.debug("Foobar DEBUG");
    AppenderForTests.hasNoLastEvent("at Debug level");
    assertFalse(logger.isDebugEnabled());

    logger.info("Foobar INFO");
    AppenderForTests.hasNoLastEvent("at Info level");
    assertFalse(logger.isInfoEnabled());

    logger.warn("Foobar WARN");
    AppenderForTests.hasLastEvent("at Warn level");
    assertTrue(logger.isWarnEnabled());

    logger.error("Foobar ERROR");
    AppenderForTests.hasLastEvent("at Error level");
    assertTrue(logger.isErrorEnabled());
  }
Exemplo n.º 11
0
  /** Filters input HTML using specified policy as white list of allowed tags. */
  @SuppressWarnings("unchecked")
  private String filter(String inputHtml, String policyFileName) {
    String filteredHtml = "";
    if (!StringUtils.isBlank(inputHtml)) {
      if (policyFileName == null) {
        LOG.warn("Provided policy file name is null.");
        policyFileName = DEFAULT_ANTISAMY_POLICY_FILE;
      }

      AntiSamy htmlScanner = getHtmlScannerByPolicyFileName(policyFileName);
      if (htmlScanner != null) {
        CleanResults scanResults;
        try {
          scanResults = htmlScanner.scan(inputHtml);
          filteredHtml = scanResults.getCleanHTML();
          ArrayList<String> scannerErrors = scanResults.getErrorMessages();
          if (!CollectionUtils.isNullOrEmpty(scannerErrors)) {
            LOG.trace("HTML input contains erorrs (" + scannerErrors.size() + "):");
            int i = 1;
            for (String error : scannerErrors) {
              LOG.trace("    " + i + ") " + error);
              i++;
            }
          }
        } catch (ScanException ex) {
          throw new HtmlScannerException(ex);
        } catch (PolicyException ex) {
          throw new HtmlScannerException(ex);
        }
      }
    }

    return filteredHtml;
  }
Exemplo n.º 12
0
  public void write(Object obj, Buffer buf) {

    if (obj == null) {
      buf.writeI8(TYPE_NULL);
    } else if (obj instanceof Float) {
      writeFloat((Float) obj, buf);
    } else if (obj instanceof Double) {
      writeDouble((Double) obj, buf);
    } else if (obj instanceof Number) {
      writeNumber((Number) obj, buf);
    } else if (obj instanceof String) {
      writeString((String) obj, buf);
    } else if (obj instanceof Boolean) {
      writeBoolean((Boolean) obj, buf);
    } else if (obj instanceof List) {
      writeList((List) obj, buf);
    } else if (obj instanceof Map) {
      writeMap((Map) obj, buf);
    } else {
      if (W) Log.warn("write variant: unknown type, writing null.");
      buf.writeI8(TYPE_NULL);
    }

    return;
  }
Exemplo n.º 13
0
  // Mostly for setting up the symlinks. Note that when we setup the distributed
  // cache, we didn't create the symlinks. This is done on a per task basis
  // by the currently executing task.
  public static void setupWorkDir(JobConf conf) throws IOException {
    File workDir = new File(".").getAbsoluteFile();
    FileUtil.fullyDelete(workDir);
    if (DistributedCache.getSymlink(conf)) {
      URI[] archives = DistributedCache.getCacheArchives(conf);
      URI[] files = DistributedCache.getCacheFiles(conf);
      Path[] localArchives = DistributedCache.getLocalCacheArchives(conf);
      Path[] localFiles = DistributedCache.getLocalCacheFiles(conf);
      if (archives != null) {
        for (int i = 0; i < archives.length; i++) {
          String link = archives[i].getFragment();
          if (link != null) {
            link = workDir.toString() + Path.SEPARATOR + link;
            File flink = new File(link);
            if (!flink.exists()) {
              FileUtil.symLink(localArchives[i].toString(), link);
            }
          }
        }
      }
      if (files != null) {
        for (int i = 0; i < files.length; i++) {
          String link = files[i].getFragment();
          if (link != null) {
            link = workDir.toString() + Path.SEPARATOR + link;
            File flink = new File(link);
            if (!flink.exists()) {
              FileUtil.symLink(localFiles[i].toString(), link);
            }
          }
        }
      }
    }
    File jobCacheDir = null;
    if (conf.getJar() != null) {
      jobCacheDir = new File(new Path(conf.getJar()).getParent().toString());
    }

    // create symlinks for all the files in job cache dir in current
    // workingdir for streaming
    try {
      DistributedCache.createAllSymlink(conf, jobCacheDir, workDir);
    } catch (IOException ie) {
      // Do not exit even if symlinks have not been created.
      LOG.warn(StringUtils.stringifyException(ie));
    }
    // add java.io.tmpdir given by mapred.child.tmp
    String tmp = conf.get("mapred.child.tmp", "./tmp");
    Path tmpDir = new Path(tmp);

    // if temp directory path is not absolute
    // prepend it with workDir.
    if (!tmpDir.isAbsolute()) {
      tmpDir = new Path(workDir.toString(), tmp);
      FileSystem localFs = FileSystem.getLocal(conf);
      if (!localFs.mkdirs(tmpDir) && !localFs.getFileStatus(tmpDir).isDir()) {
        throw new IOException("Mkdirs failed to create " + tmpDir.toString());
      }
    }
  }
Exemplo n.º 14
0
    // Register  protocol and its impl for rpc calls
    void registerProtocolAndImpl(RpcKind rpcKind, Class<?> protocolClass, Object protocolImpl) {
      String protocolName = RPC.getProtocolName(protocolClass);
      long version;

      try {
        version = RPC.getProtocolVersion(protocolClass);
      } catch (Exception ex) {
        LOG.warn("Protocol " + protocolClass + " NOT registered as cannot get protocol version ");
        return;
      }

      getProtocolImplMap(rpcKind)
          .put(
              new ProtoNameVer(protocolName, version),
              new ProtoClassProtoImpl(protocolClass, protocolImpl));
      LOG.debug(
          "RpcKind = "
              + rpcKind
              + " Protocol Name = "
              + protocolName
              + " version="
              + version
              + " ProtocolImpl="
              + protocolImpl.getClass().getName()
              + " protocolClass="
              + protocolClass.getName());
    }
Exemplo n.º 15
0
 @Override
 protected void init() {
   super.init();
   // Reject request if classification is required and response column is float
   // Argument a4class = find("classification"); // get UI control
   // String p4class = input("classification");  // get value from HTTP requests
   // if there is UI control and classification field was passed
   final boolean classificationFieldSpecified =
       true; // ROLLBACK: a4class!=null ? p4class!=null : /* we are not in UI so expect that
   // parameter is specified correctly */ true;
   if (!classificationFieldSpecified) { // can happen if a client sends a request which does not
     // specify classification parameter
     classification = response.isEnum();
     Log.warn(
         "Classification field is not specified - deriving according to response! The classification field set to "
             + classification);
   } else {
     if (classification && response.isFloat())
       throw new H2OIllegalArgumentException(
           find("classification"), "Requested classification on float column!");
     if (!classification && response.isEnum())
       throw new H2OIllegalArgumentException(
           find("classification"), "Requested regression on enum column!");
   }
 }
Exemplo n.º 16
0
 private void closeClientSocket() {
   try {
     client.close();
   } catch (IOException ex) {
     Log.warn(Thread.currentThread().getName() + ": failed to close client socket.", ex);
   }
 }
Exemplo n.º 17
0
  public Cone(Point apex, Vector axis, double radius) {
    this.apex = apex;
    this.axis = axis;
    this.radius = radius;

    Log.warn("Cone shape is not supported. This shape will be ignored.");
  }
  private int[] getSelectedRows(int[] selectedRowsNumber, Map[] selectedRowsKeys, Tab tab) {
    if (selectedRowsKeys == null || selectedRowsKeys.length == 0) return new int[0];
    // selectedRowsNumber is the most performant so we use it when possible
    else if (selectedRowsNumber.length == selectedRowsKeys.length) return selectedRowsNumber;
    else {
      // find the rows from the selectedKeys

      // This has a poor performance, but it covers the case when the selected
      // rows are not loaded for the tab, something that can occurs if the user
      // select rows and afterwards reorder the list.
      try {
        int[] s = new int[selectedRowsKeys.length];
        List selectedKeys = Arrays.asList(selectedRowsKeys);
        int end = tab.getTableModel().getTotalSize();
        int x = 0;
        for (int i = 0; i < end; i++) {
          Map key = (Map) tab.getTableModel().getObjectAt(i);
          if (selectedKeys.contains(key)) {
            s[x] = i;
            x++;
          }
        }
        return s;
      } catch (Exception ex) {
        log.warn(XavaResources.getString("fails_selected"), ex);
        throw new XavaException("fails_selected");
      }
    }
  }
Exemplo n.º 19
0
 private void writeHostAndPortToFile(File portFile) {
   String host = socket.getInetAddress().getHostName();
   int port = socket.getLocalPort();
   // The motivation for the Log.warn would be better satisfied by Bug 38.
   Log.warn("echo " + host + ":" + port + " > " + portFile);
   StringUtilities.writeFile(portFile, host + ":" + port + "\n");
 }
Exemplo n.º 20
0
 private void logSomething(boolean expectedDebug) {
   Log log = LogFactory.getLog(Object.class);
   log.warn("Warning message.");
   log.debug("Debug message.");
   log.error("Error message.");
   log.error("Error with Exception.", new Exception("Test exception."));
   assertEquals(expectedDebug, log.isDebugEnabled());
 }
Exemplo n.º 21
0
  /**
   * Instantiates a new MDC server.
   *
   * @param host the host
   * @param port the port
   */
  protected MDCServer(String host, int port) {
    _conf = Config.getConfig();

    address = (host == null) ? new InetSocketAddress(port) : new InetSocketAddress(host, port);

    /** initialize app command */
    Command.init();

    /** initialize the connection center */
    TConnCenter.init(_conf, port);

    synchronized (_conf) {
      /** load public key from database */
      TConn.pub_key = SystemConfig.s("pub_key", null);
      TConn.pri_key = SystemConfig.s("pri_key", null);

      /** initialize the RSA key, hardcode 2048 bits */
      if (TConn.pub_key == null
          || TConn.pri_key == null
          || "".equals(TConn.pub_key)
          || "".equals(TConn.pri_key)) {
        /** print out the old state */
        log.warn(
            "the pub_key or pri_key missed, the old state are pub_key:["
                + TConn.pub_key
                + "], pri_key:["
                + TConn.pri_key
                + "]");

        Key k = RSA.generate(2048);
        TConn.pri_key = k.pri_key;
        TConn.pub_key = k.pub_key;

        /** print out the new public key */
        log.warn("create new RSA key pair, pub_key:[" + TConn.pub_key + ']');

        /** set back in database */
        SystemConfig.setConfig("pri_key", TConn.pri_key);
        SystemConfig.setConfig("pub_key", TConn.pub_key);
      }

      MAX_SIZE = SystemConfig.i("mdc.max_size", MAX_SIZE);
    }
  }
Exemplo n.º 22
0
    /**
     * This method gets called everytime before any read/write to make sure that any change to
     * localDirs is reflected immediately.
     */
    private synchronized void confChanged(Configuration conf) throws IOException {
      String newLocalDirs = conf.get(contextCfgItemName);
      if (!newLocalDirs.equals(savedLocalDirs)) {
        String[] localDirs = conf.getStrings(contextCfgItemName);
        localFS = FileSystem.getLocal(conf);
        int numDirs = localDirs.length;
        ArrayList<String> dirs = new ArrayList<String>(numDirs);
        ArrayList<DF> dfList = new ArrayList<DF>(numDirs);
        for (int i = 0; i < numDirs; i++) {
          try {
            // filter problematic directories
            Path tmpDir = new Path(localDirs[i]);
            if (localFS.mkdirs(tmpDir) || localFS.exists(tmpDir)) {
              try {
                DiskChecker.checkDir(new File(localDirs[i]));
                dirs.add(localDirs[i]);
                dfList.add(new DF(new File(localDirs[i]), 30000));
              } catch (DiskErrorException de) {
                LOG.warn(localDirs[i] + "is not writable\n" + StringUtils.stringifyException(de));
              }
            } else {
              LOG.warn("Failed to create " + localDirs[i]);
            }
          } catch (IOException ie) {
            LOG.warn(
                "Failed to create "
                    + localDirs[i]
                    + ": "
                    + ie.getMessage()
                    + "\n"
                    + StringUtils.stringifyException(ie));
          } // ignore
        }
        localDirsPath = new Path[dirs.size()];
        for (int i = 0; i < localDirsPath.length; i++) {
          localDirsPath[i] = new Path(dirs.get(i));
        }
        dirDF = dfList.toArray(new DF[dirs.size()]);
        savedLocalDirs = newLocalDirs;

        // randomize the first disk picked in the round-robin selection
        dirNumLastAccessed = dirIndexRandomizer.nextInt(dirs.size());
      }
    }
Exemplo n.º 23
0
 private void acceptConnections() {
   for (; ; ) {
     try {
       String handlerName = fullName + "-Handler-" + Thread.activeCount();
       new Thread(new ClientHandler(socket.accept()), handlerName).start();
     } catch (Exception ex) {
       Log.warn(fullName + ": exception accepting connection.", ex);
     }
   }
 }
Exemplo n.º 24
0
  private void loadDatabaseMetadata() {
    if (!databaseMetadataLoaded) {
      String componentName = "UNKNOWN";
      Connection con = null;
      try {
        componentName = getMetaComponent().getName();

        con = DataSourceConnectionProvider.getByComponent(componentName).getConnection();
        DatabaseMetaData metaData = con.getMetaData();
        supportsSchemasInDataManipulation = metaData.supportsSchemasInDataManipulation();
        Collection timeDateFunctions =
            Strings.toCollection(metaData.getTimeDateFunctions().toUpperCase());

        //
        // another solution instead of the use of 'if' would be to use a xml with
        //	the information of the functions from each BBDD
        if ("DB2 UDB for AS/400".equals(metaData.getDatabaseProductName())
            || "Oracle".equals(metaData.getDatabaseProductName())
            || "PostgresSQL".equals(metaData.getDatabaseProductName())) {
          supportsTranslateFunction = true;
        }
        if ("Oracle".equals(metaData.getDatabaseProductName())
            || "PostgreSQL".equals(metaData.getDatabaseProductName())) {
          supportsYearFunction = supportsMonthFunction = false;
        } else {
          supportsYearFunction = timeDateFunctions.contains("YEAR");
          supportsMonthFunction = timeDateFunctions.contains("MONTH");
        }
        databaseMetadataLoaded = true;
      } catch (Exception ex) {
        log.warn(XavaResources.getString("load_database_metadata_warning"));
      } finally {
        try {
          if (con != null) {
            con.close();
          }
        } catch (SQLException e) {
          log.warn(XavaResources.getString("close_connection_warning"));
        }
      }
    }
  }
Exemplo n.º 25
0
 private void handleRequest() throws IOException {
   String line = in.readLine();
   if (line == null || line.length() == 0) {
     Log.warn(Thread.currentThread().getName() + ": ignoring empty request.");
     return;
   }
   if (handleCommand(line, out) == false) {
     out.println(
         Thread.currentThread().getName() + ": didn't understand request \"" + line + "\".");
   }
 }
Exemplo n.º 26
0
  // the method which actually copies the caches locally and unjars/unzips them
  // and does chmod for the files
  private static Path localizeCache(
      Configuration conf, URI cache, long confFileStamp, CacheStatus cacheStatus, boolean isArchive)
      throws IOException {
    FileSystem fs = getFileSystem(cache, conf);
    FileSystem localFs = FileSystem.getLocal(conf);
    Path parchive = null;

    if (isArchive) {
      parchive =
          new Path(
              cacheStatus.localizedLoadPath, new Path(cacheStatus.localizedLoadPath.getName()));
    } else {
      parchive = cacheStatus.localizedLoadPath;
    }
    if (!localFs.mkdirs(parchive.getParent())) {
      throw new IOException(
          "Mkdirs failed to create directory " + cacheStatus.localizedLoadPath.toString());
    }
    String cacheId = cache.getPath();

    fs.copyToLocalFile(new Path(cacheId), parchive);
    if (isArchive) {
      String tmpArchive = parchive.toString().toLowerCase();
      File srcFile = new File(parchive.toString());
      File destDir = new File(parchive.getParent().toString());
      if (tmpArchive.endsWith(".jar")) {
        RunJar.unJar(srcFile, destDir);
      } else if (tmpArchive.endsWith(".zip")) {
        FileUtil.unZip(srcFile, destDir);
      } else if (isTarFile(tmpArchive)) {
        FileUtil.unTar(srcFile, destDir);
      }
      // else will not do anyhting
      // and copy the file into the dir as it is
    }
    long cacheSize = FileUtil.getDU(new File(parchive.getParent().toString()));
    cacheStatus.size = cacheSize;
    addCacheInfoUpdate(cacheStatus);

    // do chmod here
    try {
      // Setting recursive permission to grant everyone read and execute
      Path localDir = new Path(cacheStatus.localizedBaseDir, cacheStatus.uniqueParentDir);
      LOG.info("Doing chmod on localdir :" + localDir);
      FileUtil.chmod(localDir.toString(), "ugo+rx", true);
    } catch (InterruptedException e) {
      LOG.warn("Exception in chmod" + e.toString());
    }

    // update cacheStatus to reflect the newly cached file
    cacheStatus.mtime = getTimestamp(conf, cache);
    return cacheStatus.localizedLoadPath;
  }
Exemplo n.º 27
0
 /** Servlet POST request: extracts event data from body. */
 public void doPost(HttpServletRequest request, HttpServletResponse response)
     throws ServletException, IOException {
   Event event = null;
   try {
     // Create Event by parsing XML from input stream.
     event = EventParser.parse(new InputStreamReader(request.getInputStream()));
     // Always must have an event type
     if (event.getEventType() == null) {
       Log.warn("Pushlet.doPost(): bad request, no event specified");
       response.sendError(HttpServletResponse.SC_BAD_REQUEST, "No eventType specified");
       return;
     }
   } catch (Throwable t) {
     // Error creating event
     Log.warn("Pushlet:  Error creating event in doPost(): ", t);
     response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
     return;
   }
   // Handle parsed request
   doRequest(event, request, response);
 }
Exemplo n.º 28
0
  public Object read(Buffer buf) {
    HeaderInfo info = parseHeader(buf);
    Object obj = null;

    switch (info.type) {
      case TYPE_INTEGER:
        obj = New.i8((byte) info.value);
        break;
      case TYPE_I8:
        obj = New.i8(buf.readI8());
        break;
      case TYPE_I16:
        obj = New.i16(buf.readI16());
        break;
      case TYPE_I32:
        obj = New.i32(buf.readI32());
        break;
      case TYPE_I64:
        obj = New.i64(buf.readI64());
        break;
      case TYPE_F32:
        obj = New.f32(buf.readF32());
        break;
      case TYPE_F64:
        obj = New.f64(buf.readF64());
        break;
      case TYPE_STRING:
        obj = readString(buf, info.length);
        break;
      case TYPE_TRUE:
        obj = New.bool(true);
        break;
      case TYPE_FALSE:
        obj = New.bool(false);
        break;
      case TYPE_NULL:
        obj = null;
        break;
      case TYPE_LIST:
        obj = readList(buf, info.length);
        break;
      case TYPE_MAP:
        obj = readMap(buf, info.length);
        break;
      default:
        if (W) Log.warn("read variant: unknown type, returning null.");
        obj = null;
    }

    New.release(info);
    return obj;
  }
Exemplo n.º 29
0
  /*
   * (non-Javadoc)
   *
   * @see
   * org.apache.mina.core.service.IoHandlerAdapter#sessionCreated(org.apache
   * .mina.core.session.IoSession)
   */
  public void sessionCreated(IoSession session) throws Exception {
    String remote = session.getRemoteAddress().toString();
    log.info("stub created:" + remote);

    /** check the allow ip */
    if (TConn.ALLOW_IP == null || "*".equals(TConn.ALLOW_IP) || remote.matches(TConn.ALLOW_IP)) {
      TConn d = new TConn(session);
      session.setAttribute("conn", d);
    } else {
      log.warn("deny the connection:" + remote + ", allow ip:" + TConn.ALLOW_IP);
      session.close(true);
    }
  }
Exemplo n.º 30
0
  /** Close. */
  public void close() {
    if (selector != null) {
      selector.wakeup();
      try {
        selector.close();
      } catch (IOException e1) {
        log.warn("close selector fails", e1);
      } finally {
        selector = null;
      }
    }

    if (server != null) {
      try {
        server.socket().close();
        server.close();
      } catch (IOException e) {
        log.warn("close socket server fails", e);
      } finally {
        server = null;
      }
    }
  }