Пример #1
0
  /** Create a new checkpoint */
  void doCheckpoint() throws IOException {

    LOG.info("Checkpoint starting");

    // Do the required initialization of the merge work area.
    startCheckpoint();

    // Tell the namenode to start logging transactions in a new edit file
    // Returns a token that would be used to upload the merged image.
    CheckpointSignature sig = (CheckpointSignature) namenode.rollEditLog();

    // error simulation code for junit test
    if (ErrorSimulator.getErrorSimulation(0)) {
      throw new IOException("Simulating error0 " + "after creating edits.new");
    }

    boolean loadImage = downloadCheckpointFiles(sig); // Fetch fsimage and edits
    doMerge(sig, loadImage); // Do the merge

    //
    // Upload the new image into the NameNode. Then tell the Namenode
    // to make this new uploaded image as the most current image.
    //
    putFSImage(sig);

    // error simulation code for junit test
    if (ErrorSimulator.getErrorSimulation(1)) {
      throw new IOException("Simulating error1 " + "after uploading new image to NameNode");
    }

    namenode.rollFsImage(new CheckpointSignature(checkpointImage));
    checkpointImage.endCheckpoint();

    LOG.info("Checkpoint done. New Image Size: " + checkpointImage.getFsImageName().length());
  }
Пример #2
0
  /**
   * Download <code>fsimage</code> and <code>edits</code> files from the name-node.
   *
   * @return true if a new image has been downloaded and needs to be loaded
   * @throws IOException
   */
  private boolean downloadCheckpointFiles(CheckpointSignature sig) throws IOException {

    checkpointImage.cTime = sig.cTime;
    checkpointImage.checkpointTime = sig.checkpointTime;

    boolean downloadImage = true;
    String fileid;
    File[] srcNames;
    if (sig.imageDigest.equals(checkpointImage.imageDigest)) {
      downloadImage = false;
      LOG.info("Image has not changed. Will not download image.");
    } else {
      // get fsimage
      srcNames = checkpointImage.getImageFiles();
      assert srcNames.length > 0 : "No checkpoint targets.";
      fileid = "getimage=1";
      TransferFsImage.getFileClient(fsName, fileid, srcNames, false);
      checkpointImage.imageDigest = sig.imageDigest;
      LOG.info(
          "Downloaded file " + srcNames[0].getName() + " size " + srcNames[0].length() + " bytes.");
    }
    // get edits file
    fileid = "getedit=1";
    srcNames = checkpointImage.getEditsFiles();
    assert srcNames.length > 0 : "No checkpoint targets.";
    TransferFsImage.getFileClient(fsName, fileid, srcNames, false);
    LOG.info(
        "Downloaded file " + srcNames[0].getName() + " size " + srcNames[0].length() + " bytes.");

    checkpointImage.checkpointUploadDone(null);

    return downloadImage;
  }
Пример #3
0
  /** Initialize SecondaryNameNode. */
  private void initialize(Configuration conf) throws IOException {
    // initiate Java VM metrics
    JvmMetrics.init("SecondaryNameNode", conf.get("session.id"));

    // Create connection to the namenode.
    shouldRun = true;
    nameNodeAddr = NameNode.getAddress(conf);

    this.conf = conf;
    this.namenode =
        (NamenodeProtocol)
            RPC.waitForProxy(
                NamenodeProtocol.class, NamenodeProtocol.versionID, nameNodeAddr, conf);

    // initialize checkpoint directories
    fsName = getInfoServer();
    checkpointDirs = FSImage.getCheckpointDirs(conf, "/tmp/hadoop/dfs/namesecondary");
    checkpointEditsDirs = FSImage.getCheckpointEditsDirs(conf, "/tmp/hadoop/dfs/namesecondary");
    checkpointImage = new CheckpointStorage(conf);
    checkpointImage.recoverCreate(checkpointDirs, checkpointEditsDirs);

    // Initialize other scheduling parameters from the configuration
    checkpointPeriod = conf.getLong("fs.checkpoint.period", 3600);
    checkpointSize = conf.getLong("fs.checkpoint.size", 4194304);

    // initialize the webserver for uploading files.
    String infoAddr =
        NetUtils.getServerAddress(
            conf,
            "dfs.secondary.info.bindAddress",
            "dfs.secondary.info.port",
            "dfs.secondary.http.address");
    InetSocketAddress infoSocAddr = NetUtils.createSocketAddr(infoAddr);
    infoBindAddress = infoSocAddr.getHostName();
    int tmpInfoPort = infoSocAddr.getPort();
    infoServer = new HttpServer("secondary", infoBindAddress, tmpInfoPort, tmpInfoPort == 0, conf);
    infoServer.setAttribute("name.system.image", checkpointImage);
    this.infoServer.setAttribute("name.conf", conf);
    infoServer.addInternalServlet("getimage", "/getimage", GetImageServlet.class);
    infoServer.start();

    // The web-server port can be ephemeral... ensure we have the correct info
    infoPort = infoServer.getPort();
    conf.set("dfs.secondary.http.address", infoBindAddress + ":" + infoPort);
    LOG.info("Secondary Web-server up at: " + infoBindAddress + ":" + infoPort);
    LOG.warn(
        "Checkpoint Period   :"
            + checkpointPeriod
            + " secs "
            + "("
            + checkpointPeriod / 60
            + " min)");
    LOG.warn(
        "Log Size Trigger    :"
            + checkpointSize
            + " bytes "
            + "("
            + checkpointSize / 1024
            + " KB)");
  }
Пример #4
0
  /*
   * (non-Javadoc)
   *
   * @see
   * org.apache.mina.core.service.IoHandlerAdapter#sessionClosed(org.apache
   * .mina.core.session.IoSession)
   */
  public void sessionClosed(IoSession session) throws Exception {
    log.info("closed stub: " + session.getRemoteAddress());
    TConn d = (TConn) session.getAttribute("conn");
    if (d != null) {
      d.close();

      session.removeAttribute("conn");
    }
  }
Пример #5
0
  /*
   * (non-Javadoc)
   *
   * @see
   * org.apache.mina.core.service.IoHandlerAdapter#sessionCreated(org.apache
   * .mina.core.session.IoSession)
   */
  public void sessionCreated(IoSession session) throws Exception {
    log.info("stub created:" + session.getRemoteAddress());

    Counter.add("mdc", "connection", 1);

    TConn d = new TConn(session);
    d.set("x-forwarded-for", session.getRemoteAddress().toString());

    session.setAttribute("conn", d);
  }
Пример #6
0
 /** Copy the new fsimage into the NameNode */
 private void putFSImage(CheckpointSignature sig) throws IOException {
   String fileid =
       "putimage=1&port="
           + infoPort
           + "&machine="
           + InetAddress.getLocalHost().getHostAddress()
           + "&token="
           + sig.toString();
   LOG.info("Posted URL " + fsName + fileid);
   TransferFsImage.getFileClient(fsName, fileid, (File[]) null, false);
 }
Пример #7
0
  public synchronized boolean configClearspace() {
    // If the task is running, stop it
    if (configClearspaceTask != null) {
      configClearspaceTask.cancel();
      Log.debug("Stopping previous configuration Clearspace task.");
    }

    boolean configured = false;
    try {
      doConfigClearspace();
      updateClearspaceClientSettings();
      configured = true;
    } catch (UnauthorizedException e) {
      Log.info("Unauthorized to configure Clearspace.", e);
    } catch (UnsupportedOperationException e) {
      Log.info("Error configuring Clearspace.", e);
    }

    if (!configured) {
      startClearspaceConfig();
    }
    return configured;
  }
Пример #8
0
  /**
   * Service.
   *
   * @param o the o
   * @param session the session
   */
  void service(IoBuffer o, IoSession session) {
    try {
      // System.out.println(o.remaining() + "/" + o.capacity());

      session.setAttribute("last", System.currentTimeMillis());

      SimpleIoBuffer in = (SimpleIoBuffer) session.getAttribute("buf");
      if (in == null) {
        in = SimpleIoBuffer.create(4096);
        session.setAttribute("buf", in);
      }
      byte[] data = new byte[o.remaining()];
      o.get(data);
      in.append(data);

      // log.debug("recv: " + data.length + ", " +
      // session.getRemoteAddress());

      while (in.length() > 5) {
        in.mark();
        /**
         * Byte 1: head of the package<br>
         * bit 7-6: "01", indicator of MDC<br>
         * bit 5: encrypt indicator, "0": no; "1": encrypted<br>
         * bit 4: zip indicator, "0": no, "1": ziped<br>
         * bit 0-3: reserved<br>
         * Byte 2-5: length of data<br>
         * Byte[…]: data array<br>
         */
        byte head = in.read();
        /** test the head indicator, if not correct close it */
        if ((head & 0xC0) != 0x40) {
          log.info("flag is not correct! flag:" + head + ",from: " + session.getRemoteAddress());

          session.write("error.head");
          session.close(true);
          return;
        }

        int len = in.getInt();

        if (len <= 0 || len > MAX_SIZE) {
          log.error(
              "mdcserver.Wrong lendth: "
                  + len
                  + "/"
                  + MAX_SIZE
                  + " - "
                  + session.getRemoteAddress());
          session.write("error.packet.size");
          session.close(true);
          break;
        }

        // log.info("packet.len:" + len + ", len in buffer:" +
        // in.length());
        if (in.length() < len) {
          in.reset();
          break;
        } else {
          // do it

          byte[] b = new byte[len];
          in.read(b);

          // log.info("stub.package.size: " + len + ", head:" + head +
          // ", cmd:" + Bean.toString(b));
          // log.info("stub.package.size: " + len + ", head:" + head);

          /** test the zip flag */
          if ((head & 0x10) != 0) {
            b = Zip.unzip(b);
          }

          final TConn d = (TConn) session.getAttribute("conn");
          if (d != null) {
            /** test the encrypted flag */
            if ((head & 0x20) != 0) {
              b = DES.decode(b, d.deskey);
            }

            final byte[] bb = b;

            /** test if the packet is for mdc or app */
            new WorkerTask() {

              @Override
              public void onExecute() {
                d.process(bb);
              }
            }.schedule(0);

            session.setAttribute("last", System.currentTimeMillis());
          } else {
            session.write("error.getconnection");

            log.error("error to get connection: " + session.getRemoteAddress());
            session.close(true);
          }
        }
      }
    } catch (Throwable e) {
      log.error("closing stub: " + session.getRemoteAddress(), e);
      session.write("exception." + e.getMessage());
      session.close(true);
    }
  }