Пример #1
0
 private static void run(Callable c, boolean read, int size) {
   // Count all i/o time from here, including all retry overheads
   long start_io_ms = System.currentTimeMillis();
   while (true) {
     try {
       long start_ns = System.nanoTime(); // Blocking i/o call timing - without counting repeats
       c.call();
       TimeLine.record_IOclose(start_ns, start_io_ms, read ? 1 : 0, size, Value.HDFS);
       break;
       // Explicitly ignore the following exceptions but
       // fail on the rest IOExceptions
     } catch (EOFException e) {
       ignoreAndWait(e, false);
     } catch (SocketTimeoutException e) {
       ignoreAndWait(e, false);
     } catch (S3Exception e) {
       // Preserve S3Exception before IOException
       // Since this is tricky code - we are supporting different HDFS version
       // New version declares S3Exception as IOException
       // But old versions (0.20.xxx) declares it as RuntimeException
       // So we have to catch it before IOException !!!
       ignoreAndWait(e, false);
     } catch (IOException e) {
       ignoreAndWait(e, true);
     } catch (Exception e) {
       throw Log.errRTExcept(e);
     }
   }
 }
Пример #2
0
  private void setSvnCredential(CIJob job) throws JDOMException, IOException {
    S_LOGGER.debug("Entering Method CIManagerImpl.setSvnCredential");
    try {
      String jenkinsTemplateDir = Utility.getJenkinsTemplateDir();
      String credentialFilePath = jenkinsTemplateDir + job.getRepoType() + HYPHEN + CREDENTIAL_XML;
      if (debugEnabled) {
        S_LOGGER.debug("credentialFilePath ... " + credentialFilePath);
      }
      File credentialFile = new File(credentialFilePath);

      SvnProcessor processor = new SvnProcessor(credentialFile);

      //			DataInputStream in = new DataInputStream(new FileInputStream(credentialFile));
      //			while (in.available() != 0) {
      //				System.out.println(in.readLine());
      //			}
      //			in.close();

      processor.changeNodeValue("credentials/entry//userName", job.getUserName());
      processor.changeNodeValue("credentials/entry//password", job.getPassword());
      processor.writeStream(new File(Utility.getJenkinsHome() + File.separator + job.getName()));

      // jenkins home location
      String jenkinsJobHome = System.getenv(JENKINS_HOME);
      StringBuilder builder = new StringBuilder(jenkinsJobHome);
      builder.append(File.separator);

      processor.writeStream(new File(builder.toString() + CI_CREDENTIAL_XML));
    } catch (Exception e) {
      S_LOGGER.error(
          "Entered into the catch block of CIManagerImpl.setSvnCredential "
              + e.getLocalizedMessage());
    }
  }
Пример #3
0
 private long getStartTime(Query request, String context) throws BeanValidationException {
   if (request.getStartAbsolute() != null) {
     return request.getStartAbsolute();
   } else if (request.getStartRelative() != null) {
     return request.getStartRelative().getTimeRelativeTo(System.currentTimeMillis());
   } else {
     throw new BeanValidationException(
         new SimpleConstraintViolation("start_time", "relative or absolute time must be set"),
         context);
   }
 }
Пример #4
0
 private static void run(Callable c, boolean read, int size) {
   // Count all i/o time from here, including all retry overheads
   long start_io_ms = System.currentTimeMillis();
   while (true) {
     try {
       long start_ns = System.nanoTime(); // Blocking i/o call timing - without counting repeats
       c.call();
       TimeLine.record_IOclose(start_ns, start_io_ms, read ? 1 : 0, size, Value.HDFS);
       break;
       // Explicitly ignore the following exceptions but
       // fail on the rest IOExceptions
     } catch (EOFException e) {
       ignoreAndWait(e, false);
     } catch (SocketTimeoutException e) {
       ignoreAndWait(e, false);
     } catch (IOException e) {
       ignoreAndWait(e, true);
     } catch (Exception e) {
       throw Log.errRTExcept(e);
     }
   }
 }
Пример #5
0
 @When("^I read json string from a file$")
 public void i_read_json_string_from_a_file() throws Throwable {
   FileInputStream fin =
       new FileInputStream(
           new File(System.getProperty("user.dir") + "//src//test//resources//sample.json"));
   InputStreamReader in = new InputStreamReader(fin);
   BufferedReader bufferedReader = new BufferedReader(in);
   StringBuilder sb = new StringBuilder();
   String line;
   while ((line = bufferedReader.readLine()) != null) {
     sb.append(line);
   }
   json = sb.toString();
 }
Пример #6
0
 @When("^I read the json data file \"(.*?)\"$")
 public void i_read_the_json_data_file(String arg1) throws Throwable {
   FileInputStream fin =
       new FileInputStream(
           new File(System.getProperty("user.dir") + "//src//test//resources//" + arg1));
   InputStreamReader in = new InputStreamReader(fin);
   BufferedReader bufferedReader = new BufferedReader(in);
   StringBuilder sb = new StringBuilder();
   String line;
   while ((line = bufferedReader.readLine()) != null) {
     sb.append(line);
   }
   json = sb.toString();
   System.out.println(json);
 }
Пример #7
0
  private void setMailCredential(CIJob job) {
    if (debugEnabled) {
      S_LOGGER.debug("Entering Method CIManagerImpl.setMailCredential");
    }
    try {
      String jenkinsTemplateDir = Utility.getJenkinsTemplateDir();
      String mailFilePath = jenkinsTemplateDir + MAIL + HYPHEN + CREDENTIAL_XML;
      if (debugEnabled) {
        S_LOGGER.debug("configFilePath ... " + mailFilePath);
      }
      File mailFile = new File(mailFilePath);

      SvnProcessor processor = new SvnProcessor(mailFile);

      //			DataInputStream in = new DataInputStream(new FileInputStream(mailFile));
      //			while (in.available() != 0) {
      //				System.out.println(in.readLine());
      //			}
      //			in.close();

      // Mail have to go with jenkins running email address
      InetAddress ownIP = InetAddress.getLocalHost();
      processor.changeNodeValue(
          CI_HUDSONURL,
          HTTP_PROTOCOL
              + PROTOCOL_POSTFIX
              + ownIP.getHostAddress()
              + COLON
              + job.getJenkinsPort()
              + FORWARD_SLASH
              + CI
              + FORWARD_SLASH);
      processor.changeNodeValue("smtpAuthUsername", job.getSenderEmailId());
      processor.changeNodeValue("smtpAuthPassword", job.getSenderEmailPassword());
      processor.changeNodeValue("adminAddress", job.getSenderEmailId());

      // jenkins home location
      String jenkinsJobHome = System.getenv(JENKINS_HOME);
      StringBuilder builder = new StringBuilder(jenkinsJobHome);
      builder.append(File.separator);

      processor.writeStream(new File(builder.toString() + CI_MAILER_XML));
    } catch (Exception e) {
      S_LOGGER.error(
          "Entered into the catch block of CIManagerImpl.setMailCredential "
              + e.getLocalizedMessage());
    }
  }
Пример #8
0
 private void updateClusters(
     int[] clusters, int count, long chunk, long numrows, int rpc, long updatedRow) {
   final int offset = (int) (updatedRow - (rpc * chunk));
   final Key chunkKey = ValueArray.getChunkKey(chunk, _job.dest());
   final int[] message;
   if (count == clusters.length) message = clusters;
   else {
     message = new int[count];
     System.arraycopy(clusters, 0, message, 0, message.length);
   }
   final int rows = ValueArray.rpc(chunk, rpc, numrows);
   new Atomic() {
     @Override
     public Value atomic(Value val) {
       assert val == null || val._key.equals(chunkKey);
       AutoBuffer b = new AutoBuffer(rows * ROW_SIZE);
       if (val != null) b._bb.put(val.memOrLoad());
       for (int i = 0; i < message.length; i++) b.put4((offset + i) * 4, message[i]);
       b.position(b.limit());
       return new Value(chunkKey, b.buf());
     }
   }.invoke(chunkKey);
 }
Пример #9
0
 /**
  * Start this task based on given top-level fork-join task representing job computation.
  *
  * @param fjtask top-level job computation task.
  * @return this job in {@link JobState#RUNNING} state
  * @see JobState
  * @see H2OCountedCompleter
  */
 public
 /** FIXME: should be final or at least protected */
 Job start(final H2OCountedCompleter fjtask) {
   assert state == JobState.CREATED : "Trying to run job which was already run?";
   assert fjtask != null : "Starting a job with null working task is not permitted! Fix you API";
   _fjtask = fjtask;
   start_time = System.currentTimeMillis();
   state = JobState.RUNNING;
   // Save the full state of the job
   UKV.put(self(), this);
   // Update job list
   new TAtomic<List>() {
     @Override
     public List atomic(List old) {
       if (old == null) old = new List();
       Key[] jobs = old._jobs;
       old._jobs = Arrays.copyOf(jobs, jobs.length + 1);
       old._jobs[jobs.length] = job_key;
       return old;
     }
   }.invoke(LIST);
   return this;
 }
Пример #10
0
 private long getEndTime(Query request) {
   if (request.getEndAbsolute() != null) return request.getEndAbsolute();
   else if (request.getEndRelative() != null)
     return request.getEndRelative().getTimeRelativeTo(System.currentTimeMillis());
   return -1;
 }
Пример #11
0
  @Override
  @SuppressWarnings("empty-statement")
  protected Clan doInBackground() throws Exception {
    if (bestMatch == null) { // use fallBack
      // URL URL = new
      // URL("http://worldoftanks."+gui.getServerRegion()+"/uc/clans/?type=table&offset=0&limit=10&order_by="+this.searchType+"&search="+this.clanTagName+"&echo=2&id=clans_index");
      // TODO: does API support orderBy ?!?
      URL URL =
          new URL(
              "http://worldoftanks."
                  + gui.getServerRegion()
                  + "/uc/clans/api/1.1/?source_token=Intellect_Soft-WoT_Mobile-unofficial_stats&search="
                  + this.fallbackInput.replace(" ", "%20")
                  + "&offset=0&limit=10");
      URLConnection URLConnection = URL.openConnection();
      URLConnection.setRequestProperty("Accept", "application/json, text/javascript, */*; q=0.01");
      URLConnection.setRequestProperty("Accept-Language", "en-us;q=0.5,en;q=0.3");
      URLConnection.setRequestProperty("Accept-Encoding", "paco");
      URLConnection.setRequestProperty("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7");
      URLConnection.setRequestProperty("Connection", "close");
      URLConnection.setRequestProperty("X-Requested-With", "XMLHttpRequest");
      // timeout after 10 seconds
      URLConnection.setConnectTimeout(10000);

      BufferedReader clanBufferedReader = null;
      StringBuilder data = new StringBuilder(3000);
      try {
        clanBufferedReader =
            new BufferedReader(new InputStreamReader(URLConnection.getInputStream(), "UTF8"));
        for (String line; (line = clanBufferedReader.readLine()) != null; data.append(line)) ;
      } finally {
        if (clanBufferedReader != null) clanBufferedReader.close();
      }

      JsonParser jsonParser = new JsonParser();
      JsonObject json = jsonParser.parse(data.toString()).getAsJsonObject();

      if (!"ok".equalsIgnoreCase(json.get("status").getAsString())) {
        throw new ClanAPIException(json.get("status_code").getAsString(), this.gui);
      }

      JsonArray results = json.get("data").getAsJsonObject().get("items").getAsJsonArray();
      JsonObject json_clan = null;
      for (JsonElement e : results) {
        JsonObject o = e.getAsJsonObject();
        if (fallbackInput.equalsIgnoreCase(o.get(this.searchType).getAsString())) {
          json_clan = o;
          break;
        }
      }
      if (json_clan == null) {
        throw new ClanNotFoundException(this.gui);
      }
      String name = json_clan.get("name").getAsString();
      String tag = json_clan.get("abbreviation").getAsString();
      long ID = json_clan.get("id").getAsLong();
      int member_count = json_clan.get("member_count").getAsInt();
      ImageIcon emblem =
          new ImageIcon(
              new URL("http://worldoftanks.eu" + json_clan.get("clan_emblem_url").getAsString()));
      bestMatch = new PossibleClan(name, tag, ID, member_count, emblem);
    }

    // get provinces
    GetProvinces provinceWorker = new GetProvinces(bestMatch.getID(), this.gui);
    provinceWorker.execute();

    // get members
    // URL = new
    // URL("http://worldoftanks."+gui.getServerRegion()+"/uc/clans/"+clanID+"/members/?type=table&offset=0&limit=100&order_by=name&search=&echo=1&id=clan_members_index");
    URL URL =
        new URL(
            "http://worldoftanks."
                + gui.getServerRegion()
                + "/uc/clans/"
                + bestMatch.getID()
                + "/api/1.1/?source_token=Intellect_Soft-WoT_Mobile-unofficial_stats");
    URLConnection URLConnection = URL.openConnection();
    URLConnection.setRequestProperty("Accept", "application/json, text/javascript, */*; q=0.01");
    URLConnection.setRequestProperty("Accept-Language", "en-us;q=0.5,en;q=0.3");
    URLConnection.setRequestProperty("Accept-Encoding", "paco");
    URLConnection.setRequestProperty("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7");
    URLConnection.setRequestProperty("Connection", "close");
    URLConnection.setRequestProperty("X-Requested-With", "XMLHttpRequest");
    // timeout after 15 seconds
    URLConnection.setConnectTimeout(15000);

    BufferedReader membersBufferedReader = null;
    StringBuilder members_data = new StringBuilder(10000);
    try {
      membersBufferedReader =
          new BufferedReader(new InputStreamReader(URLConnection.getInputStream(), "UTF8"));
      for (String line;
          (line = membersBufferedReader.readLine()) != null;
          members_data.append(line)) ;
    } finally {
      if (membersBufferedReader != null) membersBufferedReader.close();
    }

    JsonParser jsonParser = new JsonParser();
    JsonObject members_json = jsonParser.parse(members_data.toString()).getAsJsonObject();

    if (!"ok".equalsIgnoreCase(members_json.get("status").getAsString())) {
      throw new ClanAPIException(members_json.get("status_code").getAsString(), this.gui);
    }

    JsonArray members = members_json.get("data").getAsJsonObject().get("members").getAsJsonArray();

    GetPlayerData[] workers = new GetPlayerData[members.size()];
    players = new ArrayList<Player>(members.size());
    for (int i = 0; i < members.size(); i++) {
      JsonObject member = members.get(i).getAsJsonObject();
      workers[i] = new GetPlayerData(member.get("account_id").getAsLong(), this.gui);
      workers[i].execute();
    } // TODO: see if there's some sort of "worker pool" with a getAll() or getFirst()

    // In the meantime see if GetProvinces is ready
    this.provinces = provinceWorker.get();

    ArrayList<Vehicle> vehicles = new ArrayList<Vehicle>(5000);
    long start = System.currentTimeMillis();
    for (GetPlayerData w : workers) {
      Player p = w.get();
      players.add(p);
      vehicles.addAll(p.getVehicles());
      // TODO: update progress bar
      // TODO: sort by tier already here ?!? (implementation details)
    }
    vehicles.trimToSize();
    System.out.printf(
        "Vs:" + vehicles.size() + "\nOverall time: %dms%n", System.currentTimeMillis() - start);
    vehicles = Utils.sortVehiclesByTier(vehicles);
    vehicles = Utils.sortVehiclesByClass(vehicles);
    vehicles = Utils.sortVehiclesByNation(vehicles);

    if (players.size() > 20) { // some small logical optimizations
      double avg_top_eff = 0D, avg_eff = 0D, avg_top_wr = 0D, avg_wr = 0D;
      players = Utils.sortPlayersByEfficiency(players);
      for (int i = 0; i < players.size(); i++) {
        if (i == 20) avg_top_eff = avg_eff / 20D;
        avg_eff += players.get(i).getEfficiency();
      }
      avg_eff /= players.size();

      players = Utils.sortPlayersByWinrate(players);
      for (int i = 0; i < players.size(); i++) {
        if (i == 20) avg_top_wr = avg_wr / 20D;
        avg_wr += players.get(i).getAvg_wr();
      }
      avg_wr /= players.size();

      return new Clan(
          bestMatch.getName(),
          bestMatch.getClanTag(),
          bestMatch.getID(),
          players,
          vehicles,
          avg_wr,
          avg_top_wr,
          avg_eff,
          avg_top_eff,
          bestMatch.getEmblem());
    } else {
      double avg_eff = 0D, avg_wr = 0D; // top=all
      players = Utils.sortPlayersByEfficiency(players);
      for (int i = 0; i < players.size(); i++) {
        avg_eff += players.get(i).getEfficiency();
      }
      avg_eff /= players.size();

      players = Utils.sortPlayersByWinrate(players);
      for (int i = 0; i < players.size(); i++) {
        avg_wr += players.get(i).getAvg_wr();
      }
      avg_wr /= players.size();

      return new Clan(
          bestMatch.getName(),
          bestMatch.getClanTag(),
          bestMatch.getID(),
          players,
          vehicles,
          avg_wr,
          avg_wr,
          avg_eff,
          avg_eff,
          bestMatch.getEmblem());
    }
  }
Пример #12
0
  @Override
  public JsonElement serialize(
      WebSocketMessage src, Type typeOfSrc, JsonSerializationContext context) {

    JsonObject root = new JsonObject();
    JsonObject jsonNodeData = new JsonObject();
    JsonObject jsonRelData = new JsonObject();
    JsonArray removedProperties = new JsonArray();
    JsonArray modifiedProperties = new JsonArray();

    if (src.getCommand() != null) {

      root.add("command", new JsonPrimitive(src.getCommand()));
    }

    if (src.getId() != null) {

      root.add("id", new JsonPrimitive(src.getId()));
    }

    if (src.getPageId() != null) {

      root.add("pageId", new JsonPrimitive(src.getPageId()));
    }

    if (src.getMessage() != null) {

      root.add("message", new JsonPrimitive(src.getMessage()));
    }

    if (src.getCode() != 0) {

      root.add("code", new JsonPrimitive(src.getCode()));
    }

    if (src.getSessionId() != null) {

      root.add("sessionId", new JsonPrimitive(src.getSessionId()));
    }

    if (src.getToken() != null) {

      root.add("token", new JsonPrimitive(src.getToken()));
    }

    if (src.getCallback() != null) {

      root.add("callback", new JsonPrimitive(src.getCallback()));
    }

    if (src.getButton() != null) {

      root.add("button", new JsonPrimitive(src.getButton()));
    }

    if (src.getParent() != null) {

      root.add("parent", new JsonPrimitive(src.getParent()));
    }

    if (src.getView() != null) {

      root.add("view", new JsonPrimitive(src.getView()));
    }

    if (src.getSortKey() != null) {

      root.add("sort", new JsonPrimitive(src.getSortKey()));
    }

    if (src.getSortOrder() != null) {

      root.add("order", new JsonPrimitive(src.getSortOrder()));
    }

    if (src.getPageSize() > 0) {

      root.add("pageSize", new JsonPrimitive(src.getPageSize()));
    }

    if (src.getPage() > 0) {

      root.add("page", new JsonPrimitive(src.getPage()));
    }

    JsonArray nodesWithChildren = new JsonArray();
    Set<String> nwc = src.getNodesWithChildren();

    if ((nwc != null) && !src.getNodesWithChildren().isEmpty()) {

      for (String nodeId : nwc) {

        nodesWithChildren.add(new JsonPrimitive(nodeId));
      }

      root.add("nodesWithChildren", nodesWithChildren);
    }

    // serialize session valid flag (output only)
    root.add("sessionValid", new JsonPrimitive(src.isSessionValid()));

    // UPDATE only, serialize only removed and modified properties and use the correct values
    if ((src.getGraphObject() != null)) {

      GraphObject graphObject = src.getGraphObject();

      if (!src.getModifiedProperties().isEmpty()) {

        for (PropertyKey modifiedKey : src.getModifiedProperties()) {

          modifiedProperties.add(toJsonPrimitive(modifiedKey));

          //					Object newValue = graphObject.getProperty(modifiedKey);
          //
          //					if (newValue != null) {
          //
          //						if (graphObject instanceof AbstractNode) {
          //
          //							src.getNodeData().put(modifiedKey.jsonName(), newValue);
          //						} else {
          //
          //							src.getRelData().put(modifiedKey.jsonName(), newValue);
          //						}
          //
          //					}

        }

        root.add("modifiedProperties", modifiedProperties);
      }

      if (!src.getRemovedProperties().isEmpty()) {

        for (PropertyKey removedKey : src.getRemovedProperties()) {

          removedProperties.add(toJsonPrimitive(removedKey));
        }

        root.add("removedProperties", removedProperties);
      }
    }

    // serialize node data
    if (src.getNodeData() != null) {

      for (Entry<String, Object> entry : src.getNodeData().entrySet()) {

        Object value = entry.getValue();
        String key = entry.getKey();

        if (value != null) {

          jsonNodeData.add(key, toJsonPrimitive(value));
        }
      }

      root.add("data", jsonNodeData);
    }

    // serialize relationship data
    if (src.getRelData() != null) {

      for (Entry<String, Object> entry : src.getRelData().entrySet()) {

        Object value = entry.getValue();
        String key = entry.getKey();

        if (value != null) {

          jsonRelData.add(key, toJsonPrimitive(value));
        }
      }

      root.add("relData", jsonRelData);
    }

    // serialize result list
    if (src.getResult() != null) {

      if (src.getView() != null) {

        try {
          propertyView.set(null, src.getView());

        } catch (FrameworkException fex) {

          logger.log(Level.WARNING, "Unable to set property view", fex);
        }

      } else {

        try {
          propertyView.set(null, PropertyView.Ui);

        } catch (FrameworkException fex) {

          logger.log(Level.WARNING, "Unable to set property view", fex);
        }
      }

      JsonArray result = new JsonArray();

      for (GraphObject obj : src.getResult()) {

        result.add(graphObjectSerializer.serialize(obj, System.currentTimeMillis()));
      }

      root.add("result", result);
      root.add("rawResultCount", toJsonPrimitive(src.getRawResultCount()));
    }

    // serialize result tree
    //		if (src.getResultTree() != null) {
    //
    //			TreeNode node = src.getResultTree();
    //
    //			root.add("root", buildTree(node, context));
    //
    //		}

    return root;
  }
Пример #13
0
 /**
  * Returns job execution time in milliseconds. If job is not running then returns job execution
  * time.
  */
 public final long runTimeMs() {
   long until = end_time != 0 ? end_time : System.currentTimeMillis();
   return until - start_time;
 }
Пример #14
0
 /** Marks job as finished and records job end time. */
 public void remove() {
   end_time = System.currentTimeMillis();
   if (state == JobState.RUNNING) state = JobState.DONE;
   // Overwrite handle - copy end_time, state, msg
   replaceByJobHandle();
 }