private void createCatalogIfRequired(RetrieveEntry rent) {
   URI curURI = null;
   String addr = rent.getEffectiveAddress();
   try {
     // check if this is the first entry and the connection was redirected. If yes, then
     // store the URI as the original URI instead of the redirected URI
     String tempStr =
         URLResourceRetriever.resolveURL(rent.getBaseAddress(), rent.getCurrentAddress());
     if (!(new URI(tempStr).equals(new URI(addr)))) {
       addr = tempStr;
     }
   } catch (URISyntaxException ex) {
     // ignore
   }
   if (isSave2SingleFolder()) {
     if (!rent.getCurrentAddress().equals(rent.getEffectiveAddress()))
       addr = rent.getCurrentAddress();
   }
   try {
     curURI = new URI(addr);
   } catch (URISyntaxException ex) {
     // this is not supposed to happen. But if it does, then just return
     return;
   }
   FileObject fobj = null;
   try {
     fobj = FileUtil.toFileObject(FileUtil.normalizeFile(rent.getSaveFile()));
   } catch (Exception e) {
     return;
   }
   if (fobj == null) return;
   CatalogWriteModel dr = null;
   try {
     if (this.catalogFileObject == null) {
       Project project = FileOwnerQuery.getOwner(fobj);
       if (project == null) {
         // See issue #176769
         // In can happen if the file was saved outside of the project
         return;
       }
       dr = CatalogWriteModelFactory.getInstance().getCatalogWriteModelForProject(fobj);
     } else {
       dr =
           CatalogWriteModelFactory.getInstance()
               .getCatalogWriteModelForCatalogFile(this.catalogFileObject);
     }
   } catch (CatalogModelException ex) {
     // ignore this exception but return
     return;
   }
   // fobj = FileUtil.toFileObject(rent.getSaveFile());
   try {
     dr.addURI(curURI, fobj);
   } catch (Exception ex) {
     // ignore this exception but return
     ex = new Exception("Exception while writing in to catalog.", ex);
     handleException(rent, ex);
     return;
   }
 }
 private void updateDownloadingInfo(RetrieveEntry rent) {
   OutputWriter opt = getOPOut();
   if (rent.getBaseAddress() != null) {
     opt.println(
         NbBundle.getMessage(
             RetrieverEngineImpl.class,
             "MSG_retrieving_location_found_in",
             rent.getCurrentAddress(),
             rent.getBaseAddress())); // NOI18N
   } else {
     opt.println(
         NbBundle.getMessage(
             RetrieverEngineImpl.class,
             "MSG_retrieving_location",
             rent.getCurrentAddress())); // NOI18N
   }
   opt.flush();
 }
  private void handleException(RetrieveEntry rent, Exception ex) {
    if (audits == null) audits = new HashMap<RetrieveEntry, Exception>();
    audits.put(rent, ex);
    // System.out.println(ex instanceof UnknownHostException);
    if (ex instanceof UnknownHostException) {
      String errorMess =
          NbBundle.getMessage(RetrieverEngineImpl.class, "MSG_unknown_host_p1")
              + ex.getLocalizedMessage()
              + "\n"
              + NbBundle.getMessage(RetrieverEngineImpl.class, "MSG_unknownhost_p2");
      outputError(errorMess);
      if (showErrorPopup) {
        NotifyDescriptor.Message ndm =
            new NotifyDescriptor.Message(errorMess, NotifyDescriptor.Message.ERROR_MESSAGE);
        DialogDisplayer.getDefault().notify(ndm);
      }
      return;
    }
    if (ex instanceof URISyntaxException) {
      String errorMess = ex.getLocalizedMessage();
      outputError(errorMess);
      if (showErrorPopup) {
        NotifyDescriptor.Message ndm =
            new NotifyDescriptor.Message(errorMess, NotifyDescriptor.Message.ERROR_MESSAGE);
        DialogDisplayer.getDefault().notify(ndm);
      }
      return;
    }
    if (ex instanceof FileNotFoundException) {
      String errorMess =
          NbBundle.getMessage(RetrieverEngineImpl.class, "MSG_unknown_file", ex.getMessage());
      outputError(errorMess);
      if (showErrorPopup) {
        NotifyDescriptor.Message ndm =
            new NotifyDescriptor.Message(errorMess, NotifyDescriptor.Message.ERROR_MESSAGE);
        DialogDisplayer.getDefault().notify(ndm);
      }
      return;
    }

    if (ex instanceof IOException) {
      String exStr =
          NbBundle.getMessage(RetrieverEngineImpl.class, "EXCEPTION_CYCLIC_REFERENCE_INDICATOR");
      if (ex.getMessage().startsWith(exStr)) {
        outputMessage(
            ex.getMessage()
                + ":\n\t "
                + NbBundle.getMessage(
                    RetrieverEngineImpl.class,
                    "MSG_retrieving_location_found_in",
                    rent.getCurrentAddress(),
                    rent.getBaseAddress()));
        return;
      }
      String errorMess =
          NbBundle.getMessage(RetrieverEngineImpl.class, "MSG_general_io_error", ex.getMessage());
      if (showErrorPopup) {
        NotifyDescriptor.Message ndm =
            new NotifyDescriptor.Message(errorMess, NotifyDescriptor.Message.ERROR_MESSAGE);
        DialogDisplayer.getDefault().notify(ndm);
      }
      outputError(errorMess);
      return;
    }

    outputError(ex.getMessage());
    return;
  }
  private void pullRecursively() {
    synchronized (RetrieverEngineImpl.class) {
      while (!currentRetrievalList.isEmpty() && !STOP_PULL) {
        // System.out.println(currentRetrievalList);
        RetrieveEntry rent = currentRetrievalList.getFirst();
        // System.out.println("###"+rent.toString());
        currentRetrievalList.removeFirst();
        RetrieverTask rt = new RetrieverTask(rent, this);

        if (firstTime) {
          firstAddressParentStr =
              rent.getCurrentAddress().substring(0, rent.getCurrentAddress().lastIndexOf("/"));
          firstTime = false;
        }

        updateDownloadingInfo(rent);

        HashMap<String, File> storedFileMap = null;
        try {
          storedFileMap = rt.goGetIt();
        } catch (URISyntaxException ex) {
          // This might an error in the file. Ignore
          // ex.printStackTrace();
          handleException(rent, ex);
          continue;
        } catch (IOException ex) {
          // This might have been thrown to indicate a cyclic reference.
          // ex.printStackTrace();
          handleException(rent, ex);
          continue;
        }

        if (!rent.isRecursive()) continue;

        if (storedFileMap == null) {
          continue;
        }
        String effectiveSrcAddr = storedFileMap.keySet().iterator().next();
        File storedFile = storedFileMap.get(effectiveSrcAddr);

        rent.setSaveFile(storedFile);
        rent.setEffectiveAddress(effectiveSrcAddr);

        updateDownloadedInfo(rent);

        createCatalogIfRequired(rent);

        DocumentTypeParser dtp = DocumentParserFactory.getParser(rent.getDocType());
        List<String> thisFileRefs = null;
        try {
          thisFileRefs = dtp.getAllLocationOfReferencedEntities(storedFile);
          // System.out.println("Parsed:"+storedFile+" Got:"+thisFileRefs);
        } catch (Exception ex) {
          // was not able to parse the doc. Currently ignore this.
          // ex.printStackTrace();
          continue;
        }
        for (String ref : thisFileRefs) {
          currentRetrievalList.addLast(
              new RetrieveEntry(
                  effectiveSrcAddr, ref, storedFile, null, rent.getDocType(), rent.isRecursive()));
        }
        printList();
      }
      closeOPOuts();
    }
  }