Пример #1
0
  public static void parse(
      SpreadsheetService service,
      Project project,
      ProjectMetadata metadata,
      final ImportingJob job,
      int limit,
      JSONObject options,
      List<Exception> exceptions) {

    String docUrlString = JSONUtilities.getString(options, "docUrl", null);
    String worksheetUrlString = JSONUtilities.getString(options, "sheetUrl", null);
    if (docUrlString != null && worksheetUrlString != null) {
      try {
        parseOneWorkSheet(
            service,
            project,
            metadata,
            job,
            new URL(docUrlString),
            new URL(worksheetUrlString),
            limit,
            options,
            exceptions);
      } catch (MalformedURLException e) {
        e.printStackTrace();
        exceptions.add(e);
      }
    }
  }
Пример #2
0
 private static void setProgress(ImportingJob job, String fileSource, int percent) {
   JSONObject progress = JSONUtilities.getObject(job.config, "progress");
   if (progress == null) {
     progress = new JSONObject();
     JSONUtilities.safePut(job.config, "progress", progress);
   }
   JSONUtilities.safePut(progress, "message", "Reading " + fileSource);
   JSONUtilities.safePut(progress, "percent", percent);
 }
    @Test
    public void testElementWithMqlReadOutput(){
        String mqlOutput = "{\"code\":\"/api/status/ok\",\"result\":[{\"armed_force\":{\"id\":\"/en/wehrmacht\"},\"id\":\"/en/afrika_korps\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/m/0chtrwn\"},\"id\":\"/en/sacred_band_of_thebes\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/en/british_army\"},\"id\":\"/en/british_16_air_assault_brigade\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/en/british_army\"},\"id\":\"/en/pathfinder_platoon\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/m/0ch7qgz\"},\"id\":\"/en/sacred_band\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/en/polish_navy\"},\"id\":\"/en/3rd_ship_flotilla\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/m/0chtrwn\"},\"id\":\"/m/0c0kxn9\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/m/0chtrwn\"},\"id\":\"/m/0c0kxq9\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/m/0chtrwn\"},\"id\":\"/m/0c0kxqh\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/m/0chtrwn\"},\"id\":\"/m/0c0kxqp\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/m/0chtrwn\"},\"id\":\"/m/0c0kxqw\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/m/0chtrwn\"},\"id\":\"/m/0c1wxl3\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/m/0chtrwn\"},\"id\":\"/m/0c1wxlp\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/m/0chtrwn\"},\"id\":\"/m/0ck96kz\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/m/0chtrwn\"},\"id\":\"/m/0cm3j23\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/m/0chtrwn\"},\"id\":\"/m/0cw8hb4\",\"type\":\"/military/military_unit\"}],\"status\":\"200 OK\",\"transaction_id\":\"cache;cache01.p01.sjc1:8101;2010-10-04T15:04:33Z;0007\"}";
        
        JSONObject options = SUT.createParserUIInitializationData(
                job, new LinkedList<JSONObject>(), "text/json");
        JSONArray path = new JSONArray();
        JSONUtilities.append(path, JsonImporter.ANONYMOUS);
        JSONUtilities.append(path, "result");
        JSONUtilities.append(path, JsonImporter.ANONYMOUS);
        JSONUtilities.safePut(options, "recordPath", path);

        RunTest(mqlOutpu
Пример #4
0
  public static void parse(
      String token,
      Project project,
      ProjectMetadata metadata,
      final ImportingJob job,
      int limit,
      JSONObject options,
      List<Exception> exceptions) {

    String docType = JSONUtilities.getString(options, "docType", null);
    if ("spreadsheet".equals(docType)) {
      SpreadsheetService service = GDataExtension.getSpreadsheetService(token);
      parse(service, project, metadata, job, limit, options, exceptions);
    } else if ("table".equals(docType)) {
      GoogleService service = GDataExtension.getFusionTablesGoogleService(token);
      parse(service, project, metadata, job, limit, options, exceptions);
    }
  }
Пример #5
0
  public static void parse(
      GoogleService service,
      Project project,
      ProjectMetadata metadata,
      final ImportingJob job,
      int limit,
      JSONObject options,
      List<Exception> exceptions) {

    String docUrlString = JSONUtilities.getString(options, "docUrl", null);
    String id = getFTid(docUrlString); // Use GDataExtension.getFusionTableKey(url) ?
    // TODO: Allow arbitrary Fusion Tables URL instead of (in addition to?) constructing our own?

    try {
      List<FTColumnData> columns = new ArrayList<GDataImporter.FTColumnData>();
      List<List<String>> rows = GDataExtension.runFusionTablesSelect(service, "DESCRIBE " + id);
      if (rows.size() > 1) {
        for (int i = 1; i < rows.size(); i++) {
          List<String> row = rows.get(i);
          if (row.size() >= 2) {
            FTColumnData cd = new FTColumnData();
            cd.name = row.get(1);
            cd.type = FTColumnType.STRING;

            if (row.size() > 2) {
              String type = row.get(2).toLowerCase();
              if (type.equals("number")) {
                cd.type = FTColumnType.NUMBER;
              } else if (type.equals("datetime")) {
                cd.type = FTColumnType.DATETIME;
              } else if (type.equals("location")) {
                cd.type = FTColumnType.LOCATION;
              }
            }
            columns.add(cd);
          }
        }

        setProgress(job, docUrlString, -1);

        // Force these options for the next call because each fusion table
        // is strictly structured with a single line of headers.
        JSONUtilities.safePut(
            options, "ignoreLines", 0); // number of blank lines at the beginning to ignore
        JSONUtilities.safePut(options, "headerLines", 1); // number of header lines

        TabularImportingParserBase.readTable(
            project,
            metadata,
            job,
            new FusionTableBatchRowReader(job, docUrlString, service, id, columns, 100),
            docUrlString,
            limit,
            options,
            exceptions);
        setProgress(job, docUrlString, 100);
      }
    } catch (IOException e) {
      e.printStackTrace();
      exceptions.add(e);
    } catch (ServiceException e) {
      e.printStackTrace();
      exceptions.add(e);
    }
  }