public static void parse(
SpreadsheetService service,
Project project,
ProjectMetadata metadata,
final ImportingJob job,
int limit,
JSONObject options,
List<Exception> exceptions) {
String docUrlString = JSONUtilities.getString(options, "docUrl", null);
String worksheetUrlString = JSONUtilities.getString(options, "sheetUrl", null);
if (docUrlString != null && worksheetUrlString != null) {
try {
parseOneWorkSheet(
service,
project,
metadata,
job,
new URL(docUrlString),
new URL(worksheetUrlString),
limit,
options,
exceptions);
} catch (MalformedURLException e) {
e.printStackTrace();
exceptions.add(e);
}
}
}
private static void setProgress(ImportingJob job, String fileSource, int percent) {
JSONObject progress = JSONUtilities.getObject(job.config, "progress");
if (progress == null) {
progress = new JSONObject();
JSONUtilities.safePut(job.config, "progress", progress);
}
JSONUtilities.safePut(progress, "message", "Reading " + fileSource);
JSONUtilities.safePut(progress, "percent", percent);
}
@Test
public void testElementWithMqlReadOutput(){
String mqlOutput = "{\"code\":\"/api/status/ok\",\"result\":[{\"armed_force\":{\"id\":\"/en/wehrmacht\"},\"id\":\"/en/afrika_korps\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/m/0chtrwn\"},\"id\":\"/en/sacred_band_of_thebes\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/en/british_army\"},\"id\":\"/en/british_16_air_assault_brigade\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/en/british_army\"},\"id\":\"/en/pathfinder_platoon\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/m/0ch7qgz\"},\"id\":\"/en/sacred_band\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/en/polish_navy\"},\"id\":\"/en/3rd_ship_flotilla\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/m/0chtrwn\"},\"id\":\"/m/0c0kxn9\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/m/0chtrwn\"},\"id\":\"/m/0c0kxq9\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/m/0chtrwn\"},\"id\":\"/m/0c0kxqh\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/m/0chtrwn\"},\"id\":\"/m/0c0kxqp\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/m/0chtrwn\"},\"id\":\"/m/0c0kxqw\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/m/0chtrwn\"},\"id\":\"/m/0c1wxl3\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/m/0chtrwn\"},\"id\":\"/m/0c1wxlp\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/m/0chtrwn\"},\"id\":\"/m/0ck96kz\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/m/0chtrwn\"},\"id\":\"/m/0cm3j23\",\"type\":\"/military/military_unit\"},{\"armed_force\":{\"id\":\"/m/0chtrwn\"},\"id\":\"/m/0cw8hb4\",\"type\":\"/military/military_unit\"}],\"status\":\"200 OK\",\"transaction_id\":\"cache;cache01.p01.sjc1:8101;2010-10-04T15:04:33Z;0007\"}";
JSONObject options = SUT.createParserUIInitializationData(
job, new LinkedList<JSONObject>(), "text/json");
JSONArray path = new JSONArray();
JSONUtilities.append(path, JsonImporter.ANONYMOUS);
JSONUtilities.append(path, "result");
JSONUtilities.append(path, JsonImporter.ANONYMOUS);
JSONUtilities.safePut(options, "recordPath", path);
RunTest(mqlOutpu
public static void parse(
String token,
Project project,
ProjectMetadata metadata,
final ImportingJob job,
int limit,
JSONObject options,
List<Exception> exceptions) {
String docType = JSONUtilities.getString(options, "docType", null);
if ("spreadsheet".equals(docType)) {
SpreadsheetService service = GDataExtension.getSpreadsheetService(token);
parse(service, project, metadata, job, limit, options, exceptions);
} else if ("table".equals(docType)) {
GoogleService service = GDataExtension.getFusionTablesGoogleService(token);
parse(service, project, metadata, job, limit, options, exceptions);
}
}
public static void parse(
GoogleService service,
Project project,
ProjectMetadata metadata,
final ImportingJob job,
int limit,
JSONObject options,
List<Exception> exceptions) {
String docUrlString = JSONUtilities.getString(options, "docUrl", null);
String id = getFTid(docUrlString); // Use GDataExtension.getFusionTableKey(url) ?
// TODO: Allow arbitrary Fusion Tables URL instead of (in addition to?) constructing our own?
try {
List<FTColumnData> columns = new ArrayList<GDataImporter.FTColumnData>();
List<List<String>> rows = GDataExtension.runFusionTablesSelect(service, "DESCRIBE " + id);
if (rows.size() > 1) {
for (int i = 1; i < rows.size(); i++) {
List<String> row = rows.get(i);
if (row.size() >= 2) {
FTColumnData cd = new FTColumnData();
cd.name = row.get(1);
cd.type = FTColumnType.STRING;
if (row.size() > 2) {
String type = row.get(2).toLowerCase();
if (type.equals("number")) {
cd.type = FTColumnType.NUMBER;
} else if (type.equals("datetime")) {
cd.type = FTColumnType.DATETIME;
} else if (type.equals("location")) {
cd.type = FTColumnType.LOCATION;
}
}
columns.add(cd);
}
}
setProgress(job, docUrlString, -1);
// Force these options for the next call because each fusion table
// is strictly structured with a single line of headers.
JSONUtilities.safePut(
options, "ignoreLines", 0); // number of blank lines at the beginning to ignore
JSONUtilities.safePut(options, "headerLines", 1); // number of header lines
TabularImportingParserBase.readTable(
project,
metadata,
job,
new FusionTableBatchRowReader(job, docUrlString, service, id, columns, 100),
docUrlString,
limit,
options,
exceptions);
setProgress(job, docUrlString, 100);
}
} catch (IOException e) {
e.printStackTrace();
exceptions.add(e);
} catch (ServiceException e) {
e.printStackTrace();
exceptions.add(e);
}
}
