@Test
public void testJobNameNotNull() {
assertEquals(getNumberOfJobs(), getClient().getJobs().size());
for (MJob job : getClient().getJobs()) {
assertNotNull(job.getName());
}
MJob job = getClient().createJob(linkIdToNameMap.get(1L), linkIdToNameMap.get(1L));
assertNull(job.getName());
assertEquals(getClient().saveJob(job), Status.ERROR);
}
@Test
public void testGetJob() {
MJob testJob = job(123l, 456l);
testJob.setEnabled(true);
MJob mJobSpy = org.mockito.Mockito.spy(testJob);
when(repositoryManagerMock.getRepository()).thenReturn(jdbcRepoMock);
when(jdbcRepoMock.findJob(123l)).thenReturn(mJobSpy);
assertEquals(jobManager.getJob(123l), mJobSpy);
verify(repositoryManagerMock, times(1)).getRepository();
verify(jdbcRepoMock, times(1)).findJob(123l);
}
private void showSummary() {
List<MJob> jobs = client.getJobs();
List<String> header = new LinkedList<String>();
header.add(resourceString(Constants.RES_TABLE_HEADER_ID));
header.add(resourceString(Constants.RES_TABLE_HEADER_NAME));
header.add(resourceString(Constants.RES_TABLE_HEADER_FROM_CONNECTOR));
header.add(resourceString(Constants.RES_TABLE_HEADER_TO_CONNECTOR));
header.add(resourceString(Constants.RES_TABLE_HEADER_ENABLED));
List<String> ids = new LinkedList<String>();
List<String> names = new LinkedList<String>();
List<String> fromConnectors = new LinkedList<String>();
List<String> toConnectors = new LinkedList<String>();
List<String> availabilities = new LinkedList<String>();
for (MJob job : jobs) {
ids.add(String.valueOf(job.getPersistenceId()));
names.add(job.getName());
// From link and connnector
fromConnectors.add(job.getFromLinkName() + " (" + job.getFromConnectorName() + ")");
toConnectors.add(job.getToLinkName() + " (" + job.getToConnectorName() + ")");
availabilities.add(String.valueOf(job.getEnabled()));
}
TableDisplayer.display(header, ids, names, fromConnectors, toConnectors, availabilities);
}
protected void fillKafkaToConfig(MJob job) {
MConfigList toConfig = job.getToJobConfig();
toConfig.getStringInput("toJobConfig.topic").setValue(topic);
List<String> topics = new ArrayList<String>(1);
topics.add(topic);
testUtil.initTopicList(topics);
}
@Test
public void testDisabledJob() {
MJob testJob = job(123l, 456l);
testJob.setEnabled(false);
testJob.setPersistenceId(1111);
SqoopException exception =
new SqoopException(DriverError.DRIVER_0009, "Job id: " + testJob.getPersistenceId());
MJob mJobSpy = org.mockito.Mockito.spy(testJob);
when(repositoryManagerMock.getRepository()).thenReturn(jdbcRepoMock);
when(jdbcRepoMock.findJob(123l)).thenReturn(mJobSpy);
try {
jobManager.getJob(123l);
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
verify(repositoryManagerMock, times(1)).getRepository();
verify(jdbcRepoMock, times(1)).findJob(123l);
}
}
@Test
public void testBasic() throws Exception {
createFromFile(
"input-0001",
"1,'USA','2004-10-23','San Francisco'",
"2,'USA','2004-10-24','Sunnyvale'",
"3,'Czech Republic','2004-10-25','Brno'",
"4,'USA','2004-10-26','Palo Alto'");
// RDBMS link
MLink rdbmsLink = getClient().createLink("generic-jdbc-connector");
fillRdbmsLinkConfig(rdbmsLink);
saveLink(rdbmsLink);
// HDFS link
MLink hdfsLink = getClient().createLink("hdfs-connector");
fillHdfsLink(hdfsLink);
saveLink(hdfsLink);
// Job creation
MJob job = getClient().createJob(hdfsLink.getName(), rdbmsLink.getName());
// set hdfs "FROM" config for the job, since the connector test case base class only has
// utilities for hdfs!
fillHdfsFromConfig(job);
// set the rdbms "TO" config here
fillRdbmsToConfig(job);
// driver config
MDriverConfig driverConfig = job.getDriverConfig();
driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(3);
saveJob(job);
executeJob(job);
assertEquals(4L, provider.rowCount(getTableName()));
assertRowInCities(1, "USA", "2004-10-23", "San Francisco");
assertRowInCities(2, "USA", "2004-10-24", "Sunnyvale");
assertRowInCities(3, "Czech Republic", "2004-10-25", "Brno");
assertRowInCities(4, "USA", "2004-10-26", "Palo Alto");
}
private void displayJob(MJob job) {
DateFormat formatter = DateFormat.getDateTimeInstance(DateFormat.SHORT, DateFormat.SHORT);
printlnResource(
Constants.RES_SHOW_PROMPT_JOB_INFO,
job.getName(),
job.getEnabled(),
job.getCreationUser(),
formatter.format(job.getCreationDate()),
job.getLastUpdateUser(),
formatter.format(job.getLastUpdateDate()));
displayConfig(job.getDriverConfig().getConfigs(), client.getDriverConfigBundle());
printlnResource(Constants.RES_SHOW_PROMPT_JOB_FROM_LID_INFO, job.getFromLinkName());
displayConfig(
job.getFromJobConfig().getConfigs(),
client.getConnectorConfigBundle(job.getFromConnectorName()));
printlnResource(Constants.RES_SHOW_PROMPT_JOB_TO_LID_INFO, job.getToLinkName());
displayConfig(
job.getToJobConfig().getConfigs(),
client.getConnectorConfigBundle(job.getToConnectorName()));
}
/**
* Run given job.
*
* @param job Job object
* @throws Exception
*/
protected void executeJob(MJob job) throws Exception {
executeJob(job.getName());
}
/**
* Create job.
*
* <p>With asserts to make sure that it was created correctly.
*
* @param job
*/
protected void saveJob(MJob job) {
SqoopUtils.fillObjectName(job);
assertEquals(getClient().saveJob(job), Status.OK);
assertNotSame(job.getPersistenceId(), MPersistableEntity.PERSISTANCE_ID_DEFAULT);
}
/**
* Fill Driver config
*
* @param job
*/
protected void fillDriverConfig(MJob job) {
job.getDriverConfig().getStringInput("throttlingConfig.numExtractors").setValue("3");
}
/**
* Fill FROM config
*
* @param job MJob object to fill
*/
protected void fillHdfsFromConfig(MJob job) {
MConfigList fromConfig = job.getFromJobConfig();
fromConfig.getStringInput("fromJobConfig.inputDirectory").setValue(getMapreduceDirectory());
}
/**
* Fill TO config with specific storage and output type.
*
* @param job MJob object to fill
* @param output Output type that should be set
*/
protected void fillHdfsToConfig(MJob job, ToFormat output) {
MConfigList toConfig = job.getToJobConfig();
toConfig.getEnumInput("toJobConfig.outputFormat").setValue(output);
toConfig.getStringInput("toJobConfig.outputDirectory").setValue(getMapreduceDirectory());
}
protected void fillRdbmsToConfig(MJob job) {
MConfigList toConfig = job.getToJobConfig();
toConfig.getStringInput("toJobConfig.tableName").setValue(getTableName().getTableName());
}
protected void fillRdbmsFromConfig(MJob job, String partitionColumn) {
MConfigList fromConfig = job.getFromJobConfig();
fromConfig.getStringInput("fromJobConfig.tableName").setValue(getTableName().getTableName());
fromConfig.getStringInput("fromJobConfig.partitionColumn").setValue(partitionColumn);
}
private MJob job(long fromId, long toId) {
MJob job = new MJob(fromId, toId, 1L, 2L, null, null, null);
job.setName("Vampire");
job.setCreationUser("Buffy");
return job;
}
