@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Appender appender = (Appender) o;
return originType.equals(appender.originType) && getOuter().equals(appender.getOuter());
}
/**
* Using the given {@code appender}'s {@link Appender#append(LogSegment, ByteBuffer)} function,
* tests for various cases for append operations.
*
* @param appender the {@link Appender} to use
* @throws IOException
*/
private void doAppendTest(Appender appender) throws IOException {
String currSegmentName = "log_current";
LogSegment segment = getSegment(currSegmentName, STANDARD_SEGMENT_SIZE, true);
try {
long writeStartOffset = segment.getStartOffset();
byte[] bufOne = TestUtils.getRandomBytes(STANDARD_SEGMENT_SIZE / 2);
byte[] bufTwo = TestUtils.getRandomBytes(STANDARD_SEGMENT_SIZE / 3);
appender.append(segment, ByteBuffer.wrap(bufOne));
assertEquals(
"End offset is not as expected",
writeStartOffset + bufOne.length,
segment.getEndOffset());
appender.append(segment, ByteBuffer.wrap(bufTwo));
assertEquals(
"End offset is not as expected",
writeStartOffset + bufOne.length + bufTwo.length,
segment.getEndOffset());
// try to do a write that won't fit
ByteBuffer failBuf =
ByteBuffer.wrap(
TestUtils.getRandomBytes((int) (STANDARD_SEGMENT_SIZE - writeStartOffset + 1)));
long writeOverFlowCount = metrics.overflowWriteError.getCount();
try {
appender.append(segment, failBuf);
fail("Append should have failed because data won't fit in the segment");
} catch (IllegalArgumentException e) {
assertEquals(
"Write overflow should have been reported",
writeOverFlowCount + 1,
metrics.overflowWriteError.getCount());
assertEquals("Position of buffer has changed", 0, failBuf.position());
}
// read and ensure data matches
readAndEnsureMatch(segment, writeStartOffset, bufOne);
readAndEnsureMatch(segment, writeStartOffset + bufOne.length, bufTwo);
segment.close();
// ensure that append fails.
ByteBuffer buffer = ByteBuffer.wrap(TestUtils.getRandomBytes(1));
try {
appender.append(segment, buffer);
fail("Append should have failed because segments are closed");
} catch (ClosedChannelException e) {
assertEquals("Position of buffer has changed", 0, buffer.position());
}
} finally {
closeSegmentAndDeleteFile(segment);
}
}
private static void writeLogMessage(int msgLevel, String levelMsg, String msg) {
if (level >= msgLevel) {
try {
if (out != null) {
out.writeLogMessage(levelMsg, msg);
} else {
System.out.print(MailDateFormatter.dateToUTC(new Date()));
System.out.print(" [" + levelMsg + "] ");
System.out.println(msg);
}
} catch (DataAccessException ex) {
ex.printStackTrace();
}
}
}
@Before
public void setUp() throws Exception {
this.conf = new TajoConf();
util = new TajoTestingCluster();
catalog = util.startCatalogCluster().getCatalog();
testDir = CommonTestingUtil.getTestDir(TEST_PATH);
conf.setVar(TajoConf.ConfVars.WORKER_TEMPORAL_DIR, testDir.toString());
sm = StorageManagerFactory.getStorageManager(conf, testDir);
Schema schema = new Schema();
schema.addColumn("managerId", Type.INT4);
schema.addColumn("empId", Type.INT4);
schema.addColumn("deptName", Type.TEXT);
TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV);
Path employeePath = new Path(testDir, "employee.csv");
Appender appender =
StorageManagerFactory.getStorageManager(conf)
.getAppender(employeeMeta, schema, employeePath);
appender.enableStats();
appender.init();
Tuple tuple = new VTuple(schema.getColumnNum());
for (int i = 0; i < numTuple; i++) {
tuple.put(
new Datum[] {
DatumFactory.createInt4(rnd.nextInt(50)),
DatumFactory.createInt4(rnd.nextInt(100)),
DatumFactory.createText("dept_" + i),
});
appender.addTuple(tuple);
}
appender.flush();
appender.close();
System.out.println(
appender.getStats().getNumRows()
+ " rows ("
+ (appender.getStats().getNumBytes() / 1048576)
+ " MB)");
employee = new TableDesc("employee", schema, employeeMeta, employeePath);
catalog.addTable(employee);
analyzer = new SQLAnalyzer();
planner = new LogicalPlanner(catalog);
}
@Override
public boolean roll() throws IOException {
if (numReaders.get() > 0) {
return false;
}
if (!prev.delete() && prev.exists()) {
throw new IOException("Failed to delete " + prev);
}
synchronized (this) {
appender.close();
final boolean renamed = curr.renameTo(prev);
out =
new PrintWriter(new OutputStreamWriter(new FileOutputStream(curr, true), Charsets.UTF_8));
if (!renamed) {
throw new IOException("Failed to rename " + curr + " to " + prev);
}
}
return true;
}
void registryPut(Appender appender) {
registry.put(appender.getName(), appender);
}
void parseAppenderFilters(Properties props, String appenderName, Appender appender) {
// extract filters and filter options from props into a hashtable mapping
// the property name defining the filter class to a list of pre-parsed
// name-value pairs associated to that filter
final String filterPrefix = APPENDER_PREFIX + appenderName + ".filter.";
int fIdx = filterPrefix.length();
Hashtable filters = new Hashtable();
Enumeration e = props.keys();
String name = "";
while (e.hasMoreElements()) {
String key = (String) e.nextElement();
if (key.startsWith(filterPrefix)) {
int dotIdx = key.indexOf('.', fIdx);
String filterKey = key;
if (dotIdx != -1) {
filterKey = key.substring(0, dotIdx);
name = key.substring(dotIdx + 1);
}
Vector filterOpts = (Vector) filters.get(filterKey);
if (filterOpts == null) {
filterOpts = new Vector();
filters.put(filterKey, filterOpts);
}
if (dotIdx != -1) {
String value = OptionConverter.findAndSubst(key, props);
filterOpts.add(new NameValue(name, value));
}
}
}
// sort filters by IDs, insantiate filters, set filter options,
// add filters to the appender
Enumeration g = new SortedKeyEnumeration(filters);
while (g.hasMoreElements()) {
String key = (String) g.nextElement();
String clazz = props.getProperty(key);
if (clazz != null) {
LogLog.debug(
"Filter key: ["
+ key
+ "] class: ["
+ props.getProperty(key)
+ "] props: "
+ filters.get(key));
Filter filter = (Filter) OptionConverter.instantiateByClassName(clazz, Filter.class, null);
if (filter != null) {
PropertySetter propSetter = new PropertySetter(filter);
Vector v = (Vector) filters.get(key);
Enumeration filterProps = v.elements();
while (filterProps.hasMoreElements()) {
NameValue kv = (NameValue) filterProps.nextElement();
propSetter.setProperty(kv.key, kv.value);
}
propSetter.activate();
LogLog.debug(
"Adding filter of type ["
+ filter.getClass()
+ "] to appender named ["
+ appender.getName()
+ "].");
appender.addFilter(filter);
}
} else {
LogLog.warn("Missing class definition for filter: [" + key + "]");
}
}
}
Appender parseAppender(Properties props, String appenderName) {
Appender appender = registryGet(appenderName);
if ((appender != null)) {
LogLog.debug("Appender \"" + appenderName + "\" was already parsed.");
return appender;
}
// Appender was not previously initialized.
String prefix = APPENDER_PREFIX + appenderName;
String layoutPrefix = prefix + ".layout";
appender =
(Appender)
OptionConverter.instantiateByKey(props, prefix, org.apache.log4j.Appender.class, null);
if (appender == null) {
LogLog.error("Could not instantiate appender named \"" + appenderName + "\".");
return null;
}
appender.setName(appenderName);
if (appender instanceof OptionHandler) {
if (appender.requiresLayout()) {
Layout layout =
(Layout) OptionConverter.instantiateByKey(props, layoutPrefix, Layout.class, null);
if (layout != null) {
appender.setLayout(layout);
LogLog.debug("Parsing layout options for \"" + appenderName + "\".");
// configureOptionHandler(layout, layoutPrefix + ".", props);
PropertySetter.setProperties(layout, props, layoutPrefix + ".");
LogLog.debug("End of parsing for \"" + appenderName + "\".");
}
}
final String errorHandlerPrefix = prefix + ".errorhandler";
String errorHandlerClass = OptionConverter.findAndSubst(errorHandlerPrefix, props);
if (errorHandlerClass != null) {
ErrorHandler eh =
(ErrorHandler)
OptionConverter.instantiateByKey(
props, errorHandlerPrefix, ErrorHandler.class, null);
if (eh != null) {
appender.setErrorHandler(eh);
LogLog.debug("Parsing errorhandler options for \"" + appenderName + "\".");
parseErrorHandler(eh, errorHandlerPrefix, props, repository);
final Properties edited = new Properties();
final String[] keys =
new String[] {
errorHandlerPrefix + "." + ROOT_REF,
errorHandlerPrefix + "." + LOGGER_REF,
errorHandlerPrefix + "." + APPENDER_REF_TAG
};
for (Iterator iter = props.entrySet().iterator(); iter.hasNext(); ) {
Map.Entry entry = (Map.Entry) iter.next();
int i = 0;
for (; i < keys.length; i++) {
if (keys[i].equals(entry.getKey())) break;
}
if (i == keys.length) {
edited.put(entry.getKey(), entry.getValue());
}
}
PropertySetter.setProperties(eh, edited, errorHandlerPrefix + ".");
LogLog.debug("End of errorhandler parsing for \"" + appenderName + "\".");
}
}
// configureOptionHandler((OptionHandler) appender, prefix + ".", props);
PropertySetter.setProperties(appender, props, prefix + ".");
LogLog.debug("Parsed \"" + appenderName + "\" options.");
}
parseAppenderFilters(props, appenderName, appender);
registryPut(appender);
return appender;
}
@Before
public void setUp() throws Exception {
util = new TajoTestingCluster();
util.initTestDir();
catalog = util.startCatalogCluster().getCatalog();
testDir = CommonTestingUtil.getTestDir(TEST_PATH);
catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString());
catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME);
conf = util.getConfiguration();
// ----------------- dep3 ------------------------------
// dep_id | dep_name | loc_id
// --------------------------------
// 0 | dep_0 | 1000
// 1 | dep_1 | 1001
// 2 | dep_2 | 1002
// 3 | dep_3 | 1003
// 4 | dep_4 | 1004
// 5 | dep_5 | 1005
// 6 | dep_6 | 1006
// 7 | dep_7 | 1007
// 8 | dep_8 | 1008
// 9 | dep_9 | 1009
Schema dep3Schema = new Schema();
dep3Schema.addColumn("dep_id", Type.INT4);
dep3Schema.addColumn("dep_name", Type.TEXT);
dep3Schema.addColumn("loc_id", Type.INT4);
TableMeta dep3Meta = CatalogUtil.newTableMeta("CSV");
Path dep3Path = new Path(testDir, "dep3.csv");
Appender appender1 =
((FileTablespace) TablespaceManager.getLocalFs())
.getAppender(dep3Meta, dep3Schema, dep3Path);
appender1.init();
VTuple tuple = new VTuple(dep3Schema.size());
for (int i = 0; i < 10; i++) {
tuple.put(
new Datum[] {
DatumFactory.createInt4(i),
DatumFactory.createText("dept_" + i),
DatumFactory.createInt4(1000 + i)
});
appender1.addTuple(tuple);
}
appender1.flush();
appender1.close();
dep3 = CatalogUtil.newTableDesc(DEP3_NAME, dep3Schema, dep3Meta, dep3Path);
catalog.createTable(dep3);
// ----------------- dep4 ------------------------------
// dep_id | dep_name | loc_id
// --------------------------------
// 0 | dep_0 | 1000
// 1 | dep_1 | 1001
// 2 | dep_2 | 1002
// 3 | dep_3 | 1003
// 4 | dep_4 | 1004
// 5 | dep_5 | 1005
// 6 | dep_6 | 1006
// 7 | dep_7 | 1007
// 8 | dep_8 | 1008
// 9 | dep_9 | 1009
// 10 | dep_10 | 1010
Schema dep4Schema = new Schema();
dep4Schema.addColumn("dep_id", Type.INT4);
dep4Schema.addColumn("dep_name", Type.TEXT);
dep4Schema.addColumn("loc_id", Type.INT4);
TableMeta dep4Meta = CatalogUtil.newTableMeta("CSV");
Path dep4Path = new Path(testDir, "dep4.csv");
Appender appender4 =
((FileTablespace) TablespaceManager.getLocalFs())
.getAppender(dep4Meta, dep4Schema, dep4Path);
appender4.init();
VTuple tuple4 = new VTuple(dep4Schema.size());
for (int i = 0; i < 11; i++) {
tuple4.put(
new Datum[] {
DatumFactory.createInt4(i),
DatumFactory.createText("dept_" + i),
DatumFactory.createInt4(1000 + i)
});
appender4.addTuple(tuple4);
}
appender4.flush();
appender4.close();
dep4 = CatalogUtil.newTableDesc(DEP4_NAME, dep4Schema, dep4Meta, dep4Path);
catalog.createTable(dep4);
// ----------------- job3 ------------------------------
// job_id | job_title
// ----------------------
// 101 | job_101
// 102 | job_102
// 103 | job_103
Schema job3Schema = new Schema();
job3Schema.addColumn("job_id", Type.INT4);
job3Schema.addColumn("job_title", Type.TEXT);
TableMeta job3Meta = CatalogUtil.newTableMeta("CSV");
Path job3Path = new Path(testDir, "job3.csv");
Appender appender2 =
((FileTablespace) TablespaceManager.getLocalFs())
.getAppender(job3Meta, job3Schema, job3Path);
appender2.init();
VTuple tuple2 = new VTuple(job3Schema.size());
for (int i = 1; i < 4; i++) {
int x = 100 + i;
tuple2.put(
new Datum[] {DatumFactory.createInt4(100 + i), DatumFactory.createText("job_" + x)});
appender2.addTuple(tuple2);
}
appender2.flush();
appender2.close();
job3 = CatalogUtil.newTableDesc(JOB3_NAME, job3Schema, job3Meta, job3Path);
catalog.createTable(job3);
// ---------------------emp3 --------------------
// emp_id | first_name | last_name | dep_id | salary | job_id
// ------------------------------------------------------------
// 11 | fn_11 | ln_11 | 1 | 123 | 101
// 13 | fn_13 | ln_13 | 3 | 369 | 103
// 15 | fn_15 | ln_15 | 5 | 615 | null
// 17 | fn_17 | ln_17 | 7 | 861 | null
// 19 | fn_19 | ln_19 | 9 | 1107 | null
// 21 | fn_21 | ln_21 | 1 | 123 | 101
// 23 | fn_23 | ln_23 | 3 | 369 | 103
Schema emp3Schema = new Schema();
emp3Schema.addColumn("emp_id", Type.INT4);
emp3Schema.addColumn("first_name", Type.TEXT);
emp3Schema.addColumn("last_name", Type.TEXT);
emp3Schema.addColumn("dep_id", Type.INT4);
emp3Schema.addColumn("salary", Type.FLOAT4);
emp3Schema.addColumn("job_id", Type.INT4);
TableMeta emp3Meta = CatalogUtil.newTableMeta("CSV");
Path emp3Path = new Path(testDir, "emp3.csv");
Appender appender3 =
((FileTablespace) TablespaceManager.getLocalFs())
.getAppender(emp3Meta, emp3Schema, emp3Path);
appender3.init();
VTuple tuple3 = new VTuple(emp3Schema.size());
for (int i = 1; i < 4; i += 2) {
int x = 10 + i;
tuple3.put(
new Datum[] {
DatumFactory.createInt4(10 + i),
DatumFactory.createText("firstname_" + x),
DatumFactory.createText("lastname_" + x),
DatumFactory.createInt4(i),
DatumFactory.createFloat4(123 * i),
DatumFactory.createInt4(100 + i)
});
appender3.addTuple(tuple3);
int y = 20 + i;
tuple3.put(
new Datum[] {
DatumFactory.createInt4(20 + i),
DatumFactory.createText("firstname_" + y),
DatumFactory.createText("lastname_" + y),
DatumFactory.createInt4(i),
DatumFactory.createFloat4(123 * i),
DatumFactory.createInt4(100 + i)
});
appender3.addTuple(tuple3);
}
for (int i = 5; i < 10; i += 2) {
int x = 10 + i;
tuple3.put(
new Datum[] {
DatumFactory.createInt4(10 + i),
DatumFactory.createText("firstname_" + x),
DatumFactory.createText("lastname_" + x),
DatumFactory.createInt4(i),
DatumFactory.createFloat4(123 * i),
DatumFactory.createNullDatum()
});
appender3.addTuple(tuple3);
}
appender3.flush();
appender3.close();
emp3 = CatalogUtil.newTableDesc(EMP3_NAME, emp3Schema, emp3Meta, emp3Path);
catalog.createTable(emp3);
// ---------------------phone3 --------------------
// emp_id | phone_number
// -----------------------------------------------
// this table is empty, no rows
Schema phone3Schema = new Schema();
phone3Schema.addColumn("emp_id", Type.INT4);
phone3Schema.addColumn("phone_number", Type.TEXT);
TableMeta phone3Meta = CatalogUtil.newTableMeta("CSV");
Path phone3Path = new Path(testDir, "phone3.csv");
Appender appender5 =
((FileTablespace) TablespaceManager.getLocalFs())
.getAppender(phone3Meta, phone3Schema, phone3Path);
appender5.init();
appender5.flush();
appender5.close();
phone3 = CatalogUtil.newTableDesc(PHONE3_NAME, phone3Schema, phone3Meta, phone3Path);
catalog.createTable(phone3);
analyzer = new SQLAnalyzer();
planner = new LogicalPlanner(catalog, TablespaceManager.getInstance());
defaultContext = LocalTajoTestingUtility.createDummyContext(conf);
}
@Test
public void requiresLayout() {
Appender appender = new KettleLogChannelAppender(new MockLoggingChannel());
assertTrue(appender.requiresLayout());
}
public void initialize() {
super.initialize();
log4JProperties = new Properties();
Map configuredAppenders = new HashMap();
if (appenders == null || appenders.size() == 0) {
if (defaultAppender != null) {
throw new IllegalArgumentException(
"A default appender cant be specified without any appenders configured.");
}
defaultAppender = "anonymous";
log4JProperties.setProperty("log4j.appender.anonymous", "org.apache.log4j.ConsoleAppender");
log4JProperties.setProperty("log4j.appender.anonymous.threshold", getThresholdAsString());
log4JProperties.setProperty(
"log4j.appender.anonymous.layout", "org.apache.log4j.PatternLayout");
log4JProperties.setProperty(
"log4j.appender.anonymous.layout.conversionPattern", "%-4r [%t] %-5p %c %x - %m%n");
} else {
for (int i = 0; i < appenders.size(); i++) {
Appender appender = (Appender) appenders.get(i);
String id = appender.getId();
if (configuredAppenders.containsKey(id)) {
throw new IllegalArgumentException(
"There already exists a appender with the id '" + id + "'.");
}
if (id == null) {
throw new IllegalArgumentException("The appender must have a id.");
}
if (appender.getThreshold() == null) {
appender.setThreshold(getThresholdAsString());
}
if (appender.getConversionPattern() == null) {
throw new IllegalArgumentException("The appender must have a conversion pattern.");
}
if (appender.getType() == null) {
throw new IllegalArgumentException("The appender must have a type.");
}
try {
Class.forName(appender.getType());
} catch (ClassNotFoundException ex) {
throw new IllegalArgumentException(
"Could not find the appender class: " + appender.getType(), ex);
} catch (LinkageError ex) {
throw new IllegalArgumentException(
"Could load the appender class: " + appender.getType(), ex);
}
String base = "log4j.appender." + id;
log4JProperties.setProperty(base, appender.getType());
log4JProperties.setProperty(base + ".threshold", appender.getThreshold());
log4JProperties.setProperty(base + ".layout", "org.apache.log4j.PatternLayout");
log4JProperties.setProperty(
base + ".layout.conversionPattern", appender.getConversionPattern());
Enumeration e = appender.getProperties().keys();
while (e.hasMoreElements()) {
String key = e.nextElement().toString();
log4JProperties.setProperty(base + "." + key, appender.getProperty(key));
if ("file".equals(key)) {
File logFile = new File(appender.getProperty(key));
File logDir = logFile.getParentFile();
if (!logDir.exists()) {
logDir.mkdirs();
}
}
}
configuredAppenders.put(id, appender);
}
if (defaultAppender == null) {
if (configuredAppenders.size() == 1) {
defaultAppender = ((Appender) appenders.get(0)).getId();
} else {
throw new IllegalArgumentException(
"A default appender must be specified when having several appenders.");
}
} else {
StringTokenizer tokenizer = new StringTokenizer(defaultAppender, ",");
while (tokenizer.hasMoreTokens()) {
String appender = tokenizer.nextToken();
if (!configuredAppenders.containsKey(appender)) {
throw new IllegalArgumentException(
"Could not find the default appender: '" + defaultAppender + "'.");
}
}
}
}
if (levels != null && levels.size() > 0) {
for (Iterator it = levels.iterator(); it.hasNext(); ) {
Level level = (Level) it.next();
log4JProperties.put("log4j.logger." + level.getHierarchy(), level.getLevel());
}
}
if (getThresholdAsString() == null) {
throw new IllegalArgumentException("INTERNAL ERROR: The threshold must be set.");
}
if (defaultAppender == null) {
throw new IllegalArgumentException("INTERNAL ERROR: The default appender must be set.");
}
log4JProperties.setProperty("log4j.rootLogger", getThresholdAsString() + "," + defaultAppender);
}
public synchronized void write(String value) {
if (appender != null) {
appender.append(value);
}
}
/** Clear the current console text area. */
public synchronized void clear() {
if (appender != null) {
appender.clear();
}
}
/** Delete log file */
public static void deleteLog() {
out.deleteLogFile();
}
/**
* Ititialize log file
*
* @param object the appender object that write log file
*/
public static void initLog(Appender object) {
out = object;
out.initLogFile();
}