Beispiel #1
0
 @Override
 public boolean equals(Object o) {
   if (this == o) return true;
   if (o == null || getClass() != o.getClass()) return false;
   Appender appender = (Appender) o;
   return originType.equals(appender.originType) && getOuter().equals(appender.getOuter());
 }
Beispiel #2
0
  /**
   * Using the given {@code appender}'s {@link Appender#append(LogSegment, ByteBuffer)} function,
   * tests for various cases for append operations.
   *
   * @param appender the {@link Appender} to use
   * @throws IOException
   */
  private void doAppendTest(Appender appender) throws IOException {
    String currSegmentName = "log_current";
    LogSegment segment = getSegment(currSegmentName, STANDARD_SEGMENT_SIZE, true);
    try {
      long writeStartOffset = segment.getStartOffset();
      byte[] bufOne = TestUtils.getRandomBytes(STANDARD_SEGMENT_SIZE / 2);
      byte[] bufTwo = TestUtils.getRandomBytes(STANDARD_SEGMENT_SIZE / 3);

      appender.append(segment, ByteBuffer.wrap(bufOne));
      assertEquals(
          "End offset is not as expected",
          writeStartOffset + bufOne.length,
          segment.getEndOffset());

      appender.append(segment, ByteBuffer.wrap(bufTwo));
      assertEquals(
          "End offset is not as expected",
          writeStartOffset + bufOne.length + bufTwo.length,
          segment.getEndOffset());

      // try to do a write that won't fit
      ByteBuffer failBuf =
          ByteBuffer.wrap(
              TestUtils.getRandomBytes((int) (STANDARD_SEGMENT_SIZE - writeStartOffset + 1)));
      long writeOverFlowCount = metrics.overflowWriteError.getCount();
      try {
        appender.append(segment, failBuf);
        fail("Append should have failed because data won't fit in the segment");
      } catch (IllegalArgumentException e) {
        assertEquals(
            "Write overflow should have been reported",
            writeOverFlowCount + 1,
            metrics.overflowWriteError.getCount());
        assertEquals("Position of buffer has changed", 0, failBuf.position());
      }

      // read and ensure data matches
      readAndEnsureMatch(segment, writeStartOffset, bufOne);
      readAndEnsureMatch(segment, writeStartOffset + bufOne.length, bufTwo);

      segment.close();
      // ensure that append fails.
      ByteBuffer buffer = ByteBuffer.wrap(TestUtils.getRandomBytes(1));
      try {
        appender.append(segment, buffer);
        fail("Append should have failed because segments are closed");
      } catch (ClosedChannelException e) {
        assertEquals("Position of buffer has changed", 0, buffer.position());
      }
    } finally {
      closeSegmentAndDeleteFile(segment);
    }
  }
Beispiel #3
0
  private static void writeLogMessage(int msgLevel, String levelMsg, String msg) {

    if (level >= msgLevel) {
      try {
        if (out != null) {
          out.writeLogMessage(levelMsg, msg);
        } else {
          System.out.print(MailDateFormatter.dateToUTC(new Date()));
          System.out.print(" [" + levelMsg + "] ");
          System.out.println(msg);
        }
      } catch (DataAccessException ex) {
        ex.printStackTrace();
      }
    }
  }
  @Before
  public void setUp() throws Exception {
    this.conf = new TajoConf();
    util = new TajoTestingCluster();
    catalog = util.startCatalogCluster().getCatalog();
    testDir = CommonTestingUtil.getTestDir(TEST_PATH);
    conf.setVar(TajoConf.ConfVars.WORKER_TEMPORAL_DIR, testDir.toString());
    sm = StorageManagerFactory.getStorageManager(conf, testDir);

    Schema schema = new Schema();
    schema.addColumn("managerId", Type.INT4);
    schema.addColumn("empId", Type.INT4);
    schema.addColumn("deptName", Type.TEXT);

    TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV);
    Path employeePath = new Path(testDir, "employee.csv");
    Appender appender =
        StorageManagerFactory.getStorageManager(conf)
            .getAppender(employeeMeta, schema, employeePath);
    appender.enableStats();
    appender.init();
    Tuple tuple = new VTuple(schema.getColumnNum());
    for (int i = 0; i < numTuple; i++) {
      tuple.put(
          new Datum[] {
            DatumFactory.createInt4(rnd.nextInt(50)),
            DatumFactory.createInt4(rnd.nextInt(100)),
            DatumFactory.createText("dept_" + i),
          });
      appender.addTuple(tuple);
    }
    appender.flush();
    appender.close();

    System.out.println(
        appender.getStats().getNumRows()
            + " rows ("
            + (appender.getStats().getNumBytes() / 1048576)
            + " MB)");

    employee = new TableDesc("employee", schema, employeeMeta, employeePath);
    catalog.addTable(employee);
    analyzer = new SQLAnalyzer();
    planner = new LogicalPlanner(catalog);
  }
  @Override
  public boolean roll() throws IOException {
    if (numReaders.get() > 0) {
      return false;
    }
    if (!prev.delete() && prev.exists()) {
      throw new IOException("Failed to delete " + prev);
    }

    synchronized (this) {
      appender.close();
      final boolean renamed = curr.renameTo(prev);
      out =
          new PrintWriter(new OutputStreamWriter(new FileOutputStream(curr, true), Charsets.UTF_8));
      if (!renamed) {
        throw new IOException("Failed to rename " + curr + " to " + prev);
      }
    }
    return true;
  }
 void registryPut(Appender appender) {
   registry.put(appender.getName(), appender);
 }
  void parseAppenderFilters(Properties props, String appenderName, Appender appender) {
    // extract filters and filter options from props into a hashtable mapping
    // the property name defining the filter class to a list of pre-parsed
    // name-value pairs associated to that filter
    final String filterPrefix = APPENDER_PREFIX + appenderName + ".filter.";
    int fIdx = filterPrefix.length();
    Hashtable filters = new Hashtable();
    Enumeration e = props.keys();
    String name = "";
    while (e.hasMoreElements()) {
      String key = (String) e.nextElement();
      if (key.startsWith(filterPrefix)) {
        int dotIdx = key.indexOf('.', fIdx);
        String filterKey = key;
        if (dotIdx != -1) {
          filterKey = key.substring(0, dotIdx);
          name = key.substring(dotIdx + 1);
        }
        Vector filterOpts = (Vector) filters.get(filterKey);
        if (filterOpts == null) {
          filterOpts = new Vector();
          filters.put(filterKey, filterOpts);
        }
        if (dotIdx != -1) {
          String value = OptionConverter.findAndSubst(key, props);
          filterOpts.add(new NameValue(name, value));
        }
      }
    }

    // sort filters by IDs, insantiate filters, set filter options,
    // add filters to the appender
    Enumeration g = new SortedKeyEnumeration(filters);
    while (g.hasMoreElements()) {
      String key = (String) g.nextElement();
      String clazz = props.getProperty(key);
      if (clazz != null) {
        LogLog.debug(
            "Filter key: ["
                + key
                + "] class: ["
                + props.getProperty(key)
                + "] props: "
                + filters.get(key));
        Filter filter = (Filter) OptionConverter.instantiateByClassName(clazz, Filter.class, null);
        if (filter != null) {
          PropertySetter propSetter = new PropertySetter(filter);
          Vector v = (Vector) filters.get(key);
          Enumeration filterProps = v.elements();
          while (filterProps.hasMoreElements()) {
            NameValue kv = (NameValue) filterProps.nextElement();
            propSetter.setProperty(kv.key, kv.value);
          }
          propSetter.activate();
          LogLog.debug(
              "Adding filter of type ["
                  + filter.getClass()
                  + "] to appender named ["
                  + appender.getName()
                  + "].");
          appender.addFilter(filter);
        }
      } else {
        LogLog.warn("Missing class definition for filter: [" + key + "]");
      }
    }
  }
  Appender parseAppender(Properties props, String appenderName) {
    Appender appender = registryGet(appenderName);
    if ((appender != null)) {
      LogLog.debug("Appender \"" + appenderName + "\" was already parsed.");
      return appender;
    }
    // Appender was not previously initialized.
    String prefix = APPENDER_PREFIX + appenderName;
    String layoutPrefix = prefix + ".layout";

    appender =
        (Appender)
            OptionConverter.instantiateByKey(props, prefix, org.apache.log4j.Appender.class, null);
    if (appender == null) {
      LogLog.error("Could not instantiate appender named \"" + appenderName + "\".");
      return null;
    }
    appender.setName(appenderName);

    if (appender instanceof OptionHandler) {
      if (appender.requiresLayout()) {
        Layout layout =
            (Layout) OptionConverter.instantiateByKey(props, layoutPrefix, Layout.class, null);
        if (layout != null) {
          appender.setLayout(layout);
          LogLog.debug("Parsing layout options for \"" + appenderName + "\".");
          // configureOptionHandler(layout, layoutPrefix + ".", props);
          PropertySetter.setProperties(layout, props, layoutPrefix + ".");
          LogLog.debug("End of parsing for \"" + appenderName + "\".");
        }
      }
      final String errorHandlerPrefix = prefix + ".errorhandler";
      String errorHandlerClass = OptionConverter.findAndSubst(errorHandlerPrefix, props);
      if (errorHandlerClass != null) {
        ErrorHandler eh =
            (ErrorHandler)
                OptionConverter.instantiateByKey(
                    props, errorHandlerPrefix, ErrorHandler.class, null);
        if (eh != null) {
          appender.setErrorHandler(eh);
          LogLog.debug("Parsing errorhandler options for \"" + appenderName + "\".");
          parseErrorHandler(eh, errorHandlerPrefix, props, repository);
          final Properties edited = new Properties();
          final String[] keys =
              new String[] {
                errorHandlerPrefix + "." + ROOT_REF,
                errorHandlerPrefix + "." + LOGGER_REF,
                errorHandlerPrefix + "." + APPENDER_REF_TAG
              };
          for (Iterator iter = props.entrySet().iterator(); iter.hasNext(); ) {
            Map.Entry entry = (Map.Entry) iter.next();
            int i = 0;
            for (; i < keys.length; i++) {
              if (keys[i].equals(entry.getKey())) break;
            }
            if (i == keys.length) {
              edited.put(entry.getKey(), entry.getValue());
            }
          }
          PropertySetter.setProperties(eh, edited, errorHandlerPrefix + ".");
          LogLog.debug("End of errorhandler parsing for \"" + appenderName + "\".");
        }
      }
      // configureOptionHandler((OptionHandler) appender, prefix + ".", props);
      PropertySetter.setProperties(appender, props, prefix + ".");
      LogLog.debug("Parsed \"" + appenderName + "\" options.");
    }
    parseAppenderFilters(props, appenderName, appender);
    registryPut(appender);
    return appender;
  }
  @Before
  public void setUp() throws Exception {
    util = new TajoTestingCluster();
    util.initTestDir();
    catalog = util.startCatalogCluster().getCatalog();
    testDir = CommonTestingUtil.getTestDir(TEST_PATH);
    catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString());
    catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME);
    conf = util.getConfiguration();

    // ----------------- dep3 ------------------------------
    // dep_id | dep_name  | loc_id
    // --------------------------------
    //  0     | dep_0     | 1000
    //  1     | dep_1     | 1001
    //  2     | dep_2     | 1002
    //  3     | dep_3     | 1003
    //  4     | dep_4     | 1004
    //  5     | dep_5     | 1005
    //  6     | dep_6     | 1006
    //  7     | dep_7     | 1007
    //  8     | dep_8     | 1008
    //  9     | dep_9     | 1009
    Schema dep3Schema = new Schema();
    dep3Schema.addColumn("dep_id", Type.INT4);
    dep3Schema.addColumn("dep_name", Type.TEXT);
    dep3Schema.addColumn("loc_id", Type.INT4);

    TableMeta dep3Meta = CatalogUtil.newTableMeta("CSV");
    Path dep3Path = new Path(testDir, "dep3.csv");
    Appender appender1 =
        ((FileTablespace) TablespaceManager.getLocalFs())
            .getAppender(dep3Meta, dep3Schema, dep3Path);
    appender1.init();
    VTuple tuple = new VTuple(dep3Schema.size());
    for (int i = 0; i < 10; i++) {
      tuple.put(
          new Datum[] {
            DatumFactory.createInt4(i),
            DatumFactory.createText("dept_" + i),
            DatumFactory.createInt4(1000 + i)
          });
      appender1.addTuple(tuple);
    }

    appender1.flush();
    appender1.close();
    dep3 = CatalogUtil.newTableDesc(DEP3_NAME, dep3Schema, dep3Meta, dep3Path);
    catalog.createTable(dep3);

    // ----------------- dep4 ------------------------------
    // dep_id | dep_name  | loc_id
    // --------------------------------
    //  0     | dep_0     | 1000
    //  1     | dep_1     | 1001
    //  2     | dep_2     | 1002
    //  3     | dep_3     | 1003
    //  4     | dep_4     | 1004
    //  5     | dep_5     | 1005
    //  6     | dep_6     | 1006
    //  7     | dep_7     | 1007
    //  8     | dep_8     | 1008
    //  9     | dep_9     | 1009
    // 10     | dep_10    | 1010
    Schema dep4Schema = new Schema();
    dep4Schema.addColumn("dep_id", Type.INT4);
    dep4Schema.addColumn("dep_name", Type.TEXT);
    dep4Schema.addColumn("loc_id", Type.INT4);

    TableMeta dep4Meta = CatalogUtil.newTableMeta("CSV");
    Path dep4Path = new Path(testDir, "dep4.csv");
    Appender appender4 =
        ((FileTablespace) TablespaceManager.getLocalFs())
            .getAppender(dep4Meta, dep4Schema, dep4Path);
    appender4.init();
    VTuple tuple4 = new VTuple(dep4Schema.size());
    for (int i = 0; i < 11; i++) {
      tuple4.put(
          new Datum[] {
            DatumFactory.createInt4(i),
            DatumFactory.createText("dept_" + i),
            DatumFactory.createInt4(1000 + i)
          });
      appender4.addTuple(tuple4);
    }

    appender4.flush();
    appender4.close();
    dep4 = CatalogUtil.newTableDesc(DEP4_NAME, dep4Schema, dep4Meta, dep4Path);
    catalog.createTable(dep4);

    // ----------------- job3 ------------------------------
    //  job_id  | job_title
    // ----------------------
    //   101    |  job_101
    //   102    |  job_102
    //   103    |  job_103

    Schema job3Schema = new Schema();
    job3Schema.addColumn("job_id", Type.INT4);
    job3Schema.addColumn("job_title", Type.TEXT);

    TableMeta job3Meta = CatalogUtil.newTableMeta("CSV");
    Path job3Path = new Path(testDir, "job3.csv");
    Appender appender2 =
        ((FileTablespace) TablespaceManager.getLocalFs())
            .getAppender(job3Meta, job3Schema, job3Path);
    appender2.init();
    VTuple tuple2 = new VTuple(job3Schema.size());
    for (int i = 1; i < 4; i++) {
      int x = 100 + i;
      tuple2.put(
          new Datum[] {DatumFactory.createInt4(100 + i), DatumFactory.createText("job_" + x)});
      appender2.addTuple(tuple2);
    }

    appender2.flush();
    appender2.close();
    job3 = CatalogUtil.newTableDesc(JOB3_NAME, job3Schema, job3Meta, job3Path);
    catalog.createTable(job3);

    // ---------------------emp3 --------------------
    // emp_id  | first_name | last_name | dep_id | salary | job_id
    // ------------------------------------------------------------
    //  11     |  fn_11     |  ln_11    |  1     | 123    | 101
    //  13     |  fn_13     |  ln_13    |  3     | 369    | 103
    //  15     |  fn_15     |  ln_15    |  5     | 615    | null
    //  17     |  fn_17     |  ln_17    |  7     | 861    | null
    //  19     |  fn_19     |  ln_19    |  9     | 1107   | null
    //  21     |  fn_21     |  ln_21    |  1     | 123    | 101
    //  23     |  fn_23     |  ln_23    |  3     | 369    | 103

    Schema emp3Schema = new Schema();
    emp3Schema.addColumn("emp_id", Type.INT4);
    emp3Schema.addColumn("first_name", Type.TEXT);
    emp3Schema.addColumn("last_name", Type.TEXT);
    emp3Schema.addColumn("dep_id", Type.INT4);
    emp3Schema.addColumn("salary", Type.FLOAT4);
    emp3Schema.addColumn("job_id", Type.INT4);

    TableMeta emp3Meta = CatalogUtil.newTableMeta("CSV");
    Path emp3Path = new Path(testDir, "emp3.csv");
    Appender appender3 =
        ((FileTablespace) TablespaceManager.getLocalFs())
            .getAppender(emp3Meta, emp3Schema, emp3Path);
    appender3.init();
    VTuple tuple3 = new VTuple(emp3Schema.size());

    for (int i = 1; i < 4; i += 2) {
      int x = 10 + i;
      tuple3.put(
          new Datum[] {
            DatumFactory.createInt4(10 + i),
            DatumFactory.createText("firstname_" + x),
            DatumFactory.createText("lastname_" + x),
            DatumFactory.createInt4(i),
            DatumFactory.createFloat4(123 * i),
            DatumFactory.createInt4(100 + i)
          });
      appender3.addTuple(tuple3);

      int y = 20 + i;
      tuple3.put(
          new Datum[] {
            DatumFactory.createInt4(20 + i),
            DatumFactory.createText("firstname_" + y),
            DatumFactory.createText("lastname_" + y),
            DatumFactory.createInt4(i),
            DatumFactory.createFloat4(123 * i),
            DatumFactory.createInt4(100 + i)
          });
      appender3.addTuple(tuple3);
    }

    for (int i = 5; i < 10; i += 2) {
      int x = 10 + i;
      tuple3.put(
          new Datum[] {
            DatumFactory.createInt4(10 + i),
            DatumFactory.createText("firstname_" + x),
            DatumFactory.createText("lastname_" + x),
            DatumFactory.createInt4(i),
            DatumFactory.createFloat4(123 * i),
            DatumFactory.createNullDatum()
          });
      appender3.addTuple(tuple3);
    }

    appender3.flush();
    appender3.close();
    emp3 = CatalogUtil.newTableDesc(EMP3_NAME, emp3Schema, emp3Meta, emp3Path);
    catalog.createTable(emp3);

    // ---------------------phone3 --------------------
    // emp_id  | phone_number
    // -----------------------------------------------
    // this table is empty, no rows

    Schema phone3Schema = new Schema();
    phone3Schema.addColumn("emp_id", Type.INT4);
    phone3Schema.addColumn("phone_number", Type.TEXT);

    TableMeta phone3Meta = CatalogUtil.newTableMeta("CSV");
    Path phone3Path = new Path(testDir, "phone3.csv");
    Appender appender5 =
        ((FileTablespace) TablespaceManager.getLocalFs())
            .getAppender(phone3Meta, phone3Schema, phone3Path);
    appender5.init();

    appender5.flush();
    appender5.close();
    phone3 = CatalogUtil.newTableDesc(PHONE3_NAME, phone3Schema, phone3Meta, phone3Path);
    catalog.createTable(phone3);

    analyzer = new SQLAnalyzer();
    planner = new LogicalPlanner(catalog, TablespaceManager.getInstance());

    defaultContext = LocalTajoTestingUtility.createDummyContext(conf);
  }
 @Test
 public void requiresLayout() {
   Appender appender = new KettleLogChannelAppender(new MockLoggingChannel());
   assertTrue(appender.requiresLayout());
 }
Beispiel #11
0
  public void initialize() {
    super.initialize();

    log4JProperties = new Properties();

    Map configuredAppenders = new HashMap();

    if (appenders == null || appenders.size() == 0) {
      if (defaultAppender != null) {
        throw new IllegalArgumentException(
            "A default appender cant be specified without any appenders configured.");
      }

      defaultAppender = "anonymous";

      log4JProperties.setProperty("log4j.appender.anonymous", "org.apache.log4j.ConsoleAppender");

      log4JProperties.setProperty("log4j.appender.anonymous.threshold", getThresholdAsString());

      log4JProperties.setProperty(
          "log4j.appender.anonymous.layout", "org.apache.log4j.PatternLayout");

      log4JProperties.setProperty(
          "log4j.appender.anonymous.layout.conversionPattern", "%-4r [%t] %-5p %c %x - %m%n");
    } else {
      for (int i = 0; i < appenders.size(); i++) {
        Appender appender = (Appender) appenders.get(i);

        String id = appender.getId();

        if (configuredAppenders.containsKey(id)) {
          throw new IllegalArgumentException(
              "There already exists a appender with the id '" + id + "'.");
        }

        if (id == null) {
          throw new IllegalArgumentException("The appender must have a id.");
        }

        if (appender.getThreshold() == null) {
          appender.setThreshold(getThresholdAsString());
        }

        if (appender.getConversionPattern() == null) {
          throw new IllegalArgumentException("The appender must have a conversion pattern.");
        }

        if (appender.getType() == null) {
          throw new IllegalArgumentException("The appender must have a type.");
        }

        try {
          Class.forName(appender.getType());
        } catch (ClassNotFoundException ex) {
          throw new IllegalArgumentException(
              "Could not find the appender class: " + appender.getType(), ex);
        } catch (LinkageError ex) {
          throw new IllegalArgumentException(
              "Could load the appender class: " + appender.getType(), ex);
        }

        String base = "log4j.appender." + id;

        log4JProperties.setProperty(base, appender.getType());

        log4JProperties.setProperty(base + ".threshold", appender.getThreshold());

        log4JProperties.setProperty(base + ".layout", "org.apache.log4j.PatternLayout");

        log4JProperties.setProperty(
            base + ".layout.conversionPattern", appender.getConversionPattern());

        Enumeration e = appender.getProperties().keys();

        while (e.hasMoreElements()) {
          String key = e.nextElement().toString();

          log4JProperties.setProperty(base + "." + key, appender.getProperty(key));

          if ("file".equals(key)) {
            File logFile = new File(appender.getProperty(key));
            File logDir = logFile.getParentFile();

            if (!logDir.exists()) {
              logDir.mkdirs();
            }
          }
        }

        configuredAppenders.put(id, appender);
      }

      if (defaultAppender == null) {
        if (configuredAppenders.size() == 1) {
          defaultAppender = ((Appender) appenders.get(0)).getId();
        } else {
          throw new IllegalArgumentException(
              "A default appender must be specified when having several appenders.");
        }
      } else {
        StringTokenizer tokenizer = new StringTokenizer(defaultAppender, ",");

        while (tokenizer.hasMoreTokens()) {
          String appender = tokenizer.nextToken();

          if (!configuredAppenders.containsKey(appender)) {
            throw new IllegalArgumentException(
                "Could not find the default appender: '" + defaultAppender + "'.");
          }
        }
      }
    }

    if (levels != null && levels.size() > 0) {
      for (Iterator it = levels.iterator(); it.hasNext(); ) {
        Level level = (Level) it.next();

        log4JProperties.put("log4j.logger." + level.getHierarchy(), level.getLevel());
      }
    }

    if (getThresholdAsString() == null) {
      throw new IllegalArgumentException("INTERNAL ERROR: The threshold must be set.");
    }
    if (defaultAppender == null) {
      throw new IllegalArgumentException("INTERNAL ERROR: The default appender must be set.");
    }

    log4JProperties.setProperty("log4j.rootLogger", getThresholdAsString() + "," + defaultAppender);
  }
Beispiel #12
0
 public synchronized void write(String value) {
   if (appender != null) {
     appender.append(value);
   }
 }
Beispiel #13
0
 /** Clear the current console text area. */
 public synchronized void clear() {
   if (appender != null) {
     appender.clear();
   }
 }
Beispiel #14
0
 /** Delete log file */
 public static void deleteLog() {
   out.deleteLogFile();
 }
Beispiel #15
0
 /**
  * Ititialize log file
  *
  * @param object the appender object that write log file
  */
 public static void initLog(Appender object) {
   out = object;
   out.initLogFile();
 }