コード例 #1
0
  /**
   * @brief Test ordinary spark query.
   * @throws DDFException
   */
  @Test
  public void testLoading() throws DDFException {
    SQLDataSourceDescriptor sqlDataSourceDescriptor =
        new SQLDataSourceDescriptor(null, "SparkSQL", null, null, null);
    manager.sql("drop table if exists airline", sqlDataSourceDescriptor);

    manager.sql(
        "create table airline (Year int,Month int,DayofMonth int,"
            + "DayOfWeek int,DepTime int,CRSDepTime int,ArrTime int,"
            + "CRSArrTime int,UniqueCarrier string, FlightNum int, "
            + "TailNum string, ActualElapsedTime int, CRSElapsedTime int, "
            + "AirTime int, ArrDelay int, DepDelay int, Origin string, "
            + "Dest string, Distance int, TaxiIn int, TaxiOut int, Cancelled int, "
            + "CancellationCode string, Diverted string, CarrierDelay int, "
            + "WeatherDelay int, NASDelay int, SecurityDelay int, LateAircraftDelay int ) "
            + "ROW FORMAT DELIMITED FIELDS TERMINATED BY ','",
        sqlDataSourceDescriptor);

    manager.sql(
        "load data local inpath '../resources/test/airline.csv' " + "into table airline",
        sqlDataSourceDescriptor);

    DDF ddf =
        manager.sql2ddf(
            "select year, month, dayofweek, deptime, arrtime,origin, distance, arrdelay, "
                + "depdelay, carrierdelay, weatherdelay, nasdelay, securitydelay, lateaircraftdelay from airline",
            sqlDataSourceDescriptor);
    this.manager.setDDFName(ddf, "airlineDDF");
    // DDF sql2ddfRet = manager.sql2ddf("select * from " +
    //        "ddf://adatao/airlineDDF");
  }
コード例 #2
0
  @BeforeClass
  public static void startServer() throws Exception {
    Thread.sleep(1000);
    // LOG = LoggerFactory.getLogger(BaseTest.class);
    // manager = DDFManager.get("spark");
    /*
    manager = DDFManager.get("jdbc", new JDBCDataSourceDescriptor(new
            DataSourceURI("jdbc:mysql://localhost/testdb"), new
            JDBCDataSourceDescriptor.JDBCDataSourceCredentials("pauser",
            "papwd"), null));
    DataSourceDescriptor ds = manager.getDataSourceDescriptor();
    if (ds instanceof
            JDBCDataSourceDescriptor) {
        System.out.println("hello");
    }
    DDF ret = manager.sql2ddf("select * from testtable", "jdbc");*/
    // Add 2 test ddfs.
    manager = DDFManager.get("spark");
    manager.setEngineName("spark");
    Schema schema = new Schema("tablename1", "d  d,d  d");
    DDF ddf =
        manager.newDDF(manager, new Class<?>[] {DDFManager.class}, "spark", "adatao", "a", schema);
    Schema schema2 = new Schema("tablename2", "d  d,d  d");
    DDF ddf2 =
        manager.newDDF(manager, new Class<?>[] {DDFManager.class}, "spark", "adatao", "b", schema2);

    parser = new CCJSqlParserManager();
  }
コード例 #3
0
  @Override
  public DDF residuals() throws DDFException {
    SparkDDF predictionDDF = (SparkDDF) this.getDDF();
    JavaRDD<double[]> predictionRDD = predictionDDF.getJavaRDD(double[].class);

    JavaRDD<double[]> result = predictionRDD.map(new MetricsMapperResiduals());

    if (result == null) mLog.error(">> javaRDD result of MetricMapper residuals is null");
    if (predictionDDF.getManager() == null) mLog.error(">> predictionDDF.getManager() is null");
    if (result.rdd() == null) mLog.error(">> result.rdd() is null");
    if (predictionDDF.getSchema() == null) mLog.error(">> predictionDDF.getSchema() is null");
    if (predictionDDF.getName() == null) mLog.error(">> predictionDDF.getName() is null");

    Schema schema = new Schema("residuals double");
    DDFManager manager = this.getDDF().getManager();
    DDF residualDDF =
        manager.newDDF(
            manager, result.rdd(), new Class<?>[] {RDD.class, double[].class}, null, schema);

    if (residualDDF == null) mLog.error(">>>>>>>>>>>.residualDDF is null");

    return residualDDF;
  }
コード例 #4
0
  @AfterClass
  public static void stopServer() throws Exception {

    manager.shutdown();
  }