/** * Parses and logs the output file. * * @param listener the {@link ITestInvocationListener} * @throws DeviceNotAvailableException If the device is not available. */ private void logOutputFile(ITestInvocationListener listener) throws DeviceNotAvailableException { File outputFile = null; InputStreamSource outputSource = null; try { outputFile = mTestDevice.pullFile(mAppOutputPath); if (outputFile != null) { outputSource = new SnapshotInputStreamSource(new FileInputStream(outputFile)); listener.testLog(LAUNCH_TIME_NAME, LogDataType.TEXT, outputSource); parseOutputFile( StreamUtil.getStringFromStream( new BufferedInputStream(new FileInputStream(outputFile)))); } } catch (IOException e) { CLog.e("Got IOException: %s", e); } finally { if (outputFile != null) { outputFile.delete(); } if (outputSource != null) { outputSource.cancel(); } } if (shouldTakeBugreport()) { InputStreamSource bugreport = mTestDevice.getBugreport(); try { listener.testLog(BUGREPORT_NAME, LogDataType.TEXT, bugreport); } finally { bugreport.cancel(); } } reportMetrics(listener); }
/** * Enable btsnoop logging by changing the BtSnoopLogOutput line in /etc/bluetooth/bt_stack.conf to * true. */ private boolean enableBtsnoopLogging() throws DeviceNotAvailableException { File confFile = mTestDevice.pullFile(BTSNOOP_CONF_FILE); if (confFile == null) { return false; } BufferedReader confReader = null; try { confReader = new BufferedReader(new FileReader(confFile)); StringBuilder newConf = new StringBuilder(); String line; while ((line = confReader.readLine()) != null) { if (line.startsWith("BtSnoopLogOutput=")) { newConf.append("BtSnoopLogOutput=true\n"); } else { newConf.append(line).append("\n"); } } mTestDevice.executeAdbCommand("remount"); return mTestDevice.pushString(newConf.toString(), BTSNOOP_CONF_FILE); } catch (IOException e) { return false; } finally { confFile.delete(); StreamUtil.close(confReader); } }
/** * Wipes the device's external memory of test collateral from prior runs. * * @throws DeviceNotAvailableException If the device is unavailable or something happened while * deleting files */ private void preTestSetup() throws DeviceNotAvailableException { String extStore = mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE); mUrlsFilePath = String.format("%s/%s", extStore, URLS_FILE_NAME); mStatusFilePath = String.format("%s/%s", extStore, STATUS_FILE_NAME); if (!mTestDevice.doesFileExist(mUrlsFilePath)) { throw new RuntimeException("missing URL list file at: " + mUrlsFilePath); } mTestDevice.executeShellCommand("rm " + mStatusFilePath); }
/** {@inheritDoc} */ @Override public void tearDown(ITestDevice device, IBuildInfo buildInfo, Throwable e) throws DeviceNotAvailableException { for (String cmd : mTeardownCommands) { // If the command had any output, the executeShellCommand method will log it at the // VERBOSE level; so no need to do any logging from here. CLog.d("About to run tearDown command on device %s: %s", device.getSerialNumber(), cmd); device.executeShellCommand(cmd); } }
@Override protected void setUp() throws Exception { super.setUp(); // Get the device, this gives a handle to run commands and install APKs. mDevice = getDevice(); // Remove any previously installed versions of this APK. mDevice.uninstallPackage(PACKAGE); // Get the APK from the build. File app = mBuild.getTestApp(APK); // Get the ABI flag. String[] options = {AbiUtils.createAbiFlag(mAbi.getName())}; // Install the APK on the device. mDevice.installPackage(app, false, options); final String densityProp; if (mDevice.getSerialNumber().startsWith("emulator-")) { densityProp = DENSITY_PROP_EMULATOR; } else { densityProp = DENSITY_PROP_DEVICE; } final String zip = String.format( "/%s.zip", getDensityBucket(Integer.parseInt(mDevice.getProperty(densityProp)))); Log.logAndDisplay(LogLevel.INFO, TAG, "Loading resources from " + zip); final InputStream zipStream = this.getClass().getResourceAsStream(zip); if (zipStream != null) { final ZipInputStream in = new ZipInputStream(zipStream); try { ZipEntry ze; final byte[] buffer = new byte[1024]; while ((ze = in.getNextEntry()) != null) { final String name = ze.getName(); final File tmp = File.createTempFile("ref_" + name, ".png"); final FileOutputStream out = new FileOutputStream(tmp); int count; while ((count = in.read(buffer)) != -1) { out.write(buffer, 0, count); } out.flush(); out.close(); mReferences.put(name, tmp); } } finally { in.close(); } } mExecutionService = Executors.newFixedThreadPool(2); // 2 worker threads mCompletionService = new ExecutorCompletionService<Boolean>(mExecutionService); }
/** Run the telephony outgoing call stress test Collect results and post results to dash board */ @Override public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { CLog.d( "input options: mCallDuration(%s),mPauseTime(%s), mPhoneNumber(%s)," + "mRepeatCount(%s)", mCallDuration, mPauseTime, mPhoneNumber, mRepeatCount); Assert.assertNotNull(mTestDevice); Assert.assertNotNull(mPhoneNumber); mRadioHelper = new RadioHelper(mTestDevice); // wait for data connection if (!mRadioHelper.radioActivation() || !mRadioHelper.waitForDataSetup()) { mRadioHelper.getBugreport(listener); return; } IRemoteAndroidTestRunner runner = new RemoteAndroidTestRunner(TEST_PACKAGE_NAME, TEST_RUNNER_NAME, mTestDevice.getIDevice()); runner.setClassName(TEST_CLASS_NAME); runner.setMethodName(TEST_CLASS_NAME, TEST_METHOD); runner.addInstrumentationArg("callduration", mCallDuration); runner.addInstrumentationArg("pausetime", mPauseTime); runner.addInstrumentationArg("phonenumber", mPhoneNumber); runner.setMaxtimeToOutputResponse(TEST_TIMER); // Add bugreport listener for failed test BugreportCollector bugListener = new BugreportCollector(listener, mTestDevice); bugListener.addPredicate(BugreportCollector.AFTER_FAILED_TESTCASES); bugListener.setDescriptiveName(mTestName); CollectingTestListener collectListener = new CollectingTestListener(); int remainingCalls = Integer.parseInt(mRepeatCount); while (remainingCalls > 0) { CLog.d("remaining calls: %s", remainingCalls); runner.addInstrumentationArg("repeatcount", String.valueOf(remainingCalls)); mTestDevice.runInstrumentationTests(runner, bugListener, collectListener); if (collectListener.hasFailedTests()) { // the test failed int numCalls = logOutputFile(bugListener); remainingCalls -= numCalls; cleanOutputFiles(); } else { // the test passed remainingCalls = 0; } } reportMetrics(mMetricsName, bugListener); }
/** * Helper method for iterating through popular_urls.txt and reporting back how many pages fail to * load. * * @param runner The {@link IRemoteAndroidTestRunner} for instrumentation * @param listener The {@link CollectingTestListener} of test results * @return Number of pages that failed to load */ private int runStabilityTest(IRemoteAndroidTestRunner runner, CollectingTestListener listener) throws DeviceNotAvailableException { boolean exitConditionMet = false; int failureCounter = 0; while (!exitConditionMet) { mTestDevice.runInstrumentationTests(runner, listener); // if the status file exists, then the previous instrumentation has crashed if (!mTestDevice.doesFileExist(mStatusFilePath)) { exitConditionMet = true; } else { ++failureCounter; } } return failureCounter; }
/** * Collect number of successful calls and failure reason * * @param listener */ private int logOutputFile(ITestInvocationListener listener) throws DeviceNotAvailableException { File resFile = null; int calls = 0; resFile = mTestDevice.pullFile(mOutputFile); try { if (resFile == null) { // test failed without writing any results // either system crash, or other fails, treat as one failed iteration return 1; } BufferedReader br = new BufferedReader(new FileReader(resFile)); String line = br.readLine(); // The output file should only include one line if (line == null) { return 0; } // Get number of calls and failure reason; String[] res = line.split(" "); calls = Integer.parseInt(res[0]); int reason = Integer.parseInt(res[1]); callStatus[reason]++; } catch (IOException e) { CLog.e("IOException while reading outputfile %s", resFile.getAbsolutePath()); } if (resFile != null) { resFile.delete(); } return calls; }
/** {@inheritDoc} */ @Override protected void setUp() throws Exception { super.setUp(); mMockFlasher = EasyMock.createMock(IDeviceFlasher.class); mMockDevice = EasyMock.createMock(ITestDevice.class); EasyMock.expect(mMockDevice.getSerialNumber()).andReturn("foo").anyTimes(); mMockBuildInfo = new DeviceBuildInfo("0", "", ""); mMockBuildInfo.setBuildFlavor("flavor"); mDeviceFlashPreparer = new DeviceFlashPreparer() { @Override protected IDeviceFlasher createFlasher(ITestDevice device) { return mMockFlasher; } @Override int getDeviceBootPollTimeMs() { return 100; } }; mDeviceFlashPreparer.setDeviceBootTime(100); // expect this call mMockFlasher.setUserDataFlashOption(UserDataFlashOption.FLASH); mTmpDir = FileUtil.createTempDir("tmp"); }
/** * Pull the output file from the device, add it to the logs, and also parse out the relevant test * metrics and report them. */ private void logOutputFile(TestInfo testInfo, ITestInvocationListener listener) throws DeviceNotAvailableException { File outputFile = null; InputStreamSource outputSource = null; try { outputFile = mTestDevice.pullFileFromExternal(OUTPUT_PATH); if (outputFile != null) { CLog.d("Sending %d byte file %s into the logosphere!", outputFile.length(), outputFile); outputSource = new SnapshotInputStreamSource(new FileInputStream(outputFile)); listener.testLog( String.format("%s_output", testInfo.mTestName), LogDataType.TEXT, outputSource); parseOutputFile(testInfo, new FileInputStream(outputFile), listener); } } catch (IOException e) { CLog.e("Got an IO Exception: %s", e); } finally { if (outputFile != null) { outputFile.delete(); } if (outputSource != null) { outputSource.cancel(); } } }
public void testInstallTime() throws Exception { HostReportLog report = new HostReportLog(mDevice.getSerialNumber(), ReportLog.getClassMethodNames()); final int NUMBER_REPEAT = 10; final CtsBuildHelper build = mBuild; final ITestDevice device = mDevice; double[] result = MeasureTime.measure( NUMBER_REPEAT, new MeasureRun() { @Override public void prepare(int i) throws Exception { device.uninstallPackage(PACKAGE); } @Override public void run(int i) throws Exception { File app = build.getTestApp(APK); device.installPackage(app, false); } }); report.printArray("install time", result, ResultType.LOWER_BETTER, ResultUnit.MS); Stat.StatResult stat = Stat.getStatWithOutlierRejection(result, OUTLIER_THRESHOLD); if (stat.mDataCount != result.length) { Log.w(TAG, "rejecting " + (result.length - stat.mDataCount) + " outliers"); } report.printSummary("install time", stat.mAverage, ResultType.LOWER_BETTER, ResultUnit.MS); report.deliverReportToHost(); }
/** Verify that {@link AbiFormatter#getDefaultAbi} works as expected. */ public void testGetDefaultAbi() throws Exception { ITestDevice device = EasyMock.createMock(ITestDevice.class); EasyMock.expect(device.getProperty("ro.product.cpu.abilist32")).andReturn(null); EasyMock.replay(device); assertEquals(null, AbiFormatter.getDefaultAbi(device, "32")); EasyMock.reset(device); EasyMock.expect(device.getProperty(EasyMock.eq("ro.product.cpu.abilist32"))) .andReturn("abi,abi2,abi3"); EasyMock.replay(device); assertEquals("abi", AbiFormatter.getDefaultAbi(device, "32")); EasyMock.reset(device); EasyMock.expect(device.getProperty(EasyMock.eq("ro.product.cpu.abilist64"))).andReturn(""); EasyMock.replay(device); assertEquals(null, AbiFormatter.getDefaultAbi(device, "64")); }
@Override protected void tearDown() throws Exception { // Delete the temp files for (File ref : mReferences.values()) { ref.delete(); } mExecutionService.shutdown(); // Remove the APK. mDevice.uninstallPackage(PACKAGE); super.tearDown(); }
private boolean runCapture(int themeId, int layoutId, String imageName) throws Exception { final StringBuilder sb = new StringBuilder(START_CMD); sb.append(String.format(INTENT_INTEGER_EXTRA, EXTRA_THEME, themeId)); sb.append(String.format(INTENT_INTEGER_EXTRA, EXTRA_LAYOUT, layoutId)); sb.append(String.format(INTENT_INTEGER_EXTRA, EXTRA_TIMEOUT, CAPTURE_TIMEOUT)); final String startCommand = sb.toString(); // Clear logcat mDevice.executeAdbCommand("logcat", "-c"); // Stop any existing instances mDevice.executeShellCommand(STOP_CMD); // Start activity mDevice.executeShellCommand(startCommand); boolean success = false; boolean waiting = true; while (waiting) { // Dump logcat. final String logs = mDevice.executeAdbCommand("logcat", "-v", "brief", "-d", CLASS + ":I", "*:S"); // Search for string. final Scanner in = new Scanner(logs); while (in.hasNextLine()) { final String line = in.nextLine(); if (line.startsWith("I/" + CLASS)) { final String[] lineSplit = line.split(":"); final String s = lineSplit[1].trim(); final String imageNameGenerated = lineSplit[2].trim(); if (s.equals("OKAY") && imageNameGenerated.equals(imageName)) { success = true; waiting = false; } else if (s.equals("ERROR") && imageNameGenerated.equals(imageName)) { success = false; waiting = false; } } } in.close(); } return success; }
/** {@inheritDoc} */ @Override public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { Assert.assertNotNull(mTestDevice); mAppListPath = new File(mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE), APP_LIST_FILE) .getAbsolutePath(); mAppOutputPath = new File(mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE), APP_OUTPUT_FILE) .getAbsolutePath(); setupAppInfos(); // Setup the device mTestDevice.executeShellCommand(String.format("rm %s %s", mAppListPath, mAppOutputPath)); mTestDevice.pushString(generateAppList(), mAppListPath); mTestDevice.executeShellCommand(String.format("chmod 750 %s", APP_LAUNCH)); // Sleep 30 seconds to let device settle. RunUtil.getDefault().sleep(30 * 1000); // Run the test String output = mTestDevice.executeShellCommand(APP_LAUNCH); CLog.d("App launch output: %s", output); logOutputFile(listener); }
/** {@inheritDoc} */ @Override public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { int failCounter = 0; Map<String, String> finalMetrics = new HashMap<String, String>(); preTestSetup(); // Create and config runner for instrumentation test IRemoteAndroidTestRunner runner = new RemoteAndroidTestRunner(mTestPackage, TEST_RUNNER_NAME, mTestDevice.getIDevice()); runner.setClassName(mTestClass); runner.setMethodName(mTestClass, mTestMethod); // max timeout is the smaller of: 8 hrs or 1 minute per site runner.setMaxtimeToOutputResponse(Math.min(MAX_TIMEOUT_MS, 60 * 1000 * mTotalSites)); CollectingTestListener collectingTestListener = new CollectingTestListener(); failCounter = runStabilityTest(runner, collectingTestListener); finalMetrics.put(mSchemaKey, Integer.toString(mTotalSites - failCounter)); reportMetrics(listener, mMetricsName, finalMetrics); }
@TimeoutReq(minutes = 60) public void testHoloThemes() throws Exception { if (checkHardwareTypeSkipTest(mDevice.executeShellCommand(HARDWARE_TYPE_CMD).trim())) { Log.logAndDisplay(LogLevel.INFO, TAG, "Skipped HoloThemes test for watch and TV"); return; } if (mReferences.isEmpty()) { Log.logAndDisplay(LogLevel.INFO, TAG, "Skipped HoloThemes test due to no reference images"); return; } int numTasks = 0; for (int i = 0; i < NUM_THEMES; i++) { final String themeName = THEMES[i]; for (int j = 0; j < NUM_LAYOUTS; j++) { final String name = String.format("%s_%s", themeName, LAYOUTS[j]); if (runCapture(i, j, name)) { final File ref = mReferences.get(name + ".png"); if (!ref.exists()) { Log.logAndDisplay( LogLevel.INFO, TAG, "Skipping theme test due to missing reference for reference image " + name); continue; } mCompletionService.submit(new ComparisonTask(mDevice, ref, name)); numTasks++; } else { Log.logAndDisplay(LogLevel.ERROR, TAG, "Capture failed: " + name); } } } int failures = 0; for (int i = 0; i < numTasks; i++) { failures += mCompletionService.take().get() ? 0 : 1; } assertTrue(failures + " failures in theme test", failures == 0); }
@Override protected void tearDown() throws Exception { super.tearDown(); mDevice.uninstallPackage(PACKAGE); mReport.throwReportToHost(); }
@Override protected void tearDown() throws Exception { mDevice.uninstallPackage(PACKAGE); super.tearDown(); }
/** Test {@link DeviceSetup#setUp(ITestDevice, IBuildInfo)} when build does not boot */ public void testSetup_buildError() throws Exception { mMockDevice.setRecoveryMode(RecoveryMode.ONLINE); mMockFlasher.overrideDeviceOptions(mMockDevice); mMockFlasher.setForceSystemFlash(false); mMockFlasher.setDataWipeSkipList(Arrays.asList(new String[] {})); mMockFlasher.flash(mMockDevice, mMockBuildInfo); mMockDevice.waitForDeviceOnline(); EasyMock.expect(mMockDevice.enableAdbRoot()).andStubReturn(Boolean.TRUE); mMockDevice.setDate(null); EasyMock.expect(mMockDevice.getBuildId()).andReturn(mMockBuildInfo.getBuildId()); EasyMock.expect(mMockDevice.getBuildFlavor()).andReturn(mMockBuildInfo.getBuildFlavor()); EasyMock.expect(mMockDevice.isEncryptionSupported()).andStubReturn(Boolean.TRUE); EasyMock.expect(mMockDevice.isDeviceEncrypted()).andStubReturn(Boolean.FALSE); mMockDevice.clearLogcat(); mMockDevice.waitForDeviceAvailable(EasyMock.anyLong()); EasyMock.expectLastCall().andThrow(new DeviceUnresponsiveException("foo")); mMockDevice.setRecoveryMode(RecoveryMode.AVAILABLE); EasyMock.replay(mMockFlasher, mMockDevice); try { mDeviceFlashPreparer.setUp(mMockDevice, mMockBuildInfo); fail("DeviceFlashPreparerTest not thrown"); } catch (BuildError e) { // expected; use the general version to make absolutely sure that // DeviceFailedToBootError properly masquerades as a BuildError. assertTrue(e instanceof DeviceFailedToBootError); } EasyMock.verify(mMockFlasher, mMockDevice); }
/** Clean up the tmp output file from previous test runs */ private void cleanOutputFile() throws DeviceNotAvailableException { mTestDevice.executeShellCommand(String.format("rm ${EXTERNAL_STORAGE}/%s", OUTPUT_PATH)); if (mLogBtsnoop) { mTestDevice.executeShellCommand(String.format("rm ${EXTERNAL_STORAGE}/%s", BTSNOOP_LOG_FILE)); } }
/** Set EasyMock expectations for a normal setup call */ private void doSetupExpectations() throws TargetSetupError, DeviceNotAvailableException { mMockDevice.setRecoveryMode(RecoveryMode.ONLINE); mMockFlasher.overrideDeviceOptions(mMockDevice); mMockFlasher.setForceSystemFlash(false); mMockFlasher.setDataWipeSkipList(Arrays.asList(new String[] {})); mMockFlasher.flash(mMockDevice, mMockBuildInfo); mMockDevice.waitForDeviceOnline(); EasyMock.expect(mMockDevice.enableAdbRoot()).andStubReturn(Boolean.TRUE); mMockDevice.setDate(null); EasyMock.expect(mMockDevice.getBuildId()).andReturn(mMockBuildInfo.getBuildId()); EasyMock.expect(mMockDevice.getBuildFlavor()).andReturn(mMockBuildInfo.getBuildFlavor()); EasyMock.expect(mMockDevice.isEncryptionSupported()).andStubReturn(Boolean.TRUE); EasyMock.expect(mMockDevice.isDeviceEncrypted()).andStubReturn(Boolean.FALSE); mMockDevice.clearLogcat(); mMockDevice.waitForDeviceAvailable(EasyMock.anyLong()); mMockDevice.setRecoveryMode(RecoveryMode.AVAILABLE); mMockDevice.postBootSetup(); }
@Override public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { Assert.assertNotNull(mTestDevice); setupTests(); if (mLogBtsnoop) { if (!enableBtsnoopLogging()) { CLog.e("Unable to enable btsnoop trace logging"); throw new DeviceNotAvailableException(); } } IRemoteAndroidTestRunner runner = new RemoteAndroidTestRunner(mTestPackageName, TEST_RUNNER_NAME, mTestDevice.getIDevice()); runner.setClassName(mTestClassName); runner.setMaxTimeToOutputResponse(mTestTimeout, TimeUnit.MILLISECONDS); BugreportCollector bugListener = new BugreportCollector(listener, mTestDevice); bugListener.addPredicate(BugreportCollector.AFTER_FAILED_TESTCASES); for (TestInfo test : mTestCases) { String testName = test.mTestName; TestInfo t = test; if (t.mIterCount != null && t.mIterCount <= 0) { CLog.i("Cancelled '%s' test case with iter count %s", testName, t.mIterCount); continue; } // For semi-manual tests, print instructions and wait for user to continue. if (t.mInstructions != null) { BufferedReader br = new BufferedReader(new InputStreamReader(System.in)); System.out.println("========================================"); System.out.println(t.mInstructions); System.out.println("========================================"); try { br.readLine(); } catch (IOException e) { CLog.e("Continuing after IOException while waiting for confirmation: %s", e.getMessage()); } } // Run the test cleanOutputFile(); if (mLogHcidump) { mHcidumpCommand = new HcidumpCommand(); mHcidumpCommand.startHcidump(); } if (t.mIterKey != null && t.mIterCount != null) { runner.addInstrumentationArg(t.mIterKey, t.mIterCount.toString()); } if (t.mRemoteAddress != null) { runner.addInstrumentationArg("device_address", t.mRemoteAddress); if (t.mPairPasskey != null) { runner.addInstrumentationArg("device_pair_passkey", t.mPairPasskey); } if (t.mPairPin != null) { runner.addInstrumentationArg("device_pair_pin", t.mPairPin); } } runner.setMethodName(mTestClassName, t.mTestMethod); bugListener.setDescriptiveName(testName); mTestDevice.runInstrumentationTests(runner, bugListener); if (t.mIterKey != null && t.mIterCount != null) { runner.removeInstrumentationArg(t.mIterKey); } if (t.mRemoteAddress != null) { runner.removeInstrumentationArg("device_address"); if (t.mPairPasskey != null) { runner.removeInstrumentationArg("device_pair_passkey"); } if (t.mPairPin != null) { runner.removeInstrumentationArg("device_pair_pin"); } } // Log the output file logOutputFile(t, listener); if (mLogBtsnoop) { File logFile = null; InputStreamSource logSource = null; try { logFile = mTestDevice.pullFileFromExternal(BTSNOOP_LOG_FILE); if (logFile != null) { CLog.d("Sending %d byte file %s into the logosphere!", logFile.length(), logFile); logSource = new SnapshotInputStreamSource(new FileInputStream(logFile)); listener.testLog( String.format("%s_btsnoop", t.mTestName), LogDataType.UNKNOWN, logSource); } } catch (IOException e) { CLog.e("Got an IO Exception: %s", e); } finally { if (logFile != null) { logFile.delete(); } if (logSource != null) { logSource.cancel(); } } } if (mLogHcidump) { listener.testLog( String.format("%s_hcidump", t.mTestName), LogDataType.TEXT, mHcidumpCommand.getHcidump()); mHcidumpCommand.stopHcidump(); mHcidumpCommand = null; } cleanOutputFile(); } }
/** Clean up output files from the last test run */ private void cleanOutputFiles() throws DeviceNotAvailableException { CLog.d("Remove output file: %s", mOutputFile); mTestDevice.executeShellCommand(String.format("rm %s", mOutputFile)); }