Skip to content

Commit

Permalink
Support performance lifecycle strategy for Espresso test, Appium monk…
Browse files Browse the repository at this point in the history 
…ey test, smart test and T2C test (#303)
  • Loading branch information
@taoran6
taoran6 authored Feb 28, 2023
1 parent 3c8ee4a commit 7c80045
Show file tree
Hide file tree
Showing 6 changed files with 39 additions and 3 deletions.
2 changes: 1 addition & 1 deletion agent/src/main/java/com/microsoft/hydralab/agent/runner/appium/AppiumListener.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -238,13 +238,13 @@ public void testRunFinished(Result result) {
}

}
performanceTestListener.testRunFinished();

logEnter("testRunEnded", elapsedTime, Thread.currentThread().getName());
synchronized (this) {
if (alreadyEnd) {
return;
}
performanceTestListener.testRunFinished();
testRun.addNewTimeTag("testRunEnded", System.currentTimeMillis() - recordingStartTimeMillis);
testRun.onTestEnded();
deviceInfo.setRunningTestName(null);
Expand Down
3 changes: 2 additions & 1 deletion agent/src/main/java/com/microsoft/hydralab/agent/runner/espresso/EspressoRunner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@ protected void run(DeviceInfo deviceInfo, TestTask testTask, TestRun testRun) th
try {
/** xml report: parse listener */
reportLogger.info("Start xml report: parse listener");
EspressoTestInfoProcessorListener listener = new EspressoTestInfoProcessorListener(deviceManager, adbOperateUtil, deviceInfo, testRun, testTask.getPkgName());
EspressoTestInfoProcessorListener listener = new EspressoTestInfoProcessorListener(deviceManager,
adbOperateUtil, deviceInfo, testRun, testTask.getPkgName(), performanceTestManagementService);
instrumentationResultParser = new InstrumentationResultParser(testTask.getTestSuite(), Collections.singletonList(listener)) {
@Override
public boolean isCancelled() {
Expand Down
13 changes: 12 additions & 1 deletion .../java/com/microsoft/hydralab/agent/runner/espresso/EspressoTestInfoProcessorListener.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import com.microsoft.hydralab.common.screen.ScreenRecorder;
import com.microsoft.hydralab.common.util.ADBOperateUtil;
import com.microsoft.hydralab.common.util.Const;
import com.microsoft.hydralab.performance.PerformanceTestListener;
import org.slf4j.Logger;

import javax.imageio.ImageIO;
Expand All @@ -35,6 +36,7 @@ public class EspressoTestInfoProcessorListener extends XmlTestRunListener {
private final AnimatedGifEncoder e = new AnimatedGifEncoder();
private final String pkgName;
private final DeviceManager deviceManager;
private final PerformanceTestListener performanceTestListener;
ADBOperateUtil adbOperateUtil;
private long recordingStartTimeMillis;
private int index;
Expand All @@ -45,13 +47,16 @@ public class EspressoTestInfoProcessorListener extends XmlTestRunListener {
private int pid;
private int addedFrameCount;

public EspressoTestInfoProcessorListener(DeviceManager deviceManager, ADBOperateUtil adbOperateUtil, DeviceInfo deviceInfo, TestRun testRun, String pkgName) {
public EspressoTestInfoProcessorListener(DeviceManager deviceManager, ADBOperateUtil adbOperateUtil,
DeviceInfo deviceInfo, TestRun testRun, String pkgName,
PerformanceTestListener performanceTestListener) {
this.deviceManager = deviceManager;
this.adbOperateUtil = adbOperateUtil;
this.deviceInfo = deviceInfo;
this.testRun = testRun;
this.logger = testRun.getLogger();
this.pkgName = pkgName;
this.performanceTestListener = performanceTestListener;
adbLogcatCollector = deviceManager.getLogCollector(deviceInfo, pkgName, testRun, logger);
adbDeviceScreenRecorder = deviceManager.getScreenRecorder(deviceInfo, testRun.getResultFolder(), logger);
setReportDir(testRun.getResultFolder());
Expand Down Expand Up @@ -90,6 +95,7 @@ public void testRunStarted(String runName, int numTests) {
super.testRunStarted(runName, numTests);
logEnter(runName, numTests);
startTools(runName);
performanceTestListener.testRunStarted();
}


Expand Down Expand Up @@ -158,6 +164,8 @@ public void testStarted(TestIdentifier test) {
ioException.printStackTrace();
}
}), logger);

performanceTestListener.testStarted(ongoingTestUnit.getTitle());
}

@Override
Expand All @@ -166,6 +174,7 @@ public void testFailed(TestIdentifier test, String trace) {
super.testFailed(test, trace);
ongoingTestUnit.setStack(trace);
ongoingTestUnit.setStatusCode(AndroidTestUnit.StatusCodes.FAILURE);
performanceTestListener.testFailure(ongoingTestUnit.getTitle());
testRun.addNewTimeTag(ongoingTestUnit.getTitle() + ".fail", System.currentTimeMillis() - recordingStartTimeMillis);
testRun.oneMoreFailure();
}
Expand Down Expand Up @@ -197,6 +206,7 @@ public void testEnded(TestIdentifier test, Map<String, String> testMetrics) {
) {
ongoingTestUnit.setStatusCode(AndroidTestUnit.StatusCodes.OK);
ongoingTestUnit.setSuccess(true);
performanceTestListener.testSuccess(ongoingTestUnit.getTitle());
}
ongoingTestUnit.setEndTimeMillis(System.currentTimeMillis());
ongoingTestUnit.setRelEndTimeInVideo(ongoingTestUnit.getEndTimeMillis() - recordingStartTimeMillis);
Expand Down Expand Up @@ -240,6 +250,7 @@ public void testRunEnded(long elapsedTime, Map<String, String> runMetrics) {
if (alreadyEnd) {
return;
}
performanceTestListener.testRunFinished();
testRun.addNewTimeTag("testRunEnded", System.currentTimeMillis() - recordingStartTimeMillis);
super.testRunEnded(elapsedTime, runMetrics);
testRun.onTestEnded();
Expand Down
8 changes: 8 additions & 0 deletions agent/src/main/java/com/microsoft/hydralab/agent/runner/monkey/AppiumMonkeyRunner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ protected File runAndGetGif(File appiumJarFile, String appiumCommand, DeviceInfo

reportLogger.info(ongoingMonkeyTest.getTitle());

performanceTestManagementService.testRunStarted();

testRun.addNewTimeTag(1 + ". " + ongoingMonkeyTest.getTitle(), System.currentTimeMillis() - recordingStartTimeMillis);
deviceInfo.setRunningTestName(ongoingMonkeyTest.getTitle());
File gifFile = new File(testRun.getResultFolder(), pkgName + ".gif");
Expand All @@ -74,6 +76,9 @@ protected File runAndGetGif(File appiumJarFile, String appiumCommand, DeviceInfo
}
}), reportLogger);
testRun.setTestStartTimeMillis(System.currentTimeMillis());

performanceTestManagementService.testStarted(ongoingMonkeyTest.getTitle());

deviceManager.runAppiumMonkey(deviceInfo, pkgName, testTask.getMaxStepCount(), reportLogger);

deviceScreenRecorder.finishRecording();
Expand All @@ -86,17 +91,20 @@ protected File runAndGetGif(File appiumJarFile, String appiumCommand, DeviceInfo
ongoingMonkeyTest.setSuccess(false);
ongoingMonkeyTest.setStack(e.toString());
testRun.setSuccess(false);
performanceTestManagementService.testFailure(ongoingMonkeyTest.getTitle());
testRun.addNewTimeTagBeforeLast(ongoingMonkeyTest.getTitle() + ".fail", System.currentTimeMillis() - recordingStartTimeMillis);
testRun.oneMoreFailure();
} else {
// Pass
ongoingMonkeyTest.setStatusCode(AndroidTestUnit.StatusCodes.OK);
ongoingMonkeyTest.setSuccess(true);
testRun.setSuccess(true);
performanceTestManagementService.testSuccess(ongoingMonkeyTest.getTitle());
}

// Test finish
reportLogger.info(ongoingMonkeyTest.getTitle() + ".end");
performanceTestManagementService.testRunFinished();
ongoingMonkeyTest.setEndTimeMillis(System.currentTimeMillis());
deviceInfo.setRunningTestName(null);
testRun.addNewTestUnit(ongoingMonkeyTest);
Expand Down
8 changes: 8 additions & 0 deletions agent/src/main/java/com/microsoft/hydralab/agent/runner/smart/SmartRunner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ protected void run(DeviceInfo deviceInfo, TestTask testTask, TestRun testRun) th
reportLogger.info("Start Smart test");
checkTestTaskCancel(testTask);
testRun.setTestStartTimeMillis(System.currentTimeMillis());
performanceTestManagementService.testRunStarted();

/** init smart_test arg */
//TODO choose model before starting test task
Expand Down Expand Up @@ -138,6 +139,9 @@ public void runSmartTestOnce(int i, DeviceInfo deviceInfo, TestRun testRun, Logg
ioException.printStackTrace();
}
}), logger);

performanceTestManagementService.testStarted(ongoingSmartTest.getTitle());

Boolean isSuccess = false;
JSONObject res = new JSONObject();
JSONObject analysisRes = new JSONObject();
Expand All @@ -156,18 +160,21 @@ public void runSmartTestOnce(int i, DeviceInfo deviceInfo, TestRun testRun, Logg
ongoingSmartTest.setStatusCode(AndroidTestUnit.StatusCodes.FAILURE);
ongoingSmartTest.setSuccess(false);
ongoingSmartTest.setStack(res.getString(Const.SmartTestConfig.TASK_EXP_TAG));
performanceTestManagementService.testFailure(ongoingSmartTest.getTitle());
testRun.addNewTimeTag(ongoingSmartTest.getTitle() + ".fail", System.currentTimeMillis() - recordingStartTimeMillis);
testRun.oneMoreFailure();
} else if (crashStack != null && crashStack.size() > 0) {
ongoingSmartTest.setStatusCode(AndroidTestUnit.StatusCodes.FAILURE);
ongoingSmartTest.setSuccess(false);
ongoingSmartTest.setStack(crashStack.toJSONString());
performanceTestManagementService.testFailure(ongoingSmartTest.getTitle());
testRun.addNewTimeTag(ongoingSmartTest.getTitle() + ".fail", System.currentTimeMillis() - recordingStartTimeMillis);
testRun.oneMoreFailure();
} else {
analysisRes = smartTestUtil.analysisRes(res);
ongoingSmartTest.setStatusCode(AndroidTestUnit.StatusCodes.OK);
ongoingSmartTest.setSuccess(true);
performanceTestManagementService.testSuccess(ongoingSmartTest.getTitle());
}
ongoingSmartTest.setEndTimeMillis(System.currentTimeMillis());
logger.info(ongoingSmartTest.getTitle() + ".end");
Expand All @@ -181,6 +188,7 @@ public void runSmartTestOnce(int i, DeviceInfo deviceInfo, TestRun testRun, Logg
}

public void testRunEnded(DeviceInfo deviceInfo, TestRun testRun) {
performanceTestManagementService.testRunFinished();

testRun.addNewTimeTag("testRunEnded", System.currentTimeMillis() - recordingStartTimeMillis);
testRun.onTestEnded();
Expand Down
8 changes: 8 additions & 0 deletions agent/src/main/java/com/microsoft/hydralab/agent/runner/t2c/T2CRunner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,9 @@ protected File runAndGetGif(File initialJsonFile, String unusedSuiteName, Device
testRun.setTotalCount(testTask.testJsonFileList.size() + (initialJsonFile == null ? 0 : 1));
testRun.setTestStartTimeMillis(System.currentTimeMillis());
testRun.addNewTimeTag("testRunStarted", System.currentTimeMillis() - recordingStartTimeMillis);

performanceTestManagementService.testRunStarted();

deviceInfo.setRunningTestName(pkgName.substring(pkgName.lastIndexOf('.') + 1) + ".testRunStarted");
currentIndex = 0;

Expand All @@ -71,6 +74,7 @@ protected File runAndGetGif(File initialJsonFile, String unusedSuiteName, Device

// Test finish
reportLogger.info(pkgName + ".end");
performanceTestManagementService.testRunFinished();
testRun.addNewTimeTag("testRunEnded", System.currentTimeMillis() - recordingStartTimeMillis);
testRun.onTestEnded();
deviceInfo.setRunningTestName(null);
Expand Down Expand Up @@ -103,6 +107,8 @@ private void runT2CJsonTestCase(File jsonFile, DeviceInfo deviceInfo, TestRun te
testRun.addNewTimeTag(currentIndex + ". " + ongoingTest.getTitle(), System.currentTimeMillis() - recordingStartTimeMillis);
testRun.addNewTestUnit(ongoingTest);

performanceTestManagementService.testStarted(ongoingTest.getTitle());

deviceManager.updateScreenshotImageAsyncDelay(deviceInfo, TimeUnit.SECONDS.toMillis(5), (imagePNGFile -> {
if (imagePNGFile == null) {
return;
Expand All @@ -120,13 +126,15 @@ private void runT2CJsonTestCase(File jsonFile, DeviceInfo deviceInfo, TestRun te
// Run Test
try {
deviceManager.runAppiumT2CTest(deviceInfo, jsonFile, reportLogger);
performanceTestManagementService.testSuccess(ongoingTest.getTitle());
ongoingTest.setStatusCode(AndroidTestUnit.StatusCodes.OK);
ongoingTest.setSuccess(true);
} catch (Exception e) {
// Fail
ongoingTest.setStatusCode(AndroidTestUnit.StatusCodes.FAILURE);
ongoingTest.setSuccess(false);
ongoingTest.setStack(e.toString());
performanceTestManagementService.testFailure(ongoingTest.getTitle());
testRun.addNewTimeTag(ongoingTest.getTitle() + ".fail", System.currentTimeMillis() - recordingStartTimeMillis);
testRun.oneMoreFailure();
}
Expand Down

0 comments on commit 7c80045

Please sign in to comment.