Update screen recorder for tests of android_client, to make sure the …

…screen recording is performed correctly (#469)

* Update screen recorder for tests of android_client, to make sure the screen recording is performed correctly.

* Add compatibility in code for android client tests

* Update command for device action usage.

* Abstract retry logic into ADBOperateUtil.pullFileToDir;

* Update var name with log

* Update flag

* Update related gradle plugin config

agent/src/main/java/com/microsoft/hydralab/agent/command/DeviceScriptCommandLoader.java

@@ -64,7 +64,15 @@ private List<DeviceAction> command2Action(DeviceScriptCommand deviceCommand) {
     }
 
     private enum ActionConverter {
-        //generate action by command type
+        /**
+         * Generate action by command type.
+         * ADBShell: execute command on mobile device only
+         * AgentShell: execute command on agent only
+         *  - type = Agent: run for each Agent device, e.g. Windows, Mac, Linux. The commands would be run for only once.
+         *      Used for environment setup and build generation.
+         *  - type = Android: run for each Android device. The commands would be run on each Android device.
+         *      Used for specific device operation through PC.
+         */
         ADBShell() {
             @Override
             public DeviceAction getAction(String commandline, String deviceType) {

agent/src/main/java/com/microsoft/hydralab/agent/runner/ActionExecutor.java

@@ -40,7 +40,7 @@ public class ActionExecutor {
     private final Set<String> actionTypes =
             Set.of("setProperty", "setDefaultLauncher", "backToHome", "changeGlobalSetting",
                     "changeSystemSetting", "execCommandOnDevice", "execCommandOnAgent", "pushFileToDevice",
-                    "pullFileFromDevice");
+                    "pullFileFromDevice", "addToBatteryWhiteList");
 
     public List<Exception> doActions(@NotNull DeviceDriver deviceDriverManager,
                                      @NotNull TestRunDevice testRunDevice,

agent/src/main/java/com/microsoft/hydralab/agent/runner/TestRunner.java

@@ -13,6 +13,8 @@
 import com.microsoft.hydralab.common.entity.common.TestRunDevice;
 import com.microsoft.hydralab.common.entity.common.TestTask;
 import com.microsoft.hydralab.common.management.AgentManagementService;
+import com.microsoft.hydralab.common.screen.PhoneAppScreenRecorder;
+import com.microsoft.hydralab.common.util.Const;
 import com.microsoft.hydralab.common.util.DateUtil;
 import com.microsoft.hydralab.common.util.FlowUtil;
 import com.microsoft.hydralab.common.util.LogUtils;
@@ -25,8 +27,10 @@
 import org.slf4j.LoggerFactory;
 
 import java.io.File;
+import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.FutureTask;
 import java.util.concurrent.TimeUnit;
@@ -192,6 +196,16 @@ protected TestRun createTestRun(TestRunDevice testRunDevice, TestTask testTask)
     }
 
     protected void setUp(TestRunDevice testRunDevice, TestTask testTask, TestRun testRun) throws Exception {
+        // grant battery white list when testing android_client
+        if (PhoneAppScreenRecorder.RECORD_PACKAGE_NAME.equals(testTask.getPkgName())) {
+            Map<String, List<DeviceAction>> deviceActionsMap = testTask.getDeviceActions();
+            List<DeviceAction> setUpDeviceActions = deviceActionsMap.getOrDefault(DeviceAction.When.SET_UP, new ArrayList<>());
+            DeviceAction deviceAction1 = new DeviceAction(Const.OperatedDevice.ANDROID, "addToBatteryWhiteList");
+            deviceAction1.setArgs(List.of(PhoneAppScreenRecorder.RECORD_PACKAGE_NAME));
+            setUpDeviceActions.add(deviceAction1);
+            deviceActionsMap.put(DeviceAction.When.SET_UP, setUpDeviceActions);
+        }
+
         testRunDeviceOrchestrator.killAll(testRunDevice);
         // this key will be used to recover device status when lost the connection between agent and master
         testRunDeviceOrchestrator.addCurrentTask(testRunDevice, testTask);

common/src/main/java/com/microsoft/hydralab/common/entity/common/DeviceInfo.java

@@ -41,6 +41,7 @@ public class DeviceInfo extends MobileDevice {
     private String deviceId;
     private String runningTaskId;
     private String runningTestName;
+    private String runningTaskPackageName;
     private String agentId;
     private Set<String> deviceGroup = new HashSet<>();
     private boolean supportScreenRecording = true;
@@ -90,17 +91,20 @@ public void addCurrentTask(TestTask testTask) {
         this.currentTask.put(Thread.currentThread(), testTask);
         this.status = DeviceInfo.TESTING;
         this.runningTaskId = testTask.getId();
+        this.runningTaskPackageName = testTask.getPkgName();
     }
 
     public void finishTask() {
         this.currentTask.remove(Thread.currentThread());
         this.status = DeviceInfo.ONLINE;
         this.runningTaskId = null;
+        this.runningTaskPackageName = null;
     }
 
     public void reset() {
         this.status = DeviceInfo.ONLINE;
         this.runningTaskId = null;
+        this.runningTaskPackageName = null;
         killAll();
     }
 

common/src/main/java/com/microsoft/hydralab/common/management/device/impl/AndroidDeviceDriver.java

@@ -21,6 +21,7 @@
 import com.microsoft.hydralab.common.management.AgentManagementService;
 import com.microsoft.hydralab.common.management.AppiumServerManager;
 import com.microsoft.hydralab.common.management.device.DeviceType;
+import com.microsoft.hydralab.common.screen.ADBScreenRecorder;
 import com.microsoft.hydralab.common.screen.PhoneAppScreenRecorder;
 import com.microsoft.hydralab.common.screen.ScreenRecorder;
 import com.microsoft.hydralab.common.util.ADBOperateUtil;
@@ -43,7 +44,6 @@
 import java.awt.image.BufferedImage;
 import java.io.File;
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -61,7 +61,7 @@
 import static com.android.ddmlib.IDevice.PROP_DEVICE_CPU_ABI_LIST;
 import static com.android.ddmlib.IDevice.PROP_DEVICE_MANUFACTURER;
 import static com.android.ddmlib.IDevice.PROP_DEVICE_MODEL;
-import static com.microsoft.hydralab.common.screen.PhoneAppScreenRecorder.recordPackageName;
+import static com.microsoft.hydralab.common.screen.PhoneAppScreenRecorder.RECORD_PACKAGE_NAME;
 
 public class AndroidDeviceDriver extends AbstractDeviceDriver {
 
@@ -355,6 +355,9 @@ public void pullFileFromDevice(@NotNull DeviceInfo deviceInfo, @NotNull String p
 
     @Override
     public ScreenRecorder getScreenRecorder(DeviceInfo deviceInfo, File folder, Logger logger) {
+        if (PhoneAppScreenRecorder.RECORD_PACKAGE_NAME.equals(deviceInfo.getRunningTaskPackageName())) {
+            return new ADBScreenRecorder(this, this.adbOperateUtil, deviceInfo, logger, folder);
+        }
         return new PhoneAppScreenRecorder(this, this.adbOperateUtil, deviceInfo, folder, logger);
     }
 
@@ -652,7 +655,7 @@ private void stopPackageProcess(DeviceInfo deviceInfo, String packageName, Logge
     private void startRecordActivity(DeviceInfo deviceInfo, Logger logger) {
         try {
             adbOperateUtil.execOnDevice(Objects.requireNonNull(deviceInfo),
-                    "am start -n " + recordPackageName + "/.MainActivity",
+                    "am start -n " + RECORD_PACKAGE_NAME + "/.MainActivity",
                     new MultiLineNoCancelLoggingReceiver(logger), logger);
         } catch (Exception e) {
             logger.error(e.getMessage(), e);

common/src/main/java/com/microsoft/hydralab/common/screen/ADBScreenRecorder.java

@@ -1,9 +1,11 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
+
 package com.microsoft.hydralab.common.screen;
 
 import cn.hutool.core.thread.ThreadUtil;
 import com.microsoft.hydralab.common.entity.common.DeviceInfo;
+import com.microsoft.hydralab.common.management.device.DeviceDriver;
 import com.microsoft.hydralab.common.util.ADBOperateUtil;
 import com.microsoft.hydralab.common.util.DateUtil;
 import com.microsoft.hydralab.common.util.ThreadUtils;
@@ -25,14 +27,16 @@ public class ADBScreenRecorder implements ScreenRecorder {
     private final File baseFolder;
 
     private File mergedVideo;
+    private final DeviceDriver deviceDriver;
     public int preSleepSeconds = 0;
     ADBOperateUtil adbOperateUtil;
     private Process recordingProcess;
     private Thread recordingThread;
     private boolean shouldStop = true;
     private boolean shouldInterrupt = false;
 
-    public ADBScreenRecorder(ADBOperateUtil adbOperateUtil, DeviceInfo deviceInfo, Logger logger, File baseFolder) {
+    public ADBScreenRecorder(DeviceDriver deviceDriver, ADBOperateUtil adbOperateUtil, DeviceInfo deviceInfo, Logger logger, File baseFolder) {
+        this.deviceDriver = deviceDriver;
         this.adbOperateUtil = adbOperateUtil;
         this.deviceInfo = deviceInfo;
         this.logger = logger;
@@ -50,7 +54,6 @@ public void setPreSleepSeconds(int preSleepSeconds) {
 
     @Override
     public void setupDevice() {
-
     }
 
     @Override
@@ -71,12 +74,12 @@ public void startRecord(int maxTimeInSecond) {
                 int totalTime = 0;
                 List<File> list = new ArrayList<>();
                 while (totalTime < maxTimeInSecond && !shouldStop) {
-                    String fileName = String.format("/sdcard/scr_rec_%d_%d.mp4", totalTime, totalTime + timeSpan);
-                    String command = String.format("shell screenrecord --bit-rate 3200000 --time-limit %d %s", timeSpan, fileName);
-                    deviceInfo.addCurrentCommand(command);
+                    String pathOnDevice = String.format("/sdcard/scr_rec_%d_%d.mp4", totalTime, totalTime + timeSpan);
+                    String recordCommand = String.format("shell screenrecord --bit-rate 3200000 --time-limit %d %s", timeSpan, pathOnDevice);
+                    deviceInfo.addCurrentCommand(recordCommand);
                     // Blocking command
-                    recordingProcess = adbOperateUtil.executeDeviceCommandOnPC(deviceInfo, command, logger);
-                    logger.info("ADBDeviceScreenRecorder>> command: " + command);
+                    recordingProcess = adbOperateUtil.executeDeviceCommandOnPC(deviceInfo, recordCommand, logger);
+                    logger.info("ADBDeviceScreenRecorder>> command: " + recordCommand);
                     logger.info(IOUtils.toString(recordingProcess.getInputStream(), StandardCharsets.UTF_8));
                     logger.error(IOUtils.toString(recordingProcess.getErrorStream(), StandardCharsets.UTF_8));
                     deviceInfo.addCurrentProcess(recordingProcess);
@@ -93,18 +96,14 @@ public void startRecord(int maxTimeInSecond) {
                         recordingProcess.destroy();
                     }
                     deviceInfo.finishCommand();
+                    // make sure the recording procedure is stopped completely
+                    ThreadUtil.safeSleep(2000);
 
-                    String outputFilePrefix = new File(baseFolder, DateUtil.fileNameDateDashFormat.format(new Date())).getAbsolutePath();
-
-                    final String outFileFullPath = outputFilePrefix + "_" + totalTime + "_" + (totalTime + timeSpan) + ".mp4";
-                    String pullComm = String.format("pull %s %s", fileName, outFileFullPath);
-                    Process process = adbOperateUtil.executeDeviceCommandOnPC(deviceInfo, pullComm, logger);
-
-                    logger.info(IOUtils.toString(process.getInputStream(), StandardCharsets.UTF_8));
-                    logger.error(IOUtils.toString(process.getErrorStream(), StandardCharsets.UTF_8));
-                    process.destroy();
-
-                    list.add(new File(outFileFullPath));
+                    final String outFileName = DateUtil.fileNameDateDashFormat.format(new Date()) + "_" + totalTime + "_" + (totalTime + timeSpan) + ".mp4";
+                    String pathOnAgent = new File(baseFolder, outFileName).getAbsolutePath();
+                    adbOperateUtil.pullFileToDir(deviceInfo, pathOnAgent, pathOnDevice, logger);
+                    list.add(new File(pathOnAgent));
+                    deviceDriver.removeFileInDevice(deviceInfo, pathOnDevice, logger);
 
                     totalTime += timeSpan;
                     logger.info("ADBDeviceScreenRecorder>> Time recorded {}", totalTime);
@@ -118,7 +117,7 @@ public void startRecord(int maxTimeInSecond) {
                     list.forEach(File::delete);
                 }
 
-            } catch (IOException e) {
+            } catch (IOException | InterruptedException e) {
                 logger.warn("Exception from recordingThread {} {}", e.getClass().getName(), e.getMessage());
             } finally {
                 if (recordingProcess != null) {

common/src/main/java/com/microsoft/hydralab/common/screen/PhoneAppScreenRecorder.java

@@ -23,7 +23,7 @@
 
 
 public class PhoneAppScreenRecorder implements ScreenRecorder {
-    public static final String recordPackageName = "com.microsoft.hydralab.android.client";
+    public static final String RECORD_PACKAGE_NAME = "com.microsoft.hydralab.android.client";
     private static File recordApk;
     protected final File baseFolder;
     protected final Logger logger;
@@ -60,19 +60,19 @@ public static void copyAPK(File preAppDir) {
 
     @Override
     public void setupDevice() {
-        if (!deviceDriver.isAppInstalled(deviceInfo, recordPackageName, logger)) {
+        if (!deviceDriver.isAppInstalled(deviceInfo, RECORD_PACKAGE_NAME, logger)) {
             installRecorderServiceApp();
         }
         try {
             deviceDriver.wakeUpDevice(deviceInfo, logger);
             deviceDriver.unlockDevice(deviceInfo, logger);
-            deviceDriver.grantAllPackageNeededPermissions(deviceInfo, recordApk, recordPackageName, false, logger);
-            deviceDriver.grantPermission(deviceInfo, recordPackageName, "android.permission.FOREGROUND_SERVICE", logger);
-            deviceDriver.addToBatteryWhiteList(deviceInfo, recordPackageName, logger);
+            deviceDriver.grantAllPackageNeededPermissions(deviceInfo, recordApk, RECORD_PACKAGE_NAME, false, logger);
+            deviceDriver.grantPermission(deviceInfo, RECORD_PACKAGE_NAME, "android.permission.FOREGROUND_SERVICE", logger);
+            deviceDriver.addToBatteryWhiteList(deviceInfo, RECORD_PACKAGE_NAME, logger);
         } catch (Exception e) {
             logger.error(e.getMessage(), e);
         }
-        FlowUtil.retryWhenFalse(3, () -> deviceDriver.grantProjectionAndBatteryPermission(deviceInfo, recordPackageName, logger));
+        FlowUtil.retryWhenFalse(3, () -> deviceDriver.grantProjectionAndBatteryPermission(deviceInfo, RECORD_PACKAGE_NAME, logger));
     }
 
     @Override
@@ -107,7 +107,6 @@ public String finishRecording() {
         if (!started) {
             return null;
         }
-        boolean tag = false;
         // wait 5s to record more info after testing
         ThreadUtils.safeSleep(5000);
         stopRecordService();
@@ -121,38 +120,16 @@ public String finishRecording() {
         // wait for screen recording to finish
         ThreadUtils.safeSleep(5000);
 
-        int retryTime = 1;
-        while (retryTime < Const.AgentConfig.RETRY_TIME) {
-            logger.info("Pull file round :" + retryTime);
-            File videoFile = new File(pathOnAgent);
-            if (videoFile.exists()) {
-                videoFile.delete();
-            }
-            try {
-                adbOperateUtil.pullFileToDir(deviceInfo, pathOnAgent, pathOnDevice, logger);
-            } catch (IOException | InterruptedException e) {
-                logger.error(e.getMessage(), e);
-            }
-            ThreadUtils.safeSleep(5000);
-
-            long phoneFileSize = adbOperateUtil.getFileLength(deviceInfo, logger, pathOnDevice);
-            logger.info("PC file path:{} size:{} , Phone file path {} size {}", pathOnAgent, videoFile.length(), pathOnDevice, phoneFileSize);
-            if (videoFile.length() == phoneFileSize) {
-                logger.info("Pull video file success!");
-                tag = true;
-                break;
-            }
-            retryTime++;
-            if (retryTime == Const.AgentConfig.RETRY_TIME) {
-                logger.error("Pull video file fail!");
-            }
+        try {
+            adbOperateUtil.pullFileToDir(deviceInfo, pathOnAgent, pathOnDevice, logger);
+        } catch (IOException | InterruptedException e) {
+            logger.error(e.getMessage(), e);
+            pathOnAgent = null;
         }
         deviceDriver.removeFileInDevice(deviceInfo, pathOnDevice, logger);
         started = false;
-        if (tag) {
-            return pathOnAgent;
-        }
-        return null;
+
+        return pathOnAgent;
     }
 
     @Override
@@ -163,7 +140,8 @@ public int getPreSleepSeconds() {
     public boolean startRecordService() {
         try {
             // am startservice --es fileName test.mp4 com.microsoft.hydralab.android.client/.ScreenRecorderService
-            adbOperateUtil.execOnDevice(deviceInfo, String.format("am startservice -a %s.action.START --es fileName %s --es SNCode %s --ei width 720 --ei bitrate 1200000 %s/.ScreenRecorderService", recordPackageName, fileName, deviceInfo.getSerialNum(), recordPackageName), new MultiLineNoCancelLoggingReceiver(logger), logger);
+            adbOperateUtil.execOnDevice(deviceInfo, String.format("am startservice -a %s.action.START --es fileName %s --es SNCode %s --ei width 720 --ei bitrate 1200000 %s/.ScreenRecorderService",
+                    RECORD_PACKAGE_NAME, fileName, deviceInfo.getSerialNum(), RECORD_PACKAGE_NAME), new MultiLineNoCancelLoggingReceiver(logger), logger);
             return true;
         } catch (Exception e) {
             logger.error(e.getMessage(), e);
@@ -173,7 +151,7 @@ public boolean startRecordService() {
 
     public boolean stopRecordService() {
         try {
-            adbOperateUtil.execOnDevice(deviceInfo, "am startservice -a " + recordPackageName + ".action.STOP " + recordPackageName + "/.ScreenRecorderService", new MultiLineNoCancelLoggingReceiver(logger), logger);
+            adbOperateUtil.execOnDevice(deviceInfo, "am startservice -a " + RECORD_PACKAGE_NAME + ".action.STOP " + RECORD_PACKAGE_NAME + "/.ScreenRecorderService", new MultiLineNoCancelLoggingReceiver(logger), logger);
             return true;
         } catch (Exception e) {
             logger.error(e.getMessage(), e);
@@ -184,7 +162,7 @@ public boolean stopRecordService() {
     public boolean sendKeepAliveSignal() {
 
         try {
-            adbOperateUtil.execOnDevice(deviceInfo, "am startservice -a " + recordPackageName + ".action.SIGNAL " + recordPackageName + "/.ScreenRecorderService", new MultiLineNoCancelLoggingReceiver(logger), logger);
+            adbOperateUtil.execOnDevice(deviceInfo, "am startservice -a " + RECORD_PACKAGE_NAME + ".action.SIGNAL " + RECORD_PACKAGE_NAME + "/.ScreenRecorderService", new MultiLineNoCancelLoggingReceiver(logger), logger);
             return true;
         } catch (Exception e) {
             logger.error(e.getMessage(), e);
@@ -198,7 +176,7 @@ private void installRecorderServiceApp() {
         } catch (HydraLabRuntimeException e) {
             // if failed to install app, uninstall app and try again.
             logger.error(e.getMessage(), e);
-            deviceDriver.uninstallApp(deviceInfo, recordPackageName, logger);
+            deviceDriver.uninstallApp(deviceInfo, RECORD_PACKAGE_NAME, logger);
             deviceDriver.installApp(deviceInfo, recordApk.getAbsolutePath(), logger);
         }
     }