diff --git a/.gitignore b/.gitignore index 229d30d..78bd220 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,5 @@ out .settings # NetBeans -.nbattrs \ No newline at end of file +.nbattrs +/log.txt diff --git a/pom.xml b/pom.xml index 00967dc..fcbe114 100644 --- a/pom.xml +++ b/pom.xml @@ -1,66 +1,64 @@ - 4.0.0 + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + 4.0.0 - org.quickperf - maven-test-bench - 1.0-SNAPSHOT + org.quickperf + maven-test-bench + 1.0-SNAPSHOT - - UTF-8 - 1.8 - 1.8 - + + UTF-8 + 1.8 + 1.8 + - - - org.apache.maven.shared - maven-verifier - 1.6 - test - - - org.quickperf - quick-perf-junit4 - 1.0.0-RC4 - test - - - junit - junit - 4.12 - test - - - net.lingala.zip4j - zip4j - 2.1.1 - - - commons-io - commons-io - 2.6 - - - org.apache.commons - commons-csv - 1.7 - - - com.github.oshi - oshi-core - 4.0.0 - - - - org.slf4j - slf4j-simple - 1.7.25 - - + + + org.apache.maven.shared + maven-verifier + 1.6 + test + + + org.quickperf + quick-perf-junit4 + 1.0.0-RC4 + test + + + junit + junit + 4.12 + test + + + net.lingala.zip4j + zip4j + 2.1.1 + + + commons-io + commons-io + 2.6 + + + org.apache.commons + commons-csv + 1.7 + + + com.github.oshi + oshi-core + 4.0.0 + + + org.slf4j + slf4j-log4j12 + 1.7.28 + test + + \ No newline at end of file diff --git a/src/main/java/BenchProperties.java b/src/main/java/BenchProperties.java index 582e2d8..fead1c9 100644 --- a/src/main/java/BenchProperties.java +++ b/src/main/java/BenchProperties.java @@ -25,8 +25,16 @@ private BenchProperties() { private String mavenBinariesPath; private String projectUnderTest; + + private String commitFirstHash; + + private String commitLastHash; + + private String mavenSourceBranch; - private List maven3VersionsToMeasure; + private String mavenSourcePath; + + private List maven3VersionsToMeasure; private void initializeProperties() { try { @@ -45,6 +53,14 @@ private void initializeProperties() { this.exportPathOfMeasures = properties.getProperty("measures.export.path"); this.maven3VersionsToMeasure = findMaven3VersionsToMeasure(properties); + + this.commitFirstHash=properties.getProperty("commit.first.hash"); + + this.commitLastHash=properties.getProperty("commit.last.hash"); + + this.mavenSourcePath = properties.getProperty("maven.sources.path"); + + this.mavenSourceBranch=properties.getProperty("maven.sources.branch","master"); } catch (IOException e) { throw new IllegalStateException("Unable to load bench properties.", e); @@ -124,5 +140,22 @@ public String getPathOfProjectUnderTest() { public List getMaven3VersionsToMeasure() { return maven3VersionsToMeasure; } + + public String getCommitFirstHash() { + return commitFirstHash; + } + + public String getCommitLastHash() { + return commitLastHash; + } + + public String getMavenSourceBranch() { + return mavenSourceBranch; + } + + + public String getMavenSourcePath() { + return mavenSourcePath; + } } diff --git a/src/main/java/CmdExecutionResult.java b/src/main/java/CmdExecutionResult.java new file mode 100644 index 0000000..03e5139 --- /dev/null +++ b/src/main/java/CmdExecutionResult.java @@ -0,0 +1,4 @@ + +public class CmdExecutionResult { + +} diff --git a/src/main/java/GitDelegate.java b/src/main/java/GitDelegate.java new file mode 100644 index 0000000..15e4d5b --- /dev/null +++ b/src/main/java/GitDelegate.java @@ -0,0 +1,111 @@ +import java.io.IOException; +import java.io.StringWriter; +import java.nio.file.Path; + +import org.apache.commons.io.IOUtils; + +public class GitDelegate { + + private final static String repoUrl = "https://github.com/apache/maven.git"; + + public static void clone(Path path) { + try { + System.out.println("Cloning "+repoUrl+" into "+path.toAbsolutePath()); + ProcessBuilder pb = new ProcessBuilder("git","clone",repoUrl); + pb.directory(path.toFile()); + Process process = pb.start(); + + final StringWriter messageWriter = new StringWriter(); + final StringWriter errorWriter = new StringWriter(); + + Thread outDrainer = new Thread(new Runnable() { + public void run() { + try { + IOUtils.copy(process.getInputStream(), messageWriter); + } catch (IOException e) { + } + } + }); + + Thread errorDrainer = new Thread(new Runnable() { + public void run() { + try { + IOUtils.copy(process.getErrorStream(), errorWriter); + } catch (IOException e) { + } + } + }); + + outDrainer.start(); + errorDrainer.start(); + + int err = process.waitFor(); + + outDrainer.join(); + errorDrainer.join(); + + if (err != 0) { + throw new RuntimeException("Error during repository clone "+errorWriter.toString()); + } + + String message = messageWriter.toString(); + System.out.println("Cloning completed "+message); + } catch (IOException | InterruptedException ex) { + throw new RuntimeException("Error during repository clone "+ex.getMessage()); + } + + } + + public static void checkOut(Path path,String branch, String commitHash) { + try { + System.out.println("checkout branch "+branch+ " commit hash "+commitHash); + ProcessBuilder pb = new ProcessBuilder("git","checkout",commitHash); + pb.directory(path.toFile()); + Process process = pb.start(); + + final StringWriter messageWriter = new StringWriter(); + final StringWriter errorWriter = new StringWriter(); + + Thread outDrainer = new Thread(new Runnable() { + public void run() { + try { + IOUtils.copy(process.getInputStream(), messageWriter); + } catch (IOException e) { + } + } + }); + + Thread errorDrainer = new Thread(new Runnable() { + public void run() { + try { + IOUtils.copy(process.getErrorStream(), errorWriter); + } catch (IOException e) { + } + } + }); + + outDrainer.start(); + errorDrainer.start(); + + int err = process.waitFor(); + + outDrainer.join(); + errorDrainer.join(); + + if (err != 0) { + String errorMessage = errorWriter.toString(); + System.out.println("Errror message: "+errorMessage); + return ; + } + + String message = messageWriter.toString(); + System.out.println("Completed Cloning "+message); + + } catch (IOException | InterruptedException ex) { + System.out.println("Errror message "+ex.getMessage()); + } + } + + + +} diff --git a/src/main/java/Maven3Version.java b/src/main/java/Maven3Version.java index e31af80..23aaa3c 100644 --- a/src/main/java/Maven3Version.java +++ b/src/main/java/Maven3Version.java @@ -30,6 +30,7 @@ public enum Maven3Version { ,V_3_6_1("3.6.1") ,V_3_6_2("3.6.2") ,HEAD("head") + ,HASH("hash") ; private final String numVersion; diff --git a/src/test/java/MvnValidateAllocationByMaven3HashTest.java b/src/test/java/MvnValidateAllocationByMaven3HashTest.java new file mode 100644 index 0000000..51c98ed --- /dev/null +++ b/src/test/java/MvnValidateAllocationByMaven3HashTest.java @@ -0,0 +1,315 @@ +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.maven.it.VerificationException; +import org.apache.maven.it.Verifier; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.results.PrintableResult; +import org.junit.runner.RunWith; +import org.quickperf.junit4.QuickPerfJUnitRunner; +import org.quickperf.jvm.allocation.AllocationUnit; +import org.quickperf.jvm.annotations.HeapSize; +import org.quickperf.jvm.annotations.MeasureHeapAllocation; +import org.quickperf.repository.LongFileRepository; +import org.quickperf.repository.ObjectFileRepository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import net.lingala.zip4j.ZipFile; +import net.lingala.zip4j.exception.ZipException; +import sun.awt.OSInfo; +import sun.awt.OSInfo.OSType; + +import java.io.File; +import java.io.FilenameFilter; +import java.io.IOException; +import java.io.StringWriter; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.*; + +import static org.junit.Assert.assertNotNull; +import static org.junit.experimental.results.PrintableResult.testResult; + +public class MvnValidateAllocationByMaven3HashTest { + + private Logger logger = LoggerFactory.getLogger(MvnValidateAllocationByMaven3HashTest.class); + + @RunWith(QuickPerfJUnitRunner.class) + public static class MvnValidate { + + private final String pathOfMavenProjectUnderTest = BenchProperties.INSTANCE.getPathOfProjectUnderTest(); + + private Verifier verifier; + + private final List validate = Collections.singletonList("validate"); + + @Before + public void before() throws VerificationException, IOException { + + String mavenBuiltPath = System.getProperty("maven.built.home"); + + System.out.println(mavenBuiltPath); + System.setProperty("verifier.forkMode", "auto"); // embedded + System.setProperty("maven.home", mavenBuiltPath); + verifier = new Verifier(pathOfMavenProjectUnderTest); + verifier.setSystemProperty("maven.multiModuleProjectDirectory", pathOfMavenProjectUnderTest); + + } + + @HeapSize(value = 6, unit = AllocationUnit.GIGA_BYTE) + @MeasureHeapAllocation + @Test + public void execute_maven_validate() throws VerificationException { + verifier.executeGoals(validate); + } + + } + + public static final String MAVEN_3_VERSION_FILE_NAME = "Maven3Version"; + + private final String tempDirPath = System.getProperty("java.io.tmpdir"); + + private final FilenameFilter quickPerfDirFilter = (dir, name) -> name.contains("QuickPerf"); + + private final File tempDir = new File(tempDirPath); + + @Test + public void measure() throws IOException { + logger.debug("measure - start"); + if (!Maven3Version.V_3_6_2.alreadyDownloaded()) { + Maven3Version.V_3_6_2.download(); + System.out.println(); + } + + String mavenPath = Maven3Version.V_3_6_2.getMavenPath(); + + System.setProperty("verifier.forkMode", "auto"); // embedded + System.setProperty("maven.home", mavenPath); + + String dateTimeAsString = getDateTimeAsString(); + String resultFilePath = buildAllocationCsvExportPath(dateTimeAsString); + + String firstCommitHash = BenchProperties.INSTANCE.getCommitFirstHash(); + String lastCommitHash = BenchProperties.INSTANCE.getCommitLastHash(); + String sourcePath = BenchProperties.INSTANCE.getMavenSourcePath(); + String branch = BenchProperties.INSTANCE.getMavenSourceBranch(); + + assertNotNull(firstCommitHash, "commit.first.hash must be provided in maven-bench.properties"); + assertNotNull(lastCommitHash, "commit.last.hash must be provided in maven-bench.properties"); + + int numberOfMeasuresByVersion = BenchProperties.INSTANCE.getNumberOfMeasuresByMavenVersion(); + + cloneAndCheckoutMavenHash(firstCommitHash, sourcePath, branch); + + buildMaven(sourcePath); + + + Path hashMavenBuilt = getBuiltPath(sourcePath, "maven", "apache-maven"); + System.setProperty("maven.built.home", hashMavenBuilt.toAbsolutePath().toString()); + + Class testClass = MvnValidate.class; + + + + applyWarmMeasurements(testClass); +// AllocationTimePair[] allocations = measureAllocationSeveralTimes(testClass, numberOfMeasuresByVersion); +// AllocationCsvExporter.INSTANCE.writeAllocationsToCsv(allocations, resultFilePath); + + //ExecutionContextTextExporter.INSTANCE.writeExecutionContextToTextFile(dateTimeAsString); + logger.debug("measure - end"); + } + + private Path getBuiltPath(String sourcePath, String name, String subProjectName) throws ZipException { + Path path = Paths.get(sourcePath, name, subProjectName, "target"); + FilenameFilter filter = new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.endsWith(".zip") && name.startsWith("apache-maven-"); + } + }; + String[] res = path.toFile().list(filter); + Path zipPath = Paths.get(path.toAbsolutePath().toString(), res[0]); + System.out.println("Found build zip " + zipPath.toAbsolutePath().toString()); + ZipFile zipFile = new ZipFile(zipPath.toFile()); + zipFile.extractAll(path.toAbsolutePath().toString()); + String extractedDir = zipPath.getFileName().toString().replace("-bin.zip", ""); + Path mavenExtractedPath = Paths.get(path.toAbsolutePath().toString(), extractedDir); + System.out.println("Extracted directory " + mavenExtractedPath.toAbsolutePath().toString()); + return mavenExtractedPath; + } + +// @Test +// public void buildTest() { +// String mavenPath = Maven3Version.V_3_6_2.getMavenPath(); +// System.setProperty("verifier.forkMode", "auto"); // embedded +// System.setProperty("maven.home", mavenPath); +// String sourcePath = BenchProperties.INSTANCE.getMavenSourcePath(); +// +// buildMaven(sourcePath); +// } + +// @Test +// public void unzipTest() throws ZipException { +// String sourcePath = BenchProperties.INSTANCE.getMavenSourcePath(); +// Path hashMavenBuilt = getBuiltPath(sourcePath, "maven", "apache-maven"); +// System.out.println(); +// } + + @Test + //TODO: delete after proper implementation + public void measureTest() throws IOException { + String sourcePath = BenchProperties.INSTANCE.getMavenSourcePath(); + Path hashMavenBuilt = getBuiltPath(sourcePath, "maven", "apache-maven"); + System.setProperty("maven.built.home", hashMavenBuilt.toAbsolutePath().toString()); + + Class testClass = MvnValidate.class; + + //TODO: How to store a maven version if will be an hash ? + saveMavenVersion(Maven3Version.HASH); + applyWarmMeasurements(testClass); + } + + + private void saveMavenVersion(Maven3Version maven3Version) { + FileUtils.deleteQuietly(new File(tempDirPath + File.separator + MAVEN_3_VERSION_FILE_NAME)); + ObjectFileRepository.INSTANCE.save(tempDirPath, MAVEN_3_VERSION_FILE_NAME, maven3Version); + } + + private void applyWarmMeasurements(Class testClass) throws IOException { + int numberOfWarms = BenchProperties.INSTANCE.getNumberOfWarms(); + if (numberOfWarms != 0) { + System.out.println("First hash - Start " + numberOfWarms + " warm up"); + System.out.println("-----------------------------"); + measureAllocationSeveralTimes(testClass, numberOfWarms); + System.out.println("First hash - End warm up"); + System.out.println("----------------------------"); + } + } + + private String buildAllocationCsvExportPath(String dateTimeAsString) { + String measurementsExportPath = BenchProperties.INSTANCE.getExportPathOfMeasures(); + String fileName = "maven-memory-allocation" + "-" + dateTimeAsString + ".csv"; + return measurementsExportPath + File.separator + fileName; + } + + private String getDateTimeAsString() { + DateFormat df = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss"); + return df.format(new Date()); + } + + private AllocationTimePair[] measureAllocationSeveralTimes(Class testClass, int numberOfTimes) + throws IOException { + AllocationTimePair[] allocations = new AllocationTimePair[numberOfTimes]; + for (int i = 0; i < numberOfTimes; i++) { + allocations[i] = measureAllocation(testClass); + } + return allocations; + } + + private AllocationTimePair measureAllocation(Class testClass) throws IOException { + deleteQuickPerfFoldersInTemp(); + long startTime = System.currentTimeMillis(); + PrintableResult printableResult = testResult(testClass); + long executionTimeInMilliseconds = System.currentTimeMillis() - startTime; + if (printableResult.failureCount() != 0) { + System.out.println("Allocation can't be measured. " + printableResult.toString()); + } + Long allocationInBytes = retrieveMeasuredAllocationInBytes(); + Long lenghtInSeconds = executionTimeInMilliseconds / 1000l; + System.out.println("Allocation in bytes: " + allocationInBytes); + System.out.println("Lenght in seconds: " + lenghtInSeconds); + System.out.println("----------------"); + return new AllocationTimePair(allocationInBytes, lenghtInSeconds); + } + + private void deleteQuickPerfFoldersInTemp() throws IOException { + File[] quickPerfFoldersBeforeMeasure = tempDir.listFiles(quickPerfDirFilter); + for (File quickPerfFolder : quickPerfFoldersBeforeMeasure) { + FileUtils.deleteDirectory(quickPerfFolder); + } + } + + private Long retrieveMeasuredAllocationInBytes() { + LongFileRepository longFileRepository = new LongFileRepository(); + String[] quickPerfFolders = tempDir.list(quickPerfDirFilter); + if (quickPerfFolders.length != 1) { + throw new IllegalStateException("Several QuickPerf folders found in temp."); + } + String quickPerfFolderPath = tempDirPath + File.separator + quickPerfFolders[0]; + return longFileRepository.find(quickPerfFolderPath, "allocation.ser"); + } + + private void cloneAndCheckoutMavenHash(String commitHash, String sourcePath, String branch) throws IOException { + Path path = Paths.get(sourcePath); + if (!Files.isDirectory(path)) { + throw new IllegalArgumentException("sourcePath must be a directory"); + } + boolean empty = !Files.list(path).findAny().isPresent(); + if (!empty) { + throw new IllegalArgumentException("provided maven source directory must be empty"); + } + GitDelegate.clone(path); + GitDelegate.checkOut(Paths.get(sourcePath, "maven"), branch, commitHash); + } + + private void buildMaven(String sourcePath) { + try { + System.out.println("Mvn clean package "); + ProcessBuilder pb = new ProcessBuilder(); + String mvnExec = null; + if (OSInfo.getOSType().compareTo(OSType.WINDOWS) == 0) { + mvnExec = "mvn.cmd"; + } else { + mvnExec = "mvn"; + } + + pb.command(Paths.get(System.getProperty("maven.home"), "bin", mvnExec).toAbsolutePath().toString(), "clean", + "package"); + pb.directory(Paths.get(sourcePath, "maven").toFile()); + Process process = pb.start(); + + final StringWriter messageWriter = new StringWriter(); + final StringWriter errorWriter = new StringWriter(); + + Thread outDrainer = new Thread(new Runnable() { + public void run() { + try { + IOUtils.copy(process.getInputStream(), messageWriter); + } catch (IOException e) { + } + } + }); + + Thread errorDrainer = new Thread(new Runnable() { + public void run() { + try { + IOUtils.copy(process.getErrorStream(), errorWriter); + } catch (IOException e) { + } + } + }); + + outDrainer.start(); + errorDrainer.start(); + + int err = process.waitFor(); + + outDrainer.join(); + errorDrainer.join(); + + if (err != 0) { + throw new RuntimeException("Error mvn clean package " + errorWriter.toString()); + } + + String message = messageWriter.toString(); + System.out.println("Mvn clean package completed " + message); + } catch (IOException | InterruptedException ex) { + throw new RuntimeException("Error mvn clean package " + ex.getMessage()); + } + } + +} diff --git a/src/test/java/MvnValidateAllocationByMaven3VersionTest.java b/src/test/java/MvnValidateAllocationByMaven3VersionTest.java index 72748b4..82b825a 100644 --- a/src/test/java/MvnValidateAllocationByMaven3VersionTest.java +++ b/src/test/java/MvnValidateAllocationByMaven3VersionTest.java @@ -11,6 +11,8 @@ import org.quickperf.jvm.annotations.MeasureHeapAllocation; import org.quickperf.repository.LongFileRepository; import org.quickperf.repository.ObjectFileRepository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.File; import java.io.FilenameFilter; @@ -22,6 +24,8 @@ import static org.junit.experimental.results.PrintableResult.testResult; public class MvnValidateAllocationByMaven3VersionTest { + + private Logger logger = LoggerFactory.getLogger(MvnValidateAllocationByMaven3VersionTest.class); @RunWith(QuickPerfJUnitRunner.class) public static class MvnValidate { @@ -71,7 +75,7 @@ public void execute_maven_validate() throws VerificationException { @Test public void measure() throws IOException { - + logger.debug("measure - start"); String dateTimeAsString = getDateTimeAsString(); String resultFilePath = buildAllocationCsvExportPath(dateTimeAsString); @@ -98,7 +102,7 @@ public void measure() throws IOException { } ExecutionContextTextExporter.INSTANCE.writeExecutionContextToTextFile(dateTimeAsString); - + logger.debug("measure - end"); } private void saveMavenVersion(Maven3Version maven3Version) { diff --git a/src/test/java/MvnValidateProfilingTest.java b/src/test/java/MvnValidateProfilingTest.java index 73efa70..30e4698 100644 --- a/src/test/java/MvnValidateProfilingTest.java +++ b/src/test/java/MvnValidateProfilingTest.java @@ -7,6 +7,8 @@ import org.quickperf.jvm.allocation.AllocationUnit; import org.quickperf.jvm.annotations.HeapSize; import org.quickperf.jvm.annotations.ProfileJvm; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.Collections; @@ -14,6 +16,8 @@ @RunWith(QuickPerfJUnitRunner.class) public class MvnValidateProfilingTest { + + private Logger logger = LoggerFactory.getLogger(MvnValidateProfilingTest.class); public static Maven3Version MAVEN_3_VERSION = Maven3Version.V_3_2_5; @@ -27,7 +31,9 @@ public class MvnValidateProfilingTest { @HeapSize(value = 6, unit = AllocationUnit.GIGA_BYTE) @Test public void execute_maven_validate() throws VerificationException { + logger.debug("execute_maven_validate - start"); verifier.executeGoals(validate); + logger.debug("execute_maven_validate - end"); } @Before diff --git a/src/test/resources/log4j.properties b/src/test/resources/log4j.properties new file mode 100644 index 0000000..d8975af --- /dev/null +++ b/src/test/resources/log4j.properties @@ -0,0 +1,10 @@ +log4j.rootCategory=debug,console +log4j.logger=debug,console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.out +log4j.appender.console.immediateFlush=true +log4j.appender.console.encoding=UTF-8 + +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.conversionPattern=%d [%t] %-5p %c - %m%n \ No newline at end of file diff --git a/src/test/resources/maven-bench.properties b/src/test/resources/maven-bench.properties index aaddf29..b2c5532 100644 --- a/src/test/resources/maven-bench.properties +++ b/src/test/resources/maven-bench.properties @@ -1,18 +1,18 @@ # Path of the project on which a Maven command will be applied -project-under-test.path = C:\\code\\camel +project-under-test.path = C:\\Users\\axel\\git\\maven-test-bench-test # Path of Maven distribution. The test bench will download Maven binaries to this path -maven.binaries.path = C:\\Maven +maven.binaries.path = C:\\test\\maven # On Windows, download is not possible in C:\\Program Files #--------------------------------------------------------------------------------- # THE PROPERTIES BELOW ARE ONLY USED BY MvnValidateAllocationByMaven3VersionTest # Maven version of the first measure -maven.version.from = 3.2.5 +maven.version.from = # Maven version of the last measure -maven.version.to = 3.6.2 +maven.version.to = # The available Maven versions are given by Maven3Version @@ -20,10 +20,22 @@ maven.version.to = 3.6.2 warmup.number = 1 # Number of measures for each Maven versions -measures.number-by-maven-version = 10 +measures.number-by-maven-version = 1 # Path where measures will be exported -measures.export.path = C:\\Maven\\measures +measures.export.path = C:\\test\\maven\\measures + + #--------------------------------------------------------------------------------- +# THE PROPERTIES BELOW ARE ONLY USED BY MvnValidateAllocationByMaven3HashTest + +# Path of Maven sources. Test bench will download Maven sources to this path +maven.sources.path = C:\\test\\maven\\sources + +# Branch used by test bench (master by default) +maven.sources.branch=master +# Hash values for commit measures - must be +commit.first.hash = 55572a4eb05c495128a40f0c9080f5c17775b0b6 +commit.last.hash = eca4905fcdb8df5dc77ba0f683125643638a6b6e \ No newline at end of file