diff --git a/CMakeLists.txt b/CMakeLists.txt index 4471371..1d70414 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -56,7 +56,13 @@ set(SourceFiles Source/tests/BusTests.cpp Source/tests/ParameterFuzzTests.cpp Source/TestUtilities.cpp - Source/Validator.cpp) + Source/Validator.cpp + Source/JUnitReport.cpp + Source/JUnitReport.h + Source/JUnitListener.cpp + Source/JUnitListener.h + Source/PluginTestResult.h + ) target_sources(pluginval PRIVATE ${SourceFiles}) source_group(TREE ${CMAKE_CURRENT_SOURCE_DIR}/Source PREFIX Source FILES ${SourceFiles}) diff --git a/Source/CommandLine.cpp b/Source/CommandLine.cpp index 616c0d4..9bf25bd 100644 --- a/Source/CommandLine.cpp +++ b/Source/CommandLine.cpp @@ -49,6 +49,8 @@ CommandLineValidator::CommandLineValidator() { validator.addChangeListener (this); validator.addListener (this); + + validator.addListener(&junitListener); } CommandLineValidator::~CommandLineValidator() @@ -57,6 +59,8 @@ CommandLineValidator::~CommandLineValidator() void CommandLineValidator::validate (const StringArray& fileOrIDs, PluginTests::Options options, bool validateInProcess) { + junitListener.setReportFile(options.junitReportFile); + validator.setValidateInProcess (validateInProcess); validator.validate (fileOrIDs, options); } @@ -79,7 +83,7 @@ void CommandLineValidator::logMessage (const String& m) std::cout << m << "\n"; } -void CommandLineValidator::itemComplete (const String& id, int numItemFailures) +void CommandLineValidator::itemComplete (const String& id, int numItemFailures, const PluginTestResultArray&) { logMessage ("\nFinished validating: " + id); @@ -217,6 +221,11 @@ StringArray getDisabledTest (const ArgumentList& args) return disabledTests; } +File getJunitReportFile (const ArgumentList& args) +{ + return getOptionValue (args, "--junit-report-file", {}, "Missing junit-report-file path argument!").toString(); +} + //============================================================================== String parseCommandLineArgs (String commandLine) { @@ -332,6 +341,8 @@ static String getHelpMessage() << " If specified, sets the list of sample rates at which tests will be executed (default=44100,48000,96000)" << newLine << " --block-sizes [list of comma separated block sizes]" << newLine << " If specified, sets the list of block sizes at which tests will be executed (default=64,128,256,512,1024)" << newLine + << " --junit-report-file [output filename]" << newLine + << " If specified, creates JUnit report" << newLine << " --version" << newLine << " Print pluginval version." << newLine << newLine @@ -394,6 +405,7 @@ static void validate (CommandLineValidator& validator, const ArgumentList& args) options.disabledTests = getDisabledTest (args); options.sampleRates = getSampleRates (args); options.blockSizes = getBlockSizes (args); + options.junitReportFile = getJunitReportFile(args); validator.validate (fileOrIDs, options, diff --git a/Source/CommandLine.h b/Source/CommandLine.h index f0c7fd6..a5ec3ad 100644 --- a/Source/CommandLine.h +++ b/Source/CommandLine.h @@ -15,7 +15,9 @@ #pragma once #include +#include "PluginTestResult.h" #include "Validator.h" +#include "JUnitListener.h" struct CommandLineValidator : private ChangeListener, private Validator::Listener @@ -27,6 +29,7 @@ struct CommandLineValidator : private ChangeListener, private: Validator validator; + JUnitListener junitListener; String currentID; std::atomic numFailures { 0 }; @@ -34,7 +37,7 @@ struct CommandLineValidator : private ChangeListener, void validationStarted (const String&) override; void logMessage (const String& m) override; - void itemComplete (const String&, int numItemFailures) override; + void itemComplete (const String&, int numItemFailures, const PluginTestResultArray&) override; void allItemsComplete() override; void connectionLost() override; }; diff --git a/Source/JUnitListener.cpp b/Source/JUnitListener.cpp new file mode 100644 index 0000000..5edd271 --- /dev/null +++ b/Source/JUnitListener.cpp @@ -0,0 +1,39 @@ +#include "JUnitListener.h" + +#include "JUnitReport.h" + +JUnitListener::JUnitListener() +{ +} + +void JUnitListener::setReportFile(const File &newReportFile) +{ + reportFile = newReportFile; +} + +void JUnitListener::validationStarted(const String &) +{ +} + +void JUnitListener::logMessage(const String &) +{ +} + +void JUnitListener::itemComplete(const String &id, int, const PluginTestResultArray &itemResults) +{ + results.set(id, itemResults); +} + +void JUnitListener::allItemsComplete() +{ + if (!reportFile.getFullPathName().isEmpty()) + { + JUnitReport::write(results, reportFile); + } +} + +void JUnitListener::connectionLost() +{ + Validator::Listener::connectionLost(); + // TODO: create report when connection is lost +} diff --git a/Source/JUnitListener.h b/Source/JUnitListener.h new file mode 100644 index 0000000..988f2c9 --- /dev/null +++ b/Source/JUnitListener.h @@ -0,0 +1,21 @@ +#pragma once + +#include +#include "PluginTestResult.h" +#include "Validator.h" + +class JUnitListener: public Validator::Listener +{ +public: + JUnitListener(); + void setReportFile(const File& newReportFile); +private: + void validationStarted (const String& id) override; + void logMessage (const String& m) override; + void itemComplete (const String& id, int numItemFailures, const PluginTestResultArray& itemResults) override; + void allItemsComplete() override; + void connectionLost() override; + + File reportFile; + HashMap results; +}; diff --git a/Source/JUnitReport.cpp b/Source/JUnitReport.cpp new file mode 100644 index 0000000..0a3ce56 --- /dev/null +++ b/Source/JUnitReport.cpp @@ -0,0 +1,105 @@ +#include "JUnitReport.h" + +namespace JUnitReport +{ + +XmlElement* createTestCaseElement(const String& pluginName, int testIndex, const PluginTestResult& r) +{ + auto testcase = new XmlElement("testcase"); + + testcase->setAttribute("classname", r.result.unitTestName); + + // Adding test index here to have unique name for each test case execution +#if defined(USE_FILENAME_IN_JUNIT_REPORT) + testcase->setAttribute("name", "Test " + String(testIndex + 1) + ": " + r.subcategoryName); + testcase->setAttribute("file", pluginName); +#else + testcase->setAttribute("name", "Test " + String(testIndex + 1) + ": " + r.result.subcategoryName + " of " + pluginName); +#endif + + auto duration = (r.result.endTime - r.result.startTime).inMilliseconds() / 1000.0; + testcase->setAttribute("time", duration); + + String output = r.output.joinIntoString("\n"); + if (r.result.failures == 0) + { + auto system_out = new XmlElement("system-out"); + system_out->addTextElement(output); + + testcase->prependChildElement(system_out); + } + else + { + auto failure = new XmlElement("failure"); + failure->setAttribute("type", "ERROR"); + failure->setAttribute("message", r.result.messages.joinIntoString(" ")); + failure->addTextElement(output); + + testcase->prependChildElement(failure); + } + return testcase; +} + +XmlElement* createTestSuiteElement(const String& pluginName) +{ + auto testsuite = new XmlElement("testsuite"); + testsuite->setAttribute("package", "pluginval"); + testsuite->setAttribute("name", "pluginval of " + pluginName + " on " + SystemStats::getOperatingSystemName()); + return testsuite; +} + +void addTestsStats(XmlElement* element, int tests, int failures, int64 duration) +{ + element->setAttribute("tests", tests); + element->setAttribute("failures", failures); + element->setAttribute("time", duration / 1000.0); +} + +bool write(const HashMap &allResults, File &output) +{ + XmlElement testsuites("testsuites"); + testsuites.setAttribute("name", "pluginval test suites"); + + int total_failures = 0; + int total_tests = 0; + int64 total_duration = 0; + int test_index = 0; + for (auto it = allResults.begin(); it != allResults.end(); ++it) + { + const auto results = it.getValue(); + auto pluginName = it.getKey(); + + int suite_failures = 0; + int64 suite_duration = 0; + int suite_tests = results.size(); + + auto testsuite = createTestSuiteElement(pluginName); + + for (const auto& r: results) + { + auto testcase = createTestCaseElement(pluginName, test_index, r); + testsuite->prependChildElement(testcase); + + // calculate totals for test suite + suite_failures += r.result.failures; + suite_duration += (r.result.endTime - r.result.startTime).inMilliseconds(); + + test_index++; + } + + addTestsStats(testsuite, suite_tests, suite_failures, suite_duration); + + testsuites.prependChildElement(testsuite); + + // accumulate totals for all test suites + total_failures += suite_failures; + total_duration += suite_duration; + total_tests += suite_tests; + } + + addTestsStats(&testsuites, total_tests, total_failures, total_duration); + + return testsuites.writeTo(output); +} + +} // namespace JUnitReport diff --git a/Source/JUnitReport.h b/Source/JUnitReport.h new file mode 100644 index 0000000..97d24fb --- /dev/null +++ b/Source/JUnitReport.h @@ -0,0 +1,11 @@ +#pragma once + +#include +#include "PluginTestResult.h" + +namespace JUnitReport +{ + +bool write(const HashMap &allResults, File &output); + +} // namespace JUnitReport diff --git a/Source/MainComponent.h b/Source/MainComponent.h index 26714b6..9219436 100644 --- a/Source/MainComponent.h +++ b/Source/MainComponent.h @@ -15,6 +15,7 @@ #pragma once #include +#include "PluginTestResult.h" #include "Validator.h" #include "CrashHandler.h" @@ -91,7 +92,7 @@ struct ConnectionStatus : public Component, { } - void itemComplete (const String&, int) override + void itemComplete (const String&, int, const PluginTestResultArray&) override { } @@ -194,7 +195,7 @@ struct ConsoleComponent : public Component, std::cout << m << "\n"; } - void itemComplete (const String& id, int numFailures) override + void itemComplete (const String& id, int numFailures, const PluginTestResultArray&) override { logMessage ("\nFinished validating: " + id); diff --git a/Source/PluginTestResult.h b/Source/PluginTestResult.h new file mode 100644 index 0000000..acedd13 --- /dev/null +++ b/Source/PluginTestResult.h @@ -0,0 +1,17 @@ +#pragma once + +#include + +struct PluginTestResult +{ + PluginTestResult(const UnitTestRunner::TestResult &result, const StringArray &output): + result(result), output(output) + { + } + + UnitTestRunner::TestResult result; + StringArray output; +}; + +using PluginTestResultArray = Array; + diff --git a/Source/PluginTests.h b/Source/PluginTests.h index cf9bbe4..0ffb627 100644 --- a/Source/PluginTests.h +++ b/Source/PluginTests.h @@ -38,6 +38,7 @@ struct PluginTests : public UnitTest StringArray disabledTests; /**< List of disabled tests. */ std::vector sampleRates; /**< List of sample rates. */ std::vector blockSizes; /**< List of block sizes. */ + File junitReportFile; /**< JUnit report file. */ }; /** Creates a set of tests for a fileOrIdentifier. */ diff --git a/Source/Validator.cpp b/Source/Validator.cpp index 504b4c8..aa16d76 100644 --- a/Source/Validator.cpp +++ b/Source/Validator.cpp @@ -80,10 +80,25 @@ struct PluginsUnitTestRunner : public UnitTestRunner, if (outputStream) *outputStream << message << "\n"; + if (getNumResults() > 0) + { + auto testResult = getResult(getNumResults() - 1); + output.getReference(testResult).add(message); + } + callback (message); } } + StringArray getTestOutput(int testResultIndex) const + { + if (testResultIndex >= 0 && testResultIndex < getNumResults()) + { + return output[getResult((testResultIndex))]; + } + return {}; + } + private: std::function callback; std::unique_ptr outputStream; @@ -91,6 +106,8 @@ struct PluginsUnitTestRunner : public UnitTestRunner, std::atomic timoutTime { -1 }; std::atomic canSendLogMessage { true }; + HashMap output; + void resetTimeout() { timoutTime = (Time::getCurrentTime() + RelativeTime::milliseconds (timeoutMs)).toMilliseconds(); @@ -193,10 +210,10 @@ void updateFileNameIfPossible (PluginTests& test, PluginsUnitTestRunner& runner) //============================================================================== //============================================================================== -inline Array runTests (PluginTests& test, std::function callback) +inline PluginTestResultArray runTests (PluginTests& test, std::function callback) { const auto options = test.getOptions(); - Array results; + PluginTestResultArray results; PluginsUnitTestRunner testRunner (std::move (callback), createDestinationFileStream (test), options.timeoutMs); testRunner.setAssertOnFailure (false); @@ -205,29 +222,31 @@ inline Array runTests (PluginTests& test, std::funct testRunner.runTests (testsToRun, options.randomSeed); for (int i = 0; i < testRunner.getNumResults(); ++i) - results.add (*testRunner.getResult (i)); + { + results.add({ *testRunner.getResult(i), testRunner.getTestOutput(i) }); + } updateFileNameIfPossible (test, testRunner); return results; } -inline Array validate (const PluginDescription& pluginToValidate, PluginTests::Options options, std::function callback) +inline PluginTestResultArray validate (const PluginDescription& pluginToValidate, PluginTests::Options options, std::function callback) { PluginTests test (pluginToValidate, options); return runTests (test, std::move (callback)); } -inline Array validate (const String& fileOrIDToValidate, PluginTests::Options options, std::function callback) +inline PluginTestResultArray validate (const String& fileOrIDToValidate, PluginTests::Options options, std::function callback) { PluginTests test (fileOrIDToValidate, options); return runTests (test, std::move (callback)); } -inline int getNumFailures (Array results) +inline int getNumFailures (const PluginTestResultArray& results) { return std::accumulate (results.begin(), results.end(), 0, - [] (int count, const UnitTestRunner::TestResult& r) { return count + r.failures; }); + [] (int count, const auto& r) { return count + r.result.failures; }); } //============================================================================== @@ -258,9 +277,33 @@ namespace IDs DECLARE_ID(log) DECLARE_ID(numFailures) + DECLARE_ID(testResultArray) + DECLARE_ID(testResultItem) + DECLARE_ID(testName) + DECLARE_ID(testSubcategoryName) + DECLARE_ID(testPassCount) + DECLARE_ID(testFailureCount) + DECLARE_ID(testFailureMessageArray) + DECLARE_ID(testFailureMessageItem) + DECLARE_ID(testFailureMessageText) + DECLARE_ID(testOutputMessageArray) + DECLARE_ID(testOutputMessageItem) + DECLARE_ID(testOutputMessageText) + DECLARE_ID(testStartTime) + DECLARE_ID(testEndTime) + #undef DECLARE_ID } +namespace MessageTypes +{ + const String result = "result"; + const String log = "log"; + const String started = "started"; + const String complete = "complete"; + const String connected = "connected"; +} + //============================================================================== // This is a token that's used at both ends of our parent-child processes, to // act as a unique token in the command line arguments. @@ -319,7 +362,7 @@ class ValidatorChildProcess : public ChildProcessSlave, isConnected = isNowConnected; if (isConnected) - sendValueTreeToParent ({ IDs::MESSAGE, {{ IDs::type, "connected" }} }); + sendValueTreeToParent ({ IDs::MESSAGE, {{ IDs::type, MessageTypes::connected }} }); } void setParentProcess (ChildProcessMaster* newParent) @@ -383,7 +426,7 @@ class ValidatorChildProcess : public ChildProcessSlave, } if (owner.isConnected && ! messagesToSend.isEmpty()) - owner.sendValueTreeToParent ({ IDs::MESSAGE, {{ IDs::type, "log" }, { IDs::text, messagesToSend.joinIntoString ("\n") }} }, false); + owner.sendValueTreeToParent ({ IDs::MESSAGE, {{ IDs::type, MessageTypes::log }, { IDs::text, messagesToSend.joinIntoString ("\n") }} }, false); } ValidatorChildProcess& owner; @@ -463,6 +506,46 @@ class ValidatorChildProcess : public ChildProcessSlave, processRequest (r); } + static ValueTree serializeTestResults(const PluginTestResultArray& results) + { + ValueTree testResultArray { IDs::testResultArray }; + for (const auto& r: results) + { + auto add_messages = [](const StringArray& src, const Identifier& itemId, + const Identifier& textId, ValueTree& dest) + { + for (const auto& m: src) + { + dest.appendChild({ itemId, { { textId, m } } }, nullptr); + } + }; + + ValueTree testFailureMessageArray { IDs::testFailureMessageArray }; + + add_messages(r.result.messages, IDs::testFailureMessageItem, IDs::testFailureMessageText, testFailureMessageArray); + + ValueTree testOutputMessageArray { IDs::testOutputMessageArray }; + + add_messages(r.output, IDs::testOutputMessageItem, IDs::testOutputMessageText, testOutputMessageArray); + + ValueTree testResultItem { IDs::testResultItem, + { + { IDs::testName, r.result.unitTestName }, + { IDs::testSubcategoryName, r.result.subcategoryName }, + { IDs::testPassCount, r.result.passes }, + { IDs::testFailureCount, r.result.failures }, + { IDs::testStartTime, r.result.startTime.toMilliseconds() }, + { IDs::testEndTime, r.result.endTime.toMilliseconds() }, + }, + { testFailureMessageArray, testOutputMessageArray } + }; + + testResultArray.appendChild( testResultItem, nullptr ); + } + + return testResultArray; + } + void processRequest (MemoryBlock mb) { const ValueTree v (memoryBlockToValueTree (mb)); @@ -487,14 +570,14 @@ class ValidatorChildProcess : public ChildProcessSlave, for (auto c : v) { String fileOrID; - Array results; + PluginTestResultArray results; LOG_CHILD("processRequest - child:\n" + toXmlString (c)); if (c.hasProperty (IDs::fileOrID)) { fileOrID = c[IDs::fileOrID].toString(); sendValueTreeToParent ({ - IDs::MESSAGE, {{ IDs::type, "started" }, { IDs::fileOrID, fileOrID }} + IDs::MESSAGE, {{ IDs::type, MessageTypes::started }, { IDs::fileOrID, fileOrID }} }); results = validate (c[IDs::fileOrID].toString(), options, [this] (const String& m) { logMessage (m); }); @@ -513,7 +596,7 @@ class ValidatorChildProcess : public ChildProcessSlave, { fileOrID = pd.createIdentifierString(); sendValueTreeToParent ({ - IDs::MESSAGE, {{ IDs::type, "started" }, { IDs::fileOrID, fileOrID }} + IDs::MESSAGE, {{ IDs::type, MessageTypes::started }, { IDs::fileOrID, fileOrID }} }); results = validate (pd, options, [this] (const String& m) { logMessage (m); }); @@ -535,14 +618,22 @@ class ValidatorChildProcess : public ChildProcessSlave, } jassert (fileOrID.isNotEmpty()); - sendValueTreeToParent ({ - IDs::MESSAGE, {{ IDs::type, "result" }, { IDs::fileOrID, fileOrID }, { IDs::numFailures, getNumFailures (results) }} - }); + + ValueTree result { + IDs::MESSAGE, + { + { IDs::type, MessageTypes::result }, + { IDs::fileOrID, fileOrID }, + { IDs::numFailures, getNumFailures (results) } + }, + { serializeTestResults( results ) } + }; + sendValueTreeToParent ( result ); } } sendValueTreeToParent ({ - IDs::MESSAGE, {{ IDs::type, "complete" }} + IDs::MESSAGE, {{ IDs::type, MessageTypes::complete }} }); } }; @@ -567,7 +658,7 @@ class ValidatorParentProcess : public ChildProcessMaster std::function logMessageCallback; // Callback which can be set to be informed when a validation completes - std::function validationCompleteCallback; + std::function validationCompleteCallback; // Callback which can be set to be informed when all validations have been completed std::function completeCallback; @@ -593,6 +684,63 @@ class ValidatorParentProcess : public ChildProcessMaster return ok ? Result::ok() : Result::fail ("Error: Child failed to launch"); } + static PluginTestResultArray deserializeTestResults(const ValueTree& testResultArray) + { + if (!testResultArray.hasType(IDs::testResultArray)) + { + return {}; + } + PluginTestResultArray results; + for (const auto& testResultItem: testResultArray) + { + if (!testResultItem.hasType(IDs::testResultItem) || + !testResultItem.hasProperty(IDs::testName) || + !testResultItem.hasProperty(IDs::testSubcategoryName) || + !testResultItem.hasProperty(IDs::testPassCount) || + !testResultItem.hasProperty(IDs::testFailureCount) || + !testResultItem.hasProperty(IDs::testStartTime) || + !testResultItem.hasProperty(IDs::testEndTime) + ) + { + continue; + } + UnitTestRunner::TestResult result; + result.unitTestName = testResultItem.getProperty(IDs::testName); + result.subcategoryName = testResultItem.getProperty(IDs::testSubcategoryName); + result.passes = testResultItem.getProperty(IDs::testPassCount); + result.failures = testResultItem.getProperty(IDs::testFailureCount); + result.startTime = Time(testResultItem.getProperty(IDs::testStartTime)); + result.endTime = Time(testResultItem.getProperty(IDs::testEndTime)); + + auto add_messages = [&testResultItem](const Identifier& arrayId, const Identifier& itemId, + const Identifier& textId, StringArray& dest) + { + auto array = testResultItem.getChildWithName(arrayId); + if (array.hasType(arrayId)) + { + for (const auto& item: array) + { + if (!item.hasType(itemId) || !item.hasProperty(textId)) + { + continue; + } + dest.add(item.getProperty(textId)); + } + } + }; + + add_messages(IDs::testFailureMessageArray, IDs::testFailureMessageItem, + IDs::testFailureMessageText, result.messages); + + StringArray output; + add_messages(IDs::testOutputMessageArray, IDs::testOutputMessageItem, + IDs::testOutputMessageText, output); + + results.add( { result, output } ); + } + return results; + } + //============================================================================== void handleMessageFromSlave (const MemoryBlock& mb) override { @@ -602,20 +750,31 @@ class ValidatorParentProcess : public ChildProcessMaster { const auto type = v[IDs::type].toString(); - if (logMessageCallback && type == "log") + if (logMessageCallback && type == MessageTypes::log) + { logMessageCallback (v[IDs::text].toString()); + } - if (validationCompleteCallback && type == "result") - validationCompleteCallback (v[IDs::fileOrID].toString(), v[IDs::numFailures]); + if (validationCompleteCallback && type == MessageTypes::result) + { + auto results = deserializeTestResults(v.getChildWithName(IDs::testResultArray)); + validationCompleteCallback (v[IDs::fileOrID].toString(), v[IDs::numFailures], results); + } - if (validationStartedCallback && type == "started") + if (validationStartedCallback && type == MessageTypes::started) + { validationStartedCallback (v[IDs::fileOrID].toString()); + } - if (completeCallback && type == "complete") + if (completeCallback && type == MessageTypes::complete) + { completeCallback(); + } - if (type == "connected") + if (type == MessageTypes::connected) + { connectionWaiter.signal(); + } } logMessage ("Received: " + toXmlString (v)); @@ -757,7 +916,7 @@ bool Validator::ensureConnection() parentProcess->validationStartedCallback = [this] (const String& id) { listeners.call (&Listener::validationStarted, id); }; parentProcess->logMessageCallback = [this] (const String& m) { listeners.call (&Listener::logMessage, m); }; - parentProcess->validationCompleteCallback = [this] (const String& id, int numFailures) { listeners.call (&Listener::itemComplete, id, numFailures); }; + parentProcess->validationCompleteCallback = [this] (const String& id, int numFailures, const PluginTestResultArray& results) { listeners.call (&Listener::itemComplete, id, numFailures, results); }; parentProcess->completeCallback = [this] { listeners.call (&Listener::allItemsComplete); triggerAsyncUpdate(); }; const auto result = launchInProcess ? parentProcess->launchInProcess() diff --git a/Source/Validator.h b/Source/Validator.h index c8d06c0..9c35719 100644 --- a/Source/Validator.h +++ b/Source/Validator.h @@ -15,6 +15,7 @@ #pragma once #include +#include "PluginTestResult.h" #include "PluginTests.h" #ifndef LOG_PIPE_COMMUNICATION @@ -94,7 +95,7 @@ class Validator : public ChangeBroadcaster, virtual void validationStarted (const String& idString) = 0; virtual void logMessage (const String&) = 0; - virtual void itemComplete (const String& idString, int numFailures) = 0; + virtual void itemComplete (const String& idString, int numFailures, const PluginTestResultArray&) = 0; virtual void allItemsComplete() = 0; virtual void connectionLost() {} };