From 308a13ae273874627803e51a8c7a225d6b7789c7 Mon Sep 17 00:00:00 2001 From: Maksims Radugins Date: Sun, 17 Oct 2021 21:23:36 +0300 Subject: [PATCH 01/13] Implement junit report generation --- Source/JUnitListener.cpp | 33 +++++++++++++++ Source/JUnitListener.h | 21 ++++++++++ Source/JUnitWriter.cpp | 89 ++++++++++++++++++++++++++++++++++++++++ Source/JUnitWriter.h | 14 +++++++ 4 files changed, 157 insertions(+) create mode 100644 Source/JUnitListener.cpp create mode 100644 Source/JUnitListener.h create mode 100644 Source/JUnitWriter.cpp create mode 100644 Source/JUnitWriter.h diff --git a/Source/JUnitListener.cpp b/Source/JUnitListener.cpp new file mode 100644 index 0000000..5307515 --- /dev/null +++ b/Source/JUnitListener.cpp @@ -0,0 +1,33 @@ +#include "JUnitListener.h" + +#include "JUnitWriter.h" + +JUnitListener::JUnitListener() +{ +} + +void JUnitListener::setReportFile(const File &newReportFile) +{ + reportFile = newReportFile; +} + +void JUnitListener::validationStarted(const String &) +{ +} + +void JUnitListener::logMessage(const String &) +{ +} + +void JUnitListener::itemComplete(const String &id, int, const Array &itemResults) +{ + results.set(id, itemResults); +} + +void JUnitListener::allItemsComplete() +{ + if (!reportFile.getFullPathName().isEmpty()) + { + JUnitWriter::write(results, reportFile); + } +} diff --git a/Source/JUnitListener.h b/Source/JUnitListener.h new file mode 100644 index 0000000..7c386ec --- /dev/null +++ b/Source/JUnitListener.h @@ -0,0 +1,21 @@ +#pragma once + +#include +#include "Validator.h" + +class JUnitListener: public Validator::Listener +{ +public: + JUnitListener(); + void setReportFile(const File& newReportFile); +private: + void validationStarted (const String& id) override; + void logMessage (const String& m) override; + void itemComplete (const String& id, int numItemFailures, const Array& itemResults) override; + void allItemsComplete() override; + // void connectionLost() override; + + + File reportFile; + HashMap> results; +}; diff --git a/Source/JUnitWriter.cpp b/Source/JUnitWriter.cpp new file mode 100644 index 0000000..cef44a1 --- /dev/null +++ b/Source/JUnitWriter.cpp @@ -0,0 +1,89 @@ +#include "JUnitWriter.h" + +// +// +// +// +// +//WARNING: Use a program name that matches the source file name +//Category: COBOL Code Review – Naming Conventions +//File: /project/PROGRAM.cbl +//Line: 2 +// +// +// +// + +bool JUnitWriter::write(const HashMap> &allResults, File &output) +{ + XmlElement testsuites("testsuites"); + + int gfailures = 0; + int64 gtime = 0; + int gcount = 0; + for (auto it = allResults.begin(); it != allResults.end(); ++it) + { + const auto& results = it.getValue(); + auto pluginName = File(it.getKey()).getFileName(); + + auto testsuite = new XmlElement("testsuite"); + + int failures = 0; + int passes = 0; + int64 time = 0; + int count = 0; + + for (const auto& r: results) + { + auto testcase = new XmlElement("testcase"); + + testcase->setAttribute("id", "pluginval." + String(count)); + testcase->setAttribute("name", r.subcategoryName); + + auto test_time = (r.endTime - r.startTime).inMilliseconds(); + + testcase->setAttribute("time", test_time / 1000.0); + + for (const auto& m: r.messages) + { + auto failure = new XmlElement("failure"); + failure->setAttribute("type", "ERROR"); + failure->setAttribute("message", m); + + testcase->prependChildElement(failure); + } + + // add test case to test suite + testsuite->prependChildElement(testcase); + + // calculate totals for test suite + failures += r.failures; + passes += r.passes; + time += test_time; + + // increment count + count++; + } + + testsuite->setAttribute("id", "pluginval." + SystemStats::getOperatingSystemName() + "." + pluginName); + testsuite->setAttribute("name", "pluginval of " + pluginName + " on " + SystemStats::getOperatingSystemName()); + testsuite->setAttribute("tests", count); + testsuite->setAttribute("failures", failures); + testsuite->setAttribute("time", time / 1000.0); + + // accumulate totals for all test suites + gfailures += failures; + gtime += time; + gcount += count; + + testsuites.prependChildElement(testsuite); + } + + testsuites.setAttribute("id", "plugins"); + testsuites.setAttribute("name", "Plugin validations"); + testsuites.setAttribute("tests", gcount); + testsuites.setAttribute("failures", gfailures); + testsuites.setAttribute("time", gtime / 1000.0); + + return testsuites.writeTo(output); +} diff --git a/Source/JUnitWriter.h b/Source/JUnitWriter.h new file mode 100644 index 0000000..d27c25f --- /dev/null +++ b/Source/JUnitWriter.h @@ -0,0 +1,14 @@ +#pragma once + +#include + +class JUnitWriter +{ +public: + ~JUnitWriter() = default; + + static bool write(const HashMap > &allResults, File &output); + +private: + JUnitWriter() = default; +}; From 4d9441072bfaa4071593c451c551c6ee88dd06ee Mon Sep 17 00:00:00 2001 From: Maksims Radugins Date: Sun, 17 Oct 2021 21:29:03 +0300 Subject: [PATCH 02/13] Rename JUnitWriter to JUnitReport --- CMakeLists.txt | 7 +- Source/CommandLine.cpp | 14 +- Source/CommandLine.h | 4 +- Source/JUnitListener.cpp | 4 +- Source/{JUnitWriter.cpp => JUnitReport.cpp} | 80 ++++++++---- Source/{JUnitWriter.h => JUnitReport.h} | 6 +- Source/MainComponent.h | 4 +- Source/PluginTests.h | 1 + Source/Validator.cpp | 134 +++++++++++++++++++- Source/Validator.h | 2 +- 10 files changed, 217 insertions(+), 39 deletions(-) rename Source/{JUnitWriter.cpp => JUnitReport.cpp} (59%) rename Source/{JUnitWriter.h => JUnitReport.h} (68%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4471371..5012732 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -56,7 +56,12 @@ set(SourceFiles Source/tests/BusTests.cpp Source/tests/ParameterFuzzTests.cpp Source/TestUtilities.cpp - Source/Validator.cpp) + Source/Validator.cpp + Source/JUnitReport.cpp + Source/JUnitReport.h + Source/JUnitListener.cpp + Source/JUnitListener.h + ) target_sources(pluginval PRIVATE ${SourceFiles}) source_group(TREE ${CMAKE_CURRENT_SOURCE_DIR}/Source PREFIX Source FILES ${SourceFiles}) diff --git a/Source/CommandLine.cpp b/Source/CommandLine.cpp index 616c0d4..1bf648f 100644 --- a/Source/CommandLine.cpp +++ b/Source/CommandLine.cpp @@ -49,6 +49,8 @@ CommandLineValidator::CommandLineValidator() { validator.addChangeListener (this); validator.addListener (this); + + validator.addListener(&junitListener); } CommandLineValidator::~CommandLineValidator() @@ -57,6 +59,8 @@ CommandLineValidator::~CommandLineValidator() void CommandLineValidator::validate (const StringArray& fileOrIDs, PluginTests::Options options, bool validateInProcess) { + junitListener.setReportFile(options.junitReportFile); + validator.setValidateInProcess (validateInProcess); validator.validate (fileOrIDs, options); } @@ -79,7 +83,7 @@ void CommandLineValidator::logMessage (const String& m) std::cout << m << "\n"; } -void CommandLineValidator::itemComplete (const String& id, int numItemFailures) +void CommandLineValidator::itemComplete (const String& id, int numItemFailures, const Array&) { logMessage ("\nFinished validating: " + id); @@ -217,6 +221,11 @@ StringArray getDisabledTest (const ArgumentList& args) return disabledTests; } +File getJunitReportFile (const ArgumentList& args) +{ + return getOptionValue (args, "--junit-report-file", {}, "Missing junit-report-file path argument!").toString(); +} + //============================================================================== String parseCommandLineArgs (String commandLine) { @@ -332,6 +341,8 @@ static String getHelpMessage() << " If specified, sets the list of sample rates at which tests will be executed (default=44100,48000,96000)" << newLine << " --block-sizes [list of comma separated block sizes]" << newLine << " If specified, sets the list of block sizes at which tests will be executed (default=64,128,256,512,1024)" << newLine + << " --junit-report-file [output filename]" << newLine + << " If specified, creates JUnit report" << newLine << " --version" << newLine << " Print pluginval version." << newLine << newLine @@ -394,6 +405,7 @@ static void validate (CommandLineValidator& validator, const ArgumentList& args) options.disabledTests = getDisabledTest (args); options.sampleRates = getSampleRates (args); options.blockSizes = getBlockSizes (args); + options.junitReportFile = getJunitReportFile(args); validator.validate (fileOrIDs, options, diff --git a/Source/CommandLine.h b/Source/CommandLine.h index f0c7fd6..02d2864 100644 --- a/Source/CommandLine.h +++ b/Source/CommandLine.h @@ -16,6 +16,7 @@ #include #include "Validator.h" +#include "JUnitListener.h" struct CommandLineValidator : private ChangeListener, private Validator::Listener @@ -27,6 +28,7 @@ struct CommandLineValidator : private ChangeListener, private: Validator validator; + JUnitListener junitListener; String currentID; std::atomic numFailures { 0 }; @@ -34,7 +36,7 @@ struct CommandLineValidator : private ChangeListener, void validationStarted (const String&) override; void logMessage (const String& m) override; - void itemComplete (const String&, int numItemFailures) override; + void itemComplete (const String&, int numItemFailures, const Array&) override; void allItemsComplete() override; void connectionLost() override; }; diff --git a/Source/JUnitListener.cpp b/Source/JUnitListener.cpp index 5307515..392e6c6 100644 --- a/Source/JUnitListener.cpp +++ b/Source/JUnitListener.cpp @@ -1,6 +1,6 @@ #include "JUnitListener.h" -#include "JUnitWriter.h" +#include "JUnitReport.h" JUnitListener::JUnitListener() { @@ -28,6 +28,6 @@ void JUnitListener::allItemsComplete() { if (!reportFile.getFullPathName().isEmpty()) { - JUnitWriter::write(results, reportFile); + JUnitReport::write(results, reportFile); } } diff --git a/Source/JUnitWriter.cpp b/Source/JUnitReport.cpp similarity index 59% rename from Source/JUnitWriter.cpp rename to Source/JUnitReport.cpp index cef44a1..e9463bd 100644 --- a/Source/JUnitWriter.cpp +++ b/Source/JUnitReport.cpp @@ -1,4 +1,4 @@ -#include "JUnitWriter.h" +#include "JUnitReport.h" // // @@ -14,7 +14,60 @@ // // -bool JUnitWriter::write(const HashMap> &allResults, File &output) +namespace +{ + +String stringToId(const String& s) +{ + if (s.isEmpty()) + { + return "-"; + } + + auto firstChar = s[0]; + bool validFirstChar = + (firstChar >= 'a' && firstChar <= 'z') || + (firstChar >= 'A' && firstChar <= 'Z'); + + String id = validFirstChar ? "" : "-"; + for (auto it = s.begin(); it != s.end(); ++it) + { + auto c = *it; + bool valid = + (c >= 'a' && c <= 'z') || + (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || + c == '-' || c == '_' || c == ':' || c == '.'; + id += valid ? c : '-'; + } + return id.toLowerCase(); +} + + +XmlElement* createTestCase(const UnitTestRunner::TestResult& r) +{ + auto testcase = new XmlElement("testcase"); + + //testcase->setAttribute("id", stringToId(r.unitTestName + "." + r.subcategoryName)); + testcase->setAttribute("name", r.subcategoryName); + + auto duration = (r.endTime - r.startTime).inMilliseconds(); + testcase->setAttribute("time", duration / 1000.0); + + for (const auto& m: r.messages) + { + auto failure = new XmlElement("failure"); + failure->setAttribute("type", "ERROR"); + failure->setAttribute("message", m); + + testcase->prependChildElement(failure); + } + return testcase; +} + +} + +bool JUnitReport::write(const HashMap> &allResults, File &output) { XmlElement testsuites("testsuites"); @@ -35,37 +88,20 @@ bool JUnitWriter::write(const HashMap> for (const auto& r: results) { - auto testcase = new XmlElement("testcase"); - - testcase->setAttribute("id", "pluginval." + String(count)); - testcase->setAttribute("name", r.subcategoryName); - - auto test_time = (r.endTime - r.startTime).inMilliseconds(); - - testcase->setAttribute("time", test_time / 1000.0); - - for (const auto& m: r.messages) - { - auto failure = new XmlElement("failure"); - failure->setAttribute("type", "ERROR"); - failure->setAttribute("message", m); - - testcase->prependChildElement(failure); - } - + auto testcase = createTestCase(r); // add test case to test suite testsuite->prependChildElement(testcase); // calculate totals for test suite failures += r.failures; passes += r.passes; - time += test_time; + time += (r.endTime - r.startTime).inMilliseconds(); // increment count count++; } - testsuite->setAttribute("id", "pluginval." + SystemStats::getOperatingSystemName() + "." + pluginName); + //testsuite->setAttribute("id", stringToId("pluginval." + SystemStats::getOperatingSystemName() + "." + pluginName)); testsuite->setAttribute("name", "pluginval of " + pluginName + " on " + SystemStats::getOperatingSystemName()); testsuite->setAttribute("tests", count); testsuite->setAttribute("failures", failures); diff --git a/Source/JUnitWriter.h b/Source/JUnitReport.h similarity index 68% rename from Source/JUnitWriter.h rename to Source/JUnitReport.h index d27c25f..b4dfee9 100644 --- a/Source/JUnitWriter.h +++ b/Source/JUnitReport.h @@ -2,13 +2,13 @@ #include -class JUnitWriter +class JUnitReport { public: - ~JUnitWriter() = default; + ~JUnitReport() = default; static bool write(const HashMap > &allResults, File &output); private: - JUnitWriter() = default; + JUnitReport() = default; }; diff --git a/Source/MainComponent.h b/Source/MainComponent.h index 26714b6..207d90e 100644 --- a/Source/MainComponent.h +++ b/Source/MainComponent.h @@ -91,7 +91,7 @@ struct ConnectionStatus : public Component, { } - void itemComplete (const String&, int) override + void itemComplete (const String&, int, const Array&) override { } @@ -194,7 +194,7 @@ struct ConsoleComponent : public Component, std::cout << m << "\n"; } - void itemComplete (const String& id, int numFailures) override + void itemComplete (const String& id, int numFailures, const Array&) override { logMessage ("\nFinished validating: " + id); diff --git a/Source/PluginTests.h b/Source/PluginTests.h index cf9bbe4..0ffb627 100644 --- a/Source/PluginTests.h +++ b/Source/PluginTests.h @@ -38,6 +38,7 @@ struct PluginTests : public UnitTest StringArray disabledTests; /**< List of disabled tests. */ std::vector sampleRates; /**< List of sample rates. */ std::vector blockSizes; /**< List of block sizes. */ + File junitReportFile; /**< JUnit report file. */ }; /** Creates a set of tests for a fileOrIdentifier. */ diff --git a/Source/Validator.cpp b/Source/Validator.cpp index 504b4c8..0eec898 100644 --- a/Source/Validator.cpp +++ b/Source/Validator.cpp @@ -258,6 +258,18 @@ namespace IDs DECLARE_ID(log) DECLARE_ID(numFailures) + DECLARE_ID(testResultArray) + DECLARE_ID(testResultItem) + DECLARE_ID(testName) + DECLARE_ID(testSubcategoryName) + DECLARE_ID(testPassCount) + DECLARE_ID(testFailureCount) + DECLARE_ID(testFailureMessageArray) + DECLARE_ID(testFailureMessageItem) + DECLARE_ID(testFailureMessageText) + DECLARE_ID(testStartTime) + DECLARE_ID(testEndTime) + #undef DECLARE_ID } @@ -463,6 +475,40 @@ class ValidatorChildProcess : public ChildProcessSlave, processRequest (r); } + static ValueTree serializeTestResults(const Array& results) + { + ValueTree testResultArray { IDs::testResultArray }; + for (const auto& r: results) + { + ValueTree testFailureMessageArray { + IDs::testFailureMessageArray + }; + for (const auto& m: r.messages) + { + testFailureMessageArray.appendChild( + { IDs::testFailureMessageItem, + { { IDs::testFailureMessageText, m } } + }, nullptr); + } + + ValueTree testResultItem { IDs::testResultItem, + { + { IDs::testName, r.unitTestName }, + { IDs::testSubcategoryName, r.subcategoryName }, + { IDs::testPassCount, r.passes }, + { IDs::testFailureCount, r.failures }, + { IDs::testStartTime, r.startTime.toMilliseconds() }, + { IDs::testEndTime, r.endTime.toMilliseconds() }, + }, + { testFailureMessageArray } + }; + + testResultArray.appendChild( testResultItem, nullptr ); + } + + return testResultArray; + } + void processRequest (MemoryBlock mb) { const ValueTree v (memoryBlockToValueTree (mb)); @@ -535,9 +581,17 @@ class ValidatorChildProcess : public ChildProcessSlave, } jassert (fileOrID.isNotEmpty()); - sendValueTreeToParent ({ - IDs::MESSAGE, {{ IDs::type, "result" }, { IDs::fileOrID, fileOrID }, { IDs::numFailures, getNumFailures (results) }} - }); + + ValueTree result { + IDs::MESSAGE, + { + { IDs::type, "result" }, + { IDs::fileOrID, fileOrID }, + { IDs::numFailures, getNumFailures (results) } + }, + { serializeTestResults( results ) } + }; + sendValueTreeToParent ( result ); } } @@ -567,7 +621,7 @@ class ValidatorParentProcess : public ChildProcessMaster std::function logMessageCallback; // Callback which can be set to be informed when a validation completes - std::function validationCompleteCallback; + std::function&)> validationCompleteCallback; // Callback which can be set to be informed when all validations have been completed std::function completeCallback; @@ -593,6 +647,55 @@ class ValidatorParentProcess : public ChildProcessMaster return ok ? Result::ok() : Result::fail ("Error: Child failed to launch"); } + static Array deserializeTestResults(const ValueTree& testResultArray) + { + if (!testResultArray.hasType(IDs::testResultArray)) + { + return {}; + } + Array results; + for (const auto& testResultItem: testResultArray) + { + if (!testResultItem.hasType(IDs::testResultItem) || + !testResultItem.hasProperty(IDs::testName) || + !testResultItem.hasProperty(IDs::testSubcategoryName) || + !testResultItem.hasProperty(IDs::testPassCount) || + !testResultItem.hasProperty(IDs::testFailureCount) || + !testResultItem.hasProperty(IDs::testStartTime) || + !testResultItem.hasProperty(IDs::testEndTime) + ) + { + continue; + } + var v; + UnitTestRunner::TestResult result; + result.unitTestName = testResultItem.getProperty(IDs::testName); + result.subcategoryName = testResultItem.getProperty(IDs::testSubcategoryName); + result.passes = testResultItem.getProperty(IDs::testPassCount); + result.failures = testResultItem.getProperty(IDs::testFailureCount); + result.startTime = Time(testResultItem.getProperty(IDs::testStartTime)); + result.endTime = Time(testResultItem.getProperty(IDs::testEndTime)); + + auto testFailureMessageArray = testResultItem.getChildWithName(IDs::testFailureMessageArray); + if (!testFailureMessageArray.hasType(IDs::testFailureMessageArray)) + { + continue; + } + for (const auto& testFailureMessageItem: testFailureMessageArray) + { + if (!testFailureMessageItem.hasType(IDs::testFailureMessageItem) || + !testFailureMessageItem.hasProperty(IDs::testFailureMessageText)) + { + continue; + } + result.messages.add(testFailureMessageItem.getProperty(IDs::testFailureMessageText)); + } + + results.add(result); + } + return results; + } + //============================================================================== void handleMessageFromSlave (const MemoryBlock& mb) override { @@ -603,19 +706,38 @@ class ValidatorParentProcess : public ChildProcessMaster const auto type = v[IDs::type].toString(); if (logMessageCallback && type == "log") + { logMessageCallback (v[IDs::text].toString()); + } if (validationCompleteCallback && type == "result") - validationCompleteCallback (v[IDs::fileOrID].toString(), v[IDs::numFailures]); + { + auto results = deserializeTestResults(v.getChildWithName(IDs::testResultArray)); + for (const auto& r: results) + { + std::cout << "TEST:" << r.subcategoryName << std::endl; + for (const auto& m: r.messages) + { + std::cout << "MSG:" << m << std::endl; + } + } + validationCompleteCallback (v[IDs::fileOrID].toString(), v[IDs::numFailures], results); + } if (validationStartedCallback && type == "started") + { validationStartedCallback (v[IDs::fileOrID].toString()); + } if (completeCallback && type == "complete") + { completeCallback(); + } if (type == "connected") + { connectionWaiter.signal(); + } } logMessage ("Received: " + toXmlString (v)); @@ -757,7 +879,7 @@ bool Validator::ensureConnection() parentProcess->validationStartedCallback = [this] (const String& id) { listeners.call (&Listener::validationStarted, id); }; parentProcess->logMessageCallback = [this] (const String& m) { listeners.call (&Listener::logMessage, m); }; - parentProcess->validationCompleteCallback = [this] (const String& id, int numFailures) { listeners.call (&Listener::itemComplete, id, numFailures); }; + parentProcess->validationCompleteCallback = [this] (const String& id, int numFailures, const Array& results) { listeners.call (&Listener::itemComplete, id, numFailures, results); }; parentProcess->completeCallback = [this] { listeners.call (&Listener::allItemsComplete); triggerAsyncUpdate(); }; const auto result = launchInProcess ? parentProcess->launchInProcess() diff --git a/Source/Validator.h b/Source/Validator.h index c8d06c0..82835cc 100644 --- a/Source/Validator.h +++ b/Source/Validator.h @@ -94,7 +94,7 @@ class Validator : public ChangeBroadcaster, virtual void validationStarted (const String& idString) = 0; virtual void logMessage (const String&) = 0; - virtual void itemComplete (const String& idString, int numFailures) = 0; + virtual void itemComplete (const String& idString, int numFailures, const Array&) = 0; virtual void allItemsComplete() = 0; virtual void connectionLost() {} }; From 2c77bb69de8aebe8b885376897ee3d0f44a42e72 Mon Sep 17 00:00:00 2001 From: Maksims Radugins Date: Sun, 17 Oct 2021 21:32:39 +0300 Subject: [PATCH 03/13] Add JUnit example source URL --- Source/JUnitReport.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/Source/JUnitReport.cpp b/Source/JUnitReport.cpp index e9463bd..b0c9ea9 100644 --- a/Source/JUnitReport.cpp +++ b/Source/JUnitReport.cpp @@ -1,5 +1,6 @@ #include "JUnitReport.h" +// JUnit example from https://www.ibm.com/docs/en/adfz/developer-for-zos/9.1.1?topic=formats-junit-xml-format // // // From 83e24968356466c1cb1d08156edf9894e3574341 Mon Sep 17 00:00:00 2001 From: Maksims Radugins Date: Sun, 17 Oct 2021 21:43:48 +0300 Subject: [PATCH 04/13] Remove debug text --- Source/Validator.cpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/Source/Validator.cpp b/Source/Validator.cpp index 0eec898..0c24425 100644 --- a/Source/Validator.cpp +++ b/Source/Validator.cpp @@ -713,14 +713,6 @@ class ValidatorParentProcess : public ChildProcessMaster if (validationCompleteCallback && type == "result") { auto results = deserializeTestResults(v.getChildWithName(IDs::testResultArray)); - for (const auto& r: results) - { - std::cout << "TEST:" << r.subcategoryName << std::endl; - for (const auto& m: r.messages) - { - std::cout << "MSG:" << m << std::endl; - } - } validationCompleteCallback (v[IDs::fileOrID].toString(), v[IDs::numFailures], results); } From 079a6737b97350383a66136855146effe79d2ecc Mon Sep 17 00:00:00 2001 From: Maksims Radugins Date: Sun, 17 Oct 2021 22:21:38 +0300 Subject: [PATCH 05/13] Modify report to correctly show up in GitLab --- Source/JUnitReport.cpp | 76 ++++++++++-------------------------------- 1 file changed, 17 insertions(+), 59 deletions(-) diff --git a/Source/JUnitReport.cpp b/Source/JUnitReport.cpp index b0c9ea9..5856302 100644 --- a/Source/JUnitReport.cpp +++ b/Source/JUnitReport.cpp @@ -1,56 +1,15 @@ #include "JUnitReport.h" -// JUnit example from https://www.ibm.com/docs/en/adfz/developer-for-zos/9.1.1?topic=formats-junit-xml-format -// -// -// -// -// -//WARNING: Use a program name that matches the source file name -//Category: COBOL Code Review – Naming Conventions -//File: /project/PROGRAM.cbl -//Line: 2 -// -// -// -// - namespace { -String stringToId(const String& s) -{ - if (s.isEmpty()) - { - return "-"; - } - - auto firstChar = s[0]; - bool validFirstChar = - (firstChar >= 'a' && firstChar <= 'z') || - (firstChar >= 'A' && firstChar <= 'Z'); - - String id = validFirstChar ? "" : "-"; - for (auto it = s.begin(); it != s.end(); ++it) - { - auto c = *it; - bool valid = - (c >= 'a' && c <= 'z') || - (c >= 'A' && c <= 'Z') || - (c >= '0' && c <= '9') || - c == '-' || c == '_' || c == ':' || c == '.'; - id += valid ? c : '-'; - } - return id.toLowerCase(); -} - - -XmlElement* createTestCase(const UnitTestRunner::TestResult& r) +XmlElement* createTestCase(const String& pluginName, const UnitTestRunner::TestResult& r) { auto testcase = new XmlElement("testcase"); - //testcase->setAttribute("id", stringToId(r.unitTestName + "." + r.subcategoryName)); testcase->setAttribute("name", r.subcategoryName); + testcase->setAttribute("classname", r.unitTestName); + testcase->setAttribute("file", pluginName); auto duration = (r.endTime - r.startTime).inMilliseconds(); testcase->setAttribute("time", duration / 1000.0); @@ -72,9 +31,9 @@ bool JUnitReport::write(const HashMap> { XmlElement testsuites("testsuites"); - int gfailures = 0; - int64 gtime = 0; - int gcount = 0; + int total_failures = 0; + int64 total_time = 0; + int total_count = 0; for (auto it = allResults.begin(); it != allResults.end(); ++it) { const auto& results = it.getValue(); @@ -89,7 +48,7 @@ bool JUnitReport::write(const HashMap> for (const auto& r: results) { - auto testcase = createTestCase(r); + auto testcase = createTestCase(pluginName, r); // add test case to test suite testsuite->prependChildElement(testcase); @@ -102,25 +61,24 @@ bool JUnitReport::write(const HashMap> count++; } - //testsuite->setAttribute("id", stringToId("pluginval." + SystemStats::getOperatingSystemName() + "." + pluginName)); + testsuite->setAttribute("package", "pluginval"); testsuite->setAttribute("name", "pluginval of " + pluginName + " on " + SystemStats::getOperatingSystemName()); testsuite->setAttribute("tests", count); testsuite->setAttribute("failures", failures); testsuite->setAttribute("time", time / 1000.0); - // accumulate totals for all test suites - gfailures += failures; - gtime += time; - gcount += count; - testsuites.prependChildElement(testsuite); + + // accumulate totals for all test suites + total_failures += failures; + total_time += time; + total_count += count; } - testsuites.setAttribute("id", "plugins"); - testsuites.setAttribute("name", "Plugin validations"); - testsuites.setAttribute("tests", gcount); - testsuites.setAttribute("failures", gfailures); - testsuites.setAttribute("time", gtime / 1000.0); + testsuites.setAttribute("name", "pluginval test suites"); + testsuites.setAttribute("tests", total_count); + testsuites.setAttribute("failures", total_failures); + testsuites.setAttribute("time", total_time / 1000.0); return testsuites.writeTo(output); } From ba0914b66fff23891615ed1f8692eb0070f17e1c Mon Sep 17 00:00:00 2001 From: Maksims Radugins Date: Mon, 18 Oct 2021 00:06:50 +0300 Subject: [PATCH 06/13] Add MessageTypes constants --- Source/Validator.cpp | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/Source/Validator.cpp b/Source/Validator.cpp index 0c24425..3e6319c 100644 --- a/Source/Validator.cpp +++ b/Source/Validator.cpp @@ -273,6 +273,15 @@ namespace IDs #undef DECLARE_ID } +namespace MessageTypes +{ + const String result = "result"; + const String log = "log"; + const String started = "started"; + const String complete = "complete"; + const String connected = "connected"; +} + //============================================================================== // This is a token that's used at both ends of our parent-child processes, to // act as a unique token in the command line arguments. @@ -331,7 +340,7 @@ class ValidatorChildProcess : public ChildProcessSlave, isConnected = isNowConnected; if (isConnected) - sendValueTreeToParent ({ IDs::MESSAGE, {{ IDs::type, "connected" }} }); + sendValueTreeToParent ({ IDs::MESSAGE, {{ IDs::type, MessageTypes::connected }} }); } void setParentProcess (ChildProcessMaster* newParent) @@ -395,7 +404,7 @@ class ValidatorChildProcess : public ChildProcessSlave, } if (owner.isConnected && ! messagesToSend.isEmpty()) - owner.sendValueTreeToParent ({ IDs::MESSAGE, {{ IDs::type, "log" }, { IDs::text, messagesToSend.joinIntoString ("\n") }} }, false); + owner.sendValueTreeToParent ({ IDs::MESSAGE, {{ IDs::type, MessageTypes::log }, { IDs::text, messagesToSend.joinIntoString ("\n") }} }, false); } ValidatorChildProcess& owner; @@ -540,7 +549,7 @@ class ValidatorChildProcess : public ChildProcessSlave, { fileOrID = c[IDs::fileOrID].toString(); sendValueTreeToParent ({ - IDs::MESSAGE, {{ IDs::type, "started" }, { IDs::fileOrID, fileOrID }} + IDs::MESSAGE, {{ IDs::type, MessageTypes::started }, { IDs::fileOrID, fileOrID }} }); results = validate (c[IDs::fileOrID].toString(), options, [this] (const String& m) { logMessage (m); }); @@ -559,7 +568,7 @@ class ValidatorChildProcess : public ChildProcessSlave, { fileOrID = pd.createIdentifierString(); sendValueTreeToParent ({ - IDs::MESSAGE, {{ IDs::type, "started" }, { IDs::fileOrID, fileOrID }} + IDs::MESSAGE, {{ IDs::type, MessageTypes::started }, { IDs::fileOrID, fileOrID }} }); results = validate (pd, options, [this] (const String& m) { logMessage (m); }); @@ -585,7 +594,7 @@ class ValidatorChildProcess : public ChildProcessSlave, ValueTree result { IDs::MESSAGE, { - { IDs::type, "result" }, + { IDs::type, MessageTypes::result }, { IDs::fileOrID, fileOrID }, { IDs::numFailures, getNumFailures (results) } }, @@ -596,7 +605,7 @@ class ValidatorChildProcess : public ChildProcessSlave, } sendValueTreeToParent ({ - IDs::MESSAGE, {{ IDs::type, "complete" }} + IDs::MESSAGE, {{ IDs::type, MessageTypes::complete }} }); } }; @@ -705,28 +714,28 @@ class ValidatorParentProcess : public ChildProcessMaster { const auto type = v[IDs::type].toString(); - if (logMessageCallback && type == "log") + if (logMessageCallback && type == MessageTypes::log) { logMessageCallback (v[IDs::text].toString()); } - if (validationCompleteCallback && type == "result") + if (validationCompleteCallback && type == MessageTypes::result) { auto results = deserializeTestResults(v.getChildWithName(IDs::testResultArray)); validationCompleteCallback (v[IDs::fileOrID].toString(), v[IDs::numFailures], results); } - if (validationStartedCallback && type == "started") + if (validationStartedCallback && type == MessageTypes::started) { validationStartedCallback (v[IDs::fileOrID].toString()); } - if (completeCallback && type == "complete") + if (completeCallback && type == MessageTypes::complete) { completeCallback(); } - if (type == "connected") + if (type == MessageTypes::connected) { connectionWaiter.signal(); } From 789cd91121e10e28fdd509ec0b1c3c4b54fe4d02 Mon Sep 17 00:00:00 2001 From: Maksims Radugins Date: Mon, 18 Oct 2021 00:07:17 +0300 Subject: [PATCH 07/13] Cleanup --- Source/JUnitListener.cpp | 8 +++++++- Source/JUnitListener.h | 3 +-- Source/JUnitReport.cpp | 6 ++---- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Source/JUnitListener.cpp b/Source/JUnitListener.cpp index 392e6c6..5b904f2 100644 --- a/Source/JUnitListener.cpp +++ b/Source/JUnitListener.cpp @@ -15,7 +15,7 @@ void JUnitListener::validationStarted(const String &) { } -void JUnitListener::logMessage(const String &) +void JUnitListener::logMessage(const String &m) { } @@ -31,3 +31,9 @@ void JUnitListener::allItemsComplete() JUnitReport::write(results, reportFile); } } + +void JUnitListener::connectionLost() +{ + Validator::Listener::connectionLost(); + // TODO: create report when connection is lost +} diff --git a/Source/JUnitListener.h b/Source/JUnitListener.h index 7c386ec..dc39ec6 100644 --- a/Source/JUnitListener.h +++ b/Source/JUnitListener.h @@ -13,8 +13,7 @@ class JUnitListener: public Validator::Listener void logMessage (const String& m) override; void itemComplete (const String& id, int numItemFailures, const Array& itemResults) override; void allItemsComplete() override; - // void connectionLost() override; - + void connectionLost() override; File reportFile; HashMap> results; diff --git a/Source/JUnitReport.cpp b/Source/JUnitReport.cpp index 5856302..61b3c6e 100644 --- a/Source/JUnitReport.cpp +++ b/Source/JUnitReport.cpp @@ -3,7 +3,7 @@ namespace { -XmlElement* createTestCase(const String& pluginName, const UnitTestRunner::TestResult& r) +XmlElement* createTestCaseElement(const String& pluginName, const UnitTestRunner::TestResult& r) { auto testcase = new XmlElement("testcase"); @@ -48,7 +48,7 @@ bool JUnitReport::write(const HashMap> for (const auto& r: results) { - auto testcase = createTestCase(pluginName, r); + auto testcase = createTestCaseElement(pluginName, r); // add test case to test suite testsuite->prependChildElement(testcase); @@ -56,8 +56,6 @@ bool JUnitReport::write(const HashMap> failures += r.failures; passes += r.passes; time += (r.endTime - r.startTime).inMilliseconds(); - - // increment count count++; } From b9804413da8ac39ea4d0a3dd1c49b8764459befb Mon Sep 17 00:00:00 2001 From: Maksims Radugins Date: Mon, 18 Oct 2021 16:33:05 +0300 Subject: [PATCH 08/13] Code cleanup --- Source/JUnitListener.cpp | 2 +- Source/JUnitReport.cpp | 58 ++++++++++++++++++++++------------------ Source/JUnitReport.h | 10 +++---- Source/Validator.cpp | 1 - 4 files changed, 36 insertions(+), 35 deletions(-) diff --git a/Source/JUnitListener.cpp b/Source/JUnitListener.cpp index 5b904f2..6a14832 100644 --- a/Source/JUnitListener.cpp +++ b/Source/JUnitListener.cpp @@ -15,7 +15,7 @@ void JUnitListener::validationStarted(const String &) { } -void JUnitListener::logMessage(const String &m) +void JUnitListener::logMessage(const String &) { } diff --git a/Source/JUnitReport.cpp b/Source/JUnitReport.cpp index 61b3c6e..7c746d3 100644 --- a/Source/JUnitReport.cpp +++ b/Source/JUnitReport.cpp @@ -1,6 +1,6 @@ #include "JUnitReport.h" -namespace +namespace JUnitReport { XmlElement* createTestCaseElement(const String& pluginName, const UnitTestRunner::TestResult& r) @@ -25,26 +25,39 @@ XmlElement* createTestCaseElement(const String& pluginName, const UnitTestRunner return testcase; } +XmlElement* createTestSuiteElement(const String& pluginName) +{ + auto testsuite = new XmlElement("testsuite"); + testsuite->setAttribute("package", "pluginval"); + testsuite->setAttribute("name", "pluginval of " + pluginName + " on " + SystemStats::getOperatingSystemName()); + return testsuite; +} + +void addTestsStats(XmlElement* element, int tests, int failures, int64 duration) +{ + element->setAttribute("tests", tests); + element->setAttribute("failures", failures); + element->setAttribute("time", duration / 1000.0); } -bool JUnitReport::write(const HashMap> &allResults, File &output) +bool write(const HashMap> &allResults, File &output) { XmlElement testsuites("testsuites"); + testsuites.setAttribute("name", "pluginval test suites"); int total_failures = 0; - int64 total_time = 0; - int total_count = 0; + int total_tests = 0; + int64 total_duration = 0; for (auto it = allResults.begin(); it != allResults.end(); ++it) { - const auto& results = it.getValue(); + const auto results = it.getValue(); auto pluginName = File(it.getKey()).getFileName(); - auto testsuite = new XmlElement("testsuite"); + int suite_failures = 0; + int64 suite_duration = 0; + int suite_tests = results.size(); - int failures = 0; - int passes = 0; - int64 time = 0; - int count = 0; + auto testsuite = createTestSuiteElement(pluginName); for (const auto& r: results) { @@ -53,30 +66,23 @@ bool JUnitReport::write(const HashMap> testsuite->prependChildElement(testcase); // calculate totals for test suite - failures += r.failures; - passes += r.passes; - time += (r.endTime - r.startTime).inMilliseconds(); - count++; + suite_failures += r.failures; + suite_duration += (r.endTime - r.startTime).inMilliseconds(); } - testsuite->setAttribute("package", "pluginval"); - testsuite->setAttribute("name", "pluginval of " + pluginName + " on " + SystemStats::getOperatingSystemName()); - testsuite->setAttribute("tests", count); - testsuite->setAttribute("failures", failures); - testsuite->setAttribute("time", time / 1000.0); + addTestsStats(testsuite, suite_tests, suite_failures, suite_duration); testsuites.prependChildElement(testsuite); // accumulate totals for all test suites - total_failures += failures; - total_time += time; - total_count += count; + total_failures += suite_failures; + total_duration += suite_duration; + total_tests += suite_tests; } - testsuites.setAttribute("name", "pluginval test suites"); - testsuites.setAttribute("tests", total_count); - testsuites.setAttribute("failures", total_failures); - testsuites.setAttribute("time", total_time / 1000.0); + addTestsStats(&testsuites, total_tests, total_failures, total_duration); return testsuites.writeTo(output); } + +} // namespace JUnitReport diff --git a/Source/JUnitReport.h b/Source/JUnitReport.h index b4dfee9..fce5e6c 100644 --- a/Source/JUnitReport.h +++ b/Source/JUnitReport.h @@ -2,13 +2,9 @@ #include -class JUnitReport +namespace JUnitReport { -public: - ~JUnitReport() = default; - static bool write(const HashMap > &allResults, File &output); +bool write(const HashMap > &allResults, File &output); -private: - JUnitReport() = default; -}; +} // namespace JUnitReport diff --git a/Source/Validator.cpp b/Source/Validator.cpp index 3e6319c..00ecd9b 100644 --- a/Source/Validator.cpp +++ b/Source/Validator.cpp @@ -676,7 +676,6 @@ class ValidatorParentProcess : public ChildProcessMaster { continue; } - var v; UnitTestRunner::TestResult result; result.unitTestName = testResultItem.getProperty(IDs::testName); result.subcategoryName = testResultItem.getProperty(IDs::testSubcategoryName); From 74f91f8f46f241788a4053bd016b36cee6c988d6 Mon Sep 17 00:00:00 2001 From: Maksims Radugins Date: Sun, 31 Oct 2021 16:49:46 +0200 Subject: [PATCH 09/13] Collect unit test logs and add to junit report --- CMakeLists.txt | 1 + Source/CommandLine.cpp | 2 +- Source/CommandLine.h | 3 +- Source/CommonTypes.h | 6 ++ Source/JUnitListener.cpp | 2 +- Source/JUnitListener.h | 5 +- Source/JUnitReport.cpp | 32 ++++++++--- Source/JUnitReport.h | 3 +- Source/MainComponent.h | 5 +- Source/Validator.cpp | 119 +++++++++++++++++++++++++-------------- Source/Validator.h | 3 +- 11 files changed, 122 insertions(+), 59 deletions(-) create mode 100644 Source/CommonTypes.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 5012732..f714a8e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,6 +61,7 @@ set(SourceFiles Source/JUnitReport.h Source/JUnitListener.cpp Source/JUnitListener.h + Source/CommonTypes.h ) target_sources(pluginval PRIVATE ${SourceFiles}) diff --git a/Source/CommandLine.cpp b/Source/CommandLine.cpp index 1bf648f..3e97677 100644 --- a/Source/CommandLine.cpp +++ b/Source/CommandLine.cpp @@ -83,7 +83,7 @@ void CommandLineValidator::logMessage (const String& m) std::cout << m << "\n"; } -void CommandLineValidator::itemComplete (const String& id, int numItemFailures, const Array&) +void CommandLineValidator::itemComplete (const String& id, int numItemFailures, const UnitTestResultsWithOutput&) { logMessage ("\nFinished validating: " + id); diff --git a/Source/CommandLine.h b/Source/CommandLine.h index 02d2864..5838583 100644 --- a/Source/CommandLine.h +++ b/Source/CommandLine.h @@ -15,6 +15,7 @@ #pragma once #include +#include "CommonTypes.h" #include "Validator.h" #include "JUnitListener.h" @@ -36,7 +37,7 @@ struct CommandLineValidator : private ChangeListener, void validationStarted (const String&) override; void logMessage (const String& m) override; - void itemComplete (const String&, int numItemFailures, const Array&) override; + void itemComplete (const String&, int numItemFailures, const UnitTestResultsWithOutput&) override; void allItemsComplete() override; void connectionLost() override; }; diff --git a/Source/CommonTypes.h b/Source/CommonTypes.h new file mode 100644 index 0000000..11fb567 --- /dev/null +++ b/Source/CommonTypes.h @@ -0,0 +1,6 @@ +#pragma once + +#include + +using UnitTestResultsWithOutput = Array>; + diff --git a/Source/JUnitListener.cpp b/Source/JUnitListener.cpp index 6a14832..26a5f69 100644 --- a/Source/JUnitListener.cpp +++ b/Source/JUnitListener.cpp @@ -19,7 +19,7 @@ void JUnitListener::logMessage(const String &) { } -void JUnitListener::itemComplete(const String &id, int, const Array &itemResults) +void JUnitListener::itemComplete(const String &id, int, const UnitTestResultsWithOutput &itemResults) { results.set(id, itemResults); } diff --git a/Source/JUnitListener.h b/Source/JUnitListener.h index dc39ec6..88df87f 100644 --- a/Source/JUnitListener.h +++ b/Source/JUnitListener.h @@ -1,6 +1,7 @@ #pragma once #include +#include "CommonTypes.h" #include "Validator.h" class JUnitListener: public Validator::Listener @@ -11,10 +12,10 @@ class JUnitListener: public Validator::Listener private: void validationStarted (const String& id) override; void logMessage (const String& m) override; - void itemComplete (const String& id, int numItemFailures, const Array& itemResults) override; + void itemComplete (const String& id, int numItemFailures, const UnitTestResultsWithOutput& itemResults) override; void allItemsComplete() override; void connectionLost() override; File reportFile; - HashMap> results; + HashMap results; }; diff --git a/Source/JUnitReport.cpp b/Source/JUnitReport.cpp index 7c746d3..8827bf9 100644 --- a/Source/JUnitReport.cpp +++ b/Source/JUnitReport.cpp @@ -3,22 +3,36 @@ namespace JUnitReport { -XmlElement* createTestCaseElement(const String& pluginName, const UnitTestRunner::TestResult& r) +XmlElement* createTestCaseElement(const String& pluginName, const std::pair& r) { auto testcase = new XmlElement("testcase"); + testcase->setAttribute("classname", r.first.unitTestName); +#if defined(USE_FILENAME_IN_JUNIT_REPORT) testcase->setAttribute("name", r.subcategoryName); - testcase->setAttribute("classname", r.unitTestName); testcase->setAttribute("file", pluginName); +#else + testcase->setAttribute("name", r.first.subcategoryName + " of " + pluginName); +#endif - auto duration = (r.endTime - r.startTime).inMilliseconds(); + auto duration = (r.first.endTime - r.first.startTime).inMilliseconds(); testcase->setAttribute("time", duration / 1000.0); - for (const auto& m: r.messages) + if (r.first.messages.isEmpty()) { + // Passed test case + auto output = new XmlElement("system-out"); + output->addTextElement(r.second.joinIntoString("\n")); + + testcase->prependChildElement(output); + } + else + { + // Failing test case auto failure = new XmlElement("failure"); failure->setAttribute("type", "ERROR"); - failure->setAttribute("message", m); + failure->setAttribute("message", r.first.messages.joinIntoString(" ")); + failure->addTextElement(r.second.joinIntoString("\n")); testcase->prependChildElement(failure); } @@ -40,7 +54,7 @@ void addTestsStats(XmlElement* element, int tests, int failures, int64 duration) element->setAttribute("time", duration / 1000.0); } -bool write(const HashMap> &allResults, File &output) +bool write(const HashMap &allResults, File &output) { XmlElement testsuites("testsuites"); testsuites.setAttribute("name", "pluginval test suites"); @@ -51,7 +65,7 @@ bool write(const HashMap> &allResults, for (auto it = allResults.begin(); it != allResults.end(); ++it) { const auto results = it.getValue(); - auto pluginName = File(it.getKey()).getFileName(); + auto pluginName = it.getKey(); int suite_failures = 0; int64 suite_duration = 0; @@ -66,8 +80,8 @@ bool write(const HashMap> &allResults, testsuite->prependChildElement(testcase); // calculate totals for test suite - suite_failures += r.failures; - suite_duration += (r.endTime - r.startTime).inMilliseconds(); + suite_failures += r.first.failures; + suite_duration += (r.first.endTime - r.first.startTime).inMilliseconds(); } addTestsStats(testsuite, suite_tests, suite_failures, suite_duration); diff --git a/Source/JUnitReport.h b/Source/JUnitReport.h index fce5e6c..3d17998 100644 --- a/Source/JUnitReport.h +++ b/Source/JUnitReport.h @@ -1,10 +1,11 @@ #pragma once #include +#include "CommonTypes.h" namespace JUnitReport { -bool write(const HashMap > &allResults, File &output); +bool write(const HashMap &allResults, File &output); } // namespace JUnitReport diff --git a/Source/MainComponent.h b/Source/MainComponent.h index 207d90e..5dbdb70 100644 --- a/Source/MainComponent.h +++ b/Source/MainComponent.h @@ -15,6 +15,7 @@ #pragma once #include +#include "CommonTypes.h" #include "Validator.h" #include "CrashHandler.h" @@ -91,7 +92,7 @@ struct ConnectionStatus : public Component, { } - void itemComplete (const String&, int, const Array&) override + void itemComplete (const String&, int, const UnitTestResultsWithOutput&) override { } @@ -194,7 +195,7 @@ struct ConsoleComponent : public Component, std::cout << m << "\n"; } - void itemComplete (const String& id, int numFailures, const Array&) override + void itemComplete (const String& id, int numFailures, const UnitTestResultsWithOutput&) override { logMessage ("\nFinished validating: " + id); diff --git a/Source/Validator.cpp b/Source/Validator.cpp index 00ecd9b..c23b81c 100644 --- a/Source/Validator.cpp +++ b/Source/Validator.cpp @@ -80,10 +80,25 @@ struct PluginsUnitTestRunner : public UnitTestRunner, if (outputStream) *outputStream << message << "\n"; + if (getNumResults() > 0) + { + auto testResult = getResult(getNumResults() - 1); + output.getReference(testResult).add(message); + } + callback (message); } } + StringArray getTestOutput(int testResultIndex) const + { + if (testResultIndex >= 0 && testResultIndex < getNumResults()) + { + return output[getResult((testResultIndex))]; + } + return {}; + } + private: std::function callback; std::unique_ptr outputStream; @@ -91,6 +106,8 @@ struct PluginsUnitTestRunner : public UnitTestRunner, std::atomic timoutTime { -1 }; std::atomic canSendLogMessage { true }; + HashMap output; + void resetTimeout() { timoutTime = (Time::getCurrentTime() + RelativeTime::milliseconds (timeoutMs)).toMilliseconds(); @@ -193,10 +210,10 @@ void updateFileNameIfPossible (PluginTests& test, PluginsUnitTestRunner& runner) //============================================================================== //============================================================================== -inline Array runTests (PluginTests& test, std::function callback) +inline UnitTestResultsWithOutput runTests (PluginTests& test, std::function callback) { const auto options = test.getOptions(); - Array results; + UnitTestResultsWithOutput results; PluginsUnitTestRunner testRunner (std::move (callback), createDestinationFileStream (test), options.timeoutMs); testRunner.setAssertOnFailure (false); @@ -205,29 +222,31 @@ inline Array runTests (PluginTests& test, std::funct testRunner.runTests (testsToRun, options.randomSeed); for (int i = 0; i < testRunner.getNumResults(); ++i) - results.add (*testRunner.getResult (i)); + { + results.add(std::make_pair(*testRunner.getResult (i), testRunner.getTestOutput(i))); + } updateFileNameIfPossible (test, testRunner); return results; } -inline Array validate (const PluginDescription& pluginToValidate, PluginTests::Options options, std::function callback) +inline UnitTestResultsWithOutput validate (const PluginDescription& pluginToValidate, PluginTests::Options options, std::function callback) { PluginTests test (pluginToValidate, options); return runTests (test, std::move (callback)); } -inline Array validate (const String& fileOrIDToValidate, PluginTests::Options options, std::function callback) +inline UnitTestResultsWithOutput validate (const String& fileOrIDToValidate, PluginTests::Options options, std::function callback) { PluginTests test (fileOrIDToValidate, options); return runTests (test, std::move (callback)); } -inline int getNumFailures (Array results) +inline int getNumFailures (const UnitTestResultsWithOutput& results) { return std::accumulate (results.begin(), results.end(), 0, - [] (int count, const UnitTestRunner::TestResult& r) { return count + r.failures; }); + [] (int count, const auto& r) { return count + r.first.failures; }); } //============================================================================== @@ -267,6 +286,9 @@ namespace IDs DECLARE_ID(testFailureMessageArray) DECLARE_ID(testFailureMessageItem) DECLARE_ID(testFailureMessageText) + DECLARE_ID(testOutputMessageArray) + DECLARE_ID(testOutputMessageItem) + DECLARE_ID(testOutputMessageText) DECLARE_ID(testStartTime) DECLARE_ID(testEndTime) @@ -484,32 +506,38 @@ class ValidatorChildProcess : public ChildProcessSlave, processRequest (r); } - static ValueTree serializeTestResults(const Array& results) + static ValueTree serializeTestResults(const UnitTestResultsWithOutput& results) { ValueTree testResultArray { IDs::testResultArray }; for (const auto& r: results) { - ValueTree testFailureMessageArray { - IDs::testFailureMessageArray - }; - for (const auto& m: r.messages) + auto add_messages = [](const StringArray& src, const Identifier& itemId, + const Identifier& textId, ValueTree& dest) { - testFailureMessageArray.appendChild( - { IDs::testFailureMessageItem, - { { IDs::testFailureMessageText, m } } - }, nullptr); - } + for (const auto& m: src) + { + dest.appendChild({ itemId, { { textId, m } } }, nullptr); + } + }; + + ValueTree testFailureMessageArray { IDs::testFailureMessageArray }; + + add_messages(r.first.messages, IDs::testFailureMessageItem, IDs::testFailureMessageText, testFailureMessageArray); + + ValueTree testOutputMessageArray { IDs::testOutputMessageArray }; + + add_messages(r.second, IDs::testOutputMessageItem, IDs::testOutputMessageText, testOutputMessageArray); ValueTree testResultItem { IDs::testResultItem, { - { IDs::testName, r.unitTestName }, - { IDs::testSubcategoryName, r.subcategoryName }, - { IDs::testPassCount, r.passes }, - { IDs::testFailureCount, r.failures }, - { IDs::testStartTime, r.startTime.toMilliseconds() }, - { IDs::testEndTime, r.endTime.toMilliseconds() }, + { IDs::testName, r.first.unitTestName }, + { IDs::testSubcategoryName, r.first.subcategoryName }, + { IDs::testPassCount, r.first.passes }, + { IDs::testFailureCount, r.first.failures }, + { IDs::testStartTime, r.first.startTime.toMilliseconds() }, + { IDs::testEndTime, r.first.endTime.toMilliseconds() }, }, - { testFailureMessageArray } + { testFailureMessageArray, testOutputMessageArray } }; testResultArray.appendChild( testResultItem, nullptr ); @@ -542,7 +570,7 @@ class ValidatorChildProcess : public ChildProcessSlave, for (auto c : v) { String fileOrID; - Array results; + UnitTestResultsWithOutput results; LOG_CHILD("processRequest - child:\n" + toXmlString (c)); if (c.hasProperty (IDs::fileOrID)) @@ -630,7 +658,7 @@ class ValidatorParentProcess : public ChildProcessMaster std::function logMessageCallback; // Callback which can be set to be informed when a validation completes - std::function&)> validationCompleteCallback; + std::function validationCompleteCallback; // Callback which can be set to be informed when all validations have been completed std::function completeCallback; @@ -656,13 +684,13 @@ class ValidatorParentProcess : public ChildProcessMaster return ok ? Result::ok() : Result::fail ("Error: Child failed to launch"); } - static Array deserializeTestResults(const ValueTree& testResultArray) + static UnitTestResultsWithOutput deserializeTestResults(const ValueTree& testResultArray) { if (!testResultArray.hasType(IDs::testResultArray)) { return {}; } - Array results; + UnitTestResultsWithOutput results; for (const auto& testResultItem: testResultArray) { if (!testResultItem.hasType(IDs::testResultItem) || @@ -684,22 +712,31 @@ class ValidatorParentProcess : public ChildProcessMaster result.startTime = Time(testResultItem.getProperty(IDs::testStartTime)); result.endTime = Time(testResultItem.getProperty(IDs::testEndTime)); - auto testFailureMessageArray = testResultItem.getChildWithName(IDs::testFailureMessageArray); - if (!testFailureMessageArray.hasType(IDs::testFailureMessageArray)) - { - continue; - } - for (const auto& testFailureMessageItem: testFailureMessageArray) + auto add_messages = [&testResultItem](const Identifier& arrayId, const Identifier& itemId, + const Identifier& textId, StringArray& dest) { - if (!testFailureMessageItem.hasType(IDs::testFailureMessageItem) || - !testFailureMessageItem.hasProperty(IDs::testFailureMessageText)) + auto array = testResultItem.getChildWithName(arrayId); + if (array.hasType(arrayId)) { - continue; + for (const auto& item: array) + { + if (!item.hasType(itemId) || !item.hasProperty(textId)) + { + continue; + } + dest.add(item.getProperty(textId)); + } } - result.messages.add(testFailureMessageItem.getProperty(IDs::testFailureMessageText)); - } + }; + + add_messages(IDs::testFailureMessageArray, IDs::testFailureMessageItem, + IDs::testFailureMessageText, result.messages); + + StringArray output; + add_messages(IDs::testOutputMessageArray, IDs::testOutputMessageItem, + IDs::testOutputMessageText, output); - results.add(result); + results.add(std::make_pair(result, output)); } return results; } @@ -879,7 +916,7 @@ bool Validator::ensureConnection() parentProcess->validationStartedCallback = [this] (const String& id) { listeners.call (&Listener::validationStarted, id); }; parentProcess->logMessageCallback = [this] (const String& m) { listeners.call (&Listener::logMessage, m); }; - parentProcess->validationCompleteCallback = [this] (const String& id, int numFailures, const Array& results) { listeners.call (&Listener::itemComplete, id, numFailures, results); }; + parentProcess->validationCompleteCallback = [this] (const String& id, int numFailures, const UnitTestResultsWithOutput& results) { listeners.call (&Listener::itemComplete, id, numFailures, results); }; parentProcess->completeCallback = [this] { listeners.call (&Listener::allItemsComplete); triggerAsyncUpdate(); }; const auto result = launchInProcess ? parentProcess->launchInProcess() diff --git a/Source/Validator.h b/Source/Validator.h index 82835cc..3864779 100644 --- a/Source/Validator.h +++ b/Source/Validator.h @@ -15,6 +15,7 @@ #pragma once #include +#include "CommonTypes.h" #include "PluginTests.h" #ifndef LOG_PIPE_COMMUNICATION @@ -94,7 +95,7 @@ class Validator : public ChangeBroadcaster, virtual void validationStarted (const String& idString) = 0; virtual void logMessage (const String&) = 0; - virtual void itemComplete (const String& idString, int numFailures, const Array&) = 0; + virtual void itemComplete (const String& idString, int numFailures, const UnitTestResultsWithOutput&) = 0; virtual void allItemsComplete() = 0; virtual void connectionLost() {} }; From 6cb779ce4193a1bc6dfc0014ab5ba2e48a89de71 Mon Sep 17 00:00:00 2001 From: Maksims Radugins Date: Mon, 1 Nov 2021 11:43:17 +0200 Subject: [PATCH 10/13] Use PluginTestResult instead of std::pair, generate unique testcase names --- CMakeLists.txt | 2 +- Source/CommandLine.cpp | 2 +- Source/CommandLine.h | 4 ++-- Source/CommonTypes.h | 6 ------ Source/JUnitListener.cpp | 2 +- Source/JUnitListener.h | 6 +++--- Source/JUnitReport.cpp | 34 +++++++++++++++++------------- Source/JUnitReport.h | 4 ++-- Source/MainComponent.h | 6 +++--- Source/PluginTestResult.h | 17 +++++++++++++++ Source/Validator.cpp | 44 +++++++++++++++++++-------------------- Source/Validator.h | 4 ++-- 12 files changed, 73 insertions(+), 58 deletions(-) delete mode 100644 Source/CommonTypes.h create mode 100644 Source/PluginTestResult.h diff --git a/CMakeLists.txt b/CMakeLists.txt index f714a8e..1d70414 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,7 +61,7 @@ set(SourceFiles Source/JUnitReport.h Source/JUnitListener.cpp Source/JUnitListener.h - Source/CommonTypes.h + Source/PluginTestResult.h ) target_sources(pluginval PRIVATE ${SourceFiles}) diff --git a/Source/CommandLine.cpp b/Source/CommandLine.cpp index 3e97677..9bf25bd 100644 --- a/Source/CommandLine.cpp +++ b/Source/CommandLine.cpp @@ -83,7 +83,7 @@ void CommandLineValidator::logMessage (const String& m) std::cout << m << "\n"; } -void CommandLineValidator::itemComplete (const String& id, int numItemFailures, const UnitTestResultsWithOutput&) +void CommandLineValidator::itemComplete (const String& id, int numItemFailures, const PluginTestResultArray&) { logMessage ("\nFinished validating: " + id); diff --git a/Source/CommandLine.h b/Source/CommandLine.h index 5838583..a5ec3ad 100644 --- a/Source/CommandLine.h +++ b/Source/CommandLine.h @@ -15,7 +15,7 @@ #pragma once #include -#include "CommonTypes.h" +#include "PluginTestResult.h" #include "Validator.h" #include "JUnitListener.h" @@ -37,7 +37,7 @@ struct CommandLineValidator : private ChangeListener, void validationStarted (const String&) override; void logMessage (const String& m) override; - void itemComplete (const String&, int numItemFailures, const UnitTestResultsWithOutput&) override; + void itemComplete (const String&, int numItemFailures, const PluginTestResultArray&) override; void allItemsComplete() override; void connectionLost() override; }; diff --git a/Source/CommonTypes.h b/Source/CommonTypes.h deleted file mode 100644 index 11fb567..0000000 --- a/Source/CommonTypes.h +++ /dev/null @@ -1,6 +0,0 @@ -#pragma once - -#include - -using UnitTestResultsWithOutput = Array>; - diff --git a/Source/JUnitListener.cpp b/Source/JUnitListener.cpp index 26a5f69..5edd271 100644 --- a/Source/JUnitListener.cpp +++ b/Source/JUnitListener.cpp @@ -19,7 +19,7 @@ void JUnitListener::logMessage(const String &) { } -void JUnitListener::itemComplete(const String &id, int, const UnitTestResultsWithOutput &itemResults) +void JUnitListener::itemComplete(const String &id, int, const PluginTestResultArray &itemResults) { results.set(id, itemResults); } diff --git a/Source/JUnitListener.h b/Source/JUnitListener.h index 88df87f..988f2c9 100644 --- a/Source/JUnitListener.h +++ b/Source/JUnitListener.h @@ -1,7 +1,7 @@ #pragma once #include -#include "CommonTypes.h" +#include "PluginTestResult.h" #include "Validator.h" class JUnitListener: public Validator::Listener @@ -12,10 +12,10 @@ class JUnitListener: public Validator::Listener private: void validationStarted (const String& id) override; void logMessage (const String& m) override; - void itemComplete (const String& id, int numItemFailures, const UnitTestResultsWithOutput& itemResults) override; + void itemComplete (const String& id, int numItemFailures, const PluginTestResultArray& itemResults) override; void allItemsComplete() override; void connectionLost() override; File reportFile; - HashMap results; + HashMap results; }; diff --git a/Source/JUnitReport.cpp b/Source/JUnitReport.cpp index 8827bf9..118eb34 100644 --- a/Source/JUnitReport.cpp +++ b/Source/JUnitReport.cpp @@ -3,36 +3,37 @@ namespace JUnitReport { -XmlElement* createTestCaseElement(const String& pluginName, const std::pair& r) +XmlElement* createTestCaseElement(const String& pluginName, int testIndex, const PluginTestResult& r) { auto testcase = new XmlElement("testcase"); - testcase->setAttribute("classname", r.first.unitTestName); + testcase->setAttribute("classname", r.result.unitTestName); #if defined(USE_FILENAME_IN_JUNIT_REPORT) - testcase->setAttribute("name", r.subcategoryName); + testcase->setAttribute("name", "Test " + String(testIndex + 1) + ": " + r.subcategoryName); testcase->setAttribute("file", pluginName); #else - testcase->setAttribute("name", r.first.subcategoryName + " of " + pluginName); + testcase->setAttribute("name", "Test " + String(testIndex + 1) + ": " + r.result.subcategoryName + " of " + pluginName); #endif - auto duration = (r.first.endTime - r.first.startTime).inMilliseconds(); + auto duration = (r.result.endTime - r.result.startTime).inMilliseconds(); testcase->setAttribute("time", duration / 1000.0); - if (r.first.messages.isEmpty()) + String output = r.output.joinIntoString("\n"); + if (r.result.messages.isEmpty()) { // Passed test case - auto output = new XmlElement("system-out"); - output->addTextElement(r.second.joinIntoString("\n")); + auto system_out = new XmlElement("system-out"); + system_out->addTextElement(output); - testcase->prependChildElement(output); + testcase->prependChildElement(system_out); } else { // Failing test case auto failure = new XmlElement("failure"); failure->setAttribute("type", "ERROR"); - failure->setAttribute("message", r.first.messages.joinIntoString(" ")); - failure->addTextElement(r.second.joinIntoString("\n")); + failure->setAttribute("message", r.result.messages.joinIntoString(" ")); + failure->addTextElement(output); testcase->prependChildElement(failure); } @@ -54,7 +55,7 @@ void addTestsStats(XmlElement* element, int tests, int failures, int64 duration) element->setAttribute("time", duration / 1000.0); } -bool write(const HashMap &allResults, File &output) +bool write(const HashMap &allResults, File &output) { XmlElement testsuites("testsuites"); testsuites.setAttribute("name", "pluginval test suites"); @@ -62,6 +63,7 @@ bool write(const HashMap &allResults, File &o int total_failures = 0; int total_tests = 0; int64 total_duration = 0; + int test_index = 0; for (auto it = allResults.begin(); it != allResults.end(); ++it) { const auto results = it.getValue(); @@ -75,13 +77,15 @@ bool write(const HashMap &allResults, File &o for (const auto& r: results) { - auto testcase = createTestCaseElement(pluginName, r); + auto testcase = createTestCaseElement(pluginName, test_index, r); // add test case to test suite testsuite->prependChildElement(testcase); // calculate totals for test suite - suite_failures += r.first.failures; - suite_duration += (r.first.endTime - r.first.startTime).inMilliseconds(); + suite_failures += r.result.failures; + suite_duration += (r.result.endTime - r.result.startTime).inMilliseconds(); + + test_index++; } addTestsStats(testsuite, suite_tests, suite_failures, suite_duration); diff --git a/Source/JUnitReport.h b/Source/JUnitReport.h index 3d17998..97d24fb 100644 --- a/Source/JUnitReport.h +++ b/Source/JUnitReport.h @@ -1,11 +1,11 @@ #pragma once #include -#include "CommonTypes.h" +#include "PluginTestResult.h" namespace JUnitReport { -bool write(const HashMap &allResults, File &output); +bool write(const HashMap &allResults, File &output); } // namespace JUnitReport diff --git a/Source/MainComponent.h b/Source/MainComponent.h index 5dbdb70..9219436 100644 --- a/Source/MainComponent.h +++ b/Source/MainComponent.h @@ -15,7 +15,7 @@ #pragma once #include -#include "CommonTypes.h" +#include "PluginTestResult.h" #include "Validator.h" #include "CrashHandler.h" @@ -92,7 +92,7 @@ struct ConnectionStatus : public Component, { } - void itemComplete (const String&, int, const UnitTestResultsWithOutput&) override + void itemComplete (const String&, int, const PluginTestResultArray&) override { } @@ -195,7 +195,7 @@ struct ConsoleComponent : public Component, std::cout << m << "\n"; } - void itemComplete (const String& id, int numFailures, const UnitTestResultsWithOutput&) override + void itemComplete (const String& id, int numFailures, const PluginTestResultArray&) override { logMessage ("\nFinished validating: " + id); diff --git a/Source/PluginTestResult.h b/Source/PluginTestResult.h new file mode 100644 index 0000000..acedd13 --- /dev/null +++ b/Source/PluginTestResult.h @@ -0,0 +1,17 @@ +#pragma once + +#include + +struct PluginTestResult +{ + PluginTestResult(const UnitTestRunner::TestResult &result, const StringArray &output): + result(result), output(output) + { + } + + UnitTestRunner::TestResult result; + StringArray output; +}; + +using PluginTestResultArray = Array; + diff --git a/Source/Validator.cpp b/Source/Validator.cpp index c23b81c..aa16d76 100644 --- a/Source/Validator.cpp +++ b/Source/Validator.cpp @@ -210,10 +210,10 @@ void updateFileNameIfPossible (PluginTests& test, PluginsUnitTestRunner& runner) //============================================================================== //============================================================================== -inline UnitTestResultsWithOutput runTests (PluginTests& test, std::function callback) +inline PluginTestResultArray runTests (PluginTests& test, std::function callback) { const auto options = test.getOptions(); - UnitTestResultsWithOutput results; + PluginTestResultArray results; PluginsUnitTestRunner testRunner (std::move (callback), createDestinationFileStream (test), options.timeoutMs); testRunner.setAssertOnFailure (false); @@ -223,7 +223,7 @@ inline UnitTestResultsWithOutput runTests (PluginTests& test, std::function callback) +inline PluginTestResultArray validate (const PluginDescription& pluginToValidate, PluginTests::Options options, std::function callback) { PluginTests test (pluginToValidate, options); return runTests (test, std::move (callback)); } -inline UnitTestResultsWithOutput validate (const String& fileOrIDToValidate, PluginTests::Options options, std::function callback) +inline PluginTestResultArray validate (const String& fileOrIDToValidate, PluginTests::Options options, std::function callback) { PluginTests test (fileOrIDToValidate, options); return runTests (test, std::move (callback)); } -inline int getNumFailures (const UnitTestResultsWithOutput& results) +inline int getNumFailures (const PluginTestResultArray& results) { return std::accumulate (results.begin(), results.end(), 0, - [] (int count, const auto& r) { return count + r.first.failures; }); + [] (int count, const auto& r) { return count + r.result.failures; }); } //============================================================================== @@ -506,7 +506,7 @@ class ValidatorChildProcess : public ChildProcessSlave, processRequest (r); } - static ValueTree serializeTestResults(const UnitTestResultsWithOutput& results) + static ValueTree serializeTestResults(const PluginTestResultArray& results) { ValueTree testResultArray { IDs::testResultArray }; for (const auto& r: results) @@ -522,20 +522,20 @@ class ValidatorChildProcess : public ChildProcessSlave, ValueTree testFailureMessageArray { IDs::testFailureMessageArray }; - add_messages(r.first.messages, IDs::testFailureMessageItem, IDs::testFailureMessageText, testFailureMessageArray); + add_messages(r.result.messages, IDs::testFailureMessageItem, IDs::testFailureMessageText, testFailureMessageArray); ValueTree testOutputMessageArray { IDs::testOutputMessageArray }; - add_messages(r.second, IDs::testOutputMessageItem, IDs::testOutputMessageText, testOutputMessageArray); + add_messages(r.output, IDs::testOutputMessageItem, IDs::testOutputMessageText, testOutputMessageArray); ValueTree testResultItem { IDs::testResultItem, { - { IDs::testName, r.first.unitTestName }, - { IDs::testSubcategoryName, r.first.subcategoryName }, - { IDs::testPassCount, r.first.passes }, - { IDs::testFailureCount, r.first.failures }, - { IDs::testStartTime, r.first.startTime.toMilliseconds() }, - { IDs::testEndTime, r.first.endTime.toMilliseconds() }, + { IDs::testName, r.result.unitTestName }, + { IDs::testSubcategoryName, r.result.subcategoryName }, + { IDs::testPassCount, r.result.passes }, + { IDs::testFailureCount, r.result.failures }, + { IDs::testStartTime, r.result.startTime.toMilliseconds() }, + { IDs::testEndTime, r.result.endTime.toMilliseconds() }, }, { testFailureMessageArray, testOutputMessageArray } }; @@ -570,7 +570,7 @@ class ValidatorChildProcess : public ChildProcessSlave, for (auto c : v) { String fileOrID; - UnitTestResultsWithOutput results; + PluginTestResultArray results; LOG_CHILD("processRequest - child:\n" + toXmlString (c)); if (c.hasProperty (IDs::fileOrID)) @@ -658,7 +658,7 @@ class ValidatorParentProcess : public ChildProcessMaster std::function logMessageCallback; // Callback which can be set to be informed when a validation completes - std::function validationCompleteCallback; + std::function validationCompleteCallback; // Callback which can be set to be informed when all validations have been completed std::function completeCallback; @@ -684,13 +684,13 @@ class ValidatorParentProcess : public ChildProcessMaster return ok ? Result::ok() : Result::fail ("Error: Child failed to launch"); } - static UnitTestResultsWithOutput deserializeTestResults(const ValueTree& testResultArray) + static PluginTestResultArray deserializeTestResults(const ValueTree& testResultArray) { if (!testResultArray.hasType(IDs::testResultArray)) { return {}; } - UnitTestResultsWithOutput results; + PluginTestResultArray results; for (const auto& testResultItem: testResultArray) { if (!testResultItem.hasType(IDs::testResultItem) || @@ -736,7 +736,7 @@ class ValidatorParentProcess : public ChildProcessMaster add_messages(IDs::testOutputMessageArray, IDs::testOutputMessageItem, IDs::testOutputMessageText, output); - results.add(std::make_pair(result, output)); + results.add( { result, output } ); } return results; } @@ -916,7 +916,7 @@ bool Validator::ensureConnection() parentProcess->validationStartedCallback = [this] (const String& id) { listeners.call (&Listener::validationStarted, id); }; parentProcess->logMessageCallback = [this] (const String& m) { listeners.call (&Listener::logMessage, m); }; - parentProcess->validationCompleteCallback = [this] (const String& id, int numFailures, const UnitTestResultsWithOutput& results) { listeners.call (&Listener::itemComplete, id, numFailures, results); }; + parentProcess->validationCompleteCallback = [this] (const String& id, int numFailures, const PluginTestResultArray& results) { listeners.call (&Listener::itemComplete, id, numFailures, results); }; parentProcess->completeCallback = [this] { listeners.call (&Listener::allItemsComplete); triggerAsyncUpdate(); }; const auto result = launchInProcess ? parentProcess->launchInProcess() diff --git a/Source/Validator.h b/Source/Validator.h index 3864779..9c35719 100644 --- a/Source/Validator.h +++ b/Source/Validator.h @@ -15,7 +15,7 @@ #pragma once #include -#include "CommonTypes.h" +#include "PluginTestResult.h" #include "PluginTests.h" #ifndef LOG_PIPE_COMMUNICATION @@ -95,7 +95,7 @@ class Validator : public ChangeBroadcaster, virtual void validationStarted (const String& idString) = 0; virtual void logMessage (const String&) = 0; - virtual void itemComplete (const String& idString, int numFailures, const UnitTestResultsWithOutput&) = 0; + virtual void itemComplete (const String& idString, int numFailures, const PluginTestResultArray&) = 0; virtual void allItemsComplete() = 0; virtual void connectionLost() {} }; From a20ad444cff77998eb521dedfdc4e0a0f0eb564d Mon Sep 17 00:00:00 2001 From: Maksims Radugins Date: Thu, 4 Nov 2021 09:32:12 +0200 Subject: [PATCH 11/13] Add comment on test index in test name --- Source/JUnitReport.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Source/JUnitReport.cpp b/Source/JUnitReport.cpp index 118eb34..72345a4 100644 --- a/Source/JUnitReport.cpp +++ b/Source/JUnitReport.cpp @@ -8,6 +8,8 @@ XmlElement* createTestCaseElement(const String& pluginName, int testIndex, const auto testcase = new XmlElement("testcase"); testcase->setAttribute("classname", r.result.unitTestName); + + // Adding test index here to have unique name for each test case execution #if defined(USE_FILENAME_IN_JUNIT_REPORT) testcase->setAttribute("name", "Test " + String(testIndex + 1) + ": " + r.subcategoryName); testcase->setAttribute("file", pluginName); @@ -78,7 +80,6 @@ bool write(const HashMap &allResults, File &outpu for (const auto& r: results) { auto testcase = createTestCaseElement(pluginName, test_index, r); - // add test case to test suite testsuite->prependChildElement(testcase); // calculate totals for test suite From 53e5f8a5bff4957f3926761dac4c9c9aabe44282 Mon Sep 17 00:00:00 2001 From: Maksims Radugins Date: Thu, 4 Nov 2021 09:33:22 +0200 Subject: [PATCH 12/13] Minor cleanup --- Source/JUnitReport.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/JUnitReport.cpp b/Source/JUnitReport.cpp index 72345a4..3c341a7 100644 --- a/Source/JUnitReport.cpp +++ b/Source/JUnitReport.cpp @@ -17,8 +17,8 @@ XmlElement* createTestCaseElement(const String& pluginName, int testIndex, const testcase->setAttribute("name", "Test " + String(testIndex + 1) + ": " + r.result.subcategoryName + " of " + pluginName); #endif - auto duration = (r.result.endTime - r.result.startTime).inMilliseconds(); - testcase->setAttribute("time", duration / 1000.0); + auto duration = (r.result.endTime - r.result.startTime).inMilliseconds() / 1000.0; + testcase->setAttribute("time", duration); String output = r.output.joinIntoString("\n"); if (r.result.messages.isEmpty()) From d7476a0f9dc47e4eb3922d8fc655d7873b6c5fca Mon Sep 17 00:00:00 2001 From: Maksims Radugins Date: Thu, 23 Dec 2021 21:12:43 +0200 Subject: [PATCH 13/13] Use failure count to detect if test is failing --- Source/JUnitReport.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Source/JUnitReport.cpp b/Source/JUnitReport.cpp index 3c341a7..0a3ce56 100644 --- a/Source/JUnitReport.cpp +++ b/Source/JUnitReport.cpp @@ -21,9 +21,8 @@ XmlElement* createTestCaseElement(const String& pluginName, int testIndex, const testcase->setAttribute("time", duration); String output = r.output.joinIntoString("\n"); - if (r.result.messages.isEmpty()) + if (r.result.failures == 0) { - // Passed test case auto system_out = new XmlElement("system-out"); system_out->addTextElement(output); @@ -31,7 +30,6 @@ XmlElement* createTestCaseElement(const String& pluginName, int testIndex, const } else { - // Failing test case auto failure = new XmlElement("failure"); failure->setAttribute("type", "ERROR"); failure->setAttribute("message", r.result.messages.joinIntoString(" "));