From 97bc97f077061fa4cb244161b8bb5c590b0afe9d Mon Sep 17 00:00:00 2001 From: Jan Nedbal Date: Thu, 13 Mar 2025 14:21:28 +0100 Subject: [PATCH 1/2] Collected data: reduce memory consumption & result cache size --- src/Analyser/Analyser.php | 5 ++-- src/Analyser/AnalyserResult.php | 7 ++--- src/Analyser/FileAnalyser.php | 10 +++---- src/Analyser/FileAnalyserResult.php | 7 ++--- src/Analyser/ResultCache/ResultCache.php | 7 ++--- .../ResultCache/ResultCacheManager.php | 26 ++++++++----------- src/Command/AnalyseApplication.php | 21 ++++++++++++++- src/Command/WorkerCommand.php | 8 ++++-- src/Node/CollectedDataNode.php | 15 ++++------- src/Parallel/ParallelAnalyser.php | 9 ++++--- 10 files changed, 66 insertions(+), 49 deletions(-) diff --git a/src/Analyser/Analyser.php b/src/Analyser/Analyser.php index 4ab99fc5d3..44e3227f9f 100644 --- a/src/Analyser/Analyser.php +++ b/src/Analyser/Analyser.php @@ -3,7 +3,8 @@ namespace PHPStan\Analyser; use Closure; -use PHPStan\Collectors\CollectedData; +use PhpParser\Node; +use PHPStan\Collectors\Collector; use PHPStan\Collectors\Registry as CollectorRegistry; use PHPStan\Rules\Registry as RuleRegistry; use Throwable; @@ -59,7 +60,7 @@ public function analyse( $linesToIgnore = []; $unmatchedLineIgnores = []; - /** @var list $collectedData */ + /** @var array>, list>> $collectedData */ $collectedData = []; $internalErrorsCount = 0; diff --git a/src/Analyser/AnalyserResult.php b/src/Analyser/AnalyserResult.php index 212fdcf422..b082757dcd 100644 --- a/src/Analyser/AnalyserResult.php +++ b/src/Analyser/AnalyserResult.php @@ -2,7 +2,8 @@ namespace PHPStan\Analyser; -use PHPStan\Collectors\CollectedData; +use PhpParser\Node; +use PHPStan\Collectors\Collector; use PHPStan\Dependency\RootExportedNode; use function usort; @@ -22,7 +23,7 @@ final class AnalyserResult * @param list $locallyIgnoredErrors * @param array $linesToIgnore * @param array $unmatchedLineIgnores - * @param list $collectedData + * @param array>, list>> $collectedData * @param list $internalErrors * @param array>|null $dependencies * @param array> $exportedNodes @@ -125,7 +126,7 @@ public function getInternalErrors(): array } /** - * @return list + * @return array>, list>> */ public function getCollectedData(): array { diff --git a/src/Analyser/FileAnalyser.php b/src/Analyser/FileAnalyser.php index 969154c3c8..b3fa2b71e3 100644 --- a/src/Analyser/FileAnalyser.php +++ b/src/Analyser/FileAnalyser.php @@ -7,7 +7,7 @@ use PHPStan\BetterReflection\NodeCompiler\Exception\UnableToCompileNode; use PHPStan\BetterReflection\Reflection\Exception\CircularReference; use PHPStan\BetterReflection\Reflector\Exception\IdentifierNotFound; -use PHPStan\Collectors\CollectedData; +use PHPStan\Collectors\Collector; use PHPStan\Collectors\Registry as CollectorRegistry; use PHPStan\Dependency\DependencyResolver; use PHPStan\Node\FileNode; @@ -76,7 +76,7 @@ public function analyseFile( /** @var list $locallyIgnoredErrors */ $locallyIgnoredErrors = []; - /** @var list $fileCollectedData */ + /** @var array>, list>> $fileCollectedData */ $fileCollectedData = []; $fileDependencies = []; @@ -195,11 +195,7 @@ public function analyseFile( continue; } - $fileCollectedData[] = new CollectedData( - $collectedData, - $scope->getFile(), - get_class($collector), - ); + $fileCollectedData[$scope->getFile()][get_class($collector)][] = $collectedData; } try { diff --git a/src/Analyser/FileAnalyserResult.php b/src/Analyser/FileAnalyserResult.php index d1727f5824..068e281e29 100644 --- a/src/Analyser/FileAnalyserResult.php +++ b/src/Analyser/FileAnalyserResult.php @@ -2,7 +2,8 @@ namespace PHPStan\Analyser; -use PHPStan\Collectors\CollectedData; +use PhpParser\Node; +use PHPStan\Collectors\Collector; use PHPStan\Dependency\RootExportedNode; /** @@ -16,7 +17,7 @@ final class FileAnalyserResult * @param list $filteredPhpErrors * @param list $allPhpErrors * @param list $locallyIgnoredErrors - * @param list $collectedData + * @param array>, list>> $collectedData * @param list $dependencies * @param list $exportedNodes * @param LinesToIgnore $linesToIgnore @@ -69,7 +70,7 @@ public function getLocallyIgnoredErrors(): array } /** - * @return list + * @return array>, list>> */ public function getCollectedData(): array { diff --git a/src/Analyser/ResultCache/ResultCache.php b/src/Analyser/ResultCache/ResultCache.php index f409f9c062..889913132b 100644 --- a/src/Analyser/ResultCache/ResultCache.php +++ b/src/Analyser/ResultCache/ResultCache.php @@ -2,9 +2,10 @@ namespace PHPStan\Analyser\ResultCache; +use PhpParser\Node; use PHPStan\Analyser\Error; use PHPStan\Analyser\FileAnalyserResult; -use PHPStan\Collectors\CollectedData; +use PHPStan\Collectors\Collector; use PHPStan\Dependency\RootExportedNode; /** @@ -20,7 +21,7 @@ final class ResultCache * @param array> $locallyIgnoredErrors * @param array $linesToIgnore * @param array $unmatchedLineIgnores - * @param array> $collectedData + * @param array>, list>> $collectedData * @param array> $dependencies * @param array> $exportedNodes * @param array $projectExtensionFiles @@ -101,7 +102,7 @@ public function getUnmatchedLineIgnores(): array } /** - * @return array> + * @return array>, list>> */ public function getCollectedData(): array { diff --git a/src/Analyser/ResultCache/ResultCacheManager.php b/src/Analyser/ResultCache/ResultCacheManager.php index 0db2ffc546..154c874399 100644 --- a/src/Analyser/ResultCache/ResultCacheManager.php +++ b/src/Analyser/ResultCache/ResultCacheManager.php @@ -3,10 +3,12 @@ namespace PHPStan\Analyser\ResultCache; use Nette\Neon\Neon; +use PhpParser\Node; use PHPStan\Analyser\AnalyserResult; use PHPStan\Analyser\Error; use PHPStan\Analyser\FileAnalyserResult; use PHPStan\Collectors\CollectedData; +use PHPStan\Collectors\Collector; use PHPStan\Command\Output; use PHPStan\Dependency\ExportedNodeFetcher; use PHPStan\Dependency\RootExportedNode; @@ -406,10 +408,7 @@ public function process(AnalyserResult $analyserResult, ResultCache $resultCache $freshLocallyIgnoredErrorsByFile[$error->getFilePath()][] = $error; } - $freshCollectedDataByFile = []; - foreach ($analyserResult->getCollectedData() as $collectedData) { - $freshCollectedDataByFile[$collectedData->getFilePath()][] = $collectedData; - } + $freshCollectedDataByFile = $analyserResult->getCollectedData(); $meta = $resultCache->getMeta(); $projectConfigArray = $meta['projectConfig']; @@ -524,13 +523,6 @@ public function process(AnalyserResult $analyserResult, ResultCache $resultCache } } - $flatCollectedData = []; - foreach ($collectedDataByFile as $fileCollectedData) { - foreach ($fileCollectedData as $collectedData) { - $flatCollectedData[] = $collectedData; - } - } - return new ResultCacheProcessResult(new AnalyserResult( $flatErrors, $analyserResult->getFilteredPhpErrors(), @@ -539,7 +531,7 @@ public function process(AnalyserResult $analyserResult, ResultCache $resultCache $linesToIgnore, $unmatchedLineIgnores, $internalErrors, - $flatCollectedData, + $collectedDataByFile, $dependencies, $exportedNodes, $analyserResult->hasReachedInternalErrorsCountLimit(), @@ -584,8 +576,8 @@ private function mergeLocallyIgnoredErrors(ResultCache $resultCache, array $fres } /** - * @param array> $freshCollectedDataByFile - * @return array> + * @param array>, list>> $freshCollectedDataByFile + * @return array>, list>> */ private function mergeCollectedData(ResultCache $resultCache, array $freshCollectedDataByFile): array { @@ -704,7 +696,7 @@ private function mergeUnmatchedLineIgnores(ResultCache $resultCache, array $fres * @param array> $locallyIgnoredErrors * @param array $linesToIgnore * @param array $unmatchedLineIgnores - * @param array> $collectedData + * @param array>> $collectedData * @param array> $dependencies * @param array> $exportedNodes * @param array $projectExtensionFiles @@ -760,6 +752,10 @@ private function save( ksort($collectedData); ksort($invertedDependencies); + foreach ($collectedData as & $collectedDataPerFile) { + ksort($collectedDataPerFile); + } + foreach ($invertedDependencies as $file => $fileData) { $dependentFiles = $fileData['dependentFiles']; sort($dependentFiles); diff --git a/src/Command/AnalyseApplication.php b/src/Command/AnalyseApplication.php index 88589db6cc..5ec32bd42a 100644 --- a/src/Command/AnalyseApplication.php +++ b/src/Command/AnalyseApplication.php @@ -2,10 +2,13 @@ namespace PHPStan\Command; +use PhpParser\Node; use PHPStan\Analyser\AnalyserResult; use PHPStan\Analyser\AnalyserResultFinalizer; use PHPStan\Analyser\Ignore\IgnoredErrorHelper; use PHPStan\Analyser\ResultCache\ResultCacheManagerFactory; +use PHPStan\Collectors\CollectedData; +use PHPStan\Collectors\Collector; use PHPStan\Internal\BytesHelper; use PHPStan\PhpDoc\StubFilesProvider; use PHPStan\PhpDoc\StubValidator; @@ -150,7 +153,7 @@ public function analyse( $notFileSpecificErrors, $internalErrors, [], - $collectedData, + $this->mapCollectedData($collectedData), $defaultLevelUsed, $projectConfigFile, $savedResultCache, @@ -160,6 +163,22 @@ public function analyse( ); } + /** + * @param array>, list>> $collectedData + * + * @return list + */ + private function mapCollectedData(array $collectedData): array + { + $result = []; + foreach ($collectedData as $file => $dataPerCollector) { + foreach ($dataPerCollector as $collectorType => $rawData) { + $result[] = new CollectedData($rawData, $file, $collectorType); + } + } + return $result; + } + /** * @param string[] $files * @param string[] $allAnalysedFiles diff --git a/src/Command/WorkerCommand.php b/src/Command/WorkerCommand.php index 6e44a11217..27fdb1ce80 100644 --- a/src/Command/WorkerCommand.php +++ b/src/Command/WorkerCommand.php @@ -226,8 +226,12 @@ private function runWorker( foreach ($fileAnalyserResult->getLocallyIgnoredErrors() as $locallyIgnoredError) { $locallyIgnoredErrors[] = $locallyIgnoredError; } - foreach ($fileAnalyserResult->getCollectedData() as $data) { - $collectedData[] = $data; + foreach ($fileAnalyserResult->getCollectedData() as $collectedFile => $dataPerCollector) { + foreach ($dataPerCollector as $collectorType => $collectorData) { + foreach ($collectorData as $data) { + $collectedData[$collectedFile][$collectorType][] = $data; + } + } } } catch (Throwable $t) { $internalErrorsCount++; diff --git a/src/Node/CollectedDataNode.php b/src/Node/CollectedDataNode.php index 8c0f52dc3b..2727dd916f 100644 --- a/src/Node/CollectedDataNode.php +++ b/src/Node/CollectedDataNode.php @@ -4,9 +4,7 @@ use PhpParser\Node; use PhpParser\NodeAbstract; -use PHPStan\Collectors\CollectedData; use PHPStan\Collectors\Collector; -use function array_key_exists; /** * @api @@ -15,7 +13,7 @@ final class CollectedDataNode extends NodeAbstract implements VirtualNode { /** - * @param CollectedData[] $collectedData + * @param array>, list>> $collectedData */ public function __construct(private array $collectedData, private bool $onlyFiles) { @@ -31,17 +29,14 @@ public function __construct(private array $collectedData, private bool $onlyFile public function get(string $collectorType): array { $result = []; - foreach ($this->collectedData as $collectedData) { - if ($collectedData->getCollectorType() !== $collectorType) { + foreach ($this->collectedData as $filePath => $collectedDataPerCollector) { + if (!isset($collectedDataPerCollector[$collectorType])) { continue; } - $filePath = $collectedData->getFilePath(); - if (!array_key_exists($filePath, $result)) { - $result[$filePath] = []; + foreach ($collectedDataPerCollector[$collectorType] as $rawData) { + $result[$filePath][] = $rawData; } - - $result[$filePath][] = $collectedData->getData(); } return $result; diff --git a/src/Parallel/ParallelAnalyser.php b/src/Parallel/ParallelAnalyser.php index 4c31b63050..96dcd36820 100644 --- a/src/Parallel/ParallelAnalyser.php +++ b/src/Parallel/ParallelAnalyser.php @@ -9,7 +9,6 @@ use PHPStan\Analyser\AnalyserResult; use PHPStan\Analyser\Error; use PHPStan\Analyser\InternalError; -use PHPStan\Collectors\CollectedData; use PHPStan\Dependency\RootExportedNode; use PHPStan\Process\ProcessHelper; use React\EventLoop\LoopInterface; @@ -211,8 +210,12 @@ public function analyse( $locallyIgnoredErrors[] = $locallyIgnoredFileError; } - foreach ($json['collectedData'] as $jsonData) { - $collectedData[] = CollectedData::decode($jsonData); + foreach ($json['collectedData'] as $file => $jsonDataByCollector) { + foreach ($jsonDataByCollector as $collectorType => $listOfCollectedData) { + foreach ($listOfCollectedData as $rawCollectedData) { + $collectedData[$file][$collectorType][] = $rawCollectedData; + } + } } /** From 02fb800db9a3706c16099d53b14eaed3b4597142 Mon Sep 17 00:00:00 2001 From: Jan Nedbal Date: Tue, 18 Mar 2025 15:15:52 +0100 Subject: [PATCH 2/2] Extract CollectorData virtual type --- src/Analyser/Analyser.php | 8 +++++--- src/Analyser/AnalyserResult.php | 8 ++++---- src/Analyser/FileAnalyser.php | 7 +++++-- src/Analyser/FileAnalyserResult.php | 8 ++++---- src/Analyser/ResultCache/ResultCache.php | 8 ++++---- src/Analyser/ResultCache/ResultCacheManager.php | 7 +++---- src/Collectors/CollectedData.php | 2 ++ src/Command/AnalyseApplication.php | 7 ++++--- src/Node/CollectedDataNode.php | 4 +++- 9 files changed, 34 insertions(+), 25 deletions(-) diff --git a/src/Analyser/Analyser.php b/src/Analyser/Analyser.php index 44e3227f9f..31599aaee4 100644 --- a/src/Analyser/Analyser.php +++ b/src/Analyser/Analyser.php @@ -3,8 +3,7 @@ namespace PHPStan\Analyser; use Closure; -use PhpParser\Node; -use PHPStan\Collectors\Collector; +use PHPStan\Collectors\CollectedData; use PHPStan\Collectors\Registry as CollectorRegistry; use PHPStan\Rules\Registry as RuleRegistry; use Throwable; @@ -13,6 +12,9 @@ use function count; use function memory_get_peak_usage; +/** + * @phpstan-import-type CollectorData from CollectedData + */ final class Analyser { @@ -60,7 +62,7 @@ public function analyse( $linesToIgnore = []; $unmatchedLineIgnores = []; - /** @var array>, list>> $collectedData */ + /** @var CollectorData $collectedData */ $collectedData = []; $internalErrorsCount = 0; diff --git a/src/Analyser/AnalyserResult.php b/src/Analyser/AnalyserResult.php index b082757dcd..4226e76fbd 100644 --- a/src/Analyser/AnalyserResult.php +++ b/src/Analyser/AnalyserResult.php @@ -2,13 +2,13 @@ namespace PHPStan\Analyser; -use PhpParser\Node; -use PHPStan\Collectors\Collector; +use PHPStan\Collectors\CollectedData; use PHPStan\Dependency\RootExportedNode; use function usort; /** * @phpstan-import-type LinesToIgnore from FileAnalyserResult + * @phpstan-import-type CollectorData from CollectedData */ final class AnalyserResult { @@ -23,7 +23,7 @@ final class AnalyserResult * @param list $locallyIgnoredErrors * @param array $linesToIgnore * @param array $unmatchedLineIgnores - * @param array>, list>> $collectedData + * @param CollectorData $collectedData * @param list $internalErrors * @param array>|null $dependencies * @param array> $exportedNodes @@ -126,7 +126,7 @@ public function getInternalErrors(): array } /** - * @return array>, list>> + * @return CollectorData */ public function getCollectedData(): array { diff --git a/src/Analyser/FileAnalyser.php b/src/Analyser/FileAnalyser.php index b3fa2b71e3..80724ea18a 100644 --- a/src/Analyser/FileAnalyser.php +++ b/src/Analyser/FileAnalyser.php @@ -7,7 +7,7 @@ use PHPStan\BetterReflection\NodeCompiler\Exception\UnableToCompileNode; use PHPStan\BetterReflection\Reflection\Exception\CircularReference; use PHPStan\BetterReflection\Reflector\Exception\IdentifierNotFound; -use PHPStan\Collectors\Collector; +use PHPStan\Collectors\CollectedData; use PHPStan\Collectors\Registry as CollectorRegistry; use PHPStan\Dependency\DependencyResolver; use PHPStan\Node\FileNode; @@ -37,6 +37,9 @@ use const E_USER_WARNING; use const E_WARNING; +/** + * @phpstan-import-type CollectorData from CollectedData + */ final class FileAnalyser { @@ -76,7 +79,7 @@ public function analyseFile( /** @var list $locallyIgnoredErrors */ $locallyIgnoredErrors = []; - /** @var array>, list>> $fileCollectedData */ + /** @var CollectorData $fileCollectedData */ $fileCollectedData = []; $fileDependencies = []; diff --git a/src/Analyser/FileAnalyserResult.php b/src/Analyser/FileAnalyserResult.php index 068e281e29..2aba60730f 100644 --- a/src/Analyser/FileAnalyserResult.php +++ b/src/Analyser/FileAnalyserResult.php @@ -2,12 +2,12 @@ namespace PHPStan\Analyser; -use PhpParser\Node; -use PHPStan\Collectors\Collector; +use PHPStan\Collectors\CollectedData; use PHPStan\Dependency\RootExportedNode; /** * @phpstan-type LinesToIgnore = array|null>> + * @phpstan-import-type CollectorData from CollectedData */ final class FileAnalyserResult { @@ -17,7 +17,7 @@ final class FileAnalyserResult * @param list $filteredPhpErrors * @param list $allPhpErrors * @param list $locallyIgnoredErrors - * @param array>, list>> $collectedData + * @param CollectorData $collectedData * @param list $dependencies * @param list $exportedNodes * @param LinesToIgnore $linesToIgnore @@ -70,7 +70,7 @@ public function getLocallyIgnoredErrors(): array } /** - * @return array>, list>> + * @return CollectorData */ public function getCollectedData(): array { diff --git a/src/Analyser/ResultCache/ResultCache.php b/src/Analyser/ResultCache/ResultCache.php index 889913132b..1708a1f53b 100644 --- a/src/Analyser/ResultCache/ResultCache.php +++ b/src/Analyser/ResultCache/ResultCache.php @@ -2,14 +2,14 @@ namespace PHPStan\Analyser\ResultCache; -use PhpParser\Node; use PHPStan\Analyser\Error; use PHPStan\Analyser\FileAnalyserResult; -use PHPStan\Collectors\Collector; +use PHPStan\Collectors\CollectedData; use PHPStan\Dependency\RootExportedNode; /** * @phpstan-import-type LinesToIgnore from FileAnalyserResult + * @phpstan-import-type CollectorData from CollectedData */ final class ResultCache { @@ -21,7 +21,7 @@ final class ResultCache * @param array> $locallyIgnoredErrors * @param array $linesToIgnore * @param array $unmatchedLineIgnores - * @param array>, list>> $collectedData + * @param CollectorData $collectedData * @param array> $dependencies * @param array> $exportedNodes * @param array $projectExtensionFiles @@ -102,7 +102,7 @@ public function getUnmatchedLineIgnores(): array } /** - * @return array>, list>> + * @return CollectorData */ public function getCollectedData(): array { diff --git a/src/Analyser/ResultCache/ResultCacheManager.php b/src/Analyser/ResultCache/ResultCacheManager.php index 154c874399..80ea03ac06 100644 --- a/src/Analyser/ResultCache/ResultCacheManager.php +++ b/src/Analyser/ResultCache/ResultCacheManager.php @@ -3,12 +3,10 @@ namespace PHPStan\Analyser\ResultCache; use Nette\Neon\Neon; -use PhpParser\Node; use PHPStan\Analyser\AnalyserResult; use PHPStan\Analyser\Error; use PHPStan\Analyser\FileAnalyserResult; use PHPStan\Collectors\CollectedData; -use PHPStan\Collectors\Collector; use PHPStan\Command\Output; use PHPStan\Dependency\ExportedNodeFetcher; use PHPStan\Dependency\RootExportedNode; @@ -51,6 +49,7 @@ /** * @phpstan-import-type LinesToIgnore from FileAnalyserResult + * @phpstan-import-type CollectorData from CollectedData */ final class ResultCacheManager { @@ -576,8 +575,8 @@ private function mergeLocallyIgnoredErrors(ResultCache $resultCache, array $fres } /** - * @param array>, list>> $freshCollectedDataByFile - * @return array>, list>> + * @param CollectorData $freshCollectedDataByFile + * @return CollectorData */ private function mergeCollectedData(ResultCache $resultCache, array $freshCollectedDataByFile): array { diff --git a/src/Collectors/CollectedData.php b/src/Collectors/CollectedData.php index 1ae0078880..f6057f8cf8 100644 --- a/src/Collectors/CollectedData.php +++ b/src/Collectors/CollectedData.php @@ -8,6 +8,8 @@ /** * @api + * + * @phpstan-type CollectorData = array>, list>> */ final class CollectedData implements JsonSerializable { diff --git a/src/Command/AnalyseApplication.php b/src/Command/AnalyseApplication.php index 5ec32bd42a..81793c6ba0 100644 --- a/src/Command/AnalyseApplication.php +++ b/src/Command/AnalyseApplication.php @@ -2,13 +2,11 @@ namespace PHPStan\Command; -use PhpParser\Node; use PHPStan\Analyser\AnalyserResult; use PHPStan\Analyser\AnalyserResultFinalizer; use PHPStan\Analyser\Ignore\IgnoredErrorHelper; use PHPStan\Analyser\ResultCache\ResultCacheManagerFactory; use PHPStan\Collectors\CollectedData; -use PHPStan\Collectors\Collector; use PHPStan\Internal\BytesHelper; use PHPStan\PhpDoc\StubFilesProvider; use PHPStan\PhpDoc\StubValidator; @@ -22,6 +20,9 @@ use function sha1_file; use function sprintf; +/** + * @phpstan-import-type CollectorData from CollectedData + */ final class AnalyseApplication { @@ -164,7 +165,7 @@ public function analyse( } /** - * @param array>, list>> $collectedData + * @param CollectorData $collectedData * * @return list */ diff --git a/src/Node/CollectedDataNode.php b/src/Node/CollectedDataNode.php index 2727dd916f..acc4f24f2d 100644 --- a/src/Node/CollectedDataNode.php +++ b/src/Node/CollectedDataNode.php @@ -4,16 +4,18 @@ use PhpParser\Node; use PhpParser\NodeAbstract; +use PHPStan\Collectors\CollectedData; use PHPStan\Collectors\Collector; /** * @api + * @phpstan-import-type CollectorData from CollectedData */ final class CollectedDataNode extends NodeAbstract implements VirtualNode { /** - * @param array>, list>> $collectedData + * @param CollectorData $collectedData */ public function __construct(private array $collectedData, private bool $onlyFiles) {