From 41e87db02dd3c07a8e125314390d1819aa6f03f9 Mon Sep 17 00:00:00 2001 From: Jens Keim Date: Thu, 30 Jan 2025 16:28:03 +0100 Subject: [PATCH 1/3] feat(Provenance): Add `DirectoryProvenance` as a `LocalProvenance` In contrast to the previously added `RemoteProvenance` stands the `LocalProvenance`, which has no remote source, but instead references a local input of some kind. The `DirectoryProvenance` references a local project directory, which is lacking supported (remote) version control. It is defined by its canonical path only. Since ORT needs further refactoring until `DirectoryProvenance` can be fully utilized, the new class can not be used right now. However the presence of a new `KnownProvenance` class results in `when` conditional cases not being exhaustive anymore. To circumvent this issue, the following changes were made: 1. Wherever possible `RemoteProvenance` is used as parameter type instead of `KnownProvenance`. 2. When necessary `KnownProvenance`s are cast to `RemoteProvenance`. 3. If `Provenance` is expected, `DirectoryProvenance` is handled like `UnknownProvenance`. For instances of `Package` the new default data structure should be `RemoteProvenance`, as a `Package` by definition requires a remote. The exception being `hash` and `storageKey`, which both required a default value, which was set to the `canonicalPath`. However, since the rest of the code does not handle `DirectoryProvenance`, this should remain unused for now. See [1] for more context on the new provenance hierarchy. [1]: https://github.com/oss-review-toolkit/ort/issues/8803#issuecomment-2236958430 Signed-off-by: Jens Keim --- model/src/main/kotlin/Provenance.kt | 17 +++++++++++++++++ .../main/kotlin/ProvenanceResolutionResult.kt | 2 +- .../kotlin/utils/FileProvenanceFileStorage.kt | 2 ++ .../utils/PostgresProvenanceFileStorage.kt | 2 ++ model/src/main/kotlin/utils/PurlExtensions.kt | 7 ++++++- scanner/src/main/kotlin/ScanController.kt | 7 ++++--- scanner/src/main/kotlin/Scanner.kt | 7 ++++--- .../provenance/NestedProvenanceResolver.kt | 6 +++--- .../kotlin/provenance/ProvenanceDownloader.kt | 8 ++++---- .../storages/ProvenanceBasedFileStorage.kt | 7 ++++--- .../storages/ProvenanceBasedPostgresStorage.kt | 7 ++++++- .../src/main/kotlin/utils/FileListResolver.kt | 3 ++- scanner/src/test/kotlin/ScannerTest.kt | 5 +++-- utils/test/src/main/kotlin/Utils.kt | 3 ++- 14 files changed, 60 insertions(+), 23 deletions(-) diff --git a/model/src/main/kotlin/Provenance.kt b/model/src/main/kotlin/Provenance.kt index 5290e442ac5f6..b28497d3454f3 100644 --- a/model/src/main/kotlin/Provenance.kt +++ b/model/src/main/kotlin/Provenance.kt @@ -26,6 +26,8 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize import com.fasterxml.jackson.databind.deser.std.StdDeserializer import com.fasterxml.jackson.module.kotlin.treeToValue +import java.io.File + /** * Provenance information about the origin of source code. */ @@ -45,6 +47,8 @@ sealed interface KnownProvenance : Provenance sealed interface RemoteProvenance : KnownProvenance +sealed interface LocalProvenance : KnownProvenance + /** * Provenance information for a source artifact. */ @@ -83,6 +87,19 @@ data class RepositoryProvenance( override fun matches(pkg: Package): Boolean = vcsInfo == pkg.vcsProcessed } +/** + * Provenance information for a local directory path. + */ +data class DirectoryProvenance( + val canonicalPath: File +) : LocalProvenance { + /** + * Return false for any [pkg] as a [Package] by definition is coming from a remote, and is not stored in a local + * directory path (which would be a [Project]). + */ + override fun matches(pkg: Package): Boolean = false +} + /** * A custom deserializer for polymorphic deserialization of [Provenance] without requiring type information. */ diff --git a/model/src/main/kotlin/ProvenanceResolutionResult.kt b/model/src/main/kotlin/ProvenanceResolutionResult.kt index 2e24156299334..9988904334fe5 100644 --- a/model/src/main/kotlin/ProvenanceResolutionResult.kt +++ b/model/src/main/kotlin/ProvenanceResolutionResult.kt @@ -37,7 +37,7 @@ data class ProvenanceResolutionResult( * The resolved provenance of the package. Can only be null if a [packageProvenanceResolutionIssue] occurred. */ @JsonInclude(JsonInclude.Include.NON_NULL) - val packageProvenance: KnownProvenance? = null, + val packageProvenance: RemoteProvenance? = null, /** * The (recursive) sub-repositories of [packageProvenance]. The map can only be empty if a diff --git a/model/src/main/kotlin/utils/FileProvenanceFileStorage.kt b/model/src/main/kotlin/utils/FileProvenanceFileStorage.kt index 0d510cb1b1e19..75c8982e89df4 100644 --- a/model/src/main/kotlin/utils/FileProvenanceFileStorage.kt +++ b/model/src/main/kotlin/utils/FileProvenanceFileStorage.kt @@ -24,6 +24,7 @@ import java.io.InputStream import org.apache.logging.log4j.kotlin.logger import org.ossreviewtoolkit.model.ArtifactProvenance +import org.ossreviewtoolkit.model.DirectoryProvenance import org.ossreviewtoolkit.model.HashAlgorithm import org.ossreviewtoolkit.model.KnownProvenance import org.ossreviewtoolkit.model.RepositoryProvenance @@ -81,6 +82,7 @@ private fun KnownProvenance.hash(): String { val key = when (this) { is ArtifactProvenance -> "${sourceArtifact.url}${sourceArtifact.hash.value}" is RepositoryProvenance -> "${vcsInfo.type}${vcsInfo.url}$resolvedRevision" + is DirectoryProvenance -> "$canonicalPath" } return HashAlgorithm.SHA1.calculate(key.toByteArray()) diff --git a/model/src/main/kotlin/utils/PostgresProvenanceFileStorage.kt b/model/src/main/kotlin/utils/PostgresProvenanceFileStorage.kt index 5163bd3d18821..a9c6f643481f1 100644 --- a/model/src/main/kotlin/utils/PostgresProvenanceFileStorage.kt +++ b/model/src/main/kotlin/utils/PostgresProvenanceFileStorage.kt @@ -36,6 +36,7 @@ import org.jetbrains.exposed.sql.insert import org.jetbrains.exposed.sql.selectAll import org.ossreviewtoolkit.model.ArtifactProvenance +import org.ossreviewtoolkit.model.DirectoryProvenance import org.ossreviewtoolkit.model.KnownProvenance import org.ossreviewtoolkit.model.RepositoryProvenance import org.ossreviewtoolkit.model.utils.DatabaseUtils.checkDatabaseEncoding @@ -121,4 +122,5 @@ private fun KnownProvenance.storageKey(): String = is ArtifactProvenance -> "source-artifact|${sourceArtifact.url}|${sourceArtifact.hash}" // The trailing "|" is kept for backward compatibility because there used to be an additional parameter. is RepositoryProvenance -> "vcs|${vcsInfo.type}|${vcsInfo.url}|$resolvedRevision|" + is DirectoryProvenance -> "directory|$canonicalPath" } diff --git a/model/src/main/kotlin/utils/PurlExtensions.kt b/model/src/main/kotlin/utils/PurlExtensions.kt index 9defd45cc47f1..3f97dfd775b4a 100644 --- a/model/src/main/kotlin/utils/PurlExtensions.kt +++ b/model/src/main/kotlin/utils/PurlExtensions.kt @@ -27,6 +27,7 @@ import org.ossreviewtoolkit.model.Identifier import org.ossreviewtoolkit.model.Package import org.ossreviewtoolkit.model.Provenance import org.ossreviewtoolkit.model.RemoteArtifact +import org.ossreviewtoolkit.model.RemoteProvenance import org.ossreviewtoolkit.model.RepositoryProvenance import org.ossreviewtoolkit.model.UnknownProvenance import org.ossreviewtoolkit.model.VcsInfo @@ -101,7 +102,11 @@ fun Provenance.toPurlExtras(): PurlExtras = ) } - is UnknownProvenance -> PurlExtras() + /** + * Purls refer to packages that have been published and thus always have a remove provenance. So just return + * empty extras in all other cases. + */ + !is RemoteProvenance -> PurlExtras() } /** diff --git a/scanner/src/main/kotlin/ScanController.kt b/scanner/src/main/kotlin/ScanController.kt index 3a9cc2af5412b..074a160ad3511 100644 --- a/scanner/src/main/kotlin/ScanController.kt +++ b/scanner/src/main/kotlin/ScanController.kt @@ -24,6 +24,7 @@ import org.ossreviewtoolkit.model.Issue import org.ossreviewtoolkit.model.KnownProvenance import org.ossreviewtoolkit.model.Package import org.ossreviewtoolkit.model.Provenance +import org.ossreviewtoolkit.model.RemoteProvenance import org.ossreviewtoolkit.model.RepositoryProvenance import org.ossreviewtoolkit.model.ScanResult import org.ossreviewtoolkit.model.ScanSummary @@ -79,7 +80,7 @@ internal class ScanController( * A map of package [Identifier]s to their resolved [KnownProvenance]s. These provenances are used to filter the * scan results for a package based on the VCS path. */ - private val packageProvenances = mutableMapOf() + private val packageProvenances = mutableMapOf() /** * A map of package [Identifier]s to their resolved [KnownProvenance]s with the VCS path removed. These provenances @@ -130,7 +131,7 @@ internal class ScanController( /** * Set the [provenance] for the package denoted by [id], overwriting any existing values. */ - fun putPackageProvenance(id: Identifier, provenance: KnownProvenance) { + fun putPackageProvenance(id: Identifier, provenance: RemoteProvenance) { packageProvenances[id] = provenance packageProvenancesWithoutVcsPath[id] = when (provenance) { is RepositoryProvenance -> provenance.copy(vcsInfo = provenance.vcsInfo.copy(path = "")) @@ -267,7 +268,7 @@ internal class ScanController( fun getPackagesForProvenanceWithoutVcsPath(provenance: KnownProvenance): Set = packageProvenancesWithoutVcsPath.filter { (_, packageProvenance) -> packageProvenance == provenance }.keys - fun getPackageProvenance(id: Identifier): KnownProvenance? = packageProvenances[id] + fun getPackageProvenance(id: Identifier): RemoteProvenance? = packageProvenances[id] /** * Return the package provenanceResolutionIssue associated with the given [id]. diff --git a/scanner/src/main/kotlin/Scanner.kt b/scanner/src/main/kotlin/Scanner.kt index 2ef1dee10e2cb..ff3ec52b80d6a 100644 --- a/scanner/src/main/kotlin/Scanner.kt +++ b/scanner/src/main/kotlin/Scanner.kt @@ -44,6 +44,7 @@ import org.ossreviewtoolkit.model.OrtResult import org.ossreviewtoolkit.model.Package import org.ossreviewtoolkit.model.PackageType import org.ossreviewtoolkit.model.ProvenanceResolutionResult +import org.ossreviewtoolkit.model.RemoteProvenance import org.ossreviewtoolkit.model.ScanResult import org.ossreviewtoolkit.model.ScannerRun import org.ossreviewtoolkit.model.TextLocation @@ -235,7 +236,7 @@ class Scanner( }.awaitAll() }.forEach { (pkg, result) -> result.onSuccess { provenance -> - controller.putPackageProvenance(pkg.id, provenance) + controller.putPackageProvenance(pkg.id, provenance as RemoteProvenance) }.onFailure { controller.putPackageProvenanceResolutionIssue( pkg.id, @@ -256,7 +257,7 @@ class Scanner( controller.getPackageProvenancesWithoutVcsPath().map { provenance -> async { provenance to runCatching { - nestedProvenanceResolver.resolveNestedProvenance(provenance) + nestedProvenanceResolver.resolveNestedProvenance(provenance as RemoteProvenance) }.onFailure { if (it is CancellationException) currentCoroutineContext().ensureActive() } @@ -591,7 +592,7 @@ class Scanner( controller: ScanController ): Map { val downloadDir = try { - provenanceDownloader.download(provenance) + provenanceDownloader.download(provenance as RemoteProvenance) } catch (e: DownloadException) { val issue = createAndLogIssue( "Downloader", "Could not download provenance $provenance: ${e.collectMessages()}" diff --git a/scanner/src/main/kotlin/provenance/NestedProvenanceResolver.kt b/scanner/src/main/kotlin/provenance/NestedProvenanceResolver.kt index 29e537376bc6d..ea9e2089cb784 100644 --- a/scanner/src/main/kotlin/provenance/NestedProvenanceResolver.kt +++ b/scanner/src/main/kotlin/provenance/NestedProvenanceResolver.kt @@ -23,8 +23,8 @@ import org.apache.logging.log4j.kotlin.logger import org.ossreviewtoolkit.downloader.WorkingTreeCache import org.ossreviewtoolkit.model.ArtifactProvenance -import org.ossreviewtoolkit.model.KnownProvenance import org.ossreviewtoolkit.model.Provenance +import org.ossreviewtoolkit.model.RemoteProvenance import org.ossreviewtoolkit.model.RepositoryProvenance /** @@ -36,7 +36,7 @@ interface NestedProvenanceResolver { * [NestedProvenance] always contains only the provided [ArtifactProvenance]. For a [RepositoryProvenance] the * resolver looks for nested repositories, for example Git submodules or Mercurial subrepositories. */ - suspend fun resolveNestedProvenance(provenance: KnownProvenance): NestedProvenance + suspend fun resolveNestedProvenance(provenance: RemoteProvenance): NestedProvenance } /** @@ -46,7 +46,7 @@ class DefaultNestedProvenanceResolver( private val storage: NestedProvenanceStorage, private val workingTreeCache: WorkingTreeCache ) : NestedProvenanceResolver { - override suspend fun resolveNestedProvenance(provenance: KnownProvenance): NestedProvenance { + override suspend fun resolveNestedProvenance(provenance: RemoteProvenance): NestedProvenance { return when (provenance) { is ArtifactProvenance -> NestedProvenance(root = provenance, subRepositories = emptyMap()) is RepositoryProvenance -> resolveNestedRepository(provenance) diff --git a/scanner/src/main/kotlin/provenance/ProvenanceDownloader.kt b/scanner/src/main/kotlin/provenance/ProvenanceDownloader.kt index c57287edebba3..5f9118331a261 100644 --- a/scanner/src/main/kotlin/provenance/ProvenanceDownloader.kt +++ b/scanner/src/main/kotlin/provenance/ProvenanceDownloader.kt @@ -32,8 +32,8 @@ import org.ossreviewtoolkit.downloader.DownloadException import org.ossreviewtoolkit.downloader.Downloader import org.ossreviewtoolkit.downloader.WorkingTreeCache import org.ossreviewtoolkit.model.ArtifactProvenance -import org.ossreviewtoolkit.model.KnownProvenance import org.ossreviewtoolkit.model.Package +import org.ossreviewtoolkit.model.RemoteProvenance import org.ossreviewtoolkit.model.RepositoryProvenance import org.ossreviewtoolkit.model.config.DownloaderConfiguration import org.ossreviewtoolkit.utils.common.safeDeleteRecursively @@ -51,7 +51,7 @@ fun interface ProvenanceDownloader { * * Throws a [DownloadException] if the download fails. */ - fun download(provenance: KnownProvenance): File + fun download(provenance: RemoteProvenance): File /** * Download the source code specified by the provided [nestedProvenance] incl. sub-repositories and return the path @@ -64,7 +64,7 @@ fun interface ProvenanceDownloader { // Use the provenanceDownloader to download each provenance from nestedProvenance separately, because they are // likely already cached if a path scanner wrapper is used. - val root = download(nestedProvenance.root) + val root = download(nestedProvenance.root as RemoteProvenance) nestedProvenance.subRepositories.forEach { (path, provenance) -> val tempDir = download(provenance) @@ -86,7 +86,7 @@ class DefaultProvenanceDownloader( ) : ProvenanceDownloader { private val downloader = Downloader(config) - override fun download(provenance: KnownProvenance): File { + override fun download(provenance: RemoteProvenance): File { val downloadDir = createOrtTempDir() when (provenance) { diff --git a/scanner/src/main/kotlin/storages/ProvenanceBasedFileStorage.kt b/scanner/src/main/kotlin/storages/ProvenanceBasedFileStorage.kt index a5acedb5b45cc..7254263ce838a 100644 --- a/scanner/src/main/kotlin/storages/ProvenanceBasedFileStorage.kt +++ b/scanner/src/main/kotlin/storages/ProvenanceBasedFileStorage.kt @@ -29,6 +29,7 @@ import org.apache.logging.log4j.kotlin.logger import org.ossreviewtoolkit.model.ArtifactProvenance import org.ossreviewtoolkit.model.KnownProvenance +import org.ossreviewtoolkit.model.RemoteProvenance import org.ossreviewtoolkit.model.RepositoryProvenance import org.ossreviewtoolkit.model.ScanResult import org.ossreviewtoolkit.model.yamlMapper @@ -45,7 +46,7 @@ class ProvenanceBasedFileStorage(private val backend: FileStorage) : ProvenanceB override fun read(provenance: KnownProvenance, scannerMatcher: ScannerMatcher?): List { requireEmptyVcsPath(provenance) - val path = storagePath(provenance) + val path = storagePath(provenance as RemoteProvenance) return runCatching { backend.read(path).use { input -> @@ -97,7 +98,7 @@ class ProvenanceBasedFileStorage(private val backend: FileStorage) : ProvenanceB val scanResults = existingScanResults + scanResult - val path = storagePath(provenance) + val path = storagePath(provenance as RemoteProvenance) val yamlBytes = yamlMapper.writeValueAsBytes(scanResults) val input = ByteArrayInputStream(yamlBytes) @@ -123,7 +124,7 @@ class ProvenanceBasedFileStorage(private val backend: FileStorage) : ProvenanceB } } -private fun storagePath(provenance: KnownProvenance) = +private fun storagePath(provenance: RemoteProvenance) = when (provenance) { is ArtifactProvenance -> "artifact/${provenance.sourceArtifact.url.fileSystemEncode()}/$SCAN_RESULTS_FILE_NAME" is RepositoryProvenance -> { diff --git a/scanner/src/main/kotlin/storages/ProvenanceBasedPostgresStorage.kt b/scanner/src/main/kotlin/storages/ProvenanceBasedPostgresStorage.kt index bac29f155c249..560b8160e7a0d 100644 --- a/scanner/src/main/kotlin/storages/ProvenanceBasedPostgresStorage.kt +++ b/scanner/src/main/kotlin/storages/ProvenanceBasedPostgresStorage.kt @@ -37,6 +37,7 @@ import org.jetbrains.exposed.sql.selectAll import org.ossreviewtoolkit.model.ArtifactProvenance import org.ossreviewtoolkit.model.KnownProvenance +import org.ossreviewtoolkit.model.RemoteProvenance import org.ossreviewtoolkit.model.RepositoryProvenance import org.ossreviewtoolkit.model.ScanResult import org.ossreviewtoolkit.model.ScanSummary @@ -87,6 +88,10 @@ class ProvenanceBasedPostgresStorage( return database.transaction { val query = table.selectAll() + if (provenance !is RemoteProvenance) { + throw ScanStorageException("Scan result must have a known provenance, but it is $provenance.") + } + when (provenance) { is ArtifactProvenance -> { query.andWhere { @@ -138,7 +143,7 @@ class ProvenanceBasedPostgresStorage( requireEmptyVcsPath(provenance) - if (provenance !is KnownProvenance) { + if (provenance !is RemoteProvenance) { throw ScanStorageException("Scan result must have a known provenance, but it is $provenance.") } diff --git a/scanner/src/main/kotlin/utils/FileListResolver.kt b/scanner/src/main/kotlin/utils/FileListResolver.kt index 211b7b1a94346..15b7409526b7b 100644 --- a/scanner/src/main/kotlin/utils/FileListResolver.kt +++ b/scanner/src/main/kotlin/utils/FileListResolver.kt @@ -26,6 +26,7 @@ import java.io.File import org.ossreviewtoolkit.model.HashAlgorithm import org.ossreviewtoolkit.model.KnownProvenance +import org.ossreviewtoolkit.model.RemoteProvenance import org.ossreviewtoolkit.model.toYaml import org.ossreviewtoolkit.model.utils.ProvenanceFileStorage import org.ossreviewtoolkit.model.yamlMapper @@ -58,7 +59,7 @@ class FileListResolver( fun resolve(provenance: KnownProvenance): FileList { storage.getFileList(provenance)?.let { return it } - val dir = provenanceDownloader.download(provenance) + val dir = provenanceDownloader.download(provenance as RemoteProvenance) return createFileList(dir).also { try { diff --git a/scanner/src/test/kotlin/ScannerTest.kt b/scanner/src/test/kotlin/ScannerTest.kt index a61911176a47b..16c345d09094d 100644 --- a/scanner/src/test/kotlin/ScannerTest.kt +++ b/scanner/src/test/kotlin/ScannerTest.kt @@ -57,6 +57,7 @@ import org.ossreviewtoolkit.model.PackageType import org.ossreviewtoolkit.model.Provenance import org.ossreviewtoolkit.model.ProvenanceResolutionResult import org.ossreviewtoolkit.model.RemoteArtifact +import org.ossreviewtoolkit.model.RemoteProvenance import org.ossreviewtoolkit.model.RepositoryProvenance import org.ossreviewtoolkit.model.ScanResult import org.ossreviewtoolkit.model.ScanSummary @@ -1168,7 +1169,7 @@ private class FakePathScannerWrapper : PathScannerWrapper { * provenance, instead of actually downloading the source code. */ private class FakeProvenanceDownloader(val filename: String = "fake.txt") : ProvenanceDownloader { - override fun download(provenance: KnownProvenance): File = + override fun download(provenance: RemoteProvenance): File = createOrtTempDir().apply { resolve(filename).writeText(provenance.toYaml()) } @@ -1207,7 +1208,7 @@ private class FakePackageProvenanceResolver : PackageProvenanceResolver { * An implementation of [NestedProvenanceResolver] that always returns a non-nested provenance. */ private class FakeNestedProvenanceResolver : NestedProvenanceResolver { - override suspend fun resolveNestedProvenance(provenance: KnownProvenance): NestedProvenance = + override suspend fun resolveNestedProvenance(provenance: RemoteProvenance): NestedProvenance = NestedProvenance(root = provenance, subRepositories = emptyMap()) } diff --git a/utils/test/src/main/kotlin/Utils.kt b/utils/test/src/main/kotlin/Utils.kt index 89f683869f0cc..70f74af31848f 100644 --- a/utils/test/src/main/kotlin/Utils.kt +++ b/utils/test/src/main/kotlin/Utils.kt @@ -34,6 +34,7 @@ import org.ossreviewtoolkit.model.KnownProvenance import org.ossreviewtoolkit.model.Package import org.ossreviewtoolkit.model.ProvenanceResolutionResult import org.ossreviewtoolkit.model.RemoteArtifact +import org.ossreviewtoolkit.model.RemoteProvenance import org.ossreviewtoolkit.model.RepositoryProvenance import org.ossreviewtoolkit.model.ScanResult import org.ossreviewtoolkit.model.ScannerRun @@ -193,7 +194,7 @@ fun scannerRunOf(vararg pkgScanResults: Pair>): Sca provenances = pkgScanResultsWithKnownProvenance.mapTo(mutableSetOf()) { (id, scanResultsForId) -> ProvenanceResolutionResult( id = id, - packageProvenance = scanResultsForId.firstOrNull()?.provenance as KnownProvenance + packageProvenance = scanResultsForId.firstOrNull()?.provenance as RemoteProvenance ) }, scanResults = scanResults, From 339c154b2799f285bc23cf80ae5f08d0d329fd17 Mon Sep 17 00:00:00 2001 From: Jens Keim Date: Thu, 20 Feb 2025 13:38:34 +0100 Subject: [PATCH 2/3] refactor: Remove `RemoteProvenance` casts Casts were removed from `FileListResolver`, `ProvenanceBasedFileStorage`, `ProvenanceDownloader`, and `Scanner`. In order to avoid casting `KnownProvenance` to `RemoteProvenance`, a lot of parameters need to be changed to `RemoteProvenance`. With the exception of `ProvenanceBasedFileStorage`, which now uses `UnknownProvenance` exceptions just like `ProvenanceBasedPostgresStorage`. Signed-off-by: Jens Keim --- .../AbstractNestedProvenanceStorageFunTest.kt | 4 ++-- scanner/src/main/kotlin/ScanController.kt | 12 ++++++------ scanner/src/main/kotlin/Scanner.kt | 8 ++++---- .../src/main/kotlin/provenance/NestedProvenance.kt | 5 +++-- .../kotlin/provenance/NestedProvenanceScanResult.kt | 5 +++-- .../kotlin/provenance/PackageProvenanceResolver.kt | 6 +++--- .../main/kotlin/provenance/ProvenanceDownloader.kt | 2 +- .../kotlin/storages/ProvenanceBasedFileStorage.kt | 10 +++++++--- scanner/src/main/kotlin/utils/FileListResolver.kt | 4 ++-- scanner/src/test/kotlin/ScannerTest.kt | 6 +++--- 10 files changed, 34 insertions(+), 28 deletions(-) diff --git a/scanner/src/funTest/kotlin/provenance/AbstractNestedProvenanceStorageFunTest.kt b/scanner/src/funTest/kotlin/provenance/AbstractNestedProvenanceStorageFunTest.kt index df5eade85b1ff..0670a07b0ebe7 100644 --- a/scanner/src/funTest/kotlin/provenance/AbstractNestedProvenanceStorageFunTest.kt +++ b/scanner/src/funTest/kotlin/provenance/AbstractNestedProvenanceStorageFunTest.kt @@ -23,7 +23,7 @@ import io.kotest.core.listeners.TestListener import io.kotest.core.spec.style.WordSpec import io.kotest.matchers.shouldBe -import org.ossreviewtoolkit.model.KnownProvenance +import org.ossreviewtoolkit.model.RemoteProvenance import org.ossreviewtoolkit.model.RepositoryProvenance import org.ossreviewtoolkit.model.VcsInfo import org.ossreviewtoolkit.model.VcsType @@ -78,6 +78,6 @@ private fun createRepositoryProvenance( ) = RepositoryProvenance(vcsInfo, resolvedRevision) private fun createNestedProvenance( - root: KnownProvenance, + root: RemoteProvenance, subRepositories: Map = emptyMap() ) = NestedProvenance(root, subRepositories) diff --git a/scanner/src/main/kotlin/ScanController.kt b/scanner/src/main/kotlin/ScanController.kt index 074a160ad3511..b7b093ef48203 100644 --- a/scanner/src/main/kotlin/ScanController.kt +++ b/scanner/src/main/kotlin/ScanController.kt @@ -74,7 +74,7 @@ internal class ScanController( /** * A map of [KnownProvenance]s to their resolved [NestedProvenance]s. */ - private val nestedProvenances = mutableMapOf() + private val nestedProvenances = mutableMapOf() /** * A map of package [Identifier]s to their resolved [KnownProvenance]s. These provenances are used to filter the @@ -86,7 +86,7 @@ internal class ScanController( * A map of package [Identifier]s to their resolved [KnownProvenance]s with the VCS path removed. These provenances * are used during scanning to make sure that always the full repositories are scanned. */ - private val packageProvenancesWithoutVcsPath = mutableMapOf() + private val packageProvenancesWithoutVcsPath = mutableMapOf() /** * The [ScanResult]s for each [KnownProvenance] and [ScannerWrapper]. @@ -143,7 +143,7 @@ internal class ScanController( * Set the [nestedProvenance] corresponding to the given [package provenance][root], overwriting any existing * values. */ - fun putNestedProvenance(root: KnownProvenance, nestedProvenance: NestedProvenance) { + fun putNestedProvenance(root: RemoteProvenance, nestedProvenance: NestedProvenance) { nestedProvenances[root] = nestedProvenance } @@ -166,7 +166,7 @@ internal class ScanController( /** * Return all [KnownProvenance]s contained in [nestedProvenances]. */ - fun getAllProvenances(): Set = + fun getAllProvenances(): Set = nestedProvenances.values.flatMapTo(mutableSetOf()) { it.allProvenances } /** @@ -178,7 +178,7 @@ internal class ScanController( /** * Return all provenances including sub-repositories associated with the identifiers of the packages they belong to. */ - fun getIdsByProvenance(): Map> = + fun getIdsByProvenance(): Map> = buildMap<_, MutableSet> { getNestedProvenancesByPackage().forEach { (pkg, nestedProvenance) -> nestedProvenance.allProvenances.forEach { provenance -> @@ -278,7 +278,7 @@ internal class ScanController( /** * Return all [KnownProvenance]s for the [packages] with the VCS path removed. */ - fun getPackageProvenancesWithoutVcsPath(): Set = packageProvenancesWithoutVcsPath.values.toSet() + fun getPackageProvenancesWithoutVcsPath(): Set = packageProvenancesWithoutVcsPath.values.toSet() /** * Return all [PackageScannerWrapper]s. diff --git a/scanner/src/main/kotlin/Scanner.kt b/scanner/src/main/kotlin/Scanner.kt index ff3ec52b80d6a..b39b6b4a0ece9 100644 --- a/scanner/src/main/kotlin/Scanner.kt +++ b/scanner/src/main/kotlin/Scanner.kt @@ -236,7 +236,7 @@ class Scanner( }.awaitAll() }.forEach { (pkg, result) -> result.onSuccess { provenance -> - controller.putPackageProvenance(pkg.id, provenance as RemoteProvenance) + controller.putPackageProvenance(pkg.id, provenance) }.onFailure { controller.putPackageProvenanceResolutionIssue( pkg.id, @@ -257,7 +257,7 @@ class Scanner( controller.getPackageProvenancesWithoutVcsPath().map { provenance -> async { provenance to runCatching { - nestedProvenanceResolver.resolveNestedProvenance(provenance as RemoteProvenance) + nestedProvenanceResolver.resolveNestedProvenance(provenance) }.onFailure { if (it is CancellationException) currentCoroutineContext().ensureActive() } @@ -586,13 +586,13 @@ class Scanner( } private fun scanPath( - provenance: KnownProvenance, + provenance: RemoteProvenance, scanners: List, context: ScanContext, controller: ScanController ): Map { val downloadDir = try { - provenanceDownloader.download(provenance as RemoteProvenance) + provenanceDownloader.download(provenance) } catch (e: DownloadException) { val issue = createAndLogIssue( "Downloader", "Could not download provenance $provenance: ${e.collectMessages()}" diff --git a/scanner/src/main/kotlin/provenance/NestedProvenance.kt b/scanner/src/main/kotlin/provenance/NestedProvenance.kt index 66aea16af156a..17b43cee1e620 100644 --- a/scanner/src/main/kotlin/provenance/NestedProvenance.kt +++ b/scanner/src/main/kotlin/provenance/NestedProvenance.kt @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore import org.ossreviewtoolkit.model.ArtifactProvenance import org.ossreviewtoolkit.model.KnownProvenance +import org.ossreviewtoolkit.model.RemoteProvenance import org.ossreviewtoolkit.model.RepositoryProvenance /** @@ -32,7 +33,7 @@ data class NestedProvenance( /** * The root provenance that contains the [nested provenances][subRepositories]. */ - val root: KnownProvenance, + val root: RemoteProvenance, /** * If [root] is a [RepositoryProvenance] this map contains all paths which contain nested repositories associated @@ -45,7 +46,7 @@ data class NestedProvenance( * The set of all contained [KnownProvenance]s. */ @JsonIgnore - val allProvenances: Set = + val allProvenances: Set = buildSet { add(root) addAll(subRepositories.values) diff --git a/scanner/src/main/kotlin/provenance/NestedProvenanceScanResult.kt b/scanner/src/main/kotlin/provenance/NestedProvenanceScanResult.kt index 5205e84a848ba..8e3780cb97bc6 100644 --- a/scanner/src/main/kotlin/provenance/NestedProvenanceScanResult.kt +++ b/scanner/src/main/kotlin/provenance/NestedProvenanceScanResult.kt @@ -21,6 +21,7 @@ package org.ossreviewtoolkit.scanner.provenance import org.ossreviewtoolkit.model.KnownProvenance import org.ossreviewtoolkit.model.OrtResult +import org.ossreviewtoolkit.model.RemoteProvenance import org.ossreviewtoolkit.model.RepositoryProvenance import org.ossreviewtoolkit.model.ScanResult import org.ossreviewtoolkit.model.ScanSummary @@ -38,7 +39,7 @@ data class NestedProvenanceScanResult( /** * A map of [KnownProvenance]s from [nestedProvenance] associated with lists of [ScanResult]s. */ - val scanResults: Map> + val scanResults: Map> ) { /** * Return true if [scanResults] contains at least one scan result for each of the [KnownProvenance]s contained in @@ -107,7 +108,7 @@ data class NestedProvenanceScanResult( } } - fun KnownProvenance.withVcsPath() = + fun RemoteProvenance.withVcsPath() = when (this) { is RepositoryProvenance -> { val pathWithinProvenance = pathsWithinProvenances.getValue(this) diff --git a/scanner/src/main/kotlin/provenance/PackageProvenanceResolver.kt b/scanner/src/main/kotlin/provenance/PackageProvenanceResolver.kt index ad7ba9fcbb541..18ccb39d4712d 100644 --- a/scanner/src/main/kotlin/provenance/PackageProvenanceResolver.kt +++ b/scanner/src/main/kotlin/provenance/PackageProvenanceResolver.kt @@ -33,10 +33,10 @@ import org.apache.logging.log4j.kotlin.logger import org.ossreviewtoolkit.downloader.WorkingTreeCache import org.ossreviewtoolkit.model.ArtifactProvenance -import org.ossreviewtoolkit.model.KnownProvenance import org.ossreviewtoolkit.model.Package import org.ossreviewtoolkit.model.Provenance import org.ossreviewtoolkit.model.RemoteArtifact +import org.ossreviewtoolkit.model.RemoteProvenance import org.ossreviewtoolkit.model.RepositoryProvenance import org.ossreviewtoolkit.model.SourceCodeOrigin import org.ossreviewtoolkit.model.VcsInfo @@ -54,7 +54,7 @@ interface PackageProvenanceResolver { * * Throws an [IOException] if the provenance cannot be resolved. */ - suspend fun resolveProvenance(pkg: Package, defaultSourceCodeOrigins: List): KnownProvenance + suspend fun resolveProvenance(pkg: Package, defaultSourceCodeOrigins: List): RemoteProvenance } /** @@ -74,7 +74,7 @@ class DefaultPackageProvenanceResolver( override suspend fun resolveProvenance( pkg: Package, defaultSourceCodeOrigins: List - ): KnownProvenance { + ): RemoteProvenance { val errors = mutableMapOf() val sourceCodeOrigins = pkg.sourceCodeOrigins ?: defaultSourceCodeOrigins diff --git a/scanner/src/main/kotlin/provenance/ProvenanceDownloader.kt b/scanner/src/main/kotlin/provenance/ProvenanceDownloader.kt index 5f9118331a261..2396ddf12570f 100644 --- a/scanner/src/main/kotlin/provenance/ProvenanceDownloader.kt +++ b/scanner/src/main/kotlin/provenance/ProvenanceDownloader.kt @@ -64,7 +64,7 @@ fun interface ProvenanceDownloader { // Use the provenanceDownloader to download each provenance from nestedProvenance separately, because they are // likely already cached if a path scanner wrapper is used. - val root = download(nestedProvenance.root as RemoteProvenance) + val root = download(nestedProvenance.root) nestedProvenance.subRepositories.forEach { (path, provenance) -> val tempDir = download(provenance) diff --git a/scanner/src/main/kotlin/storages/ProvenanceBasedFileStorage.kt b/scanner/src/main/kotlin/storages/ProvenanceBasedFileStorage.kt index 7254263ce838a..c46fa78179690 100644 --- a/scanner/src/main/kotlin/storages/ProvenanceBasedFileStorage.kt +++ b/scanner/src/main/kotlin/storages/ProvenanceBasedFileStorage.kt @@ -46,7 +46,11 @@ class ProvenanceBasedFileStorage(private val backend: FileStorage) : ProvenanceB override fun read(provenance: KnownProvenance, scannerMatcher: ScannerMatcher?): List { requireEmptyVcsPath(provenance) - val path = storagePath(provenance as RemoteProvenance) + if (provenance !is RemoteProvenance) { + throw ScanStorageException("Scan result must have a known provenance, but it is $provenance.") + } + + val path = storagePath(provenance) return runCatching { backend.read(path).use { input -> @@ -81,7 +85,7 @@ class ProvenanceBasedFileStorage(private val backend: FileStorage) : ProvenanceB requireEmptyVcsPath(provenance) - if (provenance !is KnownProvenance) { + if (provenance !is RemoteProvenance) { throw ScanStorageException("Scan result must have a known provenance, but it is $provenance.") } @@ -98,7 +102,7 @@ class ProvenanceBasedFileStorage(private val backend: FileStorage) : ProvenanceB val scanResults = existingScanResults + scanResult - val path = storagePath(provenance as RemoteProvenance) + val path = storagePath(provenance) val yamlBytes = yamlMapper.writeValueAsBytes(scanResults) val input = ByteArrayInputStream(yamlBytes) diff --git a/scanner/src/main/kotlin/utils/FileListResolver.kt b/scanner/src/main/kotlin/utils/FileListResolver.kt index 15b7409526b7b..5f91a920fce2d 100644 --- a/scanner/src/main/kotlin/utils/FileListResolver.kt +++ b/scanner/src/main/kotlin/utils/FileListResolver.kt @@ -56,10 +56,10 @@ class FileListResolver( * Get the [FileList] associated with the provided [provenance]. If it is not available in the [storage], download * the provenance and create the [FileList] from it. */ - fun resolve(provenance: KnownProvenance): FileList { + fun resolve(provenance: RemoteProvenance): FileList { storage.getFileList(provenance)?.let { return it } - val dir = provenanceDownloader.download(provenance as RemoteProvenance) + val dir = provenanceDownloader.download(provenance) return createFileList(dir).also { try { diff --git a/scanner/src/test/kotlin/ScannerTest.kt b/scanner/src/test/kotlin/ScannerTest.kt index 16c345d09094d..73cec7d4f3d2b 100644 --- a/scanner/src/test/kotlin/ScannerTest.kt +++ b/scanner/src/test/kotlin/ScannerTest.kt @@ -1183,7 +1183,7 @@ private class FakePackageProvenanceResolver : PackageProvenanceResolver { override suspend fun resolveProvenance( pkg: Package, defaultSourceCodeOrigins: List - ): KnownProvenance { + ): RemoteProvenance { defaultSourceCodeOrigins.forEach { sourceCodeOrigin -> when (sourceCodeOrigin) { SourceCodeOrigin.ARTIFACT -> { @@ -1298,7 +1298,7 @@ private fun createScanResult( ) private fun createNestedScanResult( - provenance: KnownProvenance, + provenance: RemoteProvenance, scannerDetails: ScannerDetails, subRepositories: Map = emptyMap() ) = NestedProvenanceScanResult( @@ -1320,7 +1320,7 @@ private fun createStoredScanResult(provenance: Provenance, scannerDetails: Scann ) private fun createStoredNestedScanResult( - provenance: KnownProvenance, + provenance: RemoteProvenance, scannerDetails: ScannerDetails, subRepositories: Map = emptyMap() ) = NestedProvenanceScanResult( From 450b5eba58fbd1b8d42f8c860143593cba4b1a02 Mon Sep 17 00:00:00 2001 From: Jens Keim Date: Tue, 2 Sep 2025 12:58:48 +0200 Subject: [PATCH 3/3] fix(Scanner): RemoteProvenance for ORT 66.0.1 (972e3f6) Signed-off-by: Jens Keim --- scanner/src/main/kotlin/ScanController.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scanner/src/main/kotlin/ScanController.kt b/scanner/src/main/kotlin/ScanController.kt index b7b093ef48203..6da0cdf4a733f 100644 --- a/scanner/src/main/kotlin/ScanController.kt +++ b/scanner/src/main/kotlin/ScanController.kt @@ -245,8 +245,8 @@ internal class ScanController( * 'components/conanfile.txt'. This is because scanner interfaces receive packages as input, and this aims at * providing a deterministic ordering when choosing a reference package for packages with the same provenance. */ - fun getPackagesConsolidatedByProvenance(): Map> { - val packagesByProvenance = mutableMapOf>() + fun getPackagesConsolidatedByProvenance(): Map> { + val packagesByProvenance = mutableMapOf>() val comparator = compareBy(PATH_STRING_COMPARATOR) { it.id.name }.thenBy { it.id } packages.forEach { pkg ->