feat(Provenance): Add DirectoryProvenance as a LocalProvenance

In contrast to the previously added `RemoteProvenance` stands the
`LocalProvenance`, which has no remote source, but instead references
a local input of some kind.
The `DirectoryProvenance` references a local project directory,
which is lacking supported (remote) version control.
It is defined by its canonical path only.

Since ORT needs further refactoring until `DirectoryProvenance` can
be fully utilized, the new class can not be used right now.
However the presence of a new `KnownProvenance` class results in
`when` conditional cases not being exhaustive anymore.

To circumvent this issue, the following changes were made:
1. Wherever possible `RemoteProvenance` is used as parameter type
   instead of `KnownProvenance`.
2. When necessary `KnownProvenance`s are cast to `RemoteProvenance`.
3. If `Provenance` is expected, `DirectoryProvenance` is handled
   like `UnknownProvenance`.

The exception being `hash` and `storageKey`, which both required a
default value, which was set to the `canonicalPath` and its hash.
However, since the rest of the code does not handle
`DirectoryProvenance`, this should remain unused for now.

See [1] for more context on the new provenance hierarchy.

[1]: oss-review-toolkit#8803 (comment)

Signed-off-by: Jens Keim <[email protected]>

Author: pepper-jk

model/src/main/kotlin/Provenance.kt

@@ -26,6 +26,9 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize
 import com.fasterxml.jackson.databind.deser.std.StdDeserializer
 import com.fasterxml.jackson.module.kotlin.treeToValue
 
+import java.io.File
+import java.nio.file.Files
+
 /**
  * Provenance information about the origin of source code.
  */
@@ -45,6 +48,8 @@ sealed interface KnownProvenance : Provenance
 
 sealed interface RemoteProvenance : KnownProvenance
 
+sealed interface LocalProvenance : KnownProvenance
+
 /**
  * Provenance information for a source artifact.
  */
@@ -83,6 +88,23 @@ data class RepositoryProvenance(
     override fun matches(pkg: Package): Boolean = vcsInfo == pkg.vcsProcessed
 }
 
+/**
+ * Provenance information for a local directory path.
+ */
+data class DirectoryProvenance(
+    val canonicalPath: File
+) : LocalProvenance {
+    init {
+        require(canonicalPath.isDirectory) { "The directory path must exist." }
+    }
+
+    /**
+     * Return true if this provenance's directoryPath matches the package URL of the [package][pkg],
+     * as it contains the local file path for non-remote Provenances.
+     */
+    override fun matches(pkg: Package): Boolean = false
+}
+
 /**
  * A custom deserializer for polymorphic deserialization of [Provenance] without requiring type information.
  */

model/src/main/kotlin/ProvenanceResolutionResult.kt

@@ -37,7 +37,7 @@ data class ProvenanceResolutionResult(
      * The resolved provenance of the package. Can only be null if a [packageProvenanceResolutionIssue] occurred.
      */
     @JsonInclude(JsonInclude.Include.NON_NULL)
-    val packageProvenance: KnownProvenance? = null,
+    val packageProvenance: RemoteProvenance? = null,
 
     /**
      * The (recursive) sub-repositories of [packageProvenance]. The map can only be empty if a

model/src/main/kotlin/config/PackageConfiguration.kt

@@ -22,6 +22,7 @@ package org.ossreviewtoolkit.model.config
 import com.fasterxml.jackson.annotation.JsonInclude
 
 import org.ossreviewtoolkit.model.ArtifactProvenance
+import org.ossreviewtoolkit.model.DirectoryProvenance
 import org.ossreviewtoolkit.model.Identifier
 import org.ossreviewtoolkit.model.Provenance
 import org.ossreviewtoolkit.model.RepositoryProvenance
@@ -84,6 +85,7 @@ data class PackageConfiguration(
             is UnknownProvenance -> false
             is ArtifactProvenance -> sourceArtifactUrl != null && sourceArtifactUrl == provenance.sourceArtifact.url
             is RepositoryProvenance -> vcs != null && vcs.matches(provenance)
+            is DirectoryProvenance -> false
         }
     }
 }

model/src/main/kotlin/utils/FileProvenanceFileStorage.kt

@@ -24,6 +24,7 @@ import java.io.InputStream
 import org.apache.logging.log4j.kotlin.logger
 
 import org.ossreviewtoolkit.model.ArtifactProvenance
+import org.ossreviewtoolkit.model.DirectoryProvenance
 import org.ossreviewtoolkit.model.HashAlgorithm
 import org.ossreviewtoolkit.model.KnownProvenance
 import org.ossreviewtoolkit.model.RepositoryProvenance
@@ -81,6 +82,7 @@ private fun KnownProvenance.hash(): String {
     val key = when (this) {
         is ArtifactProvenance -> "${sourceArtifact.url}${sourceArtifact.hash.value}"
         is RepositoryProvenance -> "${vcsInfo.type}${vcsInfo.url}$resolvedRevision"
+        is DirectoryProvenance -> "${canonicalPath.toPath()}${canonicalPath.hashCode()}"
     }
 
     return HashAlgorithm.SHA1.calculate(key.toByteArray())

model/src/main/kotlin/utils/PostgresProvenanceFileStorage.kt

@@ -36,6 +36,7 @@ import org.jetbrains.exposed.sql.insert
 import org.jetbrains.exposed.sql.selectAll
 
 import org.ossreviewtoolkit.model.ArtifactProvenance
+import org.ossreviewtoolkit.model.DirectoryProvenance
 import org.ossreviewtoolkit.model.KnownProvenance
 import org.ossreviewtoolkit.model.RepositoryProvenance
 import org.ossreviewtoolkit.model.utils.DatabaseUtils.checkDatabaseEncoding
@@ -121,4 +122,5 @@ private fun KnownProvenance.storageKey(): String =
         is ArtifactProvenance -> "source-artifact|${sourceArtifact.url}|${sourceArtifact.hash}"
         // The trailing "|" is kept for backward compatibility because there used to be an additional parameter.
         is RepositoryProvenance -> "vcs|${vcsInfo.type}|${vcsInfo.url}|$resolvedRevision|"
+        is DirectoryProvenance -> "directory|${canonicalPath.toPath()}|${canonicalPath.hashCode()}"
     }

model/src/main/kotlin/utils/PurlExtensions.kt

@@ -27,6 +27,7 @@ import org.ossreviewtoolkit.model.Identifier
 import org.ossreviewtoolkit.model.Package
 import org.ossreviewtoolkit.model.Provenance
 import org.ossreviewtoolkit.model.RemoteArtifact
+import org.ossreviewtoolkit.model.RemoteProvenance
 import org.ossreviewtoolkit.model.RepositoryProvenance
 import org.ossreviewtoolkit.model.UnknownProvenance
 import org.ossreviewtoolkit.model.VcsInfo
@@ -101,7 +102,7 @@ fun Provenance.toPurlExtras(): PurlExtras =
             )
         }
 
-        is UnknownProvenance -> PurlExtras()
+        !is RemoteProvenance -> PurlExtras()
     }
 
 /**

scanner/src/main/kotlin/ScanController.kt

@@ -24,6 +24,7 @@ import org.ossreviewtoolkit.model.Issue
 import org.ossreviewtoolkit.model.KnownProvenance
 import org.ossreviewtoolkit.model.Package
 import org.ossreviewtoolkit.model.Provenance
+import org.ossreviewtoolkit.model.RemoteProvenance
 import org.ossreviewtoolkit.model.RepositoryProvenance
 import org.ossreviewtoolkit.model.ScanResult
 import org.ossreviewtoolkit.model.ScanSummary
@@ -79,7 +80,7 @@ internal class ScanController(
      * A map of package [Identifier]s to their resolved [KnownProvenance]s. These provenances are used to filter the
      * scan results for a package based on the VCS path.
      */
-    private val packageProvenances = mutableMapOf<Identifier, KnownProvenance>()
+    private val packageProvenances = mutableMapOf<Identifier, RemoteProvenance>()
 
     /**
      * A map of package [Identifier]s to their resolved [KnownProvenance]s with the VCS path removed. These provenances
@@ -130,7 +131,7 @@ internal class ScanController(
     /**
      * Set the [provenance] for the package denoted by [id], overwriting any existing values.
      */
-    fun putPackageProvenance(id: Identifier, provenance: KnownProvenance) {
+    fun putPackageProvenance(id: Identifier, provenance: RemoteProvenance) {
         packageProvenances[id] = provenance
         packageProvenancesWithoutVcsPath[id] = when (provenance) {
             is RepositoryProvenance -> provenance.copy(vcsInfo = provenance.vcsInfo.copy(path = ""))
@@ -262,7 +263,7 @@ internal class ScanController(
     fun getPackagesForProvenanceWithoutVcsPath(provenance: KnownProvenance): Set<Identifier> =
         packageProvenancesWithoutVcsPath.filter { (_, packageProvenance) -> packageProvenance == provenance }.keys
 
-    fun getPackageProvenance(id: Identifier): KnownProvenance? = packageProvenances[id]
+    fun getPackageProvenance(id: Identifier): RemoteProvenance? = packageProvenances[id]
 
     /**
      * Return the package provenanceResolutionIssue associated with the given [id].

scanner/src/main/kotlin/Scanner.kt

@@ -41,6 +41,7 @@ import org.ossreviewtoolkit.model.OrtResult
 import org.ossreviewtoolkit.model.Package
 import org.ossreviewtoolkit.model.PackageType
 import org.ossreviewtoolkit.model.ProvenanceResolutionResult
+import org.ossreviewtoolkit.model.RemoteProvenance
 import org.ossreviewtoolkit.model.ScanResult
 import org.ossreviewtoolkit.model.ScanSummary
 import org.ossreviewtoolkit.model.ScannerRun
@@ -258,7 +259,7 @@ class Scanner(
                 }.awaitAll()
             }.forEach { (pkg, result) ->
                 result.onSuccess { provenance ->
-                    controller.putPackageProvenance(pkg.id, provenance)
+                    controller.putPackageProvenance(pkg.id, provenance as RemoteProvenance)
                 }.onFailure {
                     controller.putPackageProvenanceResolutionIssue(
                         pkg.id,
@@ -279,7 +280,7 @@ class Scanner(
                 controller.getPackageProvenancesWithoutVcsPath().map { provenance ->
                     async {
                         provenance to runCatching {
-                            nestedProvenanceResolver.resolveNestedProvenance(provenance)
+                            nestedProvenanceResolver.resolveNestedProvenance(provenance as RemoteProvenance)
                         }
                     }
                 }.awaitAll()
@@ -574,7 +575,7 @@ class Scanner(
         context: ScanContext
     ): Map<PathScannerWrapper, ScanResult> {
         val downloadDir = try {
-            provenanceDownloader.download(provenance)
+            provenanceDownloader.download(provenance as RemoteProvenance)
         } catch (e: DownloadException) {
             val issue = createAndLogIssue(
                 "Downloader", "Could not download provenance $provenance: ${e.collectMessages()}"

scanner/src/main/kotlin/provenance/NestedProvenanceResolver.kt

@@ -23,8 +23,8 @@ import org.apache.logging.log4j.kotlin.logger
 
 import org.ossreviewtoolkit.downloader.WorkingTreeCache
 import org.ossreviewtoolkit.model.ArtifactProvenance
-import org.ossreviewtoolkit.model.KnownProvenance
 import org.ossreviewtoolkit.model.Provenance
+import org.ossreviewtoolkit.model.RemoteProvenance
 import org.ossreviewtoolkit.model.RepositoryProvenance
 
 /**
@@ -36,7 +36,7 @@ interface NestedProvenanceResolver {
      * [NestedProvenance] always contains only the provided [ArtifactProvenance]. For a [RepositoryProvenance] the
      * resolver looks for nested repositories, for example Git submodules or Mercurial subrepositories.
      */
-    suspend fun resolveNestedProvenance(provenance: KnownProvenance): NestedProvenance
+    suspend fun resolveNestedProvenance(provenance: RemoteProvenance): NestedProvenance
 }
 
 /**
@@ -46,7 +46,7 @@ class DefaultNestedProvenanceResolver(
     private val storage: NestedProvenanceStorage,
     private val workingTreeCache: WorkingTreeCache
 ) : NestedProvenanceResolver {
-    override suspend fun resolveNestedProvenance(provenance: KnownProvenance): NestedProvenance {
+    override suspend fun resolveNestedProvenance(provenance: RemoteProvenance): NestedProvenance {
         return when (provenance) {
             is ArtifactProvenance -> NestedProvenance(root = provenance, subRepositories = emptyMap())
             is RepositoryProvenance -> resolveNestedRepository(provenance)

scanner/src/main/kotlin/provenance/ProvenanceDownloader.kt

@@ -29,8 +29,8 @@ import org.ossreviewtoolkit.downloader.DownloadException
 import org.ossreviewtoolkit.downloader.Downloader
 import org.ossreviewtoolkit.downloader.WorkingTreeCache
 import org.ossreviewtoolkit.model.ArtifactProvenance
-import org.ossreviewtoolkit.model.KnownProvenance
 import org.ossreviewtoolkit.model.Package
+import org.ossreviewtoolkit.model.RemoteProvenance
 import org.ossreviewtoolkit.model.RepositoryProvenance
 import org.ossreviewtoolkit.model.config.DownloaderConfiguration
 import org.ossreviewtoolkit.utils.common.safeDeleteRecursively
@@ -48,7 +48,7 @@ fun interface ProvenanceDownloader {
      *
      * Throws a [DownloadException] if the download fails.
      */
-    fun download(provenance: KnownProvenance): File
+    fun download(provenance: RemoteProvenance): File
 
     /**
      * Download the source code specified by the provided [nestedProvenance] incl. sub-repositories and return the path
@@ -61,7 +61,7 @@ fun interface ProvenanceDownloader {
         // Use the provenanceDownloader to download each provenance from nestedProvenance separately, because they are
         // likely already cached if a path scanner wrapper is used.
 
-        val root = download(nestedProvenance.root)
+        val root = download(nestedProvenance.root as RemoteProvenance)
 
         nestedProvenance.subRepositories.forEach { (path, provenance) ->
             val tempDir = download(provenance)
@@ -83,7 +83,7 @@ class DefaultProvenanceDownloader(
 ) : ProvenanceDownloader {
     private val downloader = Downloader(config)
 
-    override fun download(provenance: KnownProvenance): File {
+    override fun download(provenance: RemoteProvenance): File {
         val downloadDir = createOrtTempDir()
 
         when (provenance) {

scanner/src/main/kotlin/storages/ProvenanceBasedFileStorage.kt

@@ -29,6 +29,7 @@ import org.apache.logging.log4j.kotlin.logger
 
 import org.ossreviewtoolkit.model.ArtifactProvenance
 import org.ossreviewtoolkit.model.KnownProvenance
+import org.ossreviewtoolkit.model.RemoteProvenance
 import org.ossreviewtoolkit.model.RepositoryProvenance
 import org.ossreviewtoolkit.model.ScanResult
 import org.ossreviewtoolkit.model.yamlMapper
@@ -45,7 +46,7 @@ class ProvenanceBasedFileStorage(private val backend: FileStorage) : ProvenanceB
     override fun read(provenance: KnownProvenance, scannerMatcher: ScannerMatcher?): List<ScanResult> {
         requireEmptyVcsPath(provenance)
 
-        val path = storagePath(provenance)
+        val path = storagePath(provenance as RemoteProvenance)
 
         return runCatching {
             backend.read(path).use { input ->
@@ -94,7 +95,7 @@ class ProvenanceBasedFileStorage(private val backend: FileStorage) : ProvenanceB
 
         val scanResults = existingScanResults + scanResult
 
-        val path = storagePath(provenance)
+        val path = storagePath(provenance as RemoteProvenance)
         val yamlBytes = yamlMapper.writeValueAsBytes(scanResults)
         val input = ByteArrayInputStream(yamlBytes)
 
@@ -117,7 +118,7 @@ class ProvenanceBasedFileStorage(private val backend: FileStorage) : ProvenanceB
     }
 }
 
-private fun storagePath(provenance: KnownProvenance) =
+private fun storagePath(provenance: RemoteProvenance) =
     when (provenance) {
         is ArtifactProvenance -> "artifact/${provenance.sourceArtifact.url.fileSystemEncode()}/$SCAN_RESULTS_FILE_NAME"
         is RepositoryProvenance -> {

scanner/src/main/kotlin/storages/ProvenanceBasedPostgresStorage.kt

@@ -36,6 +36,7 @@ import org.jetbrains.exposed.sql.selectAll
 
 import org.ossreviewtoolkit.model.ArtifactProvenance
 import org.ossreviewtoolkit.model.KnownProvenance
+import org.ossreviewtoolkit.model.RemoteProvenance
 import org.ossreviewtoolkit.model.RepositoryProvenance
 import org.ossreviewtoolkit.model.ScanResult
 import org.ossreviewtoolkit.model.ScanSummary
@@ -86,6 +87,10 @@ class ProvenanceBasedPostgresStorage(
             return database.transaction {
                 val query = table.selectAll()
 
+                if (provenance !is RemoteProvenance) {
+                    throw ScanStorageException("Scan result must have a known provenance, but it is $provenance.")
+                }
+
                 when (provenance) {
                     is ArtifactProvenance -> {
                         query.andWhere {
@@ -137,7 +142,7 @@ class ProvenanceBasedPostgresStorage(
 
         requireEmptyVcsPath(provenance)
 
-        if (provenance !is KnownProvenance) {
+        if (provenance !is RemoteProvenance) {
             throw ScanStorageException("Scan result must have a known provenance, but it is $provenance.")
         }
 

scanner/src/main/kotlin/utils/FileListResolver.kt

@@ -26,6 +26,7 @@ import java.io.File
 
 import org.ossreviewtoolkit.model.HashAlgorithm
 import org.ossreviewtoolkit.model.KnownProvenance
+import org.ossreviewtoolkit.model.RemoteProvenance
 import org.ossreviewtoolkit.model.toYaml
 import org.ossreviewtoolkit.model.utils.ProvenanceFileStorage
 import org.ossreviewtoolkit.model.yamlMapper
@@ -58,7 +59,7 @@ class FileListResolver(
     fun resolve(provenance: KnownProvenance): FileList {
         storage.getFileList(provenance)?.let { return it }
 
-        val dir = provenanceDownloader.download(provenance)
+        val dir = provenanceDownloader.download(provenance as RemoteProvenance)
 
         return createFileList(dir).also {
             try {

scanner/src/test/kotlin/ScannerTest.kt

@@ -49,6 +49,7 @@ import org.ossreviewtoolkit.model.Package
 import org.ossreviewtoolkit.model.PackageType
 import org.ossreviewtoolkit.model.Provenance
 import org.ossreviewtoolkit.model.RemoteArtifact
+import org.ossreviewtoolkit.model.RemoteProvenance
 import org.ossreviewtoolkit.model.RepositoryProvenance
 import org.ossreviewtoolkit.model.ScanResult
 import org.ossreviewtoolkit.model.ScanSummary
@@ -973,7 +974,7 @@ private class FakePathScannerWrapper : PathScannerWrapper {
  * provenance, instead of actually downloading the source code.
  */
 private class FakeProvenanceDownloader(val filename: String = "fake.txt") : ProvenanceDownloader {
-    override fun download(provenance: KnownProvenance): File =
+    override fun download(provenance: RemoteProvenance): File =
         createOrtTempDir().apply {
             resolve(filename).writeText(provenance.toYaml())
         }
@@ -1012,7 +1013,7 @@ private class FakePackageProvenanceResolver : PackageProvenanceResolver {
  * An implementation of [NestedProvenanceResolver] that always returns a non-nested provenance.
  */
 private class FakeNestedProvenanceResolver : NestedProvenanceResolver {
-    override suspend fun resolveNestedProvenance(provenance: KnownProvenance): NestedProvenance =
+    override suspend fun resolveNestedProvenance(provenance: RemoteProvenance): NestedProvenance =
         NestedProvenance(root = provenance, subRepositories = emptyMap())
 }
 

utils/test/src/main/kotlin/Utils.kt

@@ -37,6 +37,7 @@ import org.ossreviewtoolkit.model.OrtResult
 import org.ossreviewtoolkit.model.Package
 import org.ossreviewtoolkit.model.ProvenanceResolutionResult
 import org.ossreviewtoolkit.model.RemoteArtifact
+import org.ossreviewtoolkit.model.RemoteProvenance
 import org.ossreviewtoolkit.model.RepositoryProvenance
 import org.ossreviewtoolkit.model.ScanResult
 import org.ossreviewtoolkit.model.ScannerRun
@@ -198,7 +199,7 @@ fun scannerRunOf(vararg pkgScanResults: Pair<Identifier, List<ScanResult>>): Sca
 
     return ScannerRun.EMPTY.copy(
         provenances = pkgScanResultsWithKnownProvenance.mapTo(mutableSetOf()) { (id, scanResultsForId) ->
-            val packageProvenance = scanResultsForId.firstOrNull()?.provenance as KnownProvenance
+            val packageProvenance = scanResultsForId.firstOrNull()?.provenance as RemoteProvenance
 
             ProvenanceResolutionResult(
                 id = id,