Merge pull request #36 from jmatsu/support_local_file

Support collecting local files
jmatsu · Jun 12, 2020 · 8f53417 · 8f53417
2 parents 0c23f79 + 93b402a
commit 8f53417
Show file tree

Hide file tree

Showing 16 changed files with 143 additions and 78 deletions.
diff --git a/.gitignore b/.gitignore
@@ -94,7 +94,6 @@ modules.xml
 .mtj.tmp/
 
 # Package Files #
-*.jar
 *.war
 *.nar
 *.ear

diff --git a/example/app/build.gradle.kts b/example/app/build.gradle.kts
@@ -62,6 +62,10 @@ configurations.getByName("implementation").extendsFrom(sampleConfiguration)
 val orphanConfiguration = configurations.create("orphan")
 
 dependencies {
+    implementation(fileTree(project.file("lib")) {
+        include("*.jar")
+    })
+
     implementation("org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.3.71")
     implementation("androidx.core:core-ktx:1.3.0-alpha02")
     implementation("androidx.appcompat:appcompat:1.2.0-alpha03")

diff --git a/example/app/lib/ktlint-html-reporter-0.2.0.jar b/example/app/lib/ktlint-html-reporter-0.2.0.jar
diff --git a/example/app/license-list/artifact-definition.yml b/example/app/license-list/artifact-definition.yml
@@ -453,6 +453,12 @@ yellowBlueRelease:
         url: null
         copyrightHolders: []
         licenses: []
+    local-file:
+      - key: ktlint-html-reporter-0.2.0.jar
+        displayName: ktlint-html-reporter-0.2.0
+        url: null
+        copyrightHolders: []
+        licenses: []
     org.antlr:
       - key: antlr4
         displayName: ANTLR 4 Tool

diff --git a/example/fixtures/merge/expected-artifact-definition.yml b/example/fixtures/merge/expected-artifact-definition.yml
@@ -117,6 +117,12 @@ yellowBlueRelease:
         url: null
         copyrightHolders: []
         licenses: []
+    local-file:
+      - key: ktlint-html-reporter-0.2.0.jar
+        displayName: ktlint-html-reporter-0.2.0
+        url: null
+        copyrightHolders: []
+        licenses: []
     org.antlr:
       - key: antlr4
         displayName: ANTLR 4 Tool

diff --git a/example/fixtures/validate-artifact-definition.yml b/example/fixtures/validate-artifact-definition.yml
@@ -461,6 +461,14 @@ yellowBlueRelease:
       url: null
       copyrightHolders: []
       licenses: []
+  local-file:
+    - key: ktlint-html-reporter-0.2.0.jar
+      displayName: ktlint-html-reporter
+      url: https://github.com/mcassiano/ktlint-html-reporter
+      copyrightHolders:
+        - Matheus Candido
+      licenses:
+        - mit
   org.antlr:
     - key: antlr4
       displayName: ANTLR 4 Tool

diff --git a/plugin/src/main/kotlin/io/github/jmatsu/license/internal/ArtifactManagement.kt b/plugin/src/main/kotlin/io/github/jmatsu/license/internal/ArtifactManagement.kt
@@ -1,53 +1,24 @@
 package io.github.jmatsu.license.internal
 
+import io.github.jmatsu.license.LicenseListPlugin
 import io.github.jmatsu.license.ext.collectToMap
 import io.github.jmatsu.license.ext.lenientConfiguration
+import io.github.jmatsu.license.model.LicenseSeed
 import io.github.jmatsu.license.model.ResolveScope
 import io.github.jmatsu.license.model.ResolvedArtifact
+import io.github.jmatsu.license.model.ResolvedLocalFileMetadata
 import io.github.jmatsu.license.model.ResolvedModuleIdentifier
 import io.github.jmatsu.license.model.VersionString
+import io.github.jmatsu.license.model.localGroup
+import java.io.File
 import java.util.SortedMap
-import kotlin.Comparator
-import kotlin.Pair
-import kotlin.String
-import kotlin.also
-import kotlin.arrayOf
-import kotlin.collections.ArrayList
-import kotlin.collections.List
-import kotlin.collections.MutableList
-import kotlin.collections.Set
-import kotlin.collections.asSequence
+import java.util.zip.ZipFile
 import kotlin.collections.component1
 import kotlin.collections.component2
 import kotlin.collections.component3
-import kotlin.collections.contains
-import kotlin.collections.distinctBy
-import kotlin.collections.filter
-import kotlin.collections.filterNot
-import kotlin.collections.flatMap
-import kotlin.collections.getValue
-import kotlin.collections.groupBy
-import kotlin.collections.listOf
-import kotlin.collections.map
-import kotlin.collections.mapKeys
-import kotlin.collections.mapNotNull
-import kotlin.collections.mapValues
-import kotlin.collections.max
-import kotlin.collections.orEmpty
-import kotlin.collections.plus
-import kotlin.collections.plusAssign
-import kotlin.collections.reduce
-import kotlin.collections.setOf
-import kotlin.collections.toMap
-import kotlin.collections.toSet
-import kotlin.collections.toSortedMap
-import kotlin.let
-import kotlin.requireNotNull
-import kotlin.run
-import kotlin.takeIf
-import kotlin.to
 import org.gradle.api.Project
 import org.gradle.api.artifacts.Configuration
+import org.gradle.api.artifacts.SelfResolvingDependency
 import org.gradle.api.artifacts.component.ComponentIdentifier
 import org.gradle.api.artifacts.result.ResolvedArtifactResult
 import org.gradle.kotlin.dsl.withArtifacts
@@ -94,8 +65,22 @@ class ArtifactManagement(
 
         val components: MutableList<ComponentIdentifier> = ArrayList()
 
-        val scopedModules = scopedConfigurations.mapValues { (_, configurations) ->
+        val localFileMap: MutableMap<ResolveScope, List<File>> = hashMapOf()
+
+        val scopedModules = scopedConfigurations.mapValues { (scope, configurations) ->
             configurations.flatMap { configuration ->
+                runCatching {
+                    localFileMap.putIfAbsent(scope, emptyList())
+                    localFileMap[scope] = localFileMap.getValue(scope) + configuration.allDependencies
+                        .filterIsInstance<SelfResolvingDependency>()
+                        .flatMap {
+                            it.resolve()
+                        }
+                        .filter { it.name.endsWith(".jar") || it.name.endsWith(".aar") }
+                }.onFailure {
+                    LicenseListPlugin.logger?.error("could not resolve local files in ${scope.name}", it)
+                }
+
                 val newResolvedIdentifiers = configuration.toResolvedModuleIdentifiers().filterNot { id ->
                     id.id?.componentIdentifier in components
                 }
@@ -116,7 +101,41 @@ class ArtifactManagement(
 
         val allScopes = listOf(variantScope) + additionalScopes
 
-        return resolveResults.resolvedComponents.flatMap { result ->
+        val scopedLocalFiles = localFileMap.values.flatten().distinct().groupBy { file ->
+            localFileMap.asIterable().first { (_, files) -> file in files }.key
+        }.flatMap { (key, files) ->
+            files.map { file ->
+                val id = ResolvedModuleIdentifier(
+                    group = localGroup,
+                    name = file.name
+                )
+
+                val licenseCandidates = ZipFile(file).use { zip ->
+                    zip.entries().asSequence()
+                        .filter { !it.isDirectory && it.name.startsWith("META-INF/") && it.name.contains("LICENSE") }
+                        .map { it.name }
+                        .toList()
+                }
+
+                // It's hard to infer used licenses through META-INF/LICENSE*
+                val metadata = ResolvedLocalFileMetadata(
+                    displayName = file.nameWithoutExtension,
+                    licenses = licenseCandidates.mapIndexed { i, path ->
+                        LicenseSeed(
+                            name = "${file.nameWithoutExtension}-$i",
+                            url = path
+                        )
+                    }
+                )
+
+                key to ResolvedArtifact(
+                    id = id,
+                    metadata = metadata
+                )
+            }
+        }
+
+        return (resolveResults.resolvedComponents.flatMap { result ->
             result.getArtifacts(MavenPomArtifact::class.java).map {
                 it as ResolvedArtifactResult
 
@@ -143,20 +162,20 @@ class ArtifactManagement(
 
                         ResolveScope.Unknown to ResolvedArtifact(
                             id = id,
-                            pomFile = pomFile
+                            metadata = pomFile
                         )
                     }
                     else -> {
                         val (scope, modules) = e
 
                         scope to ResolvedArtifact(
                             id = modules.getValue(component),
-                            pomFile = pomFile
+                            metadata = pomFile
                         )
                     }
                 }
             }
-        }.collectToMap().toSortedMap(Comparator { o1, o2 ->
+        } + scopedLocalFiles).collectToMap().toSortedMap(Comparator { o1, o2 ->
             allScopes.indexOf(o1).compareTo(allScopes.indexOf(o2))
         })
     }

diff --git a/plugin/src/main/kotlin/io/github/jmatsu/license/internal/PomParser.kt b/plugin/src/main/kotlin/io/github/jmatsu/license/internal/PomParser.kt
@@ -2,16 +2,13 @@ package io.github.jmatsu.license.internal
 
 import groovy.util.XmlSlurper
 import groovy.util.slurpersupport.GPathResult
+import io.github.jmatsu.license.model.LicenseSeed
 import io.github.jmatsu.license.model.ResolvedPomFile
 import java.io.File
 
 class PomParser(
     private val file: File
 ) {
-    data class License(
-        val name: String?,
-        val url: String?
-    )
 
     fun parse(): ResolvedPomFile {
         val pomRoot = XmlSlurper(false, false).parse(file)
@@ -26,13 +23,13 @@ class PomParser(
             it?.trimText()
         }
 
-        val licenses: List<License> = pomRoot["licenses"]
+        val licenses: List<LicenseSeed> = pomRoot["licenses"]
             .childPaths()
             .map {
                 val name = it["name"]?.trimText()
                 val url = it["url"]?.trimText()
                 // Is distribution node required? :thinking_face:
-                License(
+                LicenseSeed(
                     name = name,
                     url = url
                 )

diff --git a/plugin/src/main/kotlin/io/github/jmatsu/license/model/LicenseSeed.kt b/plugin/src/main/kotlin/io/github/jmatsu/license/model/LicenseSeed.kt
@@ -0,0 +1,6 @@
+package io.github.jmatsu.license.model
+
+data class LicenseSeed(
+    val name: String?,
+    val url: String?
+)
diff --git a/plugin/src/main/kotlin/io/github/jmatsu/license/model/ResolvedArtifact.kt b/plugin/src/main/kotlin/io/github/jmatsu/license/model/ResolvedArtifact.kt
@@ -2,5 +2,6 @@ package io.github.jmatsu.license.model
 
 data class ResolvedArtifact(
     val id: ResolvedModuleIdentifier,
-    val pomFile: ResolvedPomFile
+    val metadata: ResolvedMetadata,
+    val local: Boolean = false
 )
diff --git a/plugin/src/main/kotlin/io/github/jmatsu/license/model/ResolvedMetadata.kt b/plugin/src/main/kotlin/io/github/jmatsu/license/model/ResolvedMetadata.kt
@@ -0,0 +1,26 @@
+package io.github.jmatsu.license.model
+
+sealed class ResolvedMetadata {
+    abstract val associatedUrl: String?
+    abstract val displayName: String
+    abstract val licenses: List<LicenseSeed>
+    abstract val copyrightHolders: List<String>
+}
+
+data class ResolvedPomFile(
+    override val associatedUrl: String?,
+    val displayNameCandidates: List<String>,
+    override val licenses: List<LicenseSeed>,
+    override val copyrightHolders: List<String>
+) : ResolvedMetadata() {
+    override val displayName: String
+        get() = displayNameCandidates.first { it.isNotBlank() }
+}
+
+data class ResolvedLocalFileMetadata(
+    override val displayName: String,
+    override val licenses: List<LicenseSeed>
+) : ResolvedMetadata() {
+    override val associatedUrl: String? = null
+    override val copyrightHolders: List<String> = emptyList()
+}
diff --git a/plugin/src/main/kotlin/io/github/jmatsu/license/model/ResolvedModuleIdentifier.kt b/plugin/src/main/kotlin/io/github/jmatsu/license/model/ResolvedModuleIdentifier.kt
@@ -6,10 +6,15 @@ data class ResolvedModuleIdentifier(
     val group: String,
     val name: String,
     val version: VersionString = VersionString("+"),
+    /**
+     * Optional. Present iff resolved artifact
+     */
     val id: ComponentArtifactIdentifier? = null
 ) : Comparable<ResolvedModuleIdentifier> {
     override fun compareTo(other: ResolvedModuleIdentifier): Int {
         return "$group:$name".compareTo("${other.group}:${other.name}")
             .takeIf { it != 0 } ?: version.compareTo(other.version)
     }
 }
+
+const val localGroup = "local-file"
diff --git a/plugin/src/main/kotlin/io/github/jmatsu/license/model/ResolvedPomFile.kt b/plugin/src/main/kotlin/io/github/jmatsu/license/model/ResolvedPomFile.kt
diff --git a/plugin/src/main/kotlin/io/github/jmatsu/license/presentation/Builder.kt b/plugin/src/main/kotlin/io/github/jmatsu/license/presentation/Builder.kt
@@ -3,7 +3,7 @@ package io.github.jmatsu.license.presentation
 import io.github.jmatsu.license.internal.LicenseClassifier
 import io.github.jmatsu.license.model.ResolveScope
 import io.github.jmatsu.license.model.ResolvedArtifact
-import io.github.jmatsu.license.model.ResolvedPomFile
+import io.github.jmatsu.license.model.ResolvedMetadata
 import io.github.jmatsu.license.poko.ArtifactDefinition
 import io.github.jmatsu.license.poko.LicenseKey
 import io.github.jmatsu.license.poko.PlainLicense
@@ -19,14 +19,14 @@ class Builder(
         fun transformToArtifact(artifact: ResolvedArtifact, licenseCapture: MutableSet<PlainLicense>): ArtifactDefinition {
             return ArtifactDefinition(
                 key = "${artifact.id.group}:${artifact.id.name}",
-                licenses = artifact.pomFile.licenses(licenseCapture),
-                copyrightHolders = artifact.pomFile.copyrightHolders,
-                url = artifact.pomFile.associatedUrl,
-                displayName = artifact.pomFile.displayName
+                licenses = artifact.metadata.licenses(licenseCapture),
+                copyrightHolders = artifact.metadata.copyrightHolders,
+                url = artifact.metadata.associatedUrl,
+                displayName = artifact.metadata.displayName
             )
         }
 
-        private fun ResolvedPomFile.licenses(licenseCapture: MutableSet<PlainLicense>): List<LicenseKey> {
+        private fun ResolvedMetadata.licenses(licenseCapture: MutableSet<PlainLicense>): List<LicenseKey> {
             return licenses.map {
                 val licenseName = it.name?.trim { c -> c.isWhitespace() || c == '\n' || c == '\r' }
 

diff --git a/plugin/src/test/kotlin/io/github/jmatsu/license/internal/PomParserTest.kt b/plugin/src/test/kotlin/io/github/jmatsu/license/internal/PomParserTest.kt
@@ -1,5 +1,6 @@
 package io.github.jmatsu.license.internal
 
+import io.github.jmatsu.license.model.LicenseSeed
 import java.io.File
 import java.util.stream.Stream
 import kotlin.test.expect
@@ -19,7 +20,7 @@ class PomParserTest {
                     "https://github.com/jmatsu",
                     listOf("jmatsu"),
                     listOf(
-                        PomParser.License(
+                        LicenseSeed(
                             name = "The Apache Software License, Version 2.0",
                             url = "http://www.apache.org/licenses/LICENSE-2.0.txt"
                         )
@@ -32,11 +33,11 @@ class PomParserTest {
                     "https://github.com/jmatsu/license-list-plugin.git",
                     listOf("jmatsu1", "jmatsu2", "jmatsu3"),
                     listOf(
-                        PomParser.License(
+                        LicenseSeed(
                             name = "license1",
                             url = "url1"
                         ),
-                        PomParser.License(
+                        LicenseSeed(
                             name = "license2",
                             url = "url2"
                         )
@@ -57,7 +58,7 @@ class PomParserTest {
                     null,
                     emptyList<String>(),
                     listOf(
-                        PomParser.License(
+                        LicenseSeed(
                             name = null,
                             url = null
                         )
@@ -75,7 +76,7 @@ class PomParserTest {
         displayNameCandidates: List<String>,
         associatedUrl: String?,
         copyrightHolders: List<String>,
-        licenses: List<PomParser.License>
+        licenses: List<LicenseSeed>
     ) {
         var pomFile: File? = null
         try {
-Original file line number
+Diff line change
@@ Expand Up / @@ -94,7 +94,6 @@ modules.xml @@
     .mtj.tmp/
     # Package Files #
-    *.jar
     *.war
     *.nar
     *.ear
@@ Expand Down @@