Remote repository scanning improvements

.github/workflows/maven-ci.yml

@@ -61,6 +61,12 @@ jobs:
           server-password: SONATYPE_PASSWORD
           gpg-private-key: ${{ secrets.SONATYPE_GPG_PRIVATE_KEY }}
           gpg-passphrase: SONATYPE_GPG_PASSPHRASE
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ secrets.GIT_USERNAME }}
+          password: ${{ secrets.GIT_PACKAGE_TOKEN }}
       - name: Set Maven Project Version
         shell: bash
         run: |
@@ -138,11 +144,12 @@ jobs:
         with:
           java-version: 8
       - name: Setup GraalVM
-        uses: DeLaGuardo/setup-graalvm@master
+        uses: graalvm/setup-graalvm@v1
         with:
-          graalvm-version: 21.2.0.java8
-      - name: Setup GraalVM Native Image Tool
-        run: gu install native-image
+          version: '21.2.0'
+          java-version: '8'
+          components: 'native-image'
+          github-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Build Mac Native Image
         if: success()
         working-directory: build
@@ -180,20 +187,16 @@ jobs:
       - name: Rename Native Image JAR
         working-directory: build
         run: ren *.jar native-image.jar
-      - name: Setup GraalVM Native Image and Visual C Build Tools
-        run: |
-          Invoke-RestMethod -Uri https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-21.3.0/graalvm-ce-java11-windows-amd64-21.3.0.zip -OutFile 'graal.zip'
-          Expand-Archive -path 'graal.zip' -destinationpath '.'
-          graalvm-ce-java11-21.3.0\bin\gu.cmd install native-image
-          choco install visualstudio2017-workload-vctools
+      - name: Setup GraalVM
+        uses: graalvm/setup-graalvm@v1
+        with:
+          version: '21.3.0'
+          java-version: '11'
+          components: 'native-image'
+          github-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Build Windows Native Image
         if: success()
-        shell: cmd
-        run: |
-          call "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvars64.bat"
-          graalvm-ce-java11-21.3.0\bin\native-image -cp .\build\native-image.jar -H:+ReportExceptionStackTraces --report-unsupported-elements-at-runtime
-        env:
-          JAVA_HOME: ./graalvm-ce-java11-21.3.0
+        run: native-image.cmd -cp .\build\native-image.jar -H:+ReportExceptionStackTraces --report-unsupported-elements-at-runtime
       - name: Archive Windows Native Image
         if: success()
         continue-on-error: true

.github/workflows/release.yml

@@ -37,6 +37,12 @@ jobs:
           server-password: SONATYPE_PASSWORD
           gpg-private-key: ${{ secrets.SONATYPE_GPG_PRIVATE_KEY }}
           gpg-passphrase: SONATYPE_GPG_PASSPHRASE
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ secrets.GIT_USERNAME }}
+          password: ${{ secrets.GIT_PACKAGE_TOKEN }}
       - name: Set Maven Project Version
         id: set_maven_project_version
         shell: bash
@@ -226,11 +232,12 @@ jobs:
         with:
           java-version: 8
       - name: Setup GraalVM
-        uses: DeLaGuardo/setup-graalvm@master
+        uses: graalvm/setup-graalvm@v1
         with:
-          graalvm-version: 21.2.0.java8
-      - name: Setup GraalVM Native Image Tool
-        run: gu install native-image
+          version: '21.2.0'
+          java-version: '8'
+          components: 'native-image'
+          github-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Build Mac Native Image
         if: success()
         working-directory: build
@@ -289,20 +296,16 @@ jobs:
       - name: Rename Native Image JAR
         working-directory: build
         run: ren *.jar native-image.jar
-      - name: Setup GraalVM Native Image and Visual C Build Tools
-        run: |
-          Invoke-RestMethod -Uri https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-21.3.0/graalvm-ce-java11-windows-amd64-21.3.0.zip -OutFile 'graal.zip'
-          Expand-Archive -path 'graal.zip' -destinationpath '.'
-          graalvm-ce-java11-21.3.0\bin\gu.cmd install native-image
-          choco install visualstudio2017-workload-vctools
+      - name: Setup GraalVM
+        uses: graalvm/setup-graalvm@v1
+        with:
+          version: '21.3.0'
+          java-version: '11'
+          components: 'native-image'
+          github-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Build Windows Native Image
         if: success()
-        shell: cmd
-        run: |
-          call "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvars64.bat"
-          graalvm-ce-java11-21.3.0\bin\native-image -cp .\build\native-image.jar -H:+ReportExceptionStackTraces --report-unsupported-elements-at-runtime
-        env:
-          JAVA_HOME: ./graalvm-ce-java11-21.3.0
+        run: native-image.cmd -cp .\build\native-image.jar -H:+ReportExceptionStackTraces --report-unsupported-elements-at-runtime
       - name: Smoke Test
         if: success()
         shell: cmd

.mvn/wrapper/maven-wrapper.properties

@@ -1 +1 @@
-distributionUrl = https://apache.claz.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.zip
+distributionUrl = https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.8.4/apache-maven-3.8.4-bin.zip

attribution.txt

@@ -390,7 +390,7 @@ https://opensource.org/licenses/BSD-2-Clause
 
 -------------------------------------------------------------------------------------------------------------------------------
 
-Package: org.spockframework:spock-core:2.0-M3-groovy-3.0
+Package: org.spockframework:spock-core:2.0-groovy-3.0
 
 License: Apache-2.0
 
@@ -1847,7 +1847,7 @@ limitations under the License.
 
 -------------------------------------------------------------------------------------------------------------------------------
 
-Package: org.codehaus.groovy:groovy:3.0.4
+Package: org.codehaus.groovy:groovy:3.0.9
 
 License: Apache-2.0
 
@@ -1955,7 +1955,7 @@ limitations under the License.
 
 -------------------------------------------------------------------------------------------------------------------------------
 
-Package: org.apache.maven:maven-model:3.6.3
+Package: org.apache.maven:maven-model:3.8.4
 
 License: Apache-2.0
 

bom/pom.xml

@@ -8,7 +8,7 @@
     <parent>
         <groupId>com.optum.sourcehawk</groupId>
         <artifactId>sourcehawk</artifactId>
-        <version>0.6.0-SNAPSHOT</version>
+        <version>0.6.1-SNAPSHOT</version>
     </parent>
 
     <artifactId>sourcehawk-bom</artifactId>

cli/pom.xml

@@ -8,7 +8,7 @@
     <parent>
         <artifactId>sourcehawk</artifactId>
         <groupId>com.optum.sourcehawk</groupId>
-        <version>0.6.0-SNAPSHOT</version>
+        <version>0.6.1-SNAPSHOT</version>
     </parent>
 
     <artifactId>sourcehawk-cli</artifactId>
@@ -23,7 +23,7 @@
         <cli.class>com.optum.sourcehawk.cli.Sourcehawk</cli.class>
 
         <!-- JACOCO OVERRIDES -->
-        <jacoco.coverage.minimum>0.94</jacoco.coverage.minimum> <!-- jacoco-maven-plugin -->
+        <jacoco.coverage.minimum>0.91</jacoco.coverage.minimum> <!-- jacoco-maven-plugin -->
 
         <!-- SONAR PROPERTIES -->
         <sonar.exclusions>**/picocli/**/*.*</sonar.exclusions>

cli/src/main/java/com/optum/sourcehawk/cli/AbstractRemoteScanCommand.java

@@ -52,21 +52,14 @@ public Integer call() {
                 .isPresent();
         val remoteRef = validateAndParseRemoteRef();
         execOptionsBuilder.remoteRef(remoteRef);
+        val repositoryFileReader = createRepositoryFileReader(remoteRef);
+        execOptionsBuilder.repositoryFileReader(repositoryFileReader);
         if (StringUtils.equals(SourcehawkConstants.DEFAULT_CONFIG_FILE_NAME, parentExecOptions.getConfigurationFileLocation()) && !configFileProvided) {
-            execOptionsBuilder.configurationFileLocation(constructRemoteConfigFileLocation(remoteRef));
+            execOptionsBuilder.configurationFileLocation(repositoryFileReader.getAbsoluteLocation(SourcehawkConstants.DEFAULT_CONFIG_FILE_NAME));
         }
-        execOptionsBuilder.repositoryFileReader(createRepositoryFileReader(remoteRef)).build();
         return parentCommand.call(execOptionsBuilder.build());
     }
 
-    /**
-     * Construct the remote config file location
-     *
-     * @param remoteRef the remote reference
-     * @return the config file remote location
-     */
-    protected abstract String constructRemoteConfigFileLocation(final RemoteRef remoteRef);
-
     /**
      * Create the repository file reader based off the remote reference
      *
@@ -76,11 +69,11 @@ public Integer call() {
     protected abstract RepositoryFileReader createRepositoryFileReader(final RemoteRef remoteRef);
 
     /**
-     * Get the remote reference descriptor
+     * Get the raw remote reference
      *
-     * @return the raw remote reference descriptor
+     * @return the raw remote reference
      */
-    protected abstract Pair<RemoteRef.Type, String> getRawRemoteReference();
+    protected abstract Pair<String, String> getRawRemoteReference();
 
     /**
      * Parse the coordinates to github options

cli/src/main/java/com/optum/sourcehawk/cli/BitbucketScanCommand.java

@@ -1,14 +1,15 @@
 package com.optum.sourcehawk.cli;
 
-import com.optum.sourcehawk.core.constants.SourcehawkConstants;
 import com.optum.sourcehawk.core.data.Pair;
 import com.optum.sourcehawk.core.data.RemoteRef;
-import com.optum.sourcehawk.core.repository.BitbucketRepositoryFileReader;
+import com.optum.sourcehawk.core.repository.RemoteRepositoryFileReader;
 import com.optum.sourcehawk.core.repository.RepositoryFileReader;
 import lombok.val;
 import picocli.CommandLine;
 
-import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.HashMap;
 import java.util.Optional;
 
 /**
@@ -25,6 +26,9 @@
 )
 public class BitbucketScanCommand extends AbstractRemoteScanCommand {
 
+    private static final String DEFAULT_BASE_URL = "https://bitbucket.org";
+    private static final String DEFAULT_REF = "main";
+
     /**
      * The Bitbucket options
      */
@@ -44,25 +48,24 @@ public static void main(final String... args) {
     /** {@inheritDoc} */
     @Override
     protected RepositoryFileReader createRepositoryFileReader(final RemoteRef remoteRef) {
-        if (bitbucket.serverUrl != null) {
-            return new BitbucketRepositoryFileReader(bitbucket.token, bitbucket.serverUrl.toString(), remoteRef);
+        val rawFileUrlTemplate = Optional.ofNullable(bitbucket.serverUrl)
+            .map(bitbucketServerUrl -> String.format("%s/rest/api/1.0/projects/%s/repos/%s/raw/%%s?at=%s",
+                bitbucketServerUrl, remoteRef.getNamespace(), remoteRef.getRepository(), remoteRef.getRef()))
+            .orElseGet(() -> String.format("%s/api/2.0/repositories/%s/%s/src/%s/%%s", DEFAULT_BASE_URL, remoteRef.getNamespace(), remoteRef.getRepository(), remoteRef.getRef()));
+        val requestProperties = new HashMap<String, String>();
+        requestProperties.put("Accept", "text/plain");
+        if (bitbucket.token != null) {
+            val authScheme = Optional.ofNullable(bitbucket.authScheme)
+                .orElse(CommandOptions.Bitbucket.DEFAULT_AUTH_SCHEME);
+            requestProperties.put("Authorization", String.format("%s %s", authScheme, bitbucket.token));
         }
-        return new BitbucketRepositoryFileReader(bitbucket.token, remoteRef);
-    }
-
-    /** {@inheritDoc} */
-    @Override
-    protected Pair<RemoteRef.Type, String> getRawRemoteReference() {
-        return Pair.of(RemoteRef.Type.BITBUCKET, bitbucket.remoteReference);
+        return new RemoteRepositoryFileReader(rawFileUrlTemplate, requestProperties);
     }
 
     /** {@inheritDoc} */
     @Override
-    protected String constructRemoteConfigFileLocation(final RemoteRef remoteRef) {
-        val bitbucketBaseUrl = Optional.ofNullable(bitbucket.serverUrl)
-                .map(URL::toString)
-                .orElseGet(RemoteRef.Type.BITBUCKET::getBaseUrl);
-        return BitbucketRepositoryFileReader.constructBaseUrl(remoteRef, bitbucketBaseUrl) + SourcehawkConstants.DEFAULT_CONFIG_FILE_NAME;
+    protected Pair<String, String> getRawRemoteReference() {
+        return Pair.of(bitbucket.remoteReference, DEFAULT_REF);
     }
 
 }

cli/src/main/java/com/optum/sourcehawk/cli/CommandOptions.java

@@ -152,6 +152,15 @@ static class Bitbucket {
         )
         String token;
 
+        @CommandLine.Option(
+                names = {"-a", "--auth-scheme"},
+                paramLabel = "auth-scheme",
+                defaultValue = DEFAULT_AUTH_SCHEME,
+                description = "The authorization scheme to use (either Bearer or Basic).  If Basic, the provided token must be base64 encoded."
+        )
+        String authScheme;
+        static final String DEFAULT_AUTH_SCHEME = "Bearer";
+
         @CommandLine.Option(
                 names = {"-S", "--server-url"},
                 paramLabel = "bitbucket-server-url",
@@ -162,7 +171,7 @@ static class Bitbucket {
         @CommandLine.Parameters(
                 paramLabel = REMOTE_REFERENCE_LABEL,
                 description = "The Bitbucket remote reference - project/repo@ref combination, "
-                        + "i.e - project/repo, project/repo@master,  project/[email protected], or project/repo@a6de43fa51c",
+                        + "i.e - project/repo, project/repo@main,  project/[email protected], or project/repo@a6de43fa51c",
                 arity = "1"
         )
         String remoteReference;

cli/src/main/java/com/optum/sourcehawk/cli/GithubScanCommand.java

@@ -1,14 +1,13 @@
 package com.optum.sourcehawk.cli;
 
-import com.optum.sourcehawk.core.constants.SourcehawkConstants;
 import com.optum.sourcehawk.core.data.Pair;
 import com.optum.sourcehawk.core.data.RemoteRef;
-import com.optum.sourcehawk.core.repository.GithubRepositoryFileReader;
+import com.optum.sourcehawk.core.repository.RemoteRepositoryFileReader;
 import com.optum.sourcehawk.core.repository.RepositoryFileReader;
 import lombok.val;
 import picocli.CommandLine;
 
-import java.net.URL;
+import java.util.HashMap;
 import java.util.Optional;
 
 /**
@@ -25,6 +24,10 @@
 )
 public class GithubScanCommand extends AbstractRemoteScanCommand {
 
+    private static final String DEFAULT_BASE_URL = "raw.githubusercontent.com";
+    private static final String DEFAULT_REF = "main";
+    private static final String AUTHORIZATION_TOKEN_PREFIX = "Bearer";
+
     /**
      * The github options
      */
@@ -44,28 +47,22 @@ public static void main(final String... args) {
     /** {@inheritDoc} */
     @Override
     protected RepositoryFileReader createRepositoryFileReader(final RemoteRef remoteRef) {
-        if (github.enterpriseUrl != null) {
-            return new GithubRepositoryFileReader(github.token, github.enterpriseUrl.toString(), remoteRef);
+        val baseUrl = Optional.ofNullable(github.enterpriseUrl)
+            .map(githubEnterpriseUrl -> String.format("%s/raw", github.enterpriseUrl))
+            .orElse(DEFAULT_BASE_URL);
+        val rawFileUrlTemplate  = String.format("%s/%s/%s/%s/%%s", baseUrl, remoteRef.getNamespace(), remoteRef.getRepository(), remoteRef.getRef());
+        val requestProperties = new HashMap<String, String>();
+        requestProperties.put("Accept", "text/plain");
+        if (github.token != null) {
+            requestProperties.put("Authorization", String.format("%s %s", AUTHORIZATION_TOKEN_PREFIX, github.token));
         }
-        return new GithubRepositoryFileReader(github.token, remoteRef);
-    }
-
-    /** {@inheritDoc} */
-    @Override
-    protected Pair<RemoteRef.Type, String> getRawRemoteReference() {
-        return Pair.of(RemoteRef.Type.GITHUB, github.remoteReference);
+        return new RemoteRepositoryFileReader(rawFileUrlTemplate, requestProperties);
     }
 
     /** {@inheritDoc} */
     @Override
-    protected String constructRemoteConfigFileLocation(final RemoteRef remoteRef) {
-        val githubEnterpriseUrl = Optional.ofNullable(github.enterpriseUrl).map(URL::toString);
-        val githubRepoBaseUrl = GithubRepositoryFileReader.constructBaseUrl(
-                githubEnterpriseUrl.orElseGet(RemoteRef.Type.GITHUB::getBaseUrl),
-                githubEnterpriseUrl.isPresent(),
-                remoteRef
-        );
-        return githubRepoBaseUrl + SourcehawkConstants.DEFAULT_CONFIG_FILE_NAME;
+    protected Pair<String, String> getRawRemoteReference() {
+        return Pair.of(github.remoteReference, DEFAULT_REF);
     }
 
 }

cli/src/main/resources/META-INF/native-image/sourcehawk-cli/native-image.properties

@@ -1,3 +1,2 @@
 Args = -H:Class=${cli.class} \
-       -H:Name=${cli.name} \
-       --no-server
+       -H:Name=${cli.name}