advanced-security · nicolaswill · Oct 15, 2021 · Oct 18, 2021 · Oct 20, 2021 · Oct 26, 2021
diff --git a/Dockerfile.example b/Dockerfile.example
@@ -0,0 +1,33 @@
+FROM ubuntu
+LABEL description="Security & Quality CodeQL Container Build for Cool Applications"
+SHELL ["/bin/bash", "-c"]
+ENV TZ=America/New_York
+
+# create directories
+RUN mkdir /tools
+
+# setup tools
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+RUN DEBIAN_FRONTEND="noninteractive" apt-get update && apt-get install -y golang zip wget
+RUN wget -q https://github.com/github/codeql-action/releases/download/codeql-bundle-20211005/codeql-bundle-linux64.tar.gz
+RUN tar xzf /codeql-bundle-linux64.tar.gz -C tools
+
+# copy source
+COPY . /usr/src/myapp
+
+# set working directory
+WORKDIR /usr/src/myapp
+
+# example repo used: https://github.com/ghas-bootcamp/ghas-bootcamp
+
+# codeql create
+RUN /tools/codeql/codeql database create db --language=javascript, java --db-cluster  --no-run-unnecessary-builds -vvvv
+
+# codeql analyze with default queries
+RUN /tools/codeql/codeql database analyze codeql-database/go go-code-scanning.qls --format=sarif-latest --output=codeql-go-results.sarif --sarif-category=goiscool
+RUN /tools/codeql/codeql database analyze db javascript-code-scanning.qls --format=sarif-latest --output=codeql-javascript-results.sarif --sarif-category=javascriptiscool
+
+# upload results
+# remember to get the MERGE commit for a PR
+RUN /tools/codeql/codeql github upload-results --github-url=<ghes-url> --repository=oreos/miniature-invention --ref=refs/pull/1/merge --commit=778337f84a5abe2cda468c7abf6038b8a193cea2 --sarif=codeql-go-results.sarif --github-auth-stdin=<github PAT>
+RUN /tools/codeql/codeql github upload-results --github-url=<ghes-url> --repository=oreos/miniature-invention --ref=refs/pull/1/merge --commit=778337f84a5abe2cda468c7abf6038b8a193cea2 --sarif=codeql-javascript-results.sarif --github-auth-stdin=<github PAT>
diff --git a/GHAS-on-GHES-feature-matrix.md b/GHAS-on-GHES-feature-matrix.md
diff --git a/README.md b/README.md
@@ -1,2 +1,46 @@
 # Advanced Security Material
-A place for resources to help you understand and use GitHub Advanced Security
+A place for resources to help you understand and use GitHub Advanced Security (GHAS).  Browse the directories in this repository for resources and documentation.  To help you get started with GHAS, we've provided some introductory documentation in this file.
+
+## Get started with GitHub Advanced Security
+The following list of links are great resources to get you started on learning how to use, deploy, and manage GitHub Advanced Security in your environment.
+
+New to GitHub Advanced Security?  Start with [GitHub security features](https://docs.github.com/en/enterprise-cloud@latest/code-security/getting-started/github-security-features) :+1:
+
+### Code Scanning
+- [About GitHub Code Scanning](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/about-code-scanning)
+- [Configuring Code Scanning](https://docs.github.com/en/enterprise-cloud@latest/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning)
+- [Integrating other tools with GHAS](https://docs.github.com/en/code-security/code-scanning/integrating-with-code-scanning)
+
+### CodeQL
+- [Meet CodeQL](https://codeql.github.com/)
+- [CodeQL Documentation](https://codeql.github.com/docs/)
+- [CWE Query Mapping Documentation](https://codeql.github.com/codeql-query-help/codeql-cwe-coverage)
+- [Running additional queries](https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#running-additional-queries)
+- [CodeQL CLI Docs](https://codeql.github.com/docs/codeql-cli/getting-started-with-the-codeql-cli)
+- [Running CodeQL in your CI System](https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/running-codeql-code-scanning-in-your-ci-system)
+
+### Secret Scanning
+- [About Secret Scanning](https://docs.github.com/en/code-security/secret-scanning/about-secret-scanning)
+- [Supported secret patterns](https://docs.github.com/en/code-security/secret-scanning/secret-scanning-patterns#supported-secrets-for-partner-patterns)
+- [Defining custom secret patterns](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning)
+
+### Supply Chain Security (Dependabot)
+- [About](https://docs.github.com/en/enterprise-cloud@latest/code-security/supply-chain-security/understanding-your-software-supply-chain/about-supply-chain-security)
+- [Dependency Graph](https://docs.github.com/en/enterprise-cloud@latest/code-security/supply-chain-security/understanding-your-software-supply-chain/about-the-dependency-graph)
+- [Dependabot Alerts](https://docs.github.com/en/enterprise-cloud@latest/code-security/dependabot/dependabot-alerts/about-dependabot-alerts)
+- [Dependabot Security Updates](https://docs.github.com/en/code-security/dependabot/dependabot-security-updates/configuring-dependabot-security-updates)
+- [GitHub Advisory Database](https://github.com/advisories)
+- [Dependabot Quickstart Guide](https://docs.github.com/en/code-security/getting-started/dependabot-quickstart-guide)
+
+### Security Overview
+- [About Security Overview](https://docs.github.com/en/code-security/security-overview/about-the-security-overview)
+- [Managing alerts in your repository](https://docs.github.com/en/enterprise-cloud@latest/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/managing-code-scanning-alerts-for-your-repository)
+
+### Other Resources
+- [SARIF Tutorials](https://github.com/microsoft/sarif-tutorials)
+- [GitHub Advanced Security Learning Path](https://docs.microsoft.com/en-us/users/githubtraining/collections/rqymc6yw8q5rey)
+- [Scaling GHAS in Your Organization](https://resources.github.com/downloads/Whitepaper-Scaling-GHAS-in-an-Enterprise.pdf)
+- [The Complete Guide to Developer-first Security](https://resources.github.com/downloads/GitHubAdvanced%20SecurityEbook.pdf)
+- [GitHub Checkout - Code Scanning (video)](https://www.youtube.com/watch?v=z0wvGf3O69E)
+- [GitHub Checkout - Secret Scanning (video)](https://www.youtube.com/watch?v=aoL7pDrXt74)
+- [GitHub Checkout - Viewing and Managing your Dependencies (video)](https://www.youtube.com/watch?v=gNd_TGdZ1xc)
diff --git a/advanced-security-material.md b/advanced-security-material.md
@@ -13,8 +13,8 @@
 - [ ] Javascript: https://www.youtube.com/watch?v=pYzfGaLTqC0
 
 #### CodeQL Resources: 
-- [ ] QL Tutorials: https://help.semmle.com/QL/learn-ql/beginner/ql-tutorials.html
-- [ ] CodeQL for VS Code: https://help.semmle.com/codeql/codeql-for-vscode/procedures/about-codeql-for-vscode.html
+- [ ] QL Tutorials: https://codeql.github.com/docs/writing-codeql-queries/ql-tutorials/
+- [ ] CodeQL for VS Code: https://codeql.github.com/docs/codeql-for-visual-studio-code/
 - [ ] VS Code starter workspace to use with the CodeQL VS extension: https://github.com/github/vscode-codeql-starter
 - [ ] CodeQL CTF: https://securitylab.github.com/ctf
 - [ ] Read about contributing to CodeQL Queries: https://github.com/github/codeql/blob/main/CONTRIBUTING.md
@@ -34,7 +34,7 @@
   - [ ] Configure code scanning: https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning
   - [ ] Configuring builds for Compiled Languages: https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-the-codeql-workflow-for-compiled-languages
   - [ ] Running additional queries: https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#running-additional-queries
-  - [ ] Built-in Queries: https://help.semmle.com/QL/ql-built-in-queries.html
+  - [ ] Built-in Queries: https://github.com/github/codeql, https://github.com/github/codeql-go
     - For example, js query suites: https://github.com/github/codeql/tree/master/javascript/ql/src/codeql-suites
 - [ ] Troubleshooting code scanning workflow:
 https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/troubleshooting-the-codeql-workflow
@@ -46,4 +46,4 @@ https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerab
   - [ ] Jenkins + CodeQL CLI: https://github.com/kllund/sample-pipeline-files/blob/main/Jenkinsfile-template-with-codeql-cli-bundle
 
 #### OSS Issue Tracking
-- [ ] GitHub Code Scanning + Jira: https://github.com/github/codescanning-jira-integration
+- [ ] GitHub Code Scanning + Jira: https://github.com/github/ghas-jira-integration
diff --git a/advanced-security-reporting.md b/advanced-security-reporting.md
diff --git a/code-scanning-guides/setup-codeql-cli.md b/code-scanning-guides/setup-codeql-cli.md
@@ -1,6 +1,6 @@
 ### Getting started with the CodeQL CLI
 
-When you want to generate a CodeQL database locally and run the pre-compiled queries against, this is the way to go.
+When you want to generate a CodeQL database locally and run the pre-compiled queries against it, this is the way to go.
 
 First let's download the CodeQL bundle! Head over [here](https://github.com/github/codeql-action/releases ) and download the approprate bundle for your operating system.
 Once it's downloaded, untar the content to a CodeQL home folder and you can add CodeQL to your path if you'd like
@@ -15,9 +15,6 @@ Check to make sure you can use the CodeQL CLI
 codeql --version
 ```
 
-You can see in this example how the CodeQL CLI is used in a [workflow](https://github.com/advanced-security/javascript-codeql-cli-test-workflow/blob/main/.github/workflows/codeql-analysis.yml).
-Note that it always downloads the latest CodeQL bundle for Linux. In your case, choose the bundle that best fits your operating system.
-
 Now we need to use the CodeQL CLI on an actual repository. Let's start here with our [GHAS training material](https://github.com/ghas-bootcamp/ghas-bootcamp)
 There's multiple languages being used here, so for the purposes of this tutorial let's try to scan the Javascript portions of the codebase. 
 
@@ -29,7 +26,7 @@ Clone this repository and `cd` into it.
 The first thing we gotta do when it comes to CodeQL analysis is to create a CodeQL database. 
 When it comes to interpreted languages and Go, CodeQL will use an autobuild.sh script that will extract the source code and create a snapshot database. 
 When it comes to compiled languages, we require to build the source code in order to trace the build and create a snapshot database of it. 
-You can rely on the autobuild.sh script as well, or you can supply your own build instructions via `--comand` flag, which can be used when invoking the `codeql database create` command.
+You can rely on the autobuild.sh script as well, or you can supply your own build instructions via the `--command` flag, which can be used when invoking the `codeql database create` command.
 Please review this [list](https://codeql.github.com/docs/codeql-overview/supported-languages-and-frameworks/) of currently supported languages and frameworks.
 
 
@@ -41,9 +38,9 @@ CodeQL will create the `db` directory and will choose the autobuild.sh script fo
 CodeQL will also finalize the database at the specified `db` directory. Within your codeql database directory (in this case `db`) 
 you should notice a db-javascript directory which contains the db schemes and a src.zip which contains the source that was extracted.
 
-#### Importing the CodeQL database to Visual Studios
+#### Optional: Importing the CodeQL database to Visual Studios
 You can actually take this database and import it to your Visual Studios workspace. 
-To get started on that, please go to this repository and follow the instructions on how to setup the CodeQL starter workspace, as well as installing the CodeQL plugin.
+To get started on that, please go to this [repository](https://github.com/github/vscode-codeql-starter) and follow the instructions on how to setup the CodeQL starter workspace, as well as installing the CodeQL plugin.
 Once you have the CodeQL plugin installed, import the database you created in this step and try to run a javascript query against the database.
 
 
@@ -77,13 +74,13 @@ Failure to do so, in particular on a pull request, can cause confusion in that C
 This step is typically used when you want to see the SARIF in the Code Scanning alerts UI. It's typically used when you want to post results to the default branch of a repository for the first time (baseline analysis) or to a pull request to see any security alert annotations.
 
 Here are some advanced things to note:
-- When posting the analysis for the first time to a default analysis, make sure you define a `--sarif-category`. That way for the analyses for subsequent pull requests can also share the same category value. 
+- When posting the analysis for the first time to a default analysis, make sure you define a `--sarif-category`. That way  the analyses for subsequent pull requests can also share the same category value. 
 Note that this kind of depends on how you're running the builds (whether or not you've broken down a monorepo into separate analyses or you have multiple scans due to multiple languaages) but typically just starting out, 
 just make sure to have the same category value for subsequent scans, so that Code Scannning can easily figure out what the basline analysis is to compare subsequent analyses.
 
 The `--ref` and `--commit` flag combinations can be one of the following:
-- `refs/pulls/<pull request number>/merge` + HEAD commit
-- `refs/heads/<branch name>` + MERGE commit
+- `refs/pulls/<pull request number>/merge` + MERGE commit
+- `refs/heads/<branch name>` + HEAD commit
   - ` curl -H "Accept: application/vnd.github.v3+json" \\n  -H "Authorization: token $GH_TOKEN" \\n  https://api.github.com/repos/<org-name>/<repo-name>/pulls/<pull-request-number> | jq '.merge_commit_sha'`
   - The merge commit is a commit created to make sure PR checks are ran; this commit doesn't exist in the actual source tree/`git log`.
 

diff --git a/code-scanning-guides/synthetic-applications/owasp-webgoat.md b/code-scanning-guides/synthetic-applications/owasp-webgoat.md
@@ -0,0 +1,15 @@
+# OWASP WebGoat
+
+[A full Actions workflow can be found here](./owasp-webgoat.yml)
+
+## Common Issues
+
+Scanning OWASP WebGoat can have some issues right out of the box where CodeQL might find very little or worse not find anything at all.
+This is due to the following:
+
+1. WebGoat uses JDK 17
+  - Action uses JDK 8 by default
+2. Uses Project Lombok
+  - Future support will be coming to CodeQL natively
+3. Dependencies are not all present in Dependency Graph
+  - Using [Submission API](https://docs.github.com/en/enterprise-cloud@latest/code-security/supply-chain-security/understanding-your-software-supply-chain/using-the-dependency-submission-api#using-pre-made-actions)    
diff --git a/code-scanning-guides/synthetic-applications/owasp-webgoat.yml b/code-scanning-guides/synthetic-applications/owasp-webgoat.yml
@@ -0,0 +1,61 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ main ]
+  workflow_dispatch:
+
+permissions:
+  actions: read
+  contents: read
+  security-events: write
+
+env:
+  # in the future, this flag will not be needed
+  CODEQL_EXTRACTOR_JAVA_RUN_ANNOTATION_PROCESSORS: true
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'java', 'javascript' ]
+
+    steps:
+    - uses: actions/checkout@v2
+
+    # WebGoat requires Java/JDK 17
+    - name: Set up JDK 17
+      if: matrix.language == 'java'
+      uses: actions/setup-java@v3
+      with:
+        distribution: 'temurin'
+        java-version: 17
+        architecture: x64
+
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # [optional] enabled extended queries
+        # queries: +security-extended,security-and-quality
+        # [optional] Field Config - standard packs, extensions, and extra packs
+        config-file: advanced-security/codeql-queries/config/codeql.yml@main
+
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
+
+    # Run the Analysis
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2
+
+    # Submit Maven Dependency Tree to GitHub
+    - name: Maven Dependency Tree Dependency Submission
+      if: matrix.language == 'java'
+      uses: advanced-security/[email protected]
diff --git a/code-scanning-scripts/README.md b/code-scanning-scripts/README.md
@@ -0,0 +1,5 @@
+### Code scanning scripts 
+
+
+- [ ] [Code scanning bulke enable](https://github.com/mario-campos/gh-code-scanning)
+- [ ] [Run CodeQL analysis on a pull request](https://github.com/advanced-security/advanced-security-material/blob/main/code-scanning-scripts/run-pr-codeql-analysis.sh)
diff --git a/code-scanning-scripts/combine-n-databases.sh b/code-scanning-scripts/combine-n-databases.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+# For n number of dirs, initialise each of them. In this example I have two dirs at the root:
+codeql database init dbs/db-js1 --language=javascript --source-root dir1 --overwrite
+codeql database init dbs/db-js2 --language=javascript --source-root dir2 --overwrite
+
+# After db skeletons are created, use the trace command to call out the extractor/add build instructions
+codeql database trace-command dbs/db-js1 -- /Users/cmboling/Projects/codeql-home/codeql-latest/javascript/tools/autobuild.sh
+codeql database trace-command dbs/db-js2 -- /Users/cmboling/Projects/codeql-home/codeql-latest/javascript/tools/autobuild.sh
+
+# Then import n unfinalized dbs to the target db
+codeql database import dbs/db-js1 db-js2
+codeql database finalize --finalize-dataset db-js1
+
+# Analyze target db as usual
+codeql database analyze db-js1 javascript-code-scanning.qls --format=sarif-latest --output=codeql-javascript-results.sarif
diff --git a/code-scanning-scripts/get-languages.sh b/code-scanning-scripts/get-languages.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+curl -H "Accept: application/vnd.github.v3+json" -H "Authorization: token $GH_TOKEN" \                                                                           
+https://api.github.com/repos/advanced-security/ghas-bootcamp-dryrun/languages |  \
+jq 'with_entries(select([.key] | inside(["Go", "Java", "JavaScript", "Python", "C++", "C#", "C", "TypeScript"])) | if .key == "C" then .key = "cpp" else . end | if .key == "C#" then .key = "csharp" else . end | if .key == "C++" then .key = "cpp" else . end | if .key == "Go" then .key = "go" else . end | if .key == "Java" then .key = "java" else . end | if .key == "JavaScript" then .key = "javascript" else . end | if .key == "Python" then .key = "python" else . end | if .key == "TypScript" then .key = "typescript" else . end)' | jq "keys"
diff --git a/code-scanning-scripts/run-pr-codeql-analysis.sh b/code-scanning-scripts/run-pr-codeql-analysis.sh
@@ -19,17 +19,14 @@ CODEQL_SARIF_CATEGORY=.github/workflows/codeql-analysis.yml:analyze/language:go
 
 # run a single language analysis for a PR
 
-# remove db
-rm -rf $CODEQL_DATABASE
-
-# get mergit commit sha
+# get merge commit sha
 GH_MERGE_COMMIT_SHA=$(curl -H "Accept: application/vnd.github.v3+json" -H "Authorization: token $GH_TOKEN" https://api.github.com/repos/$GH_ORG/$GH_REPO/pulls/$GH_PULL_REQUEST_NUMBER | jq '.merge_commit_sha' | sed -e 's/^"//' -e 's/"$//')
 
 # check codeql --version
 codeql --version
 
 # codeql database create
-codeql database create $CODEQL_DATABASE --language=$CODEQL_LANGUAGE
+codeql database create $CODEQL_DATABASE --language=$CODEQL_LANGUAGE --overwrite
 
 # codeql database analyze
 codeql database analyze $CODEQL_DATABASE $CODEQL_QUERY_SUITE --output=$CODEQL_SARIF_RESULTS --sarif-category=$CODEQL_SARIF_CATEGORY --format=sarif-latest