diff --git a/Dockerfile b/Dockerfile index b31dcdb..7504afa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,83 +1,182 @@ -FROM ubuntu:16.04 +FROM ubuntu:19.04 MAINTAINER james@gauntlt.org ARG ARACHNI_VERSION=arachni-1.5.1-0.5.12 +WORKDIR /opt -# Install Ruby and other OS stuff -RUN apt-get update && \ - apt-get install -y build-essential \ +ENV DEBIAN_FRONTEND noninteractive + +# Install Ruby, Gauntlt and everything needing build-essential +RUN apt update && \ + apt install -y build-essential \ bzip2 \ ca-certificates \ curl \ gcc \ git \ - libcurl3 \ + libcurl4 \ libcurl4-openssl-dev \ wget \ zlib1g-dev \ + libyaml-dev \ libfontconfig \ libxml2-dev \ + libxml2 \ libxslt1-dev \ + libxslt-dev \ + libsqlite3-dev \ make \ python-pip \ + xmlstarlet \ python2.7 \ python2.7-dev \ + python-pip \ + python-setuptools \ ruby \ ruby-dev \ + openjdk-8-jre \ + xvfb \ + x11vnc \ ruby-bundler && \ - rm -rf /var/lib/apt/lists/* + gem install rake && \ + wget -O dirb.tar.gz https://downloads.sourceforge.net/project/dirb/dirb/2.22/dirb222.tar.gz && \ + tar xvf dirb.tar.gz && \ + rm dirb.tar.gz && \ + cd dirb222 && \ + chmod 755 ./configure && \ + ./configure && \ + make && \ + ln -s /opt/dirb222/dirb /usr/local/bin/dirb && \ + cd /usr/src && \ + git clone --single-branch --branch ${GAUNTLT_BRANCH} ${GAUNTLT_REPO} && \ + cd gauntlt && \ + gem install bundler && \ + bundler update && \ + git submodule update --init --recursive --force && \ + rake build && \ + rake install && \ + cd && \ + rm -rf /usr/src/* && \ + apt remove -y \ + ruby-dev \ + python2.7-dev \ + libxml2-dev \ + libxslt1-dev \ + build-essential \ + libcurl4-openssl-dev \ + zlib1g-dev && \ + pip install sslyze==1.4.3 && \ + gem install zapr && \ + rm -rf /var/lib/apt/lists/* && \ + apt autoremove -y && \ + apt clean -# Install Gauntlt -RUN gem install rake -RUN gem install ffi -v 1.9.18 -RUN gem install gauntlt --no-rdoc --no-ri -# Install Attack tools -WORKDIR /opt +# Install remaining Attack tools # arachni RUN wget https://github.com/Arachni/arachni/releases/download/v1.5.1/${ARACHNI_VERSION}-linux-x86_64.tar.gz && \ tar xzvf ${ARACHNI_VERSION}-linux-x86_64.tar.gz > /dev/null && \ mv ${ARACHNI_VERSION} /usr/local && \ + rm ${ARACHNI_VERSION}-linux-x86_64.tar.gz && \ ln -s /usr/local/${ARACHNI_VERSION}/bin/* /usr/local/bin/ # Nikto RUN apt-get update && \ apt-get install -y libtimedate-perl \ libnet-ssleay-perl && \ - rm -rf /var/lib/apt/lists/* - -RUN git clone --depth=1 https://github.com/sullo/nikto.git && \ + git clone --depth=1 https://github.com/sullo/nikto.git && \ cd nikto/program && \ echo "EXECDIR=/opt/nikto/program" >> nikto.conf && \ ln -s /opt/nikto/program/nikto.conf /etc/nikto.conf && \ chmod +x nikto.pl && \ - ln -s /opt/nikto/program/nikto.pl /usr/local/bin/nikto + ln -s /opt/nikto/program/nikto.pl /usr/local/bin/nikto && \ + rm -rf /var/lib/apt/lists/* && \ + apt autoremove -y && \ + apt clean # sqlmap -WORKDIR /opt ENV SQLMAP_PATH /opt/sqlmap/sqlmap.py RUN git clone --depth=1 https://github.com/sqlmapproject/sqlmap.git -# dirb -COPY vendor/dirb222.tar.gz dirb222.tar.gz - -RUN tar xvfz dirb222.tar.gz > /dev/null && \ - cd dirb222 && \ - chmod 755 ./configure && \ - ./configure && \ - make && \ - ln -s /opt/dirb222/dirb /usr/local/bin/dirb - +# dirdb is installed with stuff needing build esentials ENV DIRB_WORDLISTS /opt/dirb222/wordlists # nmap -RUN apt-get update && \ - apt-get install -y nmap && \ - rm -rf /var/lib/apt/lists/* +RUN apt update && \ + apt install -y nmap && \ + apt clean && \ + rm -rf /var/lib/apt/lists/* && \ + apt clean -# sslyze -RUN pip install sslyze==1.3.4 +# sslyze is installed with stuff needing build esentials ENV SSLYZE_PATH /usr/local/bin/sslyze +# Heartbleed +RUN apt update && \ + apt install -y golang && \ + export GOPATH=/go && \ + go get github.com/FiloSottile/Heartbleed && \ + go install github.com/FiloSottile/Heartbleed && \ + mv /go/bin/Heartbleed /usr/local/bin/ && \ + rm -rf /go && \ + apt remove -y golang && \ + apt autoremove -y && \ + apt clean && \ + rm -rf /var/lib/apt/lists/* && \ + apt clean + +# Garmr +RUN pip install beautifulsoup && \ + git clone https://github.com/freddyb/Garmr.git && \ + cd Garmr && \ + python setup.py install + +# owasp-zap adapted from https://github.com/zaproxy/zaproxy/blob/develop/docker/Dockerfile-weekly +RUN mkdir /opt/zap && \ + cd /opt/zap && \ + wget -q https://github.com/zaproxy/zaproxy/releases/download/w2019-05-15/ZAP_WEEKLY_D-2019-05-15.zip && \ + unzip *.zip && \ + rm *.zip && \ + mv ZAP_D*/* . && \ + rmdir ZAP_D* && \ + + # Setup Webswing + curl -s -L https://bitbucket.org/meszarv/webswing/downloads/webswing-2.5.10.zip > webswing.zip && \ + unzip webswing.zip && \ + rm webswing.zip && \ + mv webswing-* webswing && \ + # Remove Webswing demos + rm -Rf webswing/demo/ && \ + # Accept ZAP license + touch AcceptedLicense && \ + pip install zapcli python-owasp-zap-v2.4 && \ + mkdir -p /opt/zap/scripts/scripts/httpsender/ + +COPY vendor/webswing.config /opt/zap/webswing/ +COPY vendor/zap-x.sh /usr/local/bin/ +COPY vendor/zap-api-scan.py /opt/zap/ +COPY vendor/zap_common.py /opt/zap/ + +COPY vendor/policies /root/.ZAP_D/policies/ +COPY vendor/scripts /root/.ZAP_D/scripts/ +COPY vendor/.xinitrc /root/ +ENV ZAP_PORT 8080 + +RUN chmod a+x /root/.xinitrc && \ + mkdir /root/.vnc + +RUN chmod 755 /usr/local/bin/zap*.sh /opt/zap/zap-api-scan.py && \ + ln -s /opt/zap/zap-api-scan.py /usr/local/bin/zap-api-scan + +ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/ +ENV PATH $JAVA_HOME/bin:/opt/zap/:$PATH +ENV ZAP_PATH /opt/zap/zap.sh +ENV ZAPCLI_PATH /usr/local/bin/zap-cli +ENV ZAPAPISCAN_PATH /usr/local/bin/zap-api-scan + +VOLUME [ "/attacks","/output" ] + ENTRYPOINT [ "/usr/local/bin/gauntlt" ] + +CMD ["/attacks/*"] diff --git a/Dockerfile-dev b/Dockerfile-dev new file mode 100644 index 0000000..956dfc3 --- /dev/null +++ b/Dockerfile-dev @@ -0,0 +1,187 @@ +FROM ubuntu:19.04 +MAINTAINER james@gauntlt.org + +ARG ARACHNI_VERSION=arachni-1.5.1-0.5.12 + +ARG GAUNTLT_BRANCH=master +# Will use this repo for development until https://github.com/gauntlt/gauntlt/pull/120 is merged +ARG GAUNTLT_REPO=https://github.com/millaguie/gauntlt.git +WORKDIR /opt + +ENV DEBIAN_FRONTEND noninteractive + +# Install Ruby, Gauntlt and everything needing build-essential +RUN apt update && \ + apt install -y build-essential \ + bzip2 \ + ca-certificates \ + curl \ + gcc \ + git \ + libcurl4 \ + libcurl4-openssl-dev \ + wget \ + zlib1g-dev \ + libyaml-dev \ + libfontconfig \ + libxml2-dev \ + libxml2 \ + libxslt1-dev \ + libxslt-dev \ + libsqlite3-dev \ + make \ + python-pip \ + xmlstarlet \ + python2.7 \ + python2.7-dev \ + python-pip \ + python-setuptools \ + ruby \ + ruby-dev \ + openjdk-8-jre \ + xvfb \ + x11vnc \ + ruby-bundler && \ + gem install rake && \ + wget -O dirb.tar.gz https://downloads.sourceforge.net/project/dirb/dirb/2.22/dirb222.tar.gz && \ + tar xvf dirb.tar.gz && \ + rm dirb.tar.gz && \ + cd dirb222 && \ + chmod 755 ./configure && \ + ./configure && \ + make && \ + ln -s /opt/dirb222/dirb /usr/local/bin/dirb && \ + cd /usr/src && \ + git clone --single-branch --branch ${GAUNTLT_BRANCH} ${GAUNTLT_REPO} && \ + cd gauntlt && \ + gem install bundler && \ + bundler update && \ + git submodule update --init --recursive --force && \ + rake build && \ + rake install && \ + cd && \ + rm -rf /usr/src/* && \ + apt remove -y \ + ruby-dev \ + python2.7-dev \ + libxml2-dev \ + libxslt1-dev \ + build-essential \ + libcurl4-openssl-dev \ + zlib1g-dev && \ + pip install sslyze==1.4.3 && \ + gem install zapr && \ + rm -rf /var/lib/apt/lists/* && \ + apt autoremove -y && \ + apt clean + + +# Install remaining Attack tools + +# arachni +RUN wget https://github.com/Arachni/arachni/releases/download/v1.5.1/${ARACHNI_VERSION}-linux-x86_64.tar.gz && \ + tar xzvf ${ARACHNI_VERSION}-linux-x86_64.tar.gz > /dev/null && \ + mv ${ARACHNI_VERSION} /usr/local && \ + rm ${ARACHNI_VERSION}-linux-x86_64.tar.gz && \ + ln -s /usr/local/${ARACHNI_VERSION}/bin/* /usr/local/bin/ + +# Nikto +RUN apt-get update && \ + apt-get install -y libtimedate-perl \ + libnet-ssleay-perl && \ + git clone --depth=1 https://github.com/sullo/nikto.git && \ + cd nikto/program && \ + echo "EXECDIR=/opt/nikto/program" >> nikto.conf && \ + ln -s /opt/nikto/program/nikto.conf /etc/nikto.conf && \ + chmod +x nikto.pl && \ + ln -s /opt/nikto/program/nikto.pl /usr/local/bin/nikto && \ + rm -rf /var/lib/apt/lists/* && \ + apt autoremove -y && \ + apt clean + +# sqlmap +ENV SQLMAP_PATH /opt/sqlmap/sqlmap.py +RUN git clone --depth=1 https://github.com/sqlmapproject/sqlmap.git + +# dirdb is installed with stuff needing build esentials +ENV DIRB_WORDLISTS /opt/dirb222/wordlists + +# nmap +RUN apt update && \ + apt install -y nmap && \ + apt clean && \ + rm -rf /var/lib/apt/lists/* && \ + apt clean + +# sslyze is installed with stuff needing build esentials +ENV SSLYZE_PATH /usr/local/bin/sslyze + +# Heartbleed +RUN apt update && \ + apt install -y golang && \ + export GOPATH=/go && \ + go get github.com/FiloSottile/Heartbleed && \ + go install github.com/FiloSottile/Heartbleed && \ + mv /go/bin/Heartbleed /usr/local/bin/ && \ + rm -rf /go && \ + apt remove -y golang && \ + apt autoremove -y && \ + apt clean && \ + rm -rf /var/lib/apt/lists/* && \ + apt clean + +# Garmr +RUN pip install beautifulsoup && \ + git clone https://github.com/freddyb/Garmr.git && \ + cd Garmr && \ + python setup.py install + +# owasp-zap adapted from https://github.com/zaproxy/zaproxy/blob/develop/docker/Dockerfile-weekly +RUN mkdir /opt/zap && \ + cd /opt/zap && \ + wget -q https://github.com/zaproxy/zaproxy/releases/download/w2019-05-15/ZAP_WEEKLY_D-2019-05-15.zip && \ + unzip *.zip && \ + rm *.zip && \ + mv ZAP_D*/* . && \ + rmdir ZAP_D* && \ + + # Setup Webswing + curl -s -L https://bitbucket.org/meszarv/webswing/downloads/webswing-2.5.10.zip > webswing.zip && \ + unzip webswing.zip && \ + rm webswing.zip && \ + mv webswing-* webswing && \ + # Remove Webswing demos + rm -Rf webswing/demo/ && \ + # Accept ZAP license + touch AcceptedLicense && \ + pip install zapcli python-owasp-zap-v2.4 && \ + mkdir -p /opt/zap/scripts/scripts/httpsender/ + +COPY vendor/webswing.config /opt/zap/webswing/ +COPY vendor/zap-x.sh /usr/local/bin/ +COPY vendor/zap-api-scan.py /opt/zap/ +COPY vendor/zap_common.py /opt/zap/ + +COPY vendor/policies /root/.ZAP_D/policies/ +COPY vendor/scripts /root/.ZAP_D/scripts/ +COPY vendor/.xinitrc /root/ +ENV ZAP_PORT 8080 + +RUN chmod a+x /root/.xinitrc && \ + mkdir /root/.vnc + +RUN chmod 755 /usr/local/bin/zap*.sh /opt/zap/zap-api-scan.py && \ + ln -s /opt/zap/zap-api-scan.py /usr/local/bin/zap-api-scan + +ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/ +ENV PATH $JAVA_HOME/bin:/opt/zap/:$PATH +ENV ZAP_PATH /opt/zap/zap.sh +ENV ZAPCLI_PATH /usr/local/bin/zap-cli +ENV ZAPAPISCAN_PATH /usr/local/bin/zap-api-scan + +VOLUME [ "/attacks","/output" ] + + +ENTRYPOINT [ "/usr/local/bin/gauntlt" ] + +CMD ["/attacks/*"] diff --git a/Makefile b/Makefile index f173cf4..cd6c935 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,10 @@ build: @echo "Building docker container..." @./build-gauntlt.sh +build-dev: + @echo "Building docker development container..." + @docker build -f Dockerfile-dev -t gauntlt-dev . + clean: @echo "Removing unused docker containers..." @./docker-clean.sh @@ -19,13 +23,18 @@ install-stub: @echo "installing gauntlt-docker to /usr/local/bin" @cp ./bin/gauntlt-docker /usr/local/bin +runexamples: + @echo "Running all examples" + @docker run --rm -it -v ${CURDIR}/examples:/attacks gauntlt help: @echo "the help menu" @echo " make build" + @echo " make build-dev" @echo " make clean" @echo " make clean-all" @echo " make help" @echo " make install-stub" @echo " make interactive" + @echo " make runexamples" .PHONY: build clean diff --git a/README.md b/README.md index f9251bb..d3bbe30 100644 --- a/README.md +++ b/README.md @@ -23,20 +23,25 @@ Feel free to fork this and customize for your needs. This container is great to $ cd /path/to/cloned/repo/gauntlt-docker $ make build ``` +3. Build the docker development container (optional) -3. Check out what `make` can do for you + ``` + $ cd /path/to/cloned/repo/gauntlt-docker + $ make build-dev + ``` +4. Check out what `make` can do for you ``` $ make help ``` -4. Copy binary stub to your $PATH (like `/usr/local/bin`) +5. Copy binary stub to your $PATH (like `/usr/local/bin`) ``` $ make install-stub ``` -5. Test it out with a `gauntlt-docker --help` +6. Test it out with a `gauntlt-docker --help` -6. You can get interactive access to the container to test attack tools installed +7. You can get interactive access to the container to test attack tools installed ``` $ make interactive ``` diff --git a/examples/dirb.attack b/examples/dirb.attack new file mode 100644 index 0000000..7bef42b --- /dev/null +++ b/examples/dirb.attack @@ -0,0 +1,19 @@ +@slow + +Feature: Run dirb scan on a URL + +Scenario: Use dirb to scan a website for basic security requirements and the DIRB_WORDLISTS environment variable must be set in your path. You can use different wordlists by changing the environment variable. + Given "dirb" is installed + And the following profile: + | name | value | + | hostname | http://scanme.nmap.org | + | dirb_wordlists_path | Overwritten by $DIRB_WORDLISTS | + | wordlist | vulns/tests.txt | + When I launch a "dirb" attack with: + """ + dirb / -wf + """ + Then the output should contain: + """ + FOUND: 0 + """ diff --git a/examples/garmr.attack b/examples/garmr.attack new file mode 100644 index 0000000..6de4d52 --- /dev/null +++ b/examples/garmr.attack @@ -0,0 +1,18 @@ +Feature: Run garmr scan on a URL + +Scenario: Use Garmr to scan a website for the Mozilla opioniated security requirements + Given "garmr" is installed + And the following profile: + | name | value | + | target_url | http://scanme.nmap.org | + When I launch a "garmr" attack with: + """ + garmr -u -o my_garmr_output.xml + """ + Then it should pass with: + """ + [Garmr.corechecks.Http200Check] Pass The request returned an HTTP 200 response. + """ + And the file "my_garmr_output.xml" should not contain XML: + | css | + | testcase[name="Http200Check"] failure | diff --git a/examples/heartbleed.attack b/examples/heartbleed.attack new file mode 100644 index 0000000..809e7fd --- /dev/null +++ b/examples/heartbleed.attack @@ -0,0 +1,15 @@ +@slow + +Feature: Test for the Heartbleed vulnerability + +Scenario: Test my website for the Heartbleed vulnerability (see heartbleed.com for more info) + + Given "Heartbleed" is installed + And the following profile: + | name | value | + | domain | www.google.com | + When I launch a "Heartbleed" attack with: + """ + Heartbleed :443 + """ + Then the output should contain "SAFE" diff --git a/examples/nmap.attack b/examples/nmap.attack new file mode 100644 index 0000000..3f824c6 --- /dev/null +++ b/examples/nmap.attack @@ -0,0 +1,48 @@ +@slow + +Feature: nmap attacks for scanme.nmap.org and to use this for your tests, change the value in the profile + Background: + Given "nmap" is installed + And the following profile: + | name | value | + | hostname | scanme.nmap.org | + | host | scanme.nmap.org | + | tcp_ping_ports | 22,25,80,443 | + + Scenario: Verify server is open on expected set of ports using the nmap-fast attack step + When I launch a "nmap-fast" attack + Then the output should match /80.tcp\s+open/ + + Scenario: Verify server is open on expected set of ports using the nmap fast flag + When I launch an "nmap" attack with: + """ + nmap -F + """ + Then the output should match: + """ + 80/tcp\s+open + """ + + Scenario: Verify that there are no unexpected ports open + When I launch an "nmap" attack with: + """ + nmap -F + """ + Then the output should not contain: + """ + 22/tcp + 25/tcp + """ + + Scenario: Output to XML + When I launch an "nmap" attack with: + """ + nmap -p 80,443 -oX foo.xml + """ + And the file "foo.xml" should contain XML: + | css | + | ports port[protocol="tcp"][portid="80"] state[state="open"] | + And the file "foo.xml" should not contain XML: + | css | + | ports port[protocol="tcp"][portid="123"] state[state="open"] | + | ports port[protocol="tcp"][portid="443"] state[state="open"] | diff --git a/examples/serveridentification.attack b/examples/serveridentification.attack new file mode 100644 index 0000000..d0a026a --- /dev/null +++ b/examples/serveridentification.attack @@ -0,0 +1,35 @@ +Feature: Evaluate if server headers are leaking software information + +Scenario: Verify server does not send any software related information + When I launch a "generic" attack with: + """ + curl -v -o /dev/null https://www.google.com + """ + Then the output should not contain "Apache" + And the output should not contain "Microsoft-IIS" + And the output should not contain "nginx" + And the output should not contain "cPanel" + And the output should not contain "ebian" + And the output should not contain "buntu" + And the output should not contain "openresty" + And the output should not contain "Werkzeug" + And the output should not contain "ython" + And the output should not contain "PHP" + And the output should not contain "Java" + And the output should not contain "java" + And the output should not contain "Jetty" + And the output should not contain "micro_httpd" + And the output should not contain "Linux" + And the output should not contain "linux" + And the output should not contain "LINUX" + And the output should not contain "Microsoft" + And the output should not contain "lighttpd" + And the output should not contain "MicroStrategy" + And the output should not contain "Oracle" + And the output should not contain "IBM" + And the output should not contain "RedHat" + And the output should not contain "Servlet" + And the output should not contain "ASP" + And the output should not contain "Passenger" + And the output should not contain "Unicorn" + And the output should not contain "unicorn" diff --git a/examples/sqlmap.attack b/examples/sqlmap.attack new file mode 100644 index 0000000..e469587 --- /dev/null +++ b/examples/sqlmap.attack @@ -0,0 +1,14 @@ +@slow + +Feature: Run sqlmap against a target + +Scenario: Identify SQL injection vulnerabilities + Given "sqlmap" is installed + And the following profile: + | name | value | + | target_url | http://scanme.nmap.org/?id=test | + When I launch a "sqlmap" attack with: + """ + python -u --dbms sqlite --batch -v 0 --tables + """ + Then the output should contain "all tested parameters do not appear to be injectable" diff --git a/examples/sslyze.attack b/examples/sslyze.attack new file mode 100644 index 0000000..4f8c1bb --- /dev/null +++ b/examples/sslyze.attack @@ -0,0 +1,17 @@ +Feature: Run sslyze against a target + +Background: + Given "sslyze" is installed + And the following profile: + | name | value | + | hostname | google.com | + +Scenario: Ensure no anonymous certificates + When I launch an "sslyze" attack with: + """ + python :443 + """ + Then the output should not contain: + """ + Anon + """ \ No newline at end of file diff --git a/examples/verbs.attack b/examples/verbs.attack new file mode 100644 index 0000000..c743ab2 --- /dev/null +++ b/examples/verbs.attack @@ -0,0 +1,23 @@ +@slow + +Feature: Evaluate responses to various HTTP methods. + +Background: + Given "curl" is installed + And the following profile: + | name | value | + | hostname | scanme.nmap.org | + +Scenario Outline: Verify server responds correctly to various HTTP methods + When I launch a "curl" attack with: + """ + curl -i -X + """ + Then the output should contain "" + Examples: + | method | response | + | delete | HTTP/1.1 501 Not Implemented | + | patch | HTTP/1.1 501 Not Implemented | + | trace | HTTP/1.1 501 Not Implemented | + | track | HTTP/1.1 501 Not Implemented | + | bogus | HTTP/1.1 501 Not Implemented | diff --git a/examples/xss.attack b/examples/xss.attack index 9e3e1c3..d71b48d 100644 --- a/examples/xss.attack +++ b/examples/xss.attack @@ -1,4 +1,5 @@ -@slow +@reallyslow + Feature: Look for cross site scripting (xss) using arachni against scanme.nmap.org Scenario: Using arachni, look for cross site scripting and verify no issues are found @@ -10,4 +11,4 @@ Scenario: Using arachni, look for cross site scripting and verify no issues are """ arachni --checks=xss --scope-directory-depth-limit=1 """ - Then the output should contain "0 issues were detected." + Then the output should contain "With issues: 0" diff --git a/examples/zap-api-scan.attack b/examples/zap-api-scan.attack new file mode 100644 index 0000000..364f473 --- /dev/null +++ b/examples/zap-api-scan.attack @@ -0,0 +1,12 @@ + @reallyslow + + Feature: Check an aplication against its API documentation + + Scenario: Using zap-apiscan, load its API documentation and check all its endpoints with zap. + When I launch a "zap-api-scan" attack on a web service whose documentation is available in "http://raw.githubusercontent.com/OAI/OpenAPI-Specification/master/examples/v2.0/json/petstore-minimal.json" + Then the output should contain "FAIL-NEW: 0" + + Scenario: Using zap-apiscan, load its API documentation and check all its endpoints with zap, API is not running in documented host. + Given I wait for 30 seconds + When I launch a "zap-api-scan" attack on the host "petstore.swagger.io" whose documentation is available in "http://raw.githubusercontent.com/OAI/OpenAPI-Specification/master/examples/v2.0/json/petstore-minimal.json" + Then the output should contain "FAIL-NEW: 0" diff --git a/examples/zap-native-xss.attack b/examples/zap-native-xss.attack new file mode 100644 index 0000000..920eabe --- /dev/null +++ b/examples/zap-native-xss.attack @@ -0,0 +1,16 @@ +@reallyslow + +Feature: Look for common vulnerabilities using owasp-zap + +Background: + Given the following profile: + | name | value | + | url | http://scanme.nmap.org | + +Scenario: Using owasp-zap, look for common vulnerailities via passive scan + Given "zap-cli" is installed + When I launch a "zap-cli" attack with: + """ + zap-cli quick-scan -sc -o "-config api.disablekey=true --self-contained -config scanner.attackOnStart=true -config view.mode=attack -config connection.dnsTtlSuccessfulQueries=-1 -config api.addrs.addr.name=.* -config api.addrs.addr.regex=true" -s xss,sqli --spider --recursive + """ + Then the output should contain "Issues found: 0" diff --git a/examples/zap-xss.attack b/examples/zap-xss.attack new file mode 100644 index 0000000..107c15f --- /dev/null +++ b/examples/zap-xss.attack @@ -0,0 +1,15 @@ +@reallyslow + +Feature: Look for common vulnerabilities using owasp-zap + +Background: + Given the following profile: + | name | value | + | url | http://scanme.nmap.org | + +Scenario: Using owasp-zap, look for common vulnerabilities via passive scan + When I launch a "generic" attack with: + """ + zap-cli quick-scan -sc -o "-config api.disablekey=true --self-contained -config scanner.attackOnStart=true -config view.mode=attack -config connection.dnsTtlSuccessfulQueries=-1 -config api.addrs.addr.name=.* -config api.addrs.addr.regex=true" -s xss,sqli --spider --recursive + """ + Then the output should contain "Issues found: 0" diff --git a/vendor/.xinitrc b/vendor/.xinitrc new file mode 100644 index 0000000..1a38c3b --- /dev/null +++ b/vendor/.xinitrc @@ -0,0 +1,3 @@ +openbox & +xsetroot -solid black +zap.sh diff --git a/vendor/dirb222.tar.gz b/vendor/dirb222.tar.gz deleted file mode 100644 index 6a8322c..0000000 Binary files a/vendor/dirb222.tar.gz and /dev/null differ diff --git a/vendor/policies/API-Minimal.policy b/vendor/policies/API-Minimal.policy new file mode 100644 index 0000000..7eb754d --- /dev/null +++ b/vendor/policies/API-Minimal.policy @@ -0,0 +1,238 @@ + + +API Minimal + +MEDIUM +MEDIUM + + + +false +OFF + + +true +MEDIUM + + +true +MEDIUM + + +false +OFF + + +false +OFF + + +true +MEDIUM + + +true +MEDIUM + + +true +MEDIUM + + +true +MEDIUM + + +true +MEDIUM + + +true +MEDIUM + + +true +MEDIUM + + +true +MEDIUM + + +true +MEDIUM + + +false +OFF + + +false +OFF + + +true +MEDIUM + + +false +OFF + + +false +OFF + + +false +OFF + + +false +OFF + + +false +OFF + + +false +OFF + + +false +OFF + + +false +OFF + + +false +OFF + + +false +OFF + + +false +OFF + + +false +OFF + + +false +OFF + + +false +OFF + + +true +MEDIUM + + +true +MEDIUM + + +false +OFF + + +false +OFF + + +false +OFF + + +false +OFF + + +false +OFF + + +false +OFF + + +false +OFF + + +false +OFF + + +false +OFF + + +false +OFF + + +false +OFF + + +false +OFF + + +false +OFF + + +false +OFF + + +false +OFF + + +true +MEDIUM + + +true +MEDIUM + + +false +OFF + + +false +OFF + + +false +OFF + + +false +OFF + + +false +OFF + + +false +OFF + + +false +OFF + + + diff --git a/vendor/policies/Default Policy.policy b/vendor/policies/Default Policy.policy new file mode 100644 index 0000000..59795e1 --- /dev/null +++ b/vendor/policies/Default Policy.policy @@ -0,0 +1,8 @@ + + + Default Policy + + MEDIUM + MEDIUM + + diff --git a/vendor/policies/St-High-Th-High.policy b/vendor/policies/St-High-Th-High.policy new file mode 100644 index 0000000..44ff0ca --- /dev/null +++ b/vendor/policies/St-High-Th-High.policy @@ -0,0 +1,8 @@ + + + St-High-Th-High + + HIGH + HIGH + + diff --git a/vendor/policies/St-High-Th-Low.policy b/vendor/policies/St-High-Th-Low.policy new file mode 100644 index 0000000..392454c --- /dev/null +++ b/vendor/policies/St-High-Th-Low.policy @@ -0,0 +1,8 @@ + + + St-High-Th-Low + + LOW + HIGH + + diff --git a/vendor/policies/St-High-Th-Med.policy b/vendor/policies/St-High-Th-Med.policy new file mode 100644 index 0000000..adc656f --- /dev/null +++ b/vendor/policies/St-High-Th-Med.policy @@ -0,0 +1,8 @@ + + + St-High-Th-Med + + MEDIUM + HIGH + + diff --git a/vendor/policies/St-Ins-Th-High.policy b/vendor/policies/St-Ins-Th-High.policy new file mode 100644 index 0000000..43b18ab --- /dev/null +++ b/vendor/policies/St-Ins-Th-High.policy @@ -0,0 +1,8 @@ + + + St-Ins-Th-High + + HIGH + INSANE + + diff --git a/vendor/policies/St-Ins-Th-Low.policy b/vendor/policies/St-Ins-Th-Low.policy new file mode 100644 index 0000000..be6aaee --- /dev/null +++ b/vendor/policies/St-Ins-Th-Low.policy @@ -0,0 +1,8 @@ + + + St-Ins-Th-Low + + LOW + INSANE + + diff --git a/vendor/policies/St-Ins-Th-Med.policy b/vendor/policies/St-Ins-Th-Med.policy new file mode 100644 index 0000000..89f782c --- /dev/null +++ b/vendor/policies/St-Ins-Th-Med.policy @@ -0,0 +1,8 @@ + + + St-Ins-Th-Med + + MEDIUM + INSANE + + diff --git a/vendor/policies/St-Low-Th-High.policy b/vendor/policies/St-Low-Th-High.policy new file mode 100644 index 0000000..eaf229c --- /dev/null +++ b/vendor/policies/St-Low-Th-High.policy @@ -0,0 +1,8 @@ + + + St-Low-Th-Med + + HIGH + LOW + + diff --git a/vendor/policies/St-Low-Th-Low.policy b/vendor/policies/St-Low-Th-Low.policy new file mode 100644 index 0000000..17e3312 --- /dev/null +++ b/vendor/policies/St-Low-Th-Low.policy @@ -0,0 +1,8 @@ + + + St-Low-Th-Low + + LOW + LOW + + diff --git a/vendor/policies/St-Low-Th-Med.policy b/vendor/policies/St-Low-Th-Med.policy new file mode 100644 index 0000000..aafe6d4 --- /dev/null +++ b/vendor/policies/St-Low-Th-Med.policy @@ -0,0 +1,8 @@ + + + St-Low-Th-Med + + MEDIUM + LOW + + diff --git a/vendor/policies/St-Med-Th-High.policy b/vendor/policies/St-Med-Th-High.policy new file mode 100644 index 0000000..08fffbd --- /dev/null +++ b/vendor/policies/St-Med-Th-High.policy @@ -0,0 +1,8 @@ + + + St-Med-Th-High + + HIGH + MEDIUM + + diff --git a/vendor/policies/St-Med-Th-Low.policy b/vendor/policies/St-Med-Th-Low.policy new file mode 100644 index 0000000..642899f --- /dev/null +++ b/vendor/policies/St-Med-Th-Low.policy @@ -0,0 +1,8 @@ + + + St-Med-Th-Low + + LOW + MEDIUM + + diff --git a/vendor/scripts/scripts/httpsender/Alert_on_HTTP_Response_Code_Errors.js b/vendor/scripts/scripts/httpsender/Alert_on_HTTP_Response_Code_Errors.js new file mode 100644 index 0000000..eee7c57 --- /dev/null +++ b/vendor/scripts/scripts/httpsender/Alert_on_HTTP_Response_Code_Errors.js @@ -0,0 +1,87 @@ +// A script which will raise alerts based on HTTP Response codes +// By default it will raise 'Info' level alerts for Client Errors (4xx) (apart from 404s) and 'Low' Level alerts for Server Errors (5xx) +// But it can be easily changed. + +pluginid = 100000 // https://github.com/zaproxy/zaproxy/blob/develop/docs/scanners.md + +function sendingRequest(msg, initiator, helper) { + // Nothing to do +} + +function responseReceived(msg, initiator, helper) { + if (initiator == 7) { // CHECK_FOR_UPDATES_INITIATOR + // Not of interest. + return + } + + var extensionAlert = org.parosproxy.paros.control.Control.getSingleton().getExtensionLoader().getExtension( + org.zaproxy.zap.extension.alert.ExtensionAlert.NAME) + if (extensionAlert != null) { + var code = msg.getResponseHeader().getStatusCode() + if (code < 400 || code >= 600) { + // Do nothing + } else { + var risk = 0 // Info + var title = "A Client Error response code was returned by the server" + if (code >= 500) { + // Server error + risk = 1 // Low + title = "A Server Error response code was returned by the server" + } + // CONFIDENCE_HIGH = 3 (we can be pretty sure we're right) + var alert = new org.parosproxy.paros.core.scanner.Alert(pluginid, risk, 3, title) + var ref = msg.getHistoryRef() + if (ref != null && org.parosproxy.paros.model.HistoryReference.getTemporaryTypes().contains( + java.lang.Integer.valueOf(ref.getHistoryType()))) { + // Dont use temporary types as they will get deleted + ref = null + } + if (ref == null) { + // map the initiator + var type + switch (initiator) { + case 1: // PROXY_INITIATOR + type = 1 // Proxied + break + case 2: // ACTIVE_SCANNER_INITIATOR + type = 3 // Scanner + break + case 3: // SPIDER_INITIATOR + type = 2 // Spider + break + case 4: // FUZZER_INITIATOR + type = 8 // Fuzzer + break + case 5: // AUTHENTICATION_INITIATOR + type = 15 // User + break + case 6: // MANUAL_REQUEST_INITIATOR + type = 15 // User + break + case 8: // BEAN_SHELL_INITIATOR + type = 15 // User + break + case 9: // ACCESS_CONTROL_SCANNER_INITIATOR + type = 13 // Access control + break + default: + type = 15 // User - fallback + break + } + ref = new org.parosproxy.paros.model.HistoryReference( + org.parosproxy.paros.model.Model.getSingleton().getSession(), type, msg) + } + alert.setMessage(msg) + alert.setUri(msg.getRequestHeader().getURI().toString()) + alert.setDescription("A response code of " + code + " was returned by the server.\n" + + "This may indicate that the application is failing to handle unexpected input correctly.\n" + + "Raised by the 'Alert on HTTP Response Code Error' script"); + // Use a regex to extract the evidence from the response header + var regex = new RegExp("^HTTP.*" + code) + alert.setEvidence(msg.getResponseHeader().toString().match(regex)) + alert.setCweId(388) // CWE CATEGORY: Error Handling + alert.setWascId(20) // WASC Improper Input Handling + extensionAlert.alertFound(alert , ref) + } + } +} diff --git a/vendor/scripts/scripts/httpsender/Alert_on_Unexpected_Content_Types.js b/vendor/scripts/scripts/httpsender/Alert_on_Unexpected_Content_Types.js new file mode 100644 index 0000000..60b3d7f --- /dev/null +++ b/vendor/scripts/scripts/httpsender/Alert_on_Unexpected_Content_Types.js @@ -0,0 +1,96 @@ +// A script which will raise alerts based on unexpected Content-Types +// By default it will raise 'Low' level alerts for content types that are not expected to be returned by APIs. +// But it can be easily changed. + +var pluginid = 100001 // https://github.com/zaproxy/zaproxy/blob/develop/docs/scanners.md + +var extensionAlert = org.parosproxy.paros.control.Control.getSingleton().getExtensionLoader().getExtension( + org.zaproxy.zap.extension.alert.ExtensionAlert.NAME) + +var expectedTypes = [ + "application/json", + "application/octet-stream", + "application/problem+json", + "application/problem+xml", + "application/soap+xml", + "application/xml", + "application/x-yaml", + "text/x-json", + "text/json" + ] + +function sendingRequest(msg, initiator, helper) { + // Nothing to do +} + +function responseReceived(msg, initiator, helper) { + if (initiator == 7) { // CHECK_FOR_UPDATES_INITIATOR + // Not of interest. + return + } + + if (extensionAlert != null) { + var ctype = msg.getResponseHeader().getHeader("Content-Type") + if (ctype != null) { + if (ctype.indexOf(";") > 0) { + ctype = ctype.substring(0, ctype.indexOf(";")) + } + if (expectedTypes.indexOf(ctype) < 0) { + // Another rule will complain if theres no type + + var risk = 1 // Low + var title = "Unexpected Content-Type was returned" + // CONFIDENCE_HIGH = 3 (we can be pretty sure we're right) + var alert = new org.parosproxy.paros.core.scanner.Alert(pluginid, risk, 3, title) + var ref = msg.getHistoryRef() + if (ref != null && org.parosproxy.paros.model.HistoryReference.getTemporaryTypes().contains( + java.lang.Integer.valueOf(ref.getHistoryType()))) { + // Dont use temporary types as they will get deleted + ref = null + } + if (ref == null) { + // map the initiator + var type + switch (initiator) { + case 1: // PROXY_INITIATOR + type = 1 // Proxied + break + case 2: // ACTIVE_SCANNER_INITIATOR + type = 3 // Scanner + break + case 3: // SPIDER_INITIATOR + type = 2 // Spider + break + case 4: // FUZZER_INITIATOR + type = 8 // Fuzzer + break + case 5: // AUTHENTICATION_INITIATOR + type = 15 // User + break + case 6: // MANUAL_REQUEST_INITIATOR + type = 15 // User + break + case 8: // BEAN_SHELL_INITIATOR + type = 15 // User + break + case 9: // ACCESS_CONTROL_SCANNER_INITIATOR + type = 13 // Access control + break + default: + type = 15 // User - fallback + break + } + ref = new org.parosproxy.paros.model.HistoryReference( + org.parosproxy.paros.model.Model.getSingleton().getSession(), type, msg) + } + alert.setMessage(msg) + alert.setUri(msg.getRequestHeader().getURI().toString()) + alert.setDescription("A Content-Type of " + ctype + " was returned by the server.\n" + + "This is not one of the types expected to be returned by an API.\n" + + "Raised by the 'Alert on Unexpected Content Types' script"); + alert.setEvidence(ctype) + extensionAlert.alertFound(alert , ref) + } + } + } +} \ No newline at end of file diff --git a/vendor/webswing.config b/vendor/webswing.config new file mode 100644 index 0000000..8a5c0aa --- /dev/null +++ b/vendor/webswing.config @@ -0,0 +1,74 @@ +{ + "/" : { + "path" : "/", + "security" : { + "module" : "EMBEDED", + "config" : { + "users" : [] + }, + "classPath" : [ ] + }, + "icon" : null, + "webFolder" : null, + "langFolder" : "${webswing.rootDir}/lang", + "homeDir" : "${user.dir}", + "swingConfig" : null + }, + "/zap" : { + "path" : "/zap", + "homeDir" : "/zap", + "webFolder" : "", + "icon" : "icon.png", + "security" : { + "module" : "NONE", + "config" : null, + "classPath" : [ ], + "authorizationConfig" : { + "users" : [ ], + "roles" : [ ] + } + }, + "swingConfig" : { + "allowUpload" : true, + "allowDownload" : true, + "isolatedFs" : false, + "autoLogout" : true, + "name" : "OWASP ZAP", + "theme" : "Murrine", + "directdraw" : false, + "javaFx" : true, + "debug" : false, + "userDir" : "", + "jreExecutable" : "${java.home}/bin/java", + "javaVersion" : "${java.version}", + "vmArgs" : null, + "launcherType" : "Desktop", + "launcherConfig" : { + "args" : "-host 0.0.0.0 -port 8090", + "mainClass" : "org.zaproxy.zap.ZAP" + }, + "maxClients" : 1, + "sessionMode" : "CONTINUE_FOR_BROWSER", + "swingSessionTimeout" : -1, + "allowStealSession" : true, + "allowDelete" : true, + "allowAutoDownload" : true, + "uploadMaxSize" : 5, + "allowJsLink" : true, + "fontConfig" : { }, + "classPathEntries" : [ "zap-*.jar" ], + "allowedCorsOrigins" : [ ], + "transferDir" : "${user}/upload", + "goodbyeUrl" : "", + "transparentFileSave" : true, + "clearTransferDir" : true, + "transparentFileOpen" : true, + "timeoutIfInactive" : false, + "monitorEdtEnabled" : false, + "allowLocalClipboard" : true, + "allowServerPrinting" : false + }, + "enabled" : true, + "langFolder" : "" + } +} \ No newline at end of file diff --git a/vendor/zap-api-scan.py b/vendor/zap-api-scan.py new file mode 100644 index 0000000..3ada146 --- /dev/null +++ b/vendor/zap-api-scan.py @@ -0,0 +1,568 @@ +#!/usr/bin/env python +# Zed Attack Proxy (ZAP) and its related class files. +# +# ZAP is an HTTP/HTTPS proxy for assessing web application security. +# +# Copyright 2017 ZAP Development Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script runs a full scan against an API defined by OpenAPI/Swagger or SOAP +# using ZAP +# +# It can either be run 'standalone', in which case depends on +# https://pypi.python.org/pypi/python-owasp-zap-v2.4 and Docker, or it can be run +# inside one of the ZAP docker containers. It automatically detects if it is +# running in docker so the parameters are the same. +# +# It currently support APIS defined by: +# OpenAPI/Swagger URL +# OpenAPI/Swagger file +# SOAP URL +# SOAP File +# It will exit with codes of: +# 0: Success +# 1: At least 1 FAIL +# 2: At least one WARN and no FAILs +# 3: Any other failure +# By default all alerts found by ZAP will be treated as WARNings. +# You can use the -c or -u parameters to specify a configuration file to override +# this. +# You can generate a template configuration file using the -g parameter. You will +# then need to change 'WARN' to 'FAIL', 'INFO' or 'IGNORE' for the rules you want +# to be handled differently. +# You can also add your own messages for the rules by appending them after a tab +# at the end of each line. +# By default the active scan rules run are hardcoded in the API-Minimal.policy +# file but you can change them by supplying a configuration file with the rules +# you dont want to be run set to IGNORE. + +import getopt +import json +import logging +import os +import os.path +import subprocess +import sys +import time +from datetime import datetime +from six.moves.urllib.parse import urljoin +from zapv2 import ZAPv2 +from zap_common import * + + +class NoUrlsException(Exception): + pass + + +config_dict = {} +config_msg = {} +out_of_scope_dict = {} +min_level = 0 + +# Scan rules that aren't really relevant, eg the examples rules in the alpha set +blacklist = ['-1', '50003', '60000', '60001'] + +# Scan rules that are being addressed +in_progress_issues = {} + +logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') +# Hide "Starting new HTTP connection" messages +logging.getLogger("requests").setLevel(logging.WARNING) + + +def usage(): + print('Usage: zap-api-scan.py -t -f [options]') + print(' -t target target API definition, OpenAPI or SOAP, local file or URL, eg https://www.example.com/openapi.json') + print(' -f format either openapi or soap') + print('Options:') + print(' -h print this help message') + print(' -c config_file config file to use to INFO, IGNORE or FAIL warnings') + print(' -u config_url URL of config file to use to INFO, IGNORE or FAIL warnings') + print(' -g gen_file generate default config file(all rules set to WARN)') + print(' -r report_html file to write the full ZAP HTML report') + print(' -w report_md file to write the full ZAP Wiki(Markdown) report') + print(' -x report_xml file to write the full ZAP XML report') + print(' -J report_json file to write the full ZAP JSON document') + print(' -a include the alpha passive scan rules as well') + print(' -d show debug messages') + print(' -P specify listen port') + print(' -D delay in seconds to wait for passive scanning ') + print(' -i default rules not in the config file to INFO') + print(' -l level minimum level to show: PASS, IGNORE, INFO, WARN or FAIL, use with -s to hide example URLs') + print(' -n context_file context file which will be loaded prior to scanning the target') + print(' -p progress_file progress file which specifies issues that are being addressed') + print(' -s short output format - dont show PASSes or example URLs') + print(' -S safe mode this will skip the active scan and perform a baseline scan') + print(' -T max time in minutes to wait for ZAP to start and the passive scan to run') + print(' -O the hostname to override in the (remote) OpenAPI spec') + print(' -z zap_options ZAP command line options e.g. -z "-config aaa=bbb -config ccc=ddd"') + print(' --hook path to python file that define your custom hooks') + print('') + print('For more details see https://github.com/zaproxy/zaproxy/wiki/ZAP-API-Scan') + + +def main(argv): + + global min_level + global in_progress_issues + cid = '' + context_file = '' + progress_file = '' + config_file = '' + config_url = '' + generate = '' + port = 0 + detailed_output = True + report_html = '' + report_md = '' + report_xml = '' + report_json = '' + target = '' + target_file = '' + target_url = '' + host_override = '' + format = '' + zap_alpha = False + baseline = False + info_unspecified = False + base_dir = '' + zap_ip = 'localhost' + zap_options = '' + delay = 0 + timeout = 0 + hook_file = None + + pass_count = 0 + warn_count = 0 + fail_count = 0 + info_count = 0 + ignore_count = 0 + warn_inprog_count = 0 + fail_inprog_count = 0 + + try: + opts, args = getopt.getopt(argv, "t:f:c:u:g:m:n:r:J:w:x:l:hdaijSp:sz:P:D:T:O:", ["hook="]) + except getopt.GetoptError as exc: + logging.warning('Invalid option ' + exc.opt + ' : ' + exc.msg) + usage() + sys.exit(3) + + for opt, arg in opts: + if opt == '-h': + usage() + sys.exit(0) + elif opt == '-t': + target = arg + logging.debug('Target: ' + target) + elif opt == '-f': + format = arg + elif opt == '-c': + config_file = arg + elif opt == '-u': + config_url = arg + elif opt == '-g': + generate = arg + elif opt == '-d': + logging.getLogger().setLevel(logging.DEBUG) + elif opt == '-P': + port = int(arg) + elif opt == '-D': + delay = int(arg) + elif opt == '-n': + context_file = arg + elif opt == '-p': + progress_file = arg + elif opt == '-r': + report_html = arg + elif opt == '-J': + report_json = arg + elif opt == '-w': + report_md = arg + elif opt == '-x': + report_xml = arg + elif opt == '-a': + zap_alpha = True + elif opt == '-i': + info_unspecified = True + elif opt == '-l': + try: + min_level = zap_conf_lvls.index(arg) + except ValueError: + logging.warning('Level must be one of ' + str(zap_conf_lvls)) + usage() + sys.exit(3) + elif opt == '-z': + zap_options = arg + elif opt == '-s': + detailed_output = False + elif opt == '-S': + baseline = True + elif opt == '-T': + timeout = int(arg) + elif opt == '-O': + host_override = arg + elif opt == '--hook': + hook_file = arg + + check_zap_client_version() + + load_custom_hooks(hook_file) + trigger_hook('cli_opts', opts) + + # Check target supplied and ok + if len(target) == 0: + usage() + sys.exit(3) + if format != 'openapi' and format != 'soap': + logging.warning('Format must be either \'openapi\' or \'soap\'') + usage() + sys.exit(3) + + if running_in_docker(): + base_dir = '/output/' + if config_file or generate or report_html or report_xml or report_json or progress_file or context_file or target_file: + # Check directory has been mounted + if not os.path.exists(base_dir): + logging.warning('A file based option has been specified but the directory \'/output\' is not mounted ') + usage() + sys.exit(3) + + if target.startswith('http://') or target.startswith('https://'): + target_url = target + else: + # assume its a file + if not os.path.exists(base_dir + target): + logging.warning('Target must either start with \'http://\' or \'https://\' or be a local file') + logging.warning('File does not exist: ' + base_dir + target) + usage() + sys.exit(3) + else: + target_file = target + + # Choose a random 'ephemeral' port and check its available if it wasn't specified with -P option + if port == 0: + port = get_free_port() + + logging.debug('Using port: ' + str(port)) + + if config_file: + # load config file from filestore + with open(base_dir + config_file) as f: + try: + load_config(f, config_dict, config_msg, out_of_scope_dict) + except ValueError as e: + logging.warning("Failed to load config file " + base_dir + config_file + " " + str(e)) + sys.exit(3) + elif config_url: + # load config file from url + try: + config_data = urlopen(config_url).read().decode('UTF-8').splitlines() + load_config(config_data, config_dict, config_msg, out_of_scope_dict) + except ValueError as e: + logging.warning("Failed to read configs from " + config_url + " " + str(e)) + sys.exit(3) + except: + logging.warning('Failed to read configs from ' + config_url) + sys.exit(3) + + if progress_file: + # load progress file from filestore + with open(base_dir + progress_file) as f: + progress = json.load(f) + # parse into something more useful... + # in_prog_issues = map of vulnid -> {object with everything in} + for issue in progress["issues"]: + if issue["state"] == "inprogress": + in_progress_issues[issue["id"]] = issue + + if running_in_docker(): + try: + params = [ + '-addonupdate', + '-addoninstall', 'pscanrulesBeta'] # In case we're running in the stable container + + if zap_alpha: + params.append('-addoninstall') + params.append('pscanrulesAlpha') + + add_zap_options(params, zap_options) + + start_zap(port, params) + + except OSError: + logging.warning('Failed to start ZAP :(') + sys.exit(3) + + else: + # Not running in docker, so start one + mount_dir = '' + if context_file: + mount_dir = os.path.dirname(os.path.abspath(context_file)) + + params = ['-addonupdate'] + + if (zap_alpha): + params.extend(['-addoninstall', 'pscanrulesAlpha']) + + add_zap_options(params, zap_options) + + try: + cid = start_docker_zap('owasp/zap2docker-weekly', port, params, mount_dir) + zap_ip = ipaddress_for_cid(cid) + logging.debug('Docker ZAP IP Addr: ' + zap_ip) + + # Copy across the files that may not be in all of the docker images + try: + subprocess.check_output(['docker', 'exec', '-t', cid, 'mkdir', '-p', '/opt/zap/scripts/scripts/httpsender/']) + cp_to_docker(cid, 'scripts/scripts/httpsender/Alert_on_HTTP_Response_Code_Errors.js', '/opt/zap/') + cp_to_docker(cid, 'scripts/scripts/httpsender/Alert_on_Unexpected_Content_Types.js', '/opt/zap/') + cp_to_docker(cid, 'policies/API-Minimal.policy', '/opt/zap/') + if target_file: + cp_to_docker(cid, target_file, '/zap/') + + except OSError: + logging.warning('Failed to copy one of the required files') + sys.exit(3) + + except OSError: + logging.warning('Failed to start ZAP in docker :(') + sys.exit(3) + + try: + zap = ZAPv2(proxies={'http': 'http://' + zap_ip + ':' + str(port), 'https': 'http://' + zap_ip + ':' + str(port)}) + + wait_for_zap_start(zap, timeout * 60) + trigger_hook('zap_started', zap, target) + + if context_file: + # handle the context file, cant use base_dir as it might not have been set up + zap_import_context(zap, '/output/' + os.path.basename(context_file)) + + # Enable scripts + zap.script.load('Alert_on_HTTP_Response_Code_Errors.js', 'httpsender', 'Oracle Nashorn', '/opt/zap/scripts/scripts/httpsender/Alert_on_HTTP_Response_Code_Errors.js') + zap.script.enable('Alert_on_HTTP_Response_Code_Errors.js') + zap.script.load('Alert_on_Unexpected_Content_Types.js', 'httpsender', 'Oracle Nashorn', '/opt/zap/scripts/scripts/httpsender/Alert_on_Unexpected_Content_Types.js') + zap.script.enable('Alert_on_Unexpected_Content_Types.js') + + # Import the API defn + if format == 'openapi': + trigger_hook('importing_openapi', target_url, target_file) + if target_url: + logging.debug('Import OpenAPI URL ' + target_url) + res = zap.openapi.import_url(target, host_override) + urls = zap.core.urls() + if host_override: + target = urljoin(target_url, '//' + host_override) + logging.info('Using host override, new target: {0}'.format(target)) + else: + logging.debug('Import OpenAPI File ' + target_file) + res = zap.openapi.import_file(base_dir + target_file) + urls = zap.core.urls() + if len(urls) > 0: + # Choose the first one - will be striping off the path below + target = urls[0] + logging.debug('Using target from imported file: {0}'.format(target)) + else: + trigger_hook('importing_soap', target_url, target_file) + if target_url: + logging.debug('Import SOAP URL ' + target_url) + res = zap._request(zap.base + 'soap/action/importUrl/', {'url':target}) + urls = zap.core.urls() + else: + logging.debug('Import SOAP File ' + target_file) + res = zap._request(zap.base + 'soap/action/importFile/', {'file': base_dir + target_file}) + urls = zap.core.urls() + if len(urls) > 0: + # Choose the first one - will be striping off the path below + target = urls[0] + logging.debug('Using target from imported file: {0}'.format(target)) + + logging.info('Number of Imported URLs: ' + str(len(urls))) + logging.debug('Import warnings: ' + str(res)) + + if len(urls) == 0: + logging.warning('Failed to import any URLs') + # No point continue, there's nothing to scan. + raise NoUrlsException() + + if target.count('/') > 2: + old_target = target + # The url can include a valid path, but always reset to scan the host + target = target[0:target.index('/', 8)+1] + logging.debug('Normalised target from {0} to {1}'.format(old_target, target)) + + # Wait for a delay if specified with -D option + if (delay): + start_scan = datetime.now() + while((datetime.now() - start_scan).seconds < delay ): + time.sleep(5) + logging.debug('Delay active scan ' + str(delay -(datetime.now() - start_scan).seconds) + ' seconds') + + # Set up the scan policy + scan_policy = 'API-Minimal' + if config_dict: + # They have supplied a config file, use this to define the ascan rules + # Use the default one as the script might not have write access to the one just copied across + scan_policy = 'Default Policy' + zap.ascan.enable_all_scanners(scanpolicyname=scan_policy) + for scanner, state in config_dict.items(): + if state == 'IGNORE': + # Dont bother checking the result - this will fail for pscan rules + zap.ascan.set_scanner_alert_threshold(id=scanner, alertthreshold='OFF', scanpolicyname=scan_policy) + + if not baseline: + zap_active_scan(zap, target, scan_policy) + + zap_wait_for_passive_scan(zap, timeout * 60) + + # Print out a count of the number of urls + num_urls = len(zap.core.urls()) + if num_urls == 0: + logging.warning('No URLs found - is the target URL accessible? Local services may not be accessible from the Docker container') + else: + if detailed_output: + print('Total of ' + str(num_urls) + ' URLs') + + alert_dict = zap_get_alerts(zap, target, blacklist, out_of_scope_dict) + + all_ascan_rules = zap.ascan.scanners('Default Policy') + all_pscan_rules = zap.pscan.scanners + all_dict = {} + for rule in all_pscan_rules: + plugin_id = rule.get('id') + if plugin_id in blacklist: + continue + all_dict[plugin_id] = rule.get('name') + ' - Passive/' + rule.get('quality') + for rule in all_ascan_rules: + plugin_id = rule.get('id') + if plugin_id in blacklist: + continue + all_dict[plugin_id] = rule.get('name') + ' - Active/' + rule.get('quality') + + if generate: + # Create the config file + with open(base_dir + generate, 'w') as f: + f.write('# zap-api-scan rule configuration file\n') + f.write('# Change WARN to IGNORE to ignore rule or FAIL to fail if rule matches\n') + f.write('# Active scan rules set to IGNORE will not be run which will speed up the scan\n') + f.write('# Only the rule identifiers are used - the names are just for info\n') + f.write('# You can add your own messages to each rule by appending them after a tab on each line.\n') + for key, rule in sorted(all_dict.iteritems()): + f.write(key + '\tWARN\t(' + rule + ')\n') + + # print out the passing rules + pass_dict = {} + for rule in all_pscan_rules: + plugin_id = rule.get('id') + if plugin_id in blacklist: + continue + if (not alert_dict.has_key(plugin_id)): + pass_dict[plugin_id] = rule.get('name') + for rule in all_ascan_rules: + plugin_id = rule.get('id') + if plugin_id in blacklist: + continue + if not alert_dict.has_key(plugin_id) and not(config_dict.has_key(plugin_id) and config_dict[plugin_id] == 'IGNORE'): + pass_dict[plugin_id] = rule.get('name') + + if min_level == zap_conf_lvls.index("PASS") and detailed_output: + for key, rule in sorted(pass_dict.iteritems()): + print('PASS: ' + rule + ' [' + key + ']') + + pass_count = len(pass_dict) + + if detailed_output: + # print out the ignored ascan rules(there will be no alerts for these as they were not run) + for rule in all_ascan_rules: + plugin_id = rule.get('id') + if plugin_id in blacklist: + continue + if config_dict.has_key(plugin_id) and config_dict[plugin_id] == 'IGNORE': + print('SKIP: ' + rule.get('name') + ' [' + plugin_id + ']') + + # print out the ignored rules + ignore_count, not_used = print_rules(alert_dict, 'IGNORE', config_dict, config_msg, min_level, + inc_ignore_rules, True, detailed_output, {}) + + # print out the info rules + info_count, not_used = print_rules(alert_dict, 'INFO', config_dict, config_msg, min_level, + inc_info_rules, info_unspecified, detailed_output, in_progress_issues) + + # print out the warning rules + warn_count, warn_inprog_count = print_rules(alert_dict, 'WARN', config_dict, config_msg, min_level, + inc_warn_rules, not info_unspecified, detailed_output, in_progress_issues) + + # print out the failing rules + fail_count, fail_inprog_count = print_rules(alert_dict, 'FAIL', config_dict, config_msg, min_level, + inc_fail_rules, True, detailed_output, in_progress_issues) + + if report_html: + # Save the report + write_report(base_dir + report_html, zap.core.htmlreport()) + + if report_json: + # Save the report + write_report(base_dir + report_json, zap.core.jsonreport()) + + if report_md: + # Save the report + write_report(base_dir + report_md, zap.core.mdreport()) + + if report_xml: + # Save the report + write_report(base_dir + report_xml, zap.core.xmlreport()) + + print('FAIL-NEW: ' + str(fail_count) + '\tFAIL-INPROG: ' + str(fail_inprog_count) + + '\tWARN-NEW: ' + str(warn_count) + '\tWARN-INPROG: ' + str(warn_inprog_count) + + '\tINFO: ' + str(info_count) + '\tIGNORE: ' + str(ignore_count) + '\tPASS: ' + str(pass_count)) + + trigger_hook('zap_pre_shutdown', zap) + # Stop ZAP + zap.core.shutdown() + + except IOError as e: + if hasattr(e, 'args') and len(e.args) > 1: + errno, strerror = e + print("ERROR " + str(strerror)) + logging.warning('I/O error(' + str(errno) + '): ' + str(strerror)) + else: + print("ERROR %s" % e) + logging.warning('I/O error: ' + str(e)) + dump_log_file(cid) + + except NoUrlsException: + dump_log_file(cid) + + except: + print("ERROR " + str(sys.exc_info()[0])) + logging.warning('Unexpected error: ' + str(sys.exc_info()[0])) + dump_log_file(cid) + + if not running_in_docker(): + stop_docker(cid) + + trigger_hook('pre_exit', fail_count, warn_count, pass_count) + + if fail_count > 0: + sys.exit(1) + elif warn_count > 0: + sys.exit(2) + elif pass_count > 0: + sys.exit(0) + else: + sys.exit(3) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/vendor/zap-x.sh b/vendor/zap-x.sh new file mode 100755 index 0000000..798555d --- /dev/null +++ b/vendor/zap-x.sh @@ -0,0 +1,11 @@ +#!/bin/sh +export DISPLAY=:1.0 +if [ ! -f /tmp/.X1-lock ] +then + Xvfb :1 -screen 0 1024x768x16 -ac & +fi +zap.sh "$@" + +# Shutdown xvfb +kill -9 `cat /tmp/.X1-lock` +rm -f /tmp/.X1-lock diff --git a/vendor/zap.sh b/vendor/zap.sh new file mode 100755 index 0000000..71f0792 --- /dev/null +++ b/vendor/zap.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash + +# Dereference from link to the real directory +SCRIPTNAME="$0" + +# While name of this script is symbolic link +while [ -L "${SCRIPTNAME}" ] ; do + cd "`dirname "${SCRIPTNAME}"`" > /dev/null + SCRIPTNAME="$(readlink "`basename "${SCRIPTNAME}"`")" +done +cd "`dirname "${SCRIPTNAME}"`" > /dev/null + +# Base directory where ZAP is installed +BASEDIR="`pwd -P`" + +# Switch to the directory where ZAP is installed +cd "$BASEDIR" + +# Get Operating System +OS=$(uname -s) + +# If we're on OS X, try to use the bundled Java; if it's not there, then the system Java +# Life would be much easier if OS X had readlink -f +if [ "$OS" = "Darwin" ]; then + if [ -e ../PlugIns/jre*/Contents/Home/bin/java ]; then + pushd ../PlugIns/jre*/Contents/Home/bin > /dev/null + JAVA_PATH=`pwd -P` + PATH="$JAVA_PATH:$PATH" + popd > /dev/null + fi +fi + +# Extract and check the Java version +JAVA_OUTPUT=$(java -version 2>&1) + +# Catch warning: Unable to find a $JAVA_HOME at "/usr", continuing with system-provided Java +if [ "`echo ${JAVA_OUTPUT} | grep "continuing with system-provided Java"`" ] ; then + echo "WARNING, \$JAVA_HOME could be set incorrectly, Java's error is:" + echo " " $JAVA_OUTPUT + echo "Unsetting JAVA_HOME and continuing with ZAP start-up" + unset JAVA_HOME +fi + +DEFAULTJAVAGC="-XX:+UseG1GC" + +JAVA_VERSION=$(java -version 2>&1 | awk -F\" '/version/ { print $2 }') +JAVA_MAJOR_VERSION=${JAVA_VERSION%%[.|-]*} +JAVA_MINOR_VERSION=$(echo $JAVA_VERSION | awk -F\. '{ print $2 }') + +# JEP 223, newer Java versions (>= 9) no longer use 1 as major version +if [ $JAVA_MAJOR_VERSION -ge 9 ]; then + DEFAULTJAVAGC="" + echo "Found Java version $JAVA_VERSION" +elif [ $JAVA_MAJOR_VERSION -ge 1 ] && [ $JAVA_MINOR_VERSION -ge 8 ]; then + echo "Found Java version $JAVA_VERSION" +else + echo "Exiting: ZAP requires a minimum of Java 8 to run, found $JAVA_VERSION" + exit 1 +fi + +if [ "$OS" = "Darwin" ]; then + JVMPROPS="$HOME/Library/Application Support/ZAP/.ZAP_JVM.properties" +else + JVMPROPS="$HOME/.ZAP/.ZAP_JVM.properties" +fi + +# Work out best memory options +if [ -f "$JVMPROPS" ]; then + # Local jvm properties file present + JMEM=$(head -1 "$JVMPROPS") +elif [ "$OS" = "Linux" ]; then + MEM=$(expr $(sed -n 's/MemTotal:[ ]\{1,\}\([0-9]\{1,\}\) kB/\1/p' /proc/meminfo) / 1024) +elif [ "$OS" = "Darwin" ]; then + MEM=$(system_profiler SPMemoryDataType | sed -n -e 's/.*Size: \([0-9]\{1,\}\) GB/\1/p' | awk '{s+=$0} END {print s*1024}') +elif [ "$OS" = "SunOS" ]; then + MEM=$(/usr/sbin/prtconf | awk '/Memory/{print $3}') +elif [ "$OS" = "FreeBSD" ]; then + MEM=$(($(sysctl -n hw.physmem)/1024/1024)) +fi + +if [ ! -z "$JMEM" ]; then + echo "Read custom JVM args from $JVMPROPS" + JAVAGC="" +elif [ -z "$MEM" ]; then + echo "Failed to obtain current memory, using jvm default memory settings" + JAVAGC=${DEFAULTJAVAGC} +else + echo "Available memory: $MEM MB" + JAVAGC=${DEFAULTJAVAGC} + if [ "$MEM" -gt 512 ]; then + # Always go with 1/4 of the available memory - specific JVMs may round this up or down + QMEM=$(($MEM/4)) + JMEM="-Xmx${QMEM}m" + fi +fi + +ARGS=() +for var in "$@"; do + if [[ "$var" == -Xmx* ]]; then + # Overridden by the user + JMEM="$var" + elif [[ $var == --jvmdebug* ]]; then + JAVADEBUGPORT=`echo "$var" | sed -e "s/--jvmdebug//g" | sed -e "s/=//g"` + if [ ! "$JAVADEBUGPORT" ]; then + JAVADEBUGPORT=1044 + fi + JAVADEBUG="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=127.0.0.1:$JAVADEBUGPORT" + elif [[ $var != -psn_* ]]; then + # Strip the automatic -psn_x_xxxxxxx argument that OS X automatically passes into apps, since + # it freaks out ZAP + ARGS+=("$var") + fi +done + +if [ -n "$JMEM" ] +then + echo "Using JVM args: $JMEM" +fi + +if [ -n "$JAVADEBUG" ] +then + echo "Setting debug: $JAVADEBUG" +fi + +# Start ZAP; it's likely that -Xdock:icon would be ignored on other platforms, but this is known to work +if [ "$OS" = "Darwin" ]; then + # It's likely that -Xdock:icon would be ignored on other platforms, but this is known to work + exec java ${JMEM} ${JAVAGC} -Xdock:icon="../Resources/ZAP.icns" -jar "/opt/zap/zap*jar" "${ARGS[@]}" +else + exec java ${JMEM} ${JAVAGC} ${JAVADEBUG} -jar "/opt/zap/zap*jar" "${ARGS[@]}" +fi \ No newline at end of file diff --git a/vendor/zap_common.py b/vendor/zap_common.py new file mode 100644 index 0000000..0525dcd --- /dev/null +++ b/vendor/zap_common.py @@ -0,0 +1,513 @@ +#!/usr/bin/env python +# Zed Attack Proxy (ZAP) and its related class files. +# +# ZAP is an HTTP/HTTPS proxy for assessing web application security. +# +# Copyright 2017 ZAP Development Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script provides a set of common functions for other scripts + +import json +import logging +import os +import re +import shlex +import socket +import subprocess +import sys +import time +import traceback +import errno +import imp +import zapv2 +from random import randint +from six.moves.urllib.request import urlopen +from six import binary_type + +try: + import pkg_resources +except ImportError: + # don't hard fail since it's just used for the version check + logging.warning('Error importing pkg_resources. Is setuptools installed?') + + +OLD_ZAP_CLIENT_WARNING = '''A newer version of python_owasp_zap_v2.4 + is available. Please run \'pip install -U python_owasp_zap_v2.4\' to update to + the latest version.'''.replace('\n', '') + +zap_conf_lvls = ["PASS", "IGNORE", "INFO", "WARN", "FAIL"] +zap_hooks = None + +def load_custom_hooks(hooks_file=None): + """ Loads a custom python module which modifies zap scripts behaviour + hooks_file - a python file which defines custom hooks + """ + global zap_hooks + hooks_file = hooks_file if hooks_file else os.environ.get('ZAP_HOOKS', '~/.zap_hooks.py') + hooks_file = os.path.expanduser(hooks_file) + + if not os.path.exists(hooks_file): + logging.debug('Could not find custom hooks file at %s ' % hooks_file) + return + + zap_hooks = imp.load_source("zap_hooks", hooks_file) + + +def hook(hook_name=None, **kwargs): + """ + Decorator method for calling hook before/after method. + Always adds a hook that runs before intercepting args and if wrap=True will create + another hook to intercept the return. + hook_name - name of hook for interactions, if None will use the name of the method it wrapped + """ + after_hook = kwargs.get('wrap', False) + def _decorator(func): + name = func.__name__ + _hook_name = hook_name if hook_name else name + def _wrap(*args, **kwargs): + hook_args = list(args) + hook_kwargs = dict(kwargs) + args = trigger_hook(_hook_name, *hook_args, **hook_kwargs) + args_list = list(args) + return_data = func(*args_list, **kwargs) + + if after_hook: + return trigger_hook('%s_wrap' % _hook_name, return_data, **hook_kwargs) + return return_data + return _wrap + return _decorator + + +def trigger_hook(name, *args, **kwargs): + """ Trigger execution of custom hook method if found """ + global zap_hooks + arg_length = len(args) + args_list = list(args) + args = args[0] if arg_length == 1 else args + + logging.debug('Trigger hook: %s, args: %s' % (name, arg_length)) + + if not zap_hooks: + return args + elif not hasattr(zap_hooks, name): + return args + + hook_fn = getattr(zap_hooks, name) + if not callable(hook_fn): + return args + + response = hook_fn(*args_list, **kwargs) + + # The number of args returned should match arguments passed + if not response: + return args + elif arg_length == 1: + return args + elif (isinstance(response, list) or isinstance(response, tuple)) and len(response) != arg_length: + return args + return response + + +@hook() +def load_config(config, config_dict, config_msg, out_of_scope_dict): + """ Loads the config file specified into: + config_dict - a dictionary which maps plugin_ids to levels (IGNORE, WARN, FAIL) + config_msg - a dictionary which maps plugin_ids to optional user specified descriptions + out_of_scope_dict - a dictionary which maps plugin_ids to out of scope regexes + """ + for line in config: + if not line.startswith('#') and len(line) > 1: + (key, val, optional) = line.rstrip().split('\t', 2) + if val == 'OUTOFSCOPE': + for plugin_id in key.split(','): + if plugin_id not in out_of_scope_dict: + out_of_scope_dict[plugin_id] = [] + out_of_scope_dict[plugin_id].append(re.compile(optional)) + elif val not in zap_conf_lvls: + raise ValueError("Level {0} is not a supported level: {1}".format(val, zap_conf_lvls)) + else: + config_dict[key] = val + if '\t' in optional: + (ignore, usermsg) = optional.rstrip().split('\t') + config_msg[key] = usermsg + else: + config_msg[key] = '' + logging.debug('Loaded config: {0}'.format(config_dict)) + + +def is_in_scope(plugin_id, url, out_of_scope_dict): + """ Returns True if the url is in scope for the specified plugin_id """ + if '*' in out_of_scope_dict: + for oos_prog in out_of_scope_dict['*']: + #print('OOS Compare ' + oos_url + ' vs ' + 'url) + if oos_prog.match(url): + #print('OOS Ignoring ' + str(plugin_id) + ' ' + url) + return False + #print 'Not in * dict' + if plugin_id in out_of_scope_dict: + for oos_prog in out_of_scope_dict[plugin_id]: + #print('OOS Compare ' + oos_url + ' vs ' + 'url) + if oos_prog.match(url): + #print('OOS Ignoring ' + str(plugin_id) + ' ' + url) + return False + #print 'Not in ' + plugin_id + ' dict' + return True + + +def print_rule(action, alert_list, detailed_output, user_msg, in_progress_issues): + id = alert_list[0].get('pluginId') + if id in in_progress_issues: + print (action + '-IN_PROGRESS: ' + alert_list[0].get('alert') + ' [' + id + '] x ' + str(len(alert_list)) + ' ' + user_msg) + if in_progress_issues[id]["link"]: + print ('\tProgress link: ' + in_progress_issues[id]["link"]) + else: + print (action + '-NEW: ' + alert_list[0].get('alert') + ' [' + id + '] x ' + str(len(alert_list)) + ' ' + user_msg) + if detailed_output: + # Show (up to) first 5 urls + for alert in alert_list[0:5]: + print ('\t' + alert.get('url')) + + +def print_rules(alert_dict, level, config_dict, config_msg, min_level, inc_rule, inc_extra, detailed_output, in_progress_issues): + # print out the ignored rules + count = 0 + inprog_count = 0 + for key, alert_list in sorted(alert_dict.items()): + #if (config_dict.has_key(key) and config_dict[key] == level): + if inc_rule(config_dict, key, inc_extra): + user_msg = '' + if key in config_msg: + user_msg = config_msg[key] + if min_level <= zap_conf_lvls.index(level): + print_rule(level, alert_list, detailed_output, user_msg, in_progress_issues) + if key in in_progress_issues: + inprog_count += 1 + else: + count += 1 + return trigger_hook('print_rules_wrap', count, inprog_count) + + +def inc_ignore_rules(config_dict, key, inc_extra): + return (key in config_dict) and config_dict[key] == 'IGNORE' + + +def inc_info_rules(config_dict, key, inc_extra): + return ((key in config_dict) and config_dict[key] == 'INFO') or ((key not in config_dict) and inc_extra) + + +def inc_warn_rules(config_dict, key, inc_extra): + return ((key in config_dict) and config_dict[key] == 'WARN') or ((key not in config_dict) and inc_extra) + + +def inc_fail_rules(config_dict, key, inc_extra): + return (key in config_dict) and config_dict[key] == 'FAIL' + + +def dump_log_file(cid): + traceback.print_exc() + # Unexpected issue - dump the zap.log file + if running_in_docker(): + zap_log = '/zap/zap.out' + if os.path.isfile(zap_log): + with open(zap_log, 'r') as zlog: + for line in zlog: + sys.stderr.write(line) + else: + logging.debug('Failed to find zap_log ' + zap_log) + else: + logging.debug('Dumping docker logs') + subprocess.call(["docker", "logs", cid], stdout=sys.stderr) + + +def cp_to_docker(cid, file, dir): + logging.debug ('Copy ' + file) + params = ['docker', 'cp', file, cid + ':' + dir + file] + logging.debug (subprocess.check_output(params)) + + +def running_in_docker(): + return os.path.exists('/.dockerenv') + + +def add_zap_options(params, zap_options): + if zap_options: + for zap_opt in shlex.split(zap_options): + params.append(zap_opt) + + +@hook() +def start_zap(port, extra_zap_params): + logging.debug('Starting ZAP') + # All of the default common params + params = [ + 'zap-x.sh', '-daemon', + '-port', str(port), + '-host', '0.0.0.0', + '-config', 'api.disablekey=true', + '-config', 'api.addrs.addr.name=.*', + '-config', 'api.addrs.addr.regex=true'] + + params.extend(extra_zap_params) + + logging.info('Params: ' + str(params)) + + with open('zap.out', "w") as outfile: + subprocess.Popen(params, stdout=outfile) + + +def wait_for_zap_start(zap, timeout_in_secs = 600): + version = None + if not timeout_in_secs: + # if ZAP doesnt start in 10 mins then its probably not going to start + timeout_in_secs = 600 + + for x in range(0, timeout_in_secs): + try: + version = zap.core.version + logging.debug('ZAP Version ' + version) + logging.debug('Took ' + str(x) + ' seconds') + break + except IOError: + time.sleep(1) + + if not version: + raise IOError( + errno.EIO, + 'Failed to connect to ZAP after {0} seconds'.format(timeout_in_secs)) + + +@hook(wrap=True) +def start_docker_zap(docker_image, port, extra_zap_params, mount_dir): + try: + logging.debug('Pulling ZAP Docker image: ' + docker_image) + ls_output = subprocess.check_output(['docker', 'pull', docker_image]) + except OSError as err: + logging.warning('Failed to run docker - is it on your path?') + raise err + + logging.debug('Starting ZAP') + params = ['docker', 'run'] + + if mount_dir: + params.extend(['-v', mount_dir + ':/zap/wrk/:rw']) + + params.extend([ + '-u', 'zap', + '-p', str(port) + ':' + str(port), + '-d', docker_image, + 'zap-x.sh', '-daemon', + '-port', str(port), + '-host', '0.0.0.0', + '-config', 'api.disablekey=true', + '-config', 'api.addrs.addr.name=.*', + '-config', 'api.addrs.addr.regex=true']) + + params.extend(extra_zap_params) + + logging.info('Params: ' + str(params)) + + cid = subprocess.check_output(params).rstrip().decode('utf-8') + logging.debug('Docker CID: ' + cid) + return cid + + +def get_free_port(): + while True: + port = randint(32768, 61000) + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + if not (sock.connect_ex(('127.0.0.1', port)) == 0): + return port + + +def ipaddress_for_cid(cid): + insp_output = subprocess.check_output(['docker', 'inspect', cid]).strip().decode('utf-8') + #logging.debug('Docker Inspect: ' + insp_output) + insp_json = json.loads(insp_output) + return str(insp_json[0]['NetworkSettings']['IPAddress']) + + +def stop_docker(cid): + # Close container - ignore failures + try: + logging.debug('Stopping Docker container') + subprocess.check_output(['docker', 'stop', cid]) + logging.debug('Docker container stopped') + except OSError: + logging.warning('Docker stop failed') + + # Remove container - ignore failures + try: + logging.debug('Removing Docker container') + subprocess.check_output(['docker', 'rm', cid]) + logging.debug('Docker container removed') + except OSError: + logging.warning('Docker rm failed') + + +@hook() +def zap_access_target(zap, target): + res = zap.urlopen(target) + if res.startswith("ZAP Error"): + raise IOError(errno.EIO, 'ZAP failed to access: {0}'.format(target)) + + +@hook(wrap=True) +def zap_spider(zap, target): + logging.debug('Spider ' + target) + spider_scan_id = zap.spider.scan(target) + time.sleep(5) + + while (int(zap.spider.status(spider_scan_id)) < 100): + logging.debug('Spider progress %: ' + zap.spider.status(spider_scan_id)) + time.sleep(5) + logging.debug('Spider complete') + + +@hook(wrap=True) +def zap_ajax_spider(zap, target, max_time): + logging.debug('AjaxSpider ' + target) + if max_time: + zap.ajaxSpider.set_option_max_duration(str(max_time)) + zap.ajaxSpider.scan(target) + time.sleep(5) + + while (zap.ajaxSpider.status == 'running'): + logging.debug('Ajax Spider running, found urls: ' + zap.ajaxSpider.number_of_results) + time.sleep(5) + logging.debug('Ajax Spider complete') + + +@hook(wrap=True) +def zap_active_scan(zap, target, policy): + logging.debug('Active Scan ' + target + ' with policy ' + policy) + ascan_scan_id = zap.ascan.scan(target, recurse=True, scanpolicyname=policy) + time.sleep(5) + + while(int(zap.ascan.status(ascan_scan_id)) < 100): + logging.debug('Active Scan progress %: ' + zap.ascan.status(ascan_scan_id)) + time.sleep(5) + logging.debug('Active Scan complete') + logging.debug(zap.ascan.scan_progress(ascan_scan_id)) + + +def zap_wait_for_passive_scan(zap, timeout_in_secs = 0): + rtc = zap.pscan.records_to_scan + logging.debug('Records to scan...') + time_taken = 0 + timed_out = False + while (int(zap.pscan.records_to_scan) > 0): + logging.debug('Records to passive scan : ' + zap.pscan.records_to_scan) + time.sleep(2) + time_taken += 2 + if timeout_in_secs and time_taken > timeout_in_secs: + timed_out = True + break + if timed_out: + logging.debug('Exceeded passive scan timeout') + else: + logging.debug('Passive scanning complete') + + +@hook(wrap=True) +def zap_get_alerts(zap, baseurl, blacklist, out_of_scope_dict): + # Retrieve the alerts using paging in case there are lots of them + st = 0 + pg = 5000 + alert_dict = {} + alert_count = 0 + alerts = zap.core.alerts(baseurl=baseurl, start=st, count=pg) + while len(alerts) > 0: + logging.debug('Reading ' + str(pg) + ' alerts from ' + str(st)) + alert_count += len(alerts) + for alert in alerts: + plugin_id = alert.get('pluginId') + if plugin_id in blacklist: + continue + if not is_in_scope(plugin_id, alert.get('url'), out_of_scope_dict): + continue + if alert.get('risk') == 'Informational': + # Ignore all info alerts - some of them may have been downgraded by security annotations + continue + if (plugin_id not in alert_dict): + alert_dict[plugin_id] = [] + alert_dict[plugin_id].append(alert) + st += pg + alerts = zap.core.alerts(start=st, count=pg) + logging.debug('Total number of alerts: ' + str(alert_count)) + return alert_dict + + +def get_latest_zap_client_version(): + version_info = None + + try: + version_info = urlopen('https://pypi.python.org/pypi/python-owasp-zap-v2.4/json', timeout=10) + except Exception as e: + logging.warning('Error fetching latest ZAP Python API client version: %s' % e) + return None + + if version_info is None: + logging.warning('Error fetching latest ZAP Python API client version: %s' % e) + return None + + version_json = json.loads(version_info.read().decode('utf-8')) + + if 'info' not in version_json: + logging.warning('No version found for latest ZAP Python API client.') + return None + if 'version' not in version_json['info']: + logging.warning('No version found for latest ZAP Python API client.') + return None + + return version_json['info']['version'] + + +def check_zap_client_version(): + # No need to check ZAP Python API client while running in Docker + if running_in_docker(): + return + + if 'pkg_resources' not in globals(): # import failed + logging.warning('Could not check ZAP Python API client without pkg_resources.') + return + + current_version = getattr(zapv2, '__version__', None) + latest_version = get_latest_zap_client_version() + parse_version = pkg_resources.parse_version + if current_version and latest_version and \ + parse_version(current_version) < parse_version(latest_version): + logging.warning(OLD_ZAP_CLIENT_WARNING) + elif current_version is None: + # the latest versions >= 0.0.9 have a __version__ + logging.warning(OLD_ZAP_CLIENT_WARNING) + # else: + # we're up to date or ahead / running a development build + # or latest_version is None and the user already got a warning + + +def write_report(file_path, report): + with open(file_path, mode='wb') as f: + if not isinstance(report, binary_type): + report = report.encode('utf-8') + + f.write(report) + +@hook(wrap=True) +def zap_import_context(zap, context_file): + res = context_id = zap.context.import_context(context_file) + if res.startswith("ZAP Error"): + context_id = None + logging.error('Failed to load context file ' + context_file + ' : ' + res) + return context_id