Skip to content

Commit 638b251

Browse files
AppVerifier adjustment (#478)
Adjustments to AppVerifier in CI based on feedback when implementing it for the aws-c-* repositories.
1 parent 1b95d70 commit 638b251

File tree

4 files changed

+194
-175
lines changed

4 files changed

+194
-175
lines changed

.github/workflows/ci.yml

Lines changed: 16 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,6 @@ env:
1717
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_DATEST_SECRET_ACCESS_KEY }}
1818
AWS_DEFAULT_REGION: us-east-1
1919

20-
APP_VERIFIER_EXE: D:\a\work\aws-iot-device-sdk-cpp-v2\build\samples\pub_sub\cycle_pub_sub\RelWithDebInfo\cycle-pub-sub.exe
21-
APP_VERIFIER_OUTPUT_XML: D:\a\work\aws-iot-device-sdk-cpp-v2\app_verifier_output.xml
22-
APP_VERIFIER_OUTPUT_XML_HELPER: D:\a\work\aws-iot-device-sdk-cpp-v2\utils\appverifier_xml_util.py
23-
# All of the tests that will be run by AppVerifier
24-
APP_VERIFIER_TESTS: Exceptions Handles Heaps Leak Locks Memory SRWLock Threadpool TLS
25-
2620
jobs:
2721
linux-compat:
2822
runs-on: ubuntu-latest
@@ -90,20 +84,6 @@ jobs:
9084
cd D:\a\work
9185
python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder.pyz')"
9286
python builder.pyz build -p ${{ env.PACKAGE_NAME }}
93-
- name: Run and check AppVerifier
94-
run: |
95-
echo "Enabling AppVerifier:"
96-
appverif -enable ${{ env.APP_VERIFIER_TESTS }} -for ${{ env.APP_VERIFIER_EXE }}
97-
echo "Starting AppVerifier utility sample run:"
98-
python -m pip install termcolor
99-
python -m pip install boto3
100-
python ${{ env.APP_VERIFIER_OUTPUT_XML_HELPER }} --launch_sample true --sample_file ${{ env.APP_VERIFIER_EXE }}
101-
echo "Exporting XML log:"
102-
appverif -export log -for ${{ env.APP_VERIFIER_EXE }} -with to=${{ env.APP_VERIFIER_OUTPUT_XML }}
103-
echo "Disabling AppVerifier:"
104-
appverif -delete settings -for ${{ env.APP_VERIFIER_EXE }}
105-
echo "Starting AppVerifier utility parsing XML:"
106-
python ${{ env.APP_VERIFIER_OUTPUT_XML_HELPER }} --parse_xml true --xml_file ${{ env.APP_VERIFIER_OUTPUT_XML }}
10787
10888
windows-vs14:
10989
runs-on: windows-2019 # windows-2019 is last env with Visual Studio 2015 (v14.0)
@@ -117,20 +97,6 @@ jobs:
11797
cd D:\a\work
11898
python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')"
11999
python builder.pyz build -p ${{ env.PACKAGE_NAME }} --cmake-extra=-Tv140 --cmake-extra=-A${{ matrix.arch }}
120-
- name: Run and check AppVerifier
121-
run: |
122-
echo "Enabling AppVerifier:"
123-
appverif -enable ${{ env.APP_VERIFIER_TESTS }} -for ${{ env.APP_VERIFIER_EXE }}
124-
echo "Starting AppVerifier utility sample run:"
125-
python -m pip install termcolor
126-
python -m pip install boto3
127-
python ${{ env.APP_VERIFIER_OUTPUT_XML_HELPER }} --launch_sample true --sample_file ${{ env.APP_VERIFIER_EXE }}
128-
echo "Exporting XML log:"
129-
appverif -export log -for ${{ env.APP_VERIFIER_EXE }} -with to=${{ env.APP_VERIFIER_OUTPUT_XML }}
130-
echo "Disabling AppVerifier:"
131-
appverif -delete settings -for ${{ env.APP_VERIFIER_EXE }}
132-
echo "Starting AppVerifier utility parsing XML:"
133-
python ${{ env.APP_VERIFIER_OUTPUT_XML_HELPER }} --parse_xml true --xml_file ${{ env.APP_VERIFIER_OUTPUT_XML }}
134100
135101
windows-no-cpu-extensions:
136102
runs-on: windows-latest
@@ -141,20 +107,22 @@ jobs:
141107
cd D:\a\work
142108
python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder.pyz')"
143109
python builder.pyz build -p ${{ env.PACKAGE_NAME }} --cmake-extra=-DUSE_CPU_EXTENSIONS=OFF
144-
- name: Run and check AppVerifier
145-
run: |
146-
echo "Enabling AppVerifier:"
147-
appverif -enable ${{ env.APP_VERIFIER_TESTS }} -for ${{ env.APP_VERIFIER_EXE }}
148-
echo "Starting AppVerifier utility sample run:"
149-
python -m pip install termcolor
150-
python -m pip install boto3
151-
python ${{ env.APP_VERIFIER_OUTPUT_XML_HELPER }} --launch_sample true --sample_file ${{ env.APP_VERIFIER_EXE }}
152-
echo "Exporting XML log:"
153-
appverif -export log -for ${{ env.APP_VERIFIER_EXE }} -with to=${{ env.APP_VERIFIER_OUTPUT_XML }}
154-
echo "Disabling AppVerifier:"
155-
appverif -delete settings -for ${{ env.APP_VERIFIER_EXE }}
156-
echo "Starting AppVerifier utility parsing XML:"
157-
python ${{ env.APP_VERIFIER_OUTPUT_XML_HELPER }} --parse_xml true --xml_file ${{ env.APP_VERIFIER_OUTPUT_XML }}
110+
111+
windows-app-verifier:
112+
runs-on: windows-2022 # latest
113+
steps:
114+
- name: Build ${{ env.PACKAGE_NAME }} + consumers
115+
run: |
116+
md D:\a\work
117+
cd D:\a\work
118+
python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder.pyz')"
119+
python builder.pyz build -p ${{ env.PACKAGE_NAME }} --cmake-extra=-DUSE_CPU_EXTENSIONS=OFF
120+
- name: Run and check AppVerifier
121+
run: |
122+
cd D:\a\work
123+
echo "Starting to run AppVerifier with cycle pub-sub sample"
124+
python -m pip install boto3
125+
python .\aws-iot-device-sdk-cpp-v2\utils\appverifier_launch_sample.py --sample_file .\aws-iot-device-sdk-cpp-v2\build\samples\pub_sub\cycle_pub_sub\RelWithDebInfo\cycle-pub-sub.exe
158126
159127
osx:
160128
runs-on: macos-latest

.github/workflows/license-check.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,4 +43,5 @@ jobs:
4343
run: for filename in $(< fileList.txt); do ./scancode-toolkit/scancode -l -n 30 --json-pp - ./new-ref/$filename | grep short_name | sort | uniq >> new-licenses.txt; done
4444
# compare
4545
- name: License test
46-
run: if ! cmp old-licenses.txt new-licenses.txt; then echo "Licenses differ! Failing."; exit -1; else echo "Licenses are the same. Success."; exit 0; fi
46+
run: |
47+
if ! cmp old-licenses.txt new-licenses.txt; then echo "Licenses differ! Failing."; exit -1; else echo "Licenses are the same. Success."; exit 0; fi

utils/appverifier_launch_sample.py

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
# SPDX-License-Identifier: Apache-2.0.
3+
4+
# Built-in
5+
import argparse
6+
import os
7+
import subprocess
8+
import pathlib
9+
import sys
10+
import tempfile
11+
import appverifier_xml
12+
# Needs to be installed via pip
13+
import boto3 # - for launching sample
14+
15+
16+
def launchSample(sample_file, sample_region, sample_secret_endpoint, sample_secret_certificate, sample_secret_private_key, sample_arguments):
17+
print("Attempting to get credentials from secrets using Boto3...")
18+
try:
19+
secrets_client = boto3.client(
20+
"secretsmanager", region_name=sample_region)
21+
sample_endpoint = secrets_client.get_secret_value(
22+
SecretId=sample_secret_endpoint)["SecretString"]
23+
sample_certificate = secrets_client.get_secret_value(
24+
SecretId=sample_secret_certificate)
25+
sample_private_key = secrets_client.get_secret_value(
26+
SecretId=sample_secret_private_key)
27+
except Exception: # lgtm [py/catch-base-exception]
28+
sys.exit("ERROR: Could not get secrets to launch sample!")
29+
30+
current_folder = pathlib.Path(__file__).resolve()
31+
# Remove the name of the python file
32+
current_folder = str(current_folder).replace("appverifier_xml_util.py", "")
33+
34+
print("Saving credentials to file...")
35+
tmp_certificate_file_path = str(current_folder) + "tmp_certificate.pem"
36+
tmp_private_key_path = str(current_folder) + "tmp_privatekey.pem.key"
37+
with open(tmp_certificate_file_path, "w") as file:
38+
file.write(sample_certificate["SecretString"]) # lgtm [py/clear-text-storage-sensitive-data]
39+
with open(tmp_private_key_path, "w") as file:
40+
file.write(sample_private_key["SecretString"]) # lgtm [py/clear-text-storage-sensitive-data]
41+
print("Saved credentials to file...")
42+
43+
print("Processing arguments...")
44+
launch_arguments = []
45+
launch_arguments.append("--endpoint")
46+
launch_arguments.append(sample_endpoint)
47+
launch_arguments.append("--cert")
48+
launch_arguments.append(tmp_certificate_file_path)
49+
launch_arguments.append("--key")
50+
launch_arguments.append(tmp_private_key_path)
51+
sample_arguments_split = sample_arguments.split(" ")
52+
for arg in sample_arguments_split:
53+
launch_arguments.append(arg)
54+
55+
print("Running sample...")
56+
exit_code = 0
57+
sample_return = subprocess.run(
58+
args=launch_arguments, executable=sample_file)
59+
exit_code = sample_return.returncode
60+
61+
print("Deleting credentials files...")
62+
os.remove(tmp_certificate_file_path)
63+
os.remove(tmp_private_key_path)
64+
65+
if (exit_code == 0):
66+
print("SUCCESS: Finished running sample! Exiting with success")
67+
else:
68+
print("ERROR: Sample did not return success! Exit code " + str(exit_code))
69+
return exit_code
70+
71+
72+
def registerAppVerifier(test_executable, app_verifier_tests_list):
73+
arguments = ["appverif", "-enable"] + app_verifier_tests_list + ["-for", test_executable]
74+
print (f'Calling AppVerifier with: {subprocess.list2cmdline(arguments)}')
75+
# NOTE: Needs elevated permissions.
76+
subprocess.run(args=arguments)
77+
78+
79+
def unregisterAppVerifier(test_executable):
80+
arguments = ["appverif", "-delete", "settings", "-for", test_executable]
81+
print (f'Calling AppVerifier with: {subprocess.list2cmdline(arguments)}')
82+
# NOTE: Needs elevated permissions.
83+
subprocess.run(args=arguments)
84+
85+
86+
def checkAppVerifierXML(test_executable, tmp_xml_file_path):
87+
appverif_xml_dump_args = ["appverif", "-export", "log", "-for", test_executable, "-with", "to="+ tmp_xml_file_path]
88+
print (f'Calling AppVerifier with: {subprocess.list2cmdline(appverif_xml_dump_args)}')
89+
# NOTE: Needs elevated permissions
90+
subprocess.run(args=appverif_xml_dump_args)
91+
92+
xml_result = appverifier_xml.parseXML(tmp_xml_file_path, True)
93+
if (xml_result != 0):
94+
print (f"ERROR: XML parse returned failure!")
95+
return xml_result
96+
97+
98+
def booleanString(string):
99+
string = string.lower()
100+
if string not in {"false", "true"}:
101+
raise ValueError("Boolean is not true or false!")
102+
return string == "true"
103+
104+
105+
def main():
106+
argument_parser = argparse.ArgumentParser(
107+
description="AppVerifier XML output util")
108+
argument_parser.add_argument("--sample_file", metavar="<C:\\example\\sample.exe>",
109+
required=True, default="", help="Sample to launch that AppVerifier is following")
110+
argument_parser.add_argument("--sample_region", metavar="<Name of region>",
111+
required=False, default="us-east-1", help="The name of the region to use for accessing secrets")
112+
argument_parser.add_argument("--sample_secret_endpoint", metavar="<Name of endpoint secret>",
113+
required=False, default="unit-test/endpoint", help="The name of the secret containing the endpoint")
114+
argument_parser.add_argument("--sample_secret_certificate", metavar="<Name of certificate secret>", required=False,
115+
default="unit-test/certificate", help="The name of the secret containing the certificate PEM file")
116+
argument_parser.add_argument("--sample_secret_private_key", metavar="<Name of private key secret>", required=False,
117+
default="unit-test/privatekey", help="The name of the secret containing the private key PEM file")
118+
argument_parser.add_argument("--sample_arguments", metavar="<Arguments here in single string!>",
119+
required=False, default="", help="Arguments to pass to sample")
120+
121+
parsed_commands = argument_parser.parse_args()
122+
123+
print ("Registering with AppVerifier")
124+
app_verifier_tests = ["Exceptions", "Handles", "Heaps", "Leak", "Locks", "Memory", "SRWLock", "Threadpool", "TLS"]
125+
registerAppVerifier(parsed_commands.sample_file, app_verifier_tests)
126+
127+
print("Starting to launch sample...")
128+
sample_result = launchSample(
129+
parsed_commands.sample_file,
130+
parsed_commands.sample_region,
131+
parsed_commands.sample_secret_endpoint,
132+
parsed_commands.sample_secret_certificate,
133+
parsed_commands.sample_secret_private_key,
134+
parsed_commands.sample_arguments)
135+
136+
print ("Parsing XML...")
137+
tmp_xml_file_path = os.path.join(tempfile.gettempdir(), "tmp.xml")
138+
xml_result = checkAppVerifierXML(parsed_commands.sample_file, tmp_xml_file_path)
139+
140+
print ("Unregistering with AppVerifier")
141+
unregisterAppVerifier(parsed_commands.sample_file)
142+
143+
if (xml_result != 0):
144+
sys.exit(xml_result)
145+
sys.exit(sample_result)
146+
147+
148+
if __name__ == "__main__":
149+
main()

0 commit comments

Comments
 (0)