Skip to content

Assert artifact: Support that an artifact can exist in multiple flows server/#3953 #559

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ToreMerkely merged 6 commits into from
Oct 13, 2025
Merged

Assert artifact: Support that an artifact can exist in multiple flows server/#3953 #559

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
a89ec5d
Cleaned up assert tests so they work with new server image
ToreMerkely Oct 13, 2025
64482f2
Works with old server on staging
ToreMerkely Oct 13, 2025
be7296d
Have separate fingerprints for the single and multiple artifact in test
ToreMerkely Oct 13, 2025
8828bb3
Fixed printing of environment
ToreMerkely Oct 13, 2025
f1f6927
Added test for specifying policy and environment at the same time is …
ToreMerkely Oct 13, 2025
316f184
Changed order in table output
ToreMerkely Oct 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 62 additions & 0 deletions bin/get_file_from_test_integration.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
#!/usr/bin/env bash


SCRIPT_NAME=get_file_from_test_integration.sh
FROM_FILE_PATH=""
TO_FILE_NAME=""

die()
{
echo "Error: $1" >&2
exit 1
}

print_help()
{
cat <<EOF
Usage: $SCRIPT_NAME [options] <DOCKER_FILE> [LOCAL_FILE]

Get a file from test_integration

Options are:
-h Print this help menu
EOF
}

check_arguments()
{
while getopts "h" opt; do
case $opt in
h)
print_help
exit 1
;;
\?)
echo "Invalid option: -$OPTARG" >&2
exit 1
;;
esac
done

# Remove options from command line
shift $((OPTIND-1))

if [ $# -lt 1 ]; then
echo "Error: Missing DOCKER_FILE argument"
exit 1
fi
FROM_FILE_PATH=$1; shift
if [ $# -eq 1 ]; then
TO_FILE_NAME=$1; shift
else
TO_FILE_NAME=$(basename "${FROM_FILE_PATH}")
fi
}

main()
{
check_arguments "$@"
docker exec cli_kosli_server sh -c "cat ${FROM_FILE_PATH}" > ${TO_FILE_NAME} || die "Failed to get file"
}

main "$@"
53 changes: 31 additions & 22 deletions cmd/kosli/assertArtifact.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,36 +149,18 @@ func printAssertAsTable(raw string, out io.Writer, page int) error {
if err != nil {
return err
}

flow, _ := evaluationResult["flow"].(string)
trail, _ := evaluationResult["trail"].(string)
scope := evaluationResult["scope"].(string)
complianceStatus, _ := evaluationResult["compliance_status"].(map[string]interface{})
attestationsStatuses, _ := complianceStatus["attestations_statuses"].([]interface{})

if evaluationResult["compliant"].(bool) {
logger.Info("COMPLIANT")
} else {
logger.Info("Error: NON-COMPLIANT")
}
logger.Info("Flow: %v\nTrail %v", flow, trail)
logger.Info("%-32v %-30v %-15v %-10v", "Attestation-name", "type", "status", "compliant")

for _, item := range attestationsStatuses {
attestation := item.(map[string]interface{})
name := attestation["attestation_name"]
attType := attestation["attestation_type"]
status := attestation["status"]
isCompliant, _ := attestation["is_compliant"].(bool)
unexpected, _ := attestation["unexpected"].(bool)
unexpectedStr := ""
if unexpected {
unexpectedStr = "unexpected"
}

logger.Info(" %-32v %-30v %-15v %-10v %-10v", name, attType, status, isCompliant, unexpectedStr)
}
if scope == "environment" || scope == "policy" {
if scope == "environment" {
logger.Info("Environment: %v", evaluationResult["environment"].(string))
}
logger.Info("%-32v %-30v", "Policy-name", "status")
policyEvaluations := evaluationResult["policy_evaluations"].([]interface{})
for _, item := range policyEvaluations {
Expand Down Expand Up @@ -220,8 +202,35 @@ func printAssertAsTable(raw string, out io.Writer, page int) error {
}
}
}
logger.Info("")
}
logger.Info("\nSee more details at %s", evaluationResult["html_url"].(string))

flows := evaluationResult["flows"].([]interface{})
for _, item := range flows {
item := item.(map[string]interface{})
flow := item["flow"].(string)
trail, _ := item["trail"].(string)
complianceStatus, _ := item["compliance_status"].(map[string]interface{})
attestationsStatuses, _ := complianceStatus["attestations_statuses"].([]interface{})

logger.Info("Flow: %v\n Trail: %v", flow, trail)
logger.Info(" %-32v %-30v %-15v %-10v", "Attestation-name", "type", "status", "compliant")

for _, item := range attestationsStatuses {
attestation := item.(map[string]interface{})
name := attestation["attestation_name"]
attType := attestation["attestation_type"]
status := attestation["status"]
isCompliant, _ := attestation["is_compliant"].(bool)
unexpected, _ := attestation["unexpected"].(bool)
unexpectedStr := ""
if unexpected {
unexpectedStr = "unexpected"
}

logger.Info(" %-32v %-30v %-15v %-10v %-10v", name, attType, status, isCompliant, unexpectedStr)
}
logger.Info(" See more details at %s", evaluationResult["html_url"].(string))
}
return nil
}
127 changes: 91 additions & 36 deletions cmd/kosli/assertArtifact_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,105 +14,160 @@ import (
type AssertArtifactCommandTestSuite struct {
suite.Suite
defaultKosliArguments string
flowName string
flowName1 string
flowName2 string
envName string
policyName1 string
policyName2 string
artifactName string
artifactPath string
fingerprint string
artifactName1 string
artifact1Path string
fingerprint1 string
artifactName2 string
artifact2Path string
fingerprint2 string
}

func (suite *AssertArtifactCommandTestSuite) SetupTest() {
suite.flowName = "assert-artifact"
suite.flowName1 = "assert-artifact-one"
suite.flowName2 = "assert-artifact-two"
suite.envName = "assert-artifact-environment"
suite.policyName1 = "assert-artifact-policy-1"
suite.policyName2 = "assert-artifact-policy-2"
suite.artifactName = "arti"
suite.artifactPath = "testdata/folder1/hello.txt"
suite.artifactName1 = "arti-for-AssertArtifactCommandTestSuite"
suite.artifact1Path = "testdata/artifacts/AssertArtifactCommandTestSuiteArtifact1.txt"
suite.artifactName2 = "arti-for-AssertArtifactCommandTestSuite2"
suite.artifact2Path = "testdata/artifacts/AssertArtifactCommandTestSuiteArtifact2.txt"
global = &GlobalOpts{
ApiToken: "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpZCI6ImNkNzg4OTg5In0.e8i_lA_QrEhFncb05Xw6E_tkCHU9QfcY4OLTVUCHffY",
Org: "docs-cmd-test-user",
Host: "http://localhost:8001",
}
suite.defaultKosliArguments = fmt.Sprintf(" --host %s --org %s --api-token %s", global.Host, global.Org, global.ApiToken)

CreateFlow(suite.flowName, suite.Suite.T())
CreateFlow(suite.flowName1, suite.Suite.T())
CreateFlow(suite.flowName2, suite.Suite.T())
fingerprintOptions := &fingerprintOptions{
artifactType: "file",
}
CreateEnv(global.Org, suite.envName, "server", suite.Suite.T())
CreatePolicy(global.Org, suite.policyName1, suite.Suite.T())
CreatePolicy(global.Org, suite.policyName2, suite.Suite.T())
var err error
suite.fingerprint, err = GetSha256Digest(suite.artifactPath, fingerprintOptions, logger)
suite.fingerprint1, err = GetSha256Digest(suite.artifact1Path, fingerprintOptions, logger)
require.NoError(suite.Suite.T(), err)
CreateArtifact(suite.flowName, suite.fingerprint, suite.artifactName, suite.Suite.T())
CreateArtifact(suite.flowName1, suite.fingerprint1, suite.artifactName1, suite.Suite.T())
suite.fingerprint2, err = GetSha256Digest(suite.artifact2Path, fingerprintOptions, logger)
require.NoError(suite.Suite.T(), err)
CreateArtifact(suite.flowName1, suite.fingerprint2, suite.artifactName2, suite.Suite.T())
CreateArtifact(suite.flowName2, suite.fingerprint2, suite.artifactName1, suite.Suite.T())
}

func (suite *AssertArtifactCommandTestSuite) TestAssertArtifactCmd() {
tests := []cmdTestCase{
{
wantError: true,
name: "missing --org fails",
cmd: fmt.Sprintf(`assert artifact --fingerprint 8e568bd886069f1290def0caabc1e97ce0e7b80c105e611258b57d76fcef234c --flow %s --api-token secret`, suite.flowName),
name: "01 missing --org fails",
cmd: fmt.Sprintf(`assert artifact --fingerprint 8e568bd886069f1290def0caabc1e97ce0e7b80c105e611258b57d76fcef234c --flow %s --api-token secret`, suite.flowName1),
golden: "Error: --org is not set\nUsage: kosli assert artifact [IMAGE-NAME | FILE-PATH | DIR-PATH] [flags]\n",
},
{
wantError: true,
name: "asserting a non existing artifact fails",
cmd: fmt.Sprintf(`assert artifact --fingerprint 8e568bd886069f1290def0caabc1e97ce0e7b80c105e611258b57d76fcef234c --flow %s %s`, suite.flowName, suite.defaultKosliArguments),
golden: "Error: Artifact with fingerprint '8e568bd886069f1290def0caabc1e97ce0e7b80c105e611258b57d76fcef234c' does not exist in flow 'assert-artifact' belonging to organization 'docs-cmd-test-user'\n",
name: "02 asserting a non existing artifact fails",
cmd: fmt.Sprintf(`assert artifact --fingerprint 8e568bd886069f1290def0caabc1e97ce0e7b80c105e611258b57d76fcef234c --flow %s %s`, suite.flowName1, suite.defaultKosliArguments),
golden: "Error: Artifact with fingerprint '8e568bd886069f1290def0caabc1e97ce0e7b80c105e611258b57d76fcef234c' does not exist in flow 'assert-artifact-one' belonging to organization 'docs-cmd-test-user'\n",
},
{
name: "03 asserting a single existing compliant artifact (using --fingerprint) results in OK and zero exit",
cmd: fmt.Sprintf(`assert artifact --fingerprint %s %s`, suite.fingerprint1, suite.defaultKosliArguments),
goldenRegex: "(?s)^COMPLIANT\n.*Attestation-name.*See more details at http://localhost(:8001)?/docs-cmd-test-user/flows/assert-artifact-one/artifacts/0089a849fce9c7c9128cd13a2e8b1c0757bdb6a7bad0fdf2800e38c19055b7fc(?:\\?artifact_id=[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{8})?\n",
},
{
name: "asserting an existing compliant artifact (using --fingerprint) results in OK and zero exit",
cmd: fmt.Sprintf(`assert artifact --fingerprint %s --flow %s %s`, suite.fingerprint, suite.flowName, suite.defaultKosliArguments),
goldenRegex: "(?s)^COMPLIANT\n.*Attestation-name.*See more details at http://localhost(:8001)?/docs-cmd-test-user/flows/assert-artifact/artifacts/fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02(?:\\?artifact_id=[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{8})?\n",
name: "04 json output of asserting a single existing compliant artifact (using --fingerprint) results in OK and zero exit",
cmd: fmt.Sprintf(`assert artifact --output json --fingerprint %s %s`, suite.fingerprint1, suite.defaultKosliArguments),
goldenJson: []jsonCheck{
{"compliant", true},
{"scope", "flow"},
{"flows", "length:1"},
},
},
{
name: "05 asserting a single existing compliant artifact (using --fingerprint) for an environment results in OK and zero exit",
cmd: fmt.Sprintf(`assert artifact --fingerprint %s --environment %s %s`, suite.fingerprint1, suite.envName, suite.defaultKosliArguments),
goldenRegex: "(?s)^COMPLIANT\n.*Policy-name.*Attestation-name.*See more details at http://localhost(:8001)?/docs-cmd-test-user/flows/assert-artifact-one/artifacts/0089a849fce9c7c9128cd13a2e8b1c0757bdb6a7bad0fdf2800e38c19055b7fc(?:\\?artifact_id=[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{8})?\n",
},
{
name: "06 json output of asserting a single existing compliant artifact (using --fingerprint) for an environment results in OK and zero exit",
cmd: fmt.Sprintf(`assert artifact --output json --fingerprint %s --environment %s %s`, suite.fingerprint1, suite.envName, suite.defaultKosliArguments),
goldenJson: []jsonCheck{
{"compliant", true},
{"scope", "environment"},
{"environment", suite.envName},
{"flows", "length:1"},
},
},
{
name: "asserting an existing compliant artifact (using --fingerprint) for an environment results in OK and zero exit",
cmd: fmt.Sprintf(`assert artifact --fingerprint %s --flow %s --environment %s %s`, suite.fingerprint, suite.flowName, suite.envName, suite.defaultKosliArguments),
goldenRegex: "(?s)^COMPLIANT\n.*Attestation-name.*Policy-name.*See more details at http://localhost(:8001)?/docs-cmd-test-user/flows/assert-artifact/artifacts/fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02(?:\\?artifact_id=[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{8})?\n",
name: "07 asserting a single existing compliant artifact (using --fingerprint) for an policy results in OK and zero exit",
cmd: fmt.Sprintf(`assert artifact --fingerprint %s --policy %s %s`, suite.fingerprint1, suite.policyName1, suite.defaultKosliArguments),
goldenRegex: "(?s)^COMPLIANT\n.*Policy-name.*assert-artifact-policy-1.*Attestation-name.*See more details at http://localhost(:8001)?/docs-cmd-test-user/flows/assert-artifact-one/artifacts/0089a849fce9c7c9128cd13a2e8b1c0757bdb6a7bad0fdf2800e38c19055b7fc(?:\\?artifact_id=[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{8})?\n.*",
},
{
name: "asserting an existing compliant artifact (using --fingerprint) for an policy results in OK and zero exit",
cmd: fmt.Sprintf(`assert artifact --fingerprint %s --flow %s --policy %s %s`, suite.fingerprint, suite.flowName, suite.policyName1, suite.defaultKosliArguments),
goldenRegex: "(?s)^COMPLIANT\n.*Attestation-name.*Policy-name.*assert-artifact-policy-1.*See more details at http://localhost(:8001)?/docs-cmd-test-user/flows/assert-artifact/artifacts/fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02(?:\\?artifact_id=[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{8})?\n",
name: "08 json output of asserting a single existing compliant artifact (using --fingerprint) for an policy results in OK and zero exit\"",
cmd: fmt.Sprintf(`assert artifact --output json --fingerprint %s --policy %s %s`, suite.fingerprint1, suite.policyName1, suite.defaultKosliArguments),
goldenJson: []jsonCheck{
{"compliant", true},
{"scope", "policy"},
{"flows", "length:1"},
},
},
{
name: "asserting an existing compliant artifact (using --artifact-type) results in OK and zero exit",
cmd: fmt.Sprintf(`assert artifact %s --artifact-type file --flow %s %s`, suite.artifactPath, suite.flowName, suite.defaultKosliArguments),
goldenRegex: "(?s)^COMPLIANT\n.*See more details at http://localhost(:8001)?/docs-cmd-test-user/flows/assert-artifact/artifacts/fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02?(?:\\?artifact_id=[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{8})?\n",
name: "09 asserting a single existing compliant artifact (using --artifact-type) results in OK and zero exit",
cmd: fmt.Sprintf(`assert artifact %s --artifact-type file %s`, suite.artifact1Path, suite.defaultKosliArguments),
goldenRegex: "(?s)^COMPLIANT\n.*See more details at http://localhost(:8001)?/docs-cmd-test-user/flows/assert-artifact-one/artifacts/0089a849fce9c7c9128cd13a2e8b1c0757bdb6a7bad0fdf2800e38c19055b7fc?(?:\\?artifact_id=[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{8})?\n",
},
{
name: "json output of asserting an existing compliant artifact (using --artifact-type) results in OK and zero exit",
cmd: fmt.Sprintf(`assert artifact %s --output json --artifact-type file --flow %s %s`, suite.artifactPath, suite.flowName, suite.defaultKosliArguments),
name: "10 json output of asserting a single existing compliant artifact (using --artifact-type) results in OK and zero exit",
cmd: fmt.Sprintf(`assert artifact %s --output json --artifact-type file %s`, suite.artifact1Path, suite.defaultKosliArguments),
goldenJson: []jsonCheck{
{"compliant", true},
{"scope", "flow"},
{"flows", "length:1"},
},
},
{
name: "json output of asserting an existing compliant artifact (using --fingerprint) for an policy results in correct json",
cmd: fmt.Sprintf(`assert artifact %s --output json --artifact-type file --flow %s --policy=%s --policy=%s %s`, suite.artifactPath, suite.flowName, suite.policyName1, suite.policyName2, suite.defaultKosliArguments),
name: "11 json output of asserting a single existing compliant artifact (using --fingerprint) for a policy results in correct json",
cmd: fmt.Sprintf(`assert artifact %s --output json --artifact-type file --policy=%s --policy=%s %s`, suite.artifact1Path, suite.policyName1, suite.policyName2, suite.defaultKosliArguments),
goldenJson: []jsonCheck{
{"compliant", true},
{"scope", "policy"},
{"policy_evaluations", "length:2"},
{"policy_evaluations.[0].policy_name", suite.policyName1},
{"policy_evaluations.[1].policy_name", suite.policyName2}},
},
{
name: "12 asserting a multi existing compliant artifact (using --fingerprint) results in OK and zero exit",
cmd: fmt.Sprintf(`assert artifact --fingerprint %s %s`, suite.fingerprint2, suite.defaultKosliArguments),
goldenRegex: "(?s)^COMPLIANT\n.*Flow: assert-artifact-one.*Flow: assert-artifact-two.*Attestation-name.*See more details at http://localhost(:8001)?/docs-cmd-test-user/flows/assert-artifact-one/artifacts/130fabe054d8d90b5d899833cfc769253a39b38854eb0c64214b68b276ef07e8(?:\\?artifact_id=[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{8})?\n",
},
{
name: "13 json output of asserting a multi existing compliant artifact (using --fingerprint) results in OK and zero exit",
cmd: fmt.Sprintf(`assert artifact --output json --fingerprint %s %s`, suite.fingerprint2, suite.defaultKosliArguments),
goldenJson: []jsonCheck{
{"compliant", true},
{"scope", "flow"},
{"flows", "length:2"},
},
},
{
wantError: true,
name: "not providing --fingerprint nor --artifact-type fails",
cmd: fmt.Sprintf(`assert artifact --flow %s %s`, suite.flowName, suite.defaultKosliArguments),
name: "14 not providing --fingerprint nor --artifact-type fails",
cmd: fmt.Sprintf(`assert artifact --flow %s %s`, suite.flowName1, suite.defaultKosliArguments),
golden: "Error: docker image name or file/dir path is required when --fingerprint is not provided\nUsage: kosli assert artifact [IMAGE-NAME | FILE-PATH | DIR-PATH] [flags]\n",
},
{
wantError: true,
name: "not providing --flow when the artifact has multiple instances fails with a server error",
cmd: fmt.Sprintf(`assert artifact --fingerprint %s %s`, suite.fingerprint, suite.defaultKosliArguments),
golden: fmt.Sprintf("Error: Fingerprint '%s' matched multiple artifacts in org '%s'. Please narrow the search to one flow.\n", suite.fingerprint, global.Org),
name: "15 providing both --environment and --polices fails",
cmd: fmt.Sprintf(`assert artifact --fingerprint %s --environment %s --policy %s %s`, suite.fingerprint1, suite.envName, suite.policyName1, suite.defaultKosliArguments),
golden: "Error: Cannot specify both 'environment_name' and 'policy_name' at the same time\n",
},
}

Expand Down
1 change: 1 addition & 0 deletions cmd/kosli/testdata/artifacts/AssertArtifactCommandTestSuiteArtifact1.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
This shall only be used by the AssertArtifactCommandTestSuite
1 change: 1 addition & 0 deletions cmd/kosli/testdata/artifacts/AssertArtifactCommandTestSuiteArtifact2.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
This shall only be used by the AssertArtifactCommandTestSuite and is the second artifact