Skip to content

Commit

Permalink
Bugfix: Support multiple messages for the same Event ID
Browse files Browse the repository at this point in the history
Some event ID have multiple messages stored in the message lists -
these are generally designed for events which have different number of
properties. So for example the message file might contain two messages
for the same event id, one with 1 expansion and one with 2
expansions. Then the application might emit an event to the log file
with 2 properties or only 1 property of the same event id.

This pr stores both the messages and the number of expasions in the
message set and is able to select the most appropriate one for each
message - we aim to maximize the number of expasions available in the
message string.
  • Loading branch information
scudette committed Jan 17, 2025
1 parent 3e4ff3d commit e31d7ff
Show file tree
Hide file tree
Showing 7 changed files with 139 additions and 57 deletions.
2 changes: 1 addition & 1 deletion evtx.go
Original file line number Diff line number Diff line change
Expand Up @@ -986,7 +986,7 @@ func GetChunks(fd io.ReadSeeker) ([]*Chunk, error) {
for offset := int64(header.HeaderBlockSize); true; offset += EVTX_CHUNK_SIZE {
chunk, err := NewChunk(fd, offset)
if err != nil {
if errors.Cause(err) == io.EOF {
if errors.Is(err, io.EOF) || errors.Is(err, os.ErrNotExist) {
break
}
continue
Expand Down
8 changes: 4 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,14 @@ require (
github.com/alecthomas/assert v1.0.0
github.com/davecgh/go-spew v1.1.1
github.com/hashicorp/golang-lru v1.0.2
github.com/mattn/go-sqlite3 v1.14.22
github.com/mattn/go-sqlite3 v1.14.24
github.com/pkg/errors v0.9.1
github.com/sebdah/goldie v1.0.0
github.com/stretchr/testify v1.9.0
golang.org/x/sys v0.22.0
golang.org/x/sys v0.29.0
gopkg.in/alecthomas/kingpin.v2 v2.2.6
www.velocidex.com/golang/binparsergen v0.1.1-0.20240404114946-8f66c7cf586e
www.velocidex.com/golang/go-pe v0.1.1-0.20230228112150-ef2eadf34bc3
www.velocidex.com/golang/go-pe v0.1.1-0.20250101153735-7a925ba8334b
)

require (
Expand All @@ -26,7 +26,7 @@ require (
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/sergi/go-diff v1.2.0 // indirect
golang.org/x/text v0.16.0 // indirect
golang.org/x/text v0.21.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

Expand Down
16 changes: 8 additions & 8 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM=
github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
Expand Down Expand Up @@ -73,16 +73,16 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
Expand All @@ -104,5 +104,5 @@ gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81
www.velocidex.com/golang/binparsergen v0.1.1-0.20220107080050-ae6122c5ed14/go.mod h1:Q/J/huOyH6IlY2aShigY1CnZnw5EO0+FZJgnGEBrT5Q=
www.velocidex.com/golang/binparsergen v0.1.1-0.20240404114946-8f66c7cf586e h1:uf1AsYiIzUMJMIdFsVdrIw/BjrGzZbrsnz9xmeZmlYU=
www.velocidex.com/golang/binparsergen v0.1.1-0.20240404114946-8f66c7cf586e/go.mod h1:jk+uZGukrJZWgnNH6q9tJLUnbugHEDPCQdIOmBBMXY4=
www.velocidex.com/golang/go-pe v0.1.1-0.20230228112150-ef2eadf34bc3 h1:W394TEIFuHFxHY8mzTJPHI5v+M+NLKEHmHn7KY/VpEM=
www.velocidex.com/golang/go-pe v0.1.1-0.20230228112150-ef2eadf34bc3/go.mod h1:agYwYzeeytVtdwkRrvxZAjgIA8SCeM/Tg7Ym2/jBwmA=
www.velocidex.com/golang/go-pe v0.1.1-0.20250101153735-7a925ba8334b h1:hOxQYDyETh4wdnCbM9Il4X+6LwonGdLnsoznqvzw48A=
www.velocidex.com/golang/go-pe v0.1.1-0.20250101153735-7a925ba8334b/go.mod h1:agYwYzeeytVtdwkRrvxZAjgIA8SCeM/Tg7Ym2/jBwmA=
79 changes: 79 additions & 0 deletions message_sets.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
package evtx

import (
"regexp"
"strconv"
"sync"
)

var (
expansionRegex = regexp.MustCompile("%[0-9]+")
)

type MessageSet struct {
mu sync.Mutex
Provider string
Channel string
Messages map[int]string
Parameters map[int]string
}

func (self *MessageSet) AddMessage(event_id int, message string) {
self.mu.Lock()
defer self.mu.Unlock()

number_of_expansions := self.getLargestExpansion(message)
key := event_id<<16 | number_of_expansions

self.Messages[key] = message
}

func (self *MessageSet) AddParameter(event_id int, message string) {
self.mu.Lock()
defer self.mu.Unlock()

self.Parameters[event_id] = message
}

func (self *MessageSet) GetParameter(id int) string {
self.mu.Lock()
defer self.mu.Unlock()

res, _ := self.Parameters[id]
return res
}

// Calculates the largest expansion number from the message string.
func (self *MessageSet) getLargestExpansion(message string) int {
res := 0

for _, m := range expansionRegex.FindAllString(message, -1) {
val, err := strconv.Atoi(m[1:])
if err == nil {
val--
if val > res {
res = val
}
}
}

return res
}

// Sometimes a number of message strings are generated for each event
// id. This function finds the most appropriate message string with
// the most expansions relevant for this event.
func (self *MessageSet) GetBestMessage(
event_id, number_of_expansions int) string {
self.mu.Lock()
defer self.mu.Unlock()

for i := number_of_expansions; i > 0; i-- {
key := event_id<<16 | i
res, pres := self.Messages[key]
if pres {
return res
}
}
return ""
}
28 changes: 14 additions & 14 deletions messages.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,14 @@ var (
)

type MessageResolver interface {
GetMessage(provider, channel string, event_id int) string
GetMessage(provider, channel string, event_id int, number_of_expansions int) string
GetParameter(provider, channel string, parameter_id int) string
Close()
}

type NullResolver struct{}

func (self NullResolver) GetMessage(provider, channel string, event_id int) string {
func (self NullResolver) GetMessage(provider, channel string, event_id, number_of_expansions int) string {
return ""
}

Expand All @@ -37,6 +37,9 @@ func (self NullResolver) Close() {}

func flatten(dict *ordereddict.Dict) []interface{} {
result := []interface{}{}
if dict == nil {
return result
}

for _, k := range dict.Keys() {
value, _ := dict.Get(k)
Expand Down Expand Up @@ -80,16 +83,23 @@ func maybeExpandObjects(provider, channel string,
func ExpandMessage(
event *ordereddict.Dict, resolver MessageResolver) string {

// Now get and flatten the user data or event data
data, pres := ordereddict.GetMap(event, "UserData")
if !pres {
data, _ = ordereddict.GetMap(event, "EventData")
}
expansions := flatten(data)

provider, _ := ordereddict.GetString(event, "System.Provider.Name")
provider_guid, _ := ordereddict.GetString(event, "System.Provider.Guid")
channel, _ := ordereddict.GetString(event, "System.Channel")
event_id, _ := ordereddict.GetInt(event, "System.EventID.Value")

// Get the raw message. First try using the GUID then using the
// name if possible.
message := resolver.GetMessage(provider_guid, channel, event_id)
message := resolver.GetMessage(provider_guid, channel, event_id, len(expansions))
if message == "" {
message = resolver.GetMessage(provider, channel, event_id)
message = resolver.GetMessage(provider, channel, event_id, len(expansions))
if message == "" {
// No raw message string, just return.
return message
Expand All @@ -99,16 +109,6 @@ func ExpandMessage(
provider = provider_guid
}

// Now get and flatten the user data or event data
data, pres := ordereddict.GetMap(event, "UserData")
if !pres {
data, pres = ordereddict.GetMap(event, "EventData")
if !pres {
return message
}
}
expansions := flatten(data)

// Replace expansions in the message with the user data.
return expansion_re.ReplaceAllStringFunc(message, func(match string) string {
switch match {
Expand Down
2 changes: 1 addition & 1 deletion messages_database.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ type DBResolver struct {

// TODO: What is happening with the channel here?
func (self *DBResolver) GetMessage(
provider, channel string, event_id int) string {
provider, channel string, event_id, number_of_expansions int) string {
rows, err := self.query.Query(provider, event_id)
if err != nil {
return ""
Expand Down
61 changes: 32 additions & 29 deletions messages_windows.go
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
//go:build windows
// +build windows

package evtx

import (
"os"
"path/filepath"
"sort"
"strings"

lru "github.com/hashicorp/golang-lru"
Expand All @@ -22,11 +24,21 @@ func NewWindowsMessageResolver() *WindowsMessageResolver {
return &WindowsMessageResolver{
// string->MessageSet
cache: cache,

// MUI files can be found in the SxS directory - we cache that
// periodically.
mui_cache: make(map[string][]string),
}
}

type WindowsMessageResolver struct {
cache *lru.Cache

mui_cache map[string][]string
}

func (self *WindowsMessageResolver) buildSxScache() {
self.mui_cache = make(map[string][]string)
}

func (self *WindowsMessageResolver) getMessageSets(
Expand All @@ -38,10 +50,10 @@ func (self *WindowsMessageResolver) getMessageSets(
message_set_any, pres := self.cache.Get(key)
if !pres {
var err error
message_set_any, err = GetMessagesByGUID(provider, channel)
message_set_any, err = self.GetMessagesByGUID(provider, channel)
if err != nil {
// Try to get the messages by provider name
message_set_any, err = GetMessages(provider, channel)
message_set_any, err = self.GetMessages(provider, channel)
if err != nil {
// Cache the failure by storing nil in the map
self.cache.Add(key, nil)
Expand All @@ -60,19 +72,15 @@ func (self *WindowsMessageResolver) getMessageSets(
}

func (self *WindowsMessageResolver) GetMessage(
provider, channel string, event_id int) string {
provider, channel string, event_id, number_of_expansions int) string {

message_set, err := self.getMessageSets(provider, channel)
if err != nil {
return ""
}

// Get the event if it is there
res, pres := message_set.Messages[event_id]
if pres {
return res.Message
}
return ""
return message_set.GetBestMessage(event_id, number_of_expansions)
}

func (self *WindowsMessageResolver) GetParameter(
Expand All @@ -87,22 +95,11 @@ func (self *WindowsMessageResolver) GetParameter(
return ""
}

res, pres := message_set.Parameters[parameter_id]
if pres {
return res.Message
}
return ""
return message_set.GetParameter(parameter_id)
}

func (self *WindowsMessageResolver) Close() {}

type MessageSet struct {
Provider string
Channel string
Messages map[int]*pe.Message
Parameters map[int]*pe.Message
}

// ExpandLocations Produces a list of possible locations the message
// file may be. We process all of them because sometimes event
// messages are split across multiple dlls. For example, a generic
Expand Down Expand Up @@ -174,11 +171,16 @@ func ExpandLocations(message_file string) []string {
}

// Message file values may be separated by ;
return include_muis(split_system32(replace_env_vars(
res := include_muis(split_system32(replace_env_vars(
strings.Split(message_file, ";"))))
sort.Slice(res, func(i, j int) bool {
return len(res[i]) > len(res[j])
})
return res
}

func GetMessagesByGUID(provider_guid, channel string) (*MessageSet, error) {
func (self *WindowsMessageResolver) GetMessagesByGUID(
provider_guid, channel string) (*MessageSet, error) {
key_path := `Software\Microsoft\Windows\CurrentVersion\WinEVT\Publishers\{` + provider_guid + "}"
provider_key, err := registry.OpenKey(registry.LOCAL_MACHINE, key_path,
registry.READ|registry.ENUMERATE_SUB_KEYS|registry.WOW64_64KEY)
Expand Down Expand Up @@ -211,19 +213,19 @@ func expandLocations(
result := &MessageSet{
Provider: provider,
Channel: channel,
Messages: make(map[int]*pe.Message),
Parameters: make(map[int]*pe.Message),
Messages: make(map[int]string),
Parameters: make(map[int]string),
}

populateMessages(message_files, result.Messages)
populateMessages(message_files, result.AddMessage)
if parameter_files != "" {
populateMessages(parameter_files, result.Parameters)
populateMessages(parameter_files, result.AddParameter)
}

return result, nil
}

func populateMessages(message_files string, set map[int]*pe.Message) {
func populateMessages(message_files string, adder func(event_id int, message string)) {
for _, message_file := range ExpandLocations(message_files) {
fd, err := os.Open(message_file)
if err != nil {
Expand All @@ -248,12 +250,13 @@ func populateMessages(message_files string, set map[int]*pe.Message) {
}

for _, msg := range messages {
set[msg.EventId] = msg
adder(msg.EventId, msg.Message)
}
}
}

func GetMessages(provider, channel string) (*MessageSet, error) {
func (self *WindowsMessageResolver) GetMessages(
provider, channel string) (*MessageSet, error) {
root_key, err := registry.OpenKey(registry.LOCAL_MACHINE,
`SYSTEM\CurrentControlSet\Services\EventLog`,
registry.READ|registry.ENUMERATE_SUB_KEYS|registry.WOW64_64KEY)
Expand Down

0 comments on commit e31d7ff

Please sign in to comment.