From 745f39f8a18c74c01d92dce095cb11900a9d7134 Mon Sep 17 00:00:00 2001 From: Batyrkhan Koshenov Date: Mon, 31 Oct 2022 13:06:46 +0600 Subject: [PATCH 01/10] decoders/netflow: split decoding ipfix and nfv9, reuse flow sets Signed-off-by: Batyrkhan Koshenov --- decoders/netflow/netflow.go | 619 ++++++++++++++++++++++-------------- 1 file changed, 387 insertions(+), 232 deletions(-) diff --git a/decoders/netflow/netflow.go b/decoders/netflow/netflow.go index 8836494..bd025ad 100644 --- a/decoders/netflow/netflow.go +++ b/decoders/netflow/netflow.go @@ -2,8 +2,8 @@ package netflow import ( "bytes" - "encoding/binary" "fmt" + "io" "sync" "github.com/cloudflare/goflow/v3/decoders/utils" @@ -16,105 +16,142 @@ type NetFlowTemplateSystem interface { AddTemplate(version uint16, obsDomainId uint32, template interface{}) } -func DecodeNFv9OptionsTemplateSet(payload *bytes.Buffer) ([]NFv9OptionsTemplateRecord, error) { - records := make([]NFv9OptionsTemplateRecord, 0) - var err error +// DecodeNFv9OptionsTemplateSet decodes the options template data that describe +// structure of IPFIX Options Data Records. +func DecodeNFv9OptionsTemplateSet(payload *bytes.Buffer, ts *NFv9OptionsTemplateFlowSet) error { + ts.Records = ts.Records[:cap(ts.Records)] + + i := 0 for payload.Len() >= 4 { - optsTemplateRecord := NFv9OptionsTemplateRecord{} - err = utils.BinaryDecoder(payload, &optsTemplateRecord.TemplateId, &optsTemplateRecord.ScopeLength, &optsTemplateRecord.OptionLength) - if err != nil { - break + if i >= len(ts.Records) { + ts.Records = append(ts.Records, NFv9OptionsTemplateRecord{}) + } + + record := &ts.Records[i] + if !record.ReadFrom(payload) { + return fmt.Errorf("decode nfv9 options templateSet: %v", io.ErrUnexpectedEOF) } - sizeScope := int(optsTemplateRecord.ScopeLength) / 4 - sizeOptions := int(optsTemplateRecord.OptionLength) / 4 + sizeScope := int(record.ScopeLength) / 4 + sizeOptions := int(record.OptionLength) / 4 if sizeScope < 0 || sizeOptions < 0 { - return records, NewErrorDecodingNetFlow("Error decoding OptionsTemplateSet: negative length.") + return NewErrorDecodingNetFlow("error decoding OptionsTemplateSet: negative length.") } - fields := make([]Field, sizeScope) - for i := 0; i < sizeScope; i++ { - field := Field{} - err = utils.BinaryDecoder(payload, &field) - fields[i] = field + record.Scopes = record.Scopes[:cap(record.Scopes)] + if len(record.Scopes) < sizeScope { + record.Scopes = append(record.Scopes, make([]Field, sizeScope-len(record.Scopes))...) } - optsTemplateRecord.Scopes = fields - fields = make([]Field, sizeOptions) - for i := 0; i < sizeOptions; i++ { - field := Field{} - err = utils.BinaryDecoder(payload, &field) - fields[i] = field + for j := 0; j < sizeScope; j++ { + if !record.Scopes[j].ReadFrom(payload) { + return fmt.Errorf("decode nfv9 options templateSet: %v", io.ErrUnexpectedEOF) + } } - optsTemplateRecord.Options = fields + record.Scopes = record.Scopes[:sizeScope] - records = append(records, optsTemplateRecord) - } + record.Options = record.Options[:cap(record.Options)] + if len(record.Options) < sizeOptions { + record.Options = append(record.Options, make([]Field, sizeOptions-len(record.Options))...) + } - return records, nil + for k := 0; k < sizeOptions; k++ { + if !record.Options[k].ReadFrom(payload) { + return fmt.Errorf("decode nfv9 options templateSet: %v", io.ErrUnexpectedEOF) + } + } + record.Options = record.Options[:sizeOptions] + + i++ + } + ts.Records = ts.Records[:i] + return nil } -func DecodeIPFIXOptionsTemplateSet(payload *bytes.Buffer) ([]IPFIXOptionsTemplateRecord, error) { - records := make([]IPFIXOptionsTemplateRecord, 0) - var err error +// DecodeIPFIXOptionsTemplateSet decodes the options template data that describe +// structure of NetFlow Options Data Records. +func DecodeIPFIXOptionsTemplateSet(payload *bytes.Buffer, ts *IPFIXOptionsTemplateFlowSet) error { + ts.Records = ts.Records[:cap(ts.Records)] + + i := 0 for payload.Len() >= 4 { - optsTemplateRecord := IPFIXOptionsTemplateRecord{} - err = utils.BinaryDecoder(payload, &optsTemplateRecord.TemplateId, &optsTemplateRecord.FieldCount, &optsTemplateRecord.ScopeFieldCount) - if err != nil { - break + if i >= len(ts.Records) { + ts.Records = append(ts.Records, IPFIXOptionsTemplateRecord{}) + } + + optsTemplateRecord := &ts.Records[i] + if !optsTemplateRecord.ReadFrom(payload) { + return fmt.Errorf("decode ipfix options templateSet: %v", io.ErrUnexpectedEOF) + } + + if len(optsTemplateRecord.Scopes) < int(optsTemplateRecord.ScopeFieldCount) { + optsTemplateRecord.Scopes = append(optsTemplateRecord.Scopes, make([]Field, int(optsTemplateRecord.ScopeFieldCount)-len(optsTemplateRecord.Scopes))...) } - fields := make([]Field, int(optsTemplateRecord.ScopeFieldCount)) - for i := 0; i < int(optsTemplateRecord.ScopeFieldCount); i++ { - field := Field{} - err = utils.BinaryDecoder(payload, &field) - fields[i] = field + for j := 0; j < int(optsTemplateRecord.ScopeFieldCount); j++ { + if !optsTemplateRecord.Scopes[j].ReadFrom(payload) { + return fmt.Errorf("decode ipfix options templateSet: %v", io.ErrUnexpectedEOF) + } } - optsTemplateRecord.Scopes = fields + optsTemplateRecord.Scopes = optsTemplateRecord.Scopes[:int(optsTemplateRecord.ScopeFieldCount)] optionsSize := int(optsTemplateRecord.FieldCount) - int(optsTemplateRecord.ScopeFieldCount) if optionsSize < 0 { - return records, NewErrorDecodingNetFlow("Error decoding OptionsTemplateSet: negative length.") + return NewErrorDecodingNetFlow("error decoding OptionsTemplateSet: negative length.") } - fields = make([]Field, optionsSize) - for i := 0; i < optionsSize; i++ { - field := Field{} - err = utils.BinaryDecoder(payload, &field) - fields[i] = field + + if len(optsTemplateRecord.Options) < optionsSize { + optsTemplateRecord.Options = append(optsTemplateRecord.Options, make([]Field, optionsSize-len(optsTemplateRecord.Options))...) } - optsTemplateRecord.Options = fields - records = append(records, optsTemplateRecord) - } + for k := 0; k < optionsSize; k++ { + if !optsTemplateRecord.Options[k].ReadFrom(payload) { + return fmt.Errorf("decode ipfix options templateSet: %v", io.ErrUnexpectedEOF) + } + } + optsTemplateRecord.Options = optsTemplateRecord.Options[:optionsSize] - return records, nil + i++ + } + ts.Records = ts.Records[:i] + return nil } -func DecodeTemplateSet(payload *bytes.Buffer) ([]TemplateRecord, error) { - records := make([]TemplateRecord, 0) - var err error +// DecodeTemplateSet decodes the template data that describe structure of Data Records (actual netflow/ipfix data). +func DecodeTemplateSet(payload *bytes.Buffer, ts *TemplateFlowSet) error { + ts.Records = ts.Records[:cap(ts.Records)] + + i := 0 for payload.Len() >= 4 { - templateRecord := TemplateRecord{} - err = utils.BinaryDecoder(payload, &templateRecord.TemplateId, &templateRecord.FieldCount) - if err != nil { - break + if i >= len(ts.Records) { + ts.Records = append(ts.Records, TemplateRecord{}) } - if templateRecord.FieldCount == 0 { - return records, NewErrorDecodingNetFlow("Error decoding TemplateSet: zero count.") + record := &ts.Records[i] + if !record.ReadFrom(payload) { + return fmt.Errorf("decode TemplateSet: %v", io.ErrUnexpectedEOF) } - fields := make([]Field, int(templateRecord.FieldCount)) - for i := 0; i < int(templateRecord.FieldCount); i++ { - field := Field{} - err = utils.BinaryDecoder(payload, &field) - fields[i] = field + if record.FieldCount == 0 { + return NewErrorDecodingNetFlow("error decoding TemplateSet: zero count.") } - templateRecord.Fields = fields - records = append(records, templateRecord) - } - return records, nil + record.Fields = record.Fields[:cap(record.Fields)] + + if len(record.Fields) < int(record.FieldCount) { + record.Fields = append(record.Fields, make([]Field, int(record.FieldCount)-len(record.Fields))...) + } + + for j := 0; j < int(record.FieldCount); j++ { + if !record.Fields[j].ReadFrom(payload) { + return fmt.Errorf("decode TemplateSet: %v", io.ErrUnexpectedEOF) + } + } + record.Fields = record.Fields[:int(record.FieldCount)] + i++ + } + ts.Records = ts.Records[:i] + return nil } func GetTemplateSize(template []Field) int { @@ -125,22 +162,26 @@ func GetTemplateSize(template []Field) int { return sum } -func DecodeDataSetUsingFields(payload *bytes.Buffer, listFields []Field) []DataField { - for payload.Len() >= GetTemplateSize(listFields) { +// DecodeDataRecordFields decodes the fields(type and value) of DataRecord. +func DecodeDataRecordFields(payload *bytes.Buffer, listFields []Field, record *[]DataField) error { + size := GetTemplateSize(listFields) + if payload.Len() < size { + return fmt.Errorf("decode dataset: there are fewer than %d bytes in the buffer", size) + } - dataFields := make([]DataField, len(listFields)) + *record = (*record)[:cap(*record)] + if len(*record) < len(listFields) { + *record = append(*record, make([]DataField, len(listFields)-len(*record))...) + } - for i, templateField := range listFields { - value := payload.Next(int(templateField.Length)) - nfvalue := DataField{ - Type: templateField.Type, - Value: value, - } - dataFields[i] = nfvalue - } - return dataFields + for i, templateField := range listFields { + value := payload.Next(int(templateField.Length)) + + (*record)[i].Type = templateField.Type + (*record)[i].Value = value } - return []DataField{} + *record = (*record)[:len(listFields)] + return nil } type ErrorTemplateNotFound struct { @@ -160,7 +201,7 @@ func NewErrorTemplateNotFound(version uint16, obsDomainId uint32, templateId uin } func (e *ErrorTemplateNotFound) Error() string { - return fmt.Sprintf("No %v template %v found for and domain id %v", e.typeTemplate, e.templateId, e.obsDomainId) + return fmt.Sprintf("no %v template %v found for and domain id %v", e.typeTemplate, e.templateId, e.obsDomainId) } type ErrorVersion struct { @@ -174,7 +215,7 @@ func NewErrorVersion(version uint16) *ErrorVersion { } func (e *ErrorVersion) Error() string { - return fmt.Sprintf("Unknown NetFlow version %v (only decodes v9 and v10/IPFIX)", e.version) + return fmt.Sprintf("unknown NetFlow version %v (only decodes v9 and v10/IPFIX)", e.version) } type ErrorFlowId struct { @@ -188,7 +229,7 @@ func NewErrorFlowId(id uint16) *ErrorFlowId { } func (e *ErrorFlowId) Error() string { - return fmt.Sprintf("Unknown flow id %v (templates < 256, data >= 256)", e.id) + return fmt.Sprintf("unknown flow id %v (templates < 256, data >= 256)", e.id) } type ErrorDecodingNetFlow struct { @@ -202,46 +243,64 @@ func NewErrorDecodingNetFlow(msg string) *ErrorDecodingNetFlow { } func (e *ErrorDecodingNetFlow) Error() string { - return fmt.Sprintf("Error decoding NetFlow: %v", e.msg) + return fmt.Sprintf("error decoding NetFlow: %v", e.msg) } -func DecodeOptionsDataSet(payload *bytes.Buffer, listFieldsScopes, listFieldsOption []Field) ([]OptionsDataRecord, error) { - records := make([]OptionsDataRecord, 0) +// DecodeOptionsDataSet decodes the options data record ("meta-data" about the netflow/ipfix process itself). +func DecodeOptionsDataSet(payload *bytes.Buffer, fs *OptionsDataFlowSet, listFieldsScopes, listFieldsOption []Field) error { + fs.Records = fs.Records[:cap(fs.Records)] listFieldsScopesSize := GetTemplateSize(listFieldsScopes) listFieldsOptionSize := GetTemplateSize(listFieldsOption) + i := 0 for payload.Len() >= listFieldsScopesSize+listFieldsOptionSize { + if i >= len(fs.Records) { + fs.Records = append(fs.Records, OptionsDataRecord{}) + } + + record := &fs.Records[i] payloadLim := bytes.NewBuffer(payload.Next(listFieldsScopesSize)) - scopeValues := DecodeDataSetUsingFields(payloadLim, listFieldsScopes) - payloadLim = bytes.NewBuffer(payload.Next(listFieldsOptionSize)) - optionValues := DecodeDataSetUsingFields(payloadLim, listFieldsOption) - record := OptionsDataRecord{ - ScopesValues: scopeValues, - OptionsValues: optionValues, + if err := DecodeDataRecordFields(payloadLim, listFieldsScopes, &record.ScopesValues); err != nil { + return fmt.Errorf("decode options dataSet: %v", err) } - records = append(records, record) + payloadLim = bytes.NewBuffer(payload.Next(listFieldsOptionSize)) + if err := DecodeDataRecordFields(payloadLim, listFieldsOption, &record.OptionsValues); err != nil { + return fmt.Errorf("decode options dataSet: %v", err) + } + + i++ } - return records, nil + fs.Records = fs.Records[:i] + return nil } -func DecodeDataSet(payload *bytes.Buffer, listFields []Field) ([]DataRecord, error) { - records := make([]DataRecord, 0) +// DecodeDataSet decodes the Data Records (actual netflow/ipfix data). +func DecodeDataSet(payload *bytes.Buffer, listFields []Field, flowSet *DataFlowSet) error { + flowSet.Records = flowSet.Records[:cap(flowSet.Records)] listFieldsSize := GetTemplateSize(listFields) + + i := 0 for payload.Len() >= listFieldsSize { payloadLim := bytes.NewBuffer(payload.Next(listFieldsSize)) - values := DecodeDataSetUsingFields(payloadLim, listFields) - record := DataRecord{ - Values: values, + if i >= len(flowSet.Records) { + flowSet.Records = append(flowSet.Records, DataRecord{}) } - records = append(records, record) + datafields := &flowSet.Records[i].Values + if err := DecodeDataRecordFields(payloadLim, listFields, datafields); err != nil { + return err + } + + i++ } - return records, nil + + flowSet.Records = flowSet.Records[:i] + return nil } func (ts *BasicTemplateSystem) GetTemplates() map[uint16]map[uint32]map[uint16]interface{} { @@ -254,11 +313,11 @@ func (ts *BasicTemplateSystem) GetTemplates() map[uint16]map[uint32]map[uint16]i func (ts *BasicTemplateSystem) AddTemplate(version uint16, obsDomainId uint32, template interface{}) { ts.templateslock.Lock() _, exists := ts.templates[version] - if exists != true { + if !exists { ts.templates[version] = make(map[uint32]map[uint16]interface{}) } _, exists = ts.templates[version][obsDomainId] - if exists != true { + if !exists { ts.templates[version][obsDomainId] = make(map[uint16]interface{}) } var templateId uint16 @@ -308,179 +367,275 @@ func CreateTemplateSystem() *BasicTemplateSystem { return ts } -func DecodeMessage(payload *bytes.Buffer, templates NetFlowTemplateSystem) (interface{}, error) { - var size uint16 - packetNFv9 := NFv9Packet{} - packetIPFIX := IPFIXPacket{} - var returnItem interface{} - - var version uint16 - var obsDomainId uint32 - binary.Read(payload, binary.BigEndian, &version) - - if version == 9 { - utils.BinaryDecoder(payload, &packetNFv9.Count, &packetNFv9.SystemUptime, &packetNFv9.UnixSeconds, &packetNFv9.SequenceNumber, &packetNFv9.SourceId) - size = packetNFv9.Count - packetNFv9.Version = version - returnItem = *(&packetNFv9) - obsDomainId = packetNFv9.SourceId - } else if version == 10 { - utils.BinaryDecoder(payload, &packetIPFIX.Length, &packetIPFIX.ExportTime, &packetIPFIX.SequenceNumber, &packetIPFIX.ObservationDomainId) - size = packetIPFIX.Length - packetIPFIX.Version = version - returnItem = *(&packetIPFIX) - obsDomainId = packetIPFIX.ObservationDomainId +// FlowMessage processes the message(parses, and stores network information). +type FlowMessage struct { + // Version is version of netflow/ipfix records exported in this packet. + Version uint16 + PacketNFv9 NFv9Packet + PacketIPFIX IPFIXPacket + + fsheader FlowSetHeader + buf *bytes.Buffer +} + +// Decode decodes and collects the message in netflow/ipfix protocol format. +func (f *FlowMessage) Decode(payload *bytes.Buffer, templates NetFlowTemplateSystem) error { + if f.buf == nil { + f.buf = &bytes.Buffer{} + } + + if !utils.ReadUint16FromBuffer(payload, &f.Version) { + return io.ErrUnexpectedEOF + } + + if f.Version == netflow { + if !f.PacketNFv9.ReadFrom(payload) { + return fmt.Errorf("decode packet version: %v", io.ErrUnexpectedEOF) + } + f.PacketNFv9.Version = f.Version + + if err := f.DecodeNFv9Packet(payload, templates); err != nil { + return err + } + } else if f.Version == ipfix { + if !f.PacketIPFIX.ReadFrom(payload) { + return fmt.Errorf("decode packet header: %v", io.ErrUnexpectedEOF) + } + f.PacketIPFIX.Version = f.Version + + if err := f.DecodeIPFIXPacket(payload, templates); err != nil { + return err + } } else { - return nil, NewErrorVersion(version) + return NewErrorVersion(f.Version) } + return nil +} + +// DecodeNFv9Packet decodes and collects the message in netflow protocol format. +func (f *FlowMessage) DecodeNFv9Packet(payload *bytes.Buffer, templates NetFlowTemplateSystem) error { + var ( + nfDataFSidx, nfTemplateFSidx, nfOptsTemplateFSidx, nfOptsDataFSidx int + ) - for i := 0; ((i < int(size) && version == 9) || version == 10) && payload.Len() > 0; i++ { - fsheader := FlowSetHeader{} - utils.BinaryDecoder(payload, &fsheader) + for i := 0; i < int(f.PacketNFv9.Count) && payload.Len() > 0; i++ { + f.fsheader.ReadFrom(payload) - nextrelpos := int(fsheader.Length) - binary.Size(fsheader) + nextrelpos := int(f.fsheader.Length) - flowSetHeaderSize if nextrelpos < 0 { - return returnItem, NewErrorDecodingNetFlow("Error decoding packet: non-terminated stream.") + return NewErrorDecodingNetFlow("error decoding packet: non-terminated stream.") } - var flowSet interface{} + if f.fsheader.Id == nfv9TemplateFlowSetID { + f.buf.Reset() + f.buf.Write(payload.Next(nextrelpos)) - if fsheader.Id == 0 && version == 9 { - templateReader := bytes.NewBuffer(payload.Next(nextrelpos)) - records, err := DecodeTemplateSet(templateReader) - if err != nil { - return returnItem, err + f.PacketNFv9.TemplateFS = f.PacketNFv9.TemplateFS[:cap(f.PacketNFv9.TemplateFS)] + if nfTemplateFSidx >= len(f.PacketNFv9.TemplateFS) { + f.PacketNFv9.TemplateFS = append(f.PacketNFv9.TemplateFS, TemplateFlowSet{}) } - templatefs := TemplateFlowSet{ - FlowSetHeader: fsheader, - Records: records, + + ts := &f.PacketNFv9.TemplateFS[nfTemplateFSidx] + if err := DecodeTemplateSet(f.buf, ts); err != nil { + return fmt.Errorf("decode netflow packet: %v", err) } - flowSet = templatefs + ts.FlowSetHeader = f.fsheader if templates != nil { - for _, record := range records { - templates.AddTemplate(version, obsDomainId, record) + for _, record := range ts.Records { + templates.AddTemplate(f.Version, f.PacketNFv9.SourceId, record) } } - - } else if fsheader.Id == 1 && version == 9 { - templateReader := bytes.NewBuffer(payload.Next(nextrelpos)) - records, err := DecodeNFv9OptionsTemplateSet(templateReader) - if err != nil { - return returnItem, err + nfTemplateFSidx++ + } else if f.fsheader.Id == nfv9OptionsTemplateFlowSetID { + f.buf.Reset() + f.buf.Write(payload.Next(nextrelpos)) + + f.PacketNFv9.NFv9OptionsTemplateFS = f.PacketNFv9.NFv9OptionsTemplateFS[:cap(f.PacketNFv9.NFv9OptionsTemplateFS)] + if nfOptsTemplateFSidx >= len(f.PacketNFv9.NFv9OptionsTemplateFS) { + f.PacketNFv9.NFv9OptionsTemplateFS = append(f.PacketNFv9.NFv9OptionsTemplateFS, NFv9OptionsTemplateFlowSet{}) } - optsTemplatefs := NFv9OptionsTemplateFlowSet{ - FlowSetHeader: fsheader, - Records: records, + + ts := &f.PacketNFv9.NFv9OptionsTemplateFS[nfOptsTemplateFSidx] + + ts.FlowSetHeader = f.fsheader + if err := DecodeNFv9OptionsTemplateSet(f.buf, ts); err != nil { + return fmt.Errorf("decode ipfix packet: %v", err) } - flowSet = optsTemplatefs if templates != nil { - for _, record := range records { - templates.AddTemplate(version, obsDomainId, record) + for _, record := range ts.Records { + templates.AddTemplate(f.Version, f.PacketNFv9.SourceId, record) } } + nfOptsTemplateFSidx++ + } else if f.fsheader.Id >= dataTemplateFlowSetID { + f.buf.Reset() + f.buf.Write(payload.Next(nextrelpos)) - } else if fsheader.Id == 2 && version == 10 { - templateReader := bytes.NewBuffer(payload.Next(nextrelpos)) - records, err := DecodeTemplateSet(templateReader) + if templates == nil { + continue + } + + template, err := templates.GetTemplate(f.Version, f.PacketNFv9.SourceId, f.fsheader.Id) if err != nil { - return returnItem, err + return err + } + + switch templatec := template.(type) { + case TemplateRecord: + f.PacketNFv9.DataFS = f.PacketNFv9.DataFS[:cap(f.PacketNFv9.DataFS)] + + if nfDataFSidx >= len(f.PacketNFv9.DataFS) { + f.PacketNFv9.DataFS = append(f.PacketNFv9.DataFS, DataFlowSet{}) + } + + f.PacketNFv9.DataFS[nfDataFSidx].FlowSetHeader = f.fsheader + if err := DecodeDataSet(f.buf, templatec.Fields, &f.PacketNFv9.DataFS[nfDataFSidx]); err != nil { + return err + } + nfDataFSidx++ + case NFv9OptionsTemplateRecord: + f.PacketNFv9.OptionsDataFS = f.PacketNFv9.OptionsDataFS[:cap(f.PacketNFv9.OptionsDataFS)] + + if nfOptsDataFSidx >= len(f.PacketNFv9.OptionsDataFS) { + f.PacketNFv9.OptionsDataFS = append(f.PacketNFv9.OptionsDataFS, OptionsDataFlowSet{}) + } + + optFS := &f.PacketNFv9.OptionsDataFS[nfOptsDataFSidx] + + optFS.FlowSetHeader = f.fsheader + if err := DecodeOptionsDataSet(f.buf, optFS, templatec.Scopes, templatec.Options); err != nil { + return err + } + nfOptsDataFSidx++ + default: + return fmt.Errorf("unknown record type: %T", templatec) + } + } else { + return NewErrorFlowId(f.fsheader.Id) + } + } + + f.PacketNFv9.DataFS = f.PacketNFv9.DataFS[:nfDataFSidx] + f.PacketNFv9.TemplateFS = f.PacketNFv9.TemplateFS[:nfTemplateFSidx] + f.PacketNFv9.NFv9OptionsTemplateFS = f.PacketNFv9.NFv9OptionsTemplateFS[:nfOptsTemplateFSidx] + f.PacketNFv9.OptionsDataFS = f.PacketNFv9.OptionsDataFS[:nfOptsDataFSidx] + + return nil +} + +// DecodeIPFIXPacket decodes and collects the message in ipfix protocol format. +func (f *FlowMessage) DecodeIPFIXPacket(payload *bytes.Buffer, templates NetFlowTemplateSystem) error { + var ( + ipxDataFSidx, ipxTemplateFSidx, ipxOptsTemplateFSidx, ipxOptsDataFSidx int + ) + + for i := 0; i < int(f.PacketIPFIX.Length) && payload.Len() > 0; i++ { + f.fsheader.ReadFrom(payload) + + nextrelpos := int(f.fsheader.Length) - flowSetHeaderSize + if nextrelpos < 0 { + return NewErrorDecodingNetFlow("error decoding packet: non-terminated stream.") + } + + if f.fsheader.Id == ipfixTemplateFlowSetID { + f.buf.Reset() + f.buf.Write(payload.Next(nextrelpos)) + + f.PacketIPFIX.TemplateFS = f.PacketIPFIX.TemplateFS[:cap(f.PacketIPFIX.TemplateFS)] + if ipxTemplateFSidx >= len(f.PacketIPFIX.TemplateFS) { + f.PacketIPFIX.TemplateFS = append(f.PacketIPFIX.TemplateFS, TemplateFlowSet{}) } - templatefs := TemplateFlowSet{ - FlowSetHeader: fsheader, - Records: records, + + ts := &f.PacketIPFIX.TemplateFS[ipxTemplateFSidx] + if err := DecodeTemplateSet(f.buf, ts); err != nil { + return err } - flowSet = templatefs + ts.FlowSetHeader = f.fsheader if templates != nil { - for _, record := range records { - templates.AddTemplate(version, obsDomainId, record) + for _, record := range ts.Records { + templates.AddTemplate(f.Version, f.PacketIPFIX.ObservationDomainId, record) } } + ipxTemplateFSidx++ + } else if f.fsheader.Id == ipfixOptionsTemplateFlowSetID { + f.buf.Reset() + f.buf.Write(payload.Next(nextrelpos)) - } else if fsheader.Id == 3 && version == 10 { - templateReader := bytes.NewBuffer(payload.Next(nextrelpos)) - records, err := DecodeIPFIXOptionsTemplateSet(templateReader) - if err != nil { - return returnItem, err + f.PacketIPFIX.IPFIXOptionsTemplateFS = f.PacketIPFIX.IPFIXOptionsTemplateFS[:cap(f.PacketIPFIX.IPFIXOptionsTemplateFS)] + + if ipxOptsTemplateFSidx >= len(f.PacketIPFIX.IPFIXOptionsTemplateFS) { + f.PacketIPFIX.IPFIXOptionsTemplateFS = append(f.PacketIPFIX.IPFIXOptionsTemplateFS, IPFIXOptionsTemplateFlowSet{}) } - optsTemplatefs := IPFIXOptionsTemplateFlowSet{ - FlowSetHeader: fsheader, - Records: records, + + opt := &f.PacketIPFIX.IPFIXOptionsTemplateFS[ipxOptsTemplateFSidx] + + opt.FlowSetHeader = f.fsheader + if err := DecodeIPFIXOptionsTemplateSet(f.buf, opt); err != nil { + return err } - flowSet = optsTemplatefs if templates != nil { - for _, record := range records { - templates.AddTemplate(version, obsDomainId, record) + for _, record := range opt.Records { + templates.AddTemplate(f.Version, f.PacketIPFIX.ObservationDomainId, record) } } - - } else if fsheader.Id >= 256 { - dataReader := bytes.NewBuffer(payload.Next(nextrelpos)) + ipxOptsTemplateFSidx++ + } else if f.fsheader.Id >= dataTemplateFlowSetID { + f.buf.Reset() + f.buf.Write(payload.Next(nextrelpos)) if templates == nil { continue } - template, err := templates.GetTemplate(version, obsDomainId, fsheader.Id) - - if err == nil { - switch templatec := template.(type) { - case TemplateRecord: - records, err := DecodeDataSet(dataReader, templatec.Fields) - if err != nil { - return returnItem, err - } - datafs := DataFlowSet{ - FlowSetHeader: fsheader, - Records: records, - } - flowSet = datafs - case IPFIXOptionsTemplateRecord: - records, err := DecodeOptionsDataSet(dataReader, templatec.Scopes, templatec.Options) - if err != nil { - return returnItem, err - } - - datafs := OptionsDataFlowSet{ - FlowSetHeader: fsheader, - Records: records, - } - flowSet = datafs - case NFv9OptionsTemplateRecord: - records, err := DecodeOptionsDataSet(dataReader, templatec.Scopes, templatec.Options) - if err != nil { - return returnItem, err - } - - datafs := OptionsDataFlowSet{ - FlowSetHeader: fsheader, - Records: records, - } - flowSet = datafs + template, err := templates.GetTemplate(f.Version, f.PacketIPFIX.ObservationDomainId, f.fsheader.Id) + if err != nil { + return err + } + + switch templatec := template.(type) { + case TemplateRecord: + f.PacketIPFIX.DataFS = f.PacketIPFIX.DataFS[:cap(f.PacketIPFIX.DataFS)] + + if ipxDataFSidx >= len(f.PacketIPFIX.DataFS) { + f.PacketIPFIX.DataFS = append(f.PacketIPFIX.DataFS, DataFlowSet{}) + } + + f.PacketIPFIX.DataFS[ipxDataFSidx].FlowSetHeader = f.fsheader + if err := DecodeDataSet(f.buf, templatec.Fields, &f.PacketIPFIX.DataFS[ipxDataFSidx]); err != nil { + return err + } + ipxDataFSidx++ + case IPFIXOptionsTemplateRecord: + f.PacketIPFIX.OptionsDataFS = f.PacketIPFIX.OptionsDataFS[:cap(f.PacketIPFIX.OptionsDataFS)] + + if ipxOptsDataFSidx >= len(f.PacketIPFIX.OptionsDataFS) { + f.PacketIPFIX.OptionsDataFS = append(f.PacketIPFIX.OptionsDataFS, OptionsDataFlowSet{}) + } + + optFS := &f.PacketIPFIX.OptionsDataFS[ipxOptsDataFSidx] + + optFS.FlowSetHeader = f.fsheader + if err := DecodeOptionsDataSet(f.buf, optFS, templatec.Scopes, templatec.Options); err != nil { + return err } - } else { - return returnItem, err + ipxOptsDataFSidx++ + default: + return fmt.Errorf("unknown record type: %T", templatec) } } else { - return returnItem, NewErrorFlowId(fsheader.Id) - } - - if version == 9 && flowSet != nil { - packetNFv9.FlowSets = append(packetNFv9.FlowSets, flowSet) - } else if version == 10 && flowSet != nil { - packetIPFIX.FlowSets = append(packetIPFIX.FlowSets, flowSet) + return NewErrorFlowId(f.fsheader.Id) } } - if version == 9 { - return packetNFv9, nil - } else if version == 10 { - return packetIPFIX, nil - } else { - return returnItem, NewErrorVersion(version) - } -} + f.PacketIPFIX.DataFS = f.PacketIPFIX.DataFS[:ipxDataFSidx] + f.PacketIPFIX.TemplateFS = f.PacketIPFIX.TemplateFS[:ipxTemplateFSidx] + f.PacketIPFIX.IPFIXOptionsTemplateFS = f.PacketIPFIX.IPFIXOptionsTemplateFS[:ipxOptsTemplateFSidx] + f.PacketIPFIX.OptionsDataFS = f.PacketIPFIX.OptionsDataFS[:ipxOptsDataFSidx] + + return nil +} \ No newline at end of file From edb5e0934b6b0edbfaa92e2bcc6fa1b2f1f82dbd Mon Sep 17 00:00:00 2001 From: Batyrkhan Koshenov Date: Mon, 31 Oct 2022 13:13:18 +0600 Subject: [PATCH 02/10] decoders/netflow: add ReadFrom methods, change FlowSets field to avoid allocs; refactor String() method Signed-off-by: Batyrkhan Koshenov --- decoders/netflow/ipfix.go | 132 +++++++++++++++++++++++++++-------- decoders/netflow/nfv9.go | 137 +++++++++++++++++++++++++++++-------- decoders/netflow/packet.go | 78 ++++++++++++++++++++- 3 files changed, 287 insertions(+), 60 deletions(-) diff --git a/decoders/netflow/ipfix.go b/decoders/netflow/ipfix.go index 954b7d3..1a6d90d 100644 --- a/decoders/netflow/ipfix.go +++ b/decoders/netflow/ipfix.go @@ -1,8 +1,11 @@ package netflow import ( + "bytes" "fmt" "time" + + "github.com/cloudflare/goflow/v3/decoders/utils" ) const ( @@ -445,26 +448,98 @@ const ( IPFIX_FIELD_natThresholdEvent = 467 ) +// IPFIXPacket is a representation of ipfix(netflow v10) protocol packet. type IPFIXPacket struct { - Version uint16 - Length uint16 - ExportTime uint32 - SequenceNumber uint32 + // Version is version of ipfix records exported in this packet. + Version uint16 + + // Length is a total length of the IPFIX packet, measured in octets, + // including packet Header and Set(s). + Length uint16 + + // ExportTime is a time at which the IPFIX Message Header leaves the Exporter, + // expressed in seconds since the UNIX epoch of 1 January 1970 at + // 00:00 UTC, encoded as an unsigned 32-bit integer. + ExportTime uint32 + + // SequenceNumber is incremental sequence counter of all export packets sent by this export + // device; This value is cumulative, and it can be used to identify whether + // any export packets have been missed. + SequenceNumber uint32 + + // ObservationDomainId is a 32-bit identifier of the Observation Domain that + // is locally unique to the Exporting Process. ObservationDomainId uint32 - FlowSets []interface{} + + FlowSets } +// ReadFrom reads into receiver's fields Uint values from buffer and returns +// boolean flag telling if it was a success. +// +// Value is treated as big endian. +func (x *IPFIXPacket) ReadFrom(b *bytes.Buffer) bool { + if ok := utils.ReadUint16FromBuffer(b, &x.Length); !ok { + return false + } + if ok := utils.ReadUint32FromBuffer(b, &x.ExportTime); !ok { + return false + } + if ok := utils.ReadUint32FromBuffer(b, &x.SequenceNumber); !ok { + return false + } + if ok := utils.ReadUint32FromBuffer(b, &x.ObservationDomainId); !ok { + return false + } + return true +} + +// IPFIXOptionsTemplateFlowSet is a collection of Options Template Records. type IPFIXOptionsTemplateFlowSet struct { FlowSetHeader Records []IPFIXOptionsTemplateRecord } +// IPFIXOptionsTemplateRecord contains any combination of IANA-assigned and/or +// enterprise-specific Information Element identifiers. type IPFIXOptionsTemplateRecord struct { - TemplateId uint16 - FieldCount uint16 + // TemplateId is a unique number in the range 256 to 65535 used for matching + // the type of IPFIX data it will be exporting. + TemplateId uint16 + + // FieldCount is a number of all fields in this Options Template Record, + // including the Scope Fields. + FieldCount uint16 + + // ScopeFieldCount is a number of scope fields in this Options Template + // Record. The Scope Fields are normal Fields, except that they are + // interpreted as scope at the Collector. ScopeFieldCount uint16 - Options []Field - Scopes []Field + + // Options represents the type and length(in bytes) of the field that + // appears in the options record. + Options []Field + + // Scopes is one or more Information Elements, specified in the Options + // Template Record. + Scopes []Field +} + +// ReadFrom reads into receiver's fields Uint values from buffer and returns +// boolean flag telling if it was a success. +// +// Value is treated as big endian. +func (x *IPFIXOptionsTemplateRecord) ReadFrom(b *bytes.Buffer) bool { + if ok := utils.ReadUint16FromBuffer(b, &x.TemplateId); !ok { + return false + } + if ok := utils.ReadUint16FromBuffer(b, &x.FieldCount); !ok { + return false + } + if ok := utils.ReadUint16FromBuffer(b, &x.ScopeFieldCount); !ok { + return false + } + return true } func IPFIXTypeToString(typeId uint16) string { @@ -950,7 +1025,6 @@ func (flowSet IPFIXOptionsTemplateFlowSet) String(TypeToString func(uint16) stri } } - return str } @@ -964,26 +1038,26 @@ func (p IPFIXPacket) String() string { str += fmt.Sprintf(" ExportTime: %v\n", exportTime.String()) str += fmt.Sprintf(" SequenceNumber: %v\n", p.SequenceNumber) str += fmt.Sprintf(" ObservationDomainId: %v\n", p.ObservationDomainId) - str += fmt.Sprintf(" FlowSets (%v):\n", len(p.FlowSets)) - - for i, flowSet := range p.FlowSets { - switch flowSet := flowSet.(type) { - case TemplateFlowSet: - str += fmt.Sprintf(" - TemplateFlowSet %v:\n", i) - str += flowSet.String(IPFIXTypeToString) - case IPFIXOptionsTemplateFlowSet: - str += fmt.Sprintf(" - OptionsTemplateFlowSet %v:\n", i) - str += flowSet.String(IPFIXTypeToString) - case DataFlowSet: - str += fmt.Sprintf(" - DataFlowSet %v:\n", i) - str += flowSet.String(IPFIXTypeToString) - case OptionsDataFlowSet: - str += fmt.Sprintf(" - OptionsDataFlowSet %v:\n", i) - str += flowSet.String(IPFIXTypeToString, IPFIXTypeToString) - default: - str += fmt.Sprintf(" - (unknown type) %v: %v\n", i, flowSet) - } + str += fmt.Sprintf(" FlowSets (%v):\n", len(p.DataFS)+len(p.IPFIXOptionsTemplateFS)+len(p.OptionsDataFS)+len(p.TemplateFS)) + + for i, fs := range p.TemplateFS { + str += fmt.Sprintf(" - TemplateFlowSet %v:\n", i) + str += fs.String(IPFIXTypeToString) + } + + for i, fs := range p.IPFIXOptionsTemplateFS { + str += fmt.Sprintf(" - OptionsTemplateFlowSet %v:\n", i) + str += fs.String(IPFIXTypeToString) } + for i, fs := range p.DataFS { + str += fmt.Sprintf(" - DataFlowSet %v:\n", i) + str += fs.String(IPFIXTypeToString) + } + + for i, fs := range p.OptionsDataFS { + str += fmt.Sprintf(" - OptionsDataFlowSet %v:\n", i) + str += fs.String(IPFIXTypeToString, IPFIXTypeToString) + } return str } diff --git a/decoders/netflow/nfv9.go b/decoders/netflow/nfv9.go index d46f106..2dc0072 100644 --- a/decoders/netflow/nfv9.go +++ b/decoders/netflow/nfv9.go @@ -1,8 +1,11 @@ package netflow import ( + "bytes" "fmt" "time" + + "github.com/cloudflare/goflow/v3/decoders/utils" ) const ( @@ -101,27 +104,101 @@ const ( NFV9_FIELD_layer2packetSectionData = 104 ) +// NFv9Packet is a representation of netflow(v9) protocol packet type NFv9Packet struct { - Version uint16 - Count uint16 - SystemUptime uint32 - UnixSeconds uint32 + // Version is version of NetFlow records exported in this packet. + Version uint16 + + // Count is a number of FlowSet records (both template and data) contained + // within this packet. + Count uint16 + + // SystemUptime is a time in milliseconds since this device was first booted. + SystemUptime uint32 + + // UnixSeconds is seconds since 0000 Coordinated Universal Time (UTC) 1970. + UnixSeconds uint32 + + // SequenceNumber is incremental sequence counter of all export packets sent by this export + // device; This value is cumulative, and it can be used to identify whether + // any export packets have been missed. SequenceNumber uint32 - SourceId uint32 - FlowSets []interface{} + + // SourceId is a field is a 32-bit value that is used to guarantee + // uniqueness for all flows exported from a particular device. + SourceId uint32 + + FlowSets +} + +// ReadFrom reads into receiver's fields Uint values from buffer and returns +// boolean flag telling if it was a success. +// +// Value is treated as big endian. +func (x *NFv9Packet) ReadFrom(b *bytes.Buffer) bool { + if ok := utils.ReadUint16FromBuffer(b, &x.Count); !ok { + return false + } + if ok := utils.ReadUint32FromBuffer(b, &x.SystemUptime); !ok { + return false + } + if ok := utils.ReadUint32FromBuffer(b, &x.UnixSeconds); !ok { + return false + } + if ok := utils.ReadUint32FromBuffer(b, &x.SequenceNumber); !ok { + return false + } + if ok := utils.ReadUint32FromBuffer(b, &x.SourceId); !ok { + return false + } + return true } +// NFv9OptionsTemplateFlowSet is a collection of Options Template Records. type NFv9OptionsTemplateFlowSet struct { FlowSetHeader Records []NFv9OptionsTemplateRecord } +// NFv9OptionsTemplateRecord is a special type of template record used to +// communicate the format of data related to the NetFlow process. type NFv9OptionsTemplateRecord struct { - TemplateId uint16 - ScopeLength uint16 + // TemplateId is a unique number in the range 256 to 65535 used for matching + // the type of NetFlow data it will be exporting. + TemplateId uint16 + + // ScopeLength is the length in bytes of any scope fields contained in + // this options template. + ScopeLength uint16 + + // OptionLength is the length (in bytes) of any Options field definitions + // contained in this options template. OptionLength uint16 - Scopes []Field - Options []Field + + // Scopes is one or more Information Elements, specified in the Options + // Template Record. + Scopes []Field + + // Options represents the type and length(in bytes) of the field that + // appears in the options record. + Options []Field +} + +// ReadFrom reads into receiver's fields Uint values from buffer and returns +// boolean flag telling if it was a success. +// +// Value is treated as big endian. +func (x *NFv9OptionsTemplateRecord) ReadFrom(b *bytes.Buffer) bool { + if ok := utils.ReadUint16FromBuffer(b, &x.TemplateId); !ok { + return false + } + if ok := utils.ReadUint16FromBuffer(b, &x.ScopeLength); !ok { + return false + } + if ok := utils.ReadUint16FromBuffer(b, &x.OptionLength); !ok { + return false + } + return true } func NFv9TypeToString(typeId uint16) string { @@ -293,25 +370,27 @@ func (p NFv9Packet) String() string { str += fmt.Sprintf(" UnixSeconds: %v\n", unixSeconds.String()) str += fmt.Sprintf(" SequenceNumber: %v\n", p.SequenceNumber) str += fmt.Sprintf(" SourceId: %v\n", p.SourceId) - str += fmt.Sprintf(" FlowSets (%v):\n", len(p.FlowSets)) - - for i, flowSet := range p.FlowSets { - switch flowSet := flowSet.(type) { - case TemplateFlowSet: - str += fmt.Sprintf(" - TemplateFlowSet %v:\n", i) - str += flowSet.String(NFv9TypeToString) - case NFv9OptionsTemplateFlowSet: - str += fmt.Sprintf(" - OptionsTemplateFlowSet %v:\n", i) - str += flowSet.String(NFv9TypeToString) - case DataFlowSet: - str += fmt.Sprintf(" - DataFlowSet %v:\n", i) - str += flowSet.String(NFv9TypeToString) - case OptionsDataFlowSet: - str += fmt.Sprintf(" - OptionsDataFlowSet %v:\n", i) - str += flowSet.String(NFv9TypeToString, NFv9ScopeToString) - default: - str += fmt.Sprintf(" - (unknown type) %v: %v\n", i, flowSet) - } + str += fmt.Sprintf(" FlowSets (%v):\n", len(p.DataFS)+len(p.NFv9OptionsTemplateFS)+len(p.OptionsDataFS)+len(p.TemplateFS)) + + for i, fs := range p.TemplateFS { + str += fmt.Sprintf(" - TemplateFlowSet %v:\n", i) + str += fs.String(NFv9TypeToString) + } + + for i, fs := range p.NFv9OptionsTemplateFS { + str += fmt.Sprintf(" - OptionsTemplateFlowSet %v:\n", i) + str += fs.String(NFv9TypeToString) + } + + for i, fs := range p.OptionsDataFS { + str += fmt.Sprintf(" - OptionsDataFlowSet %v:\n", i) + str += fs.String(NFv9TypeToString, NFv9ScopeToString) + } + + for i, fs := range p.DataFS { + str += fmt.Sprintf(" - DataFlowSet %v:\n", i) + str += fs.String(NFv9TypeToString) } + return str } diff --git a/decoders/netflow/packet.go b/decoders/netflow/packet.go index 5384eb5..9c9945f 100644 --- a/decoders/netflow/packet.go +++ b/decoders/netflow/packet.go @@ -1,9 +1,34 @@ package netflow import ( + "bytes" "fmt" + + "github.com/cloudflare/goflow/v3/decoders/utils" ) +const ( + netflow = 9 + ipfix = 10 + nfv9TemplateFlowSetID = 0 + nfv9OptionsTemplateFlowSetID = 1 + ipfixTemplateFlowSetID = 2 + ipfixOptionsTemplateFlowSetID = 3 + dataTemplateFlowSetID = 256 + + flowSetHeaderSize = 4 +) + +// FlowSets is a wrapper that contains different types of FlowSets that must be +// parsed and interpreted by the collector device. +type FlowSets struct { + TemplateFS []TemplateFlowSet + DataFS []DataFlowSet + OptionsDataFS []OptionsDataFlowSet + NFv9OptionsTemplateFS []NFv9OptionsTemplateFlowSet + IPFIXOptionsTemplateFS []IPFIXOptionsTemplateFlowSet +} + // FlowSetHeader contains fields shared by all Flow Sets (DataFlowSet, // TemplateFlowSet, OptionsTemplateFlowSet). type FlowSetHeader struct { @@ -31,12 +56,15 @@ type TemplateFlowSet struct { type DataFlowSet struct { FlowSetHeader + // List of Data Records Records []DataRecord } +// OptionsDataFlowSet is a collection of Options Data Records. type OptionsDataFlowSet struct { FlowSetHeader + // List of Options Data Records Records []OptionsDataRecord } @@ -59,7 +87,11 @@ type TemplateRecord struct { Fields []Field } +// DataRecord is is a collection of field values. +// The type and length of the fields have been previously defined in the +// template record referenced by the FlowSet ID/template ID. type DataRecord struct { + // List of fields in this Data Record. Values []DataField } @@ -84,13 +116,55 @@ type Field struct { Length uint16 } +// DataField contains the type and record value itself. type DataField struct { // A numeric value that represents the type of field. Type uint16 // The value (in bytes) of the field. - Value interface{} - //Value []byte + Value []byte +} + +// ReadFrom reads into receiver's fields Uint values from buffer and returns +// boolean flag telling if it was a success. +// +// Value is treated as big endian. +func (f *FlowSetHeader) ReadFrom(b *bytes.Buffer) bool { + if ok := utils.ReadUint16FromBuffer(b, &f.Id); !ok { + return false + } + if ok := utils.ReadUint16FromBuffer(b, &f.Length); !ok { + return false + } + return true +} + +// ReadFrom reads into receiver's fields Uint values from buffer and returns +// boolean flag telling if it was a success. +// +// Value is treated as big endian. +func (t *TemplateRecord) ReadFrom(b *bytes.Buffer) bool { + if ok := utils.ReadUint16FromBuffer(b, &t.TemplateId); !ok { + return false + } + if ok := utils.ReadUint16FromBuffer(b, &t.FieldCount); !ok { + return false + } + return true +} + +// ReadFrom reads into receiver's fields Uint values from buffer and returns +// boolean flag telling if it was a success. +// +// Value is treated as big endian. +func (f *Field) ReadFrom(b *bytes.Buffer) bool { + if ok := utils.ReadUint16FromBuffer(b, &f.Type); !ok { + return false + } + if ok := utils.ReadUint16FromBuffer(b, &f.Length); !ok { + return false + } + return true } func (flowSet OptionsDataFlowSet) String(TypeToString func(uint16) string, ScopeToString func(uint16) string) string { From 0ac99911da5fa783a65c6ad1a08c6fa282e2627d Mon Sep 17 00:00:00 2001 From: Batyrkhan Koshenov Date: Mon, 31 Oct 2022 13:14:52 +0600 Subject: [PATCH 03/10] decoders/netflow: test packet decoding funcs Signed-off-by: Batyrkhan Koshenov --- .../netflow/netflow_ipfix_test_packets.go | 124 ++++++ decoders/netflow/netflow_test.go | 288 ++++++++++++++ decoders/netflow/netflow_test_packets.go | 358 ++++++++++++++++++ 3 files changed, 770 insertions(+) create mode 100644 decoders/netflow/netflow_ipfix_test_packets.go create mode 100644 decoders/netflow/netflow_test.go create mode 100644 decoders/netflow/netflow_test_packets.go diff --git a/decoders/netflow/netflow_ipfix_test_packets.go b/decoders/netflow/netflow_ipfix_test_packets.go new file mode 100644 index 0000000..bd17783 --- /dev/null +++ b/decoders/netflow/netflow_ipfix_test_packets.go @@ -0,0 +1,124 @@ +package netflow + +// +// Code generated automatically. DO NOT EDIT. +// Generated on 2020-10-21 12:58:41.259310782 +0600 +06 m=+0.002264378 +// +import "time" +import "github.com/google/gopacket" + +var ipfixTestPackets = []struct { + Data []byte + gopacket.CaptureInfo +}{ + // + // Frame 1 + { + []byte{ + // + // Ethernet, off: 0, len: 14 + 0x00, 0x50, 0x56, 0x84, 0x63, 0xC9, 0x84, 0x78, + 0xAC, 0x00, 0x0B, 0x0A, 0x08, 0x00, + // + // IPv4, off: 14, len: 20 + 0x45, 0x00, 0x00, 0x90, 0x00, 0x00, 0x00, 0x00, + 0xFD, 0x11, 0x47, 0x04, 0xCC, 0x2A, 0x6E, 0x1E, + 0xCC, 0x2A, 0x6F, 0xE5, + // + // UDP, off: 34, len: 8 + 0x81, 0x44, 0x27, 0x0B, 0x00, 0x7C, 0x00, 0x00, + // + // Payload, off: 42, len: 116 + 0x00, 0x0A, 0x00, 0x74, 0x58, 0x3D, 0xE0, 0x57, + 0x00, 0x00, 0x0E, 0xCF, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x02, 0x00, 0x64, 0x01, 0x33, 0x00, 0x17, + 0x00, 0x08, 0x00, 0x04, 0x00, 0x0C, 0x00, 0x04, + 0x00, 0x05, 0x00, 0x01, 0x00, 0x04, 0x00, 0x01, + 0x00, 0x07, 0x00, 0x02, 0x00, 0x0B, 0x00, 0x02, + 0x00, 0x20, 0x00, 0x02, 0x00, 0x0A, 0x00, 0x04, + 0x00, 0x10, 0x00, 0x04, 0x00, 0x11, 0x00, 0x04, + 0x00, 0x12, 0x00, 0x04, 0x00, 0x0E, 0x00, 0x04, + 0x00, 0x01, 0x00, 0x04, 0x00, 0x02, 0x00, 0x04, + 0x00, 0x16, 0x00, 0x04, 0x00, 0x15, 0x00, 0x04, + 0x00, 0x0F, 0x00, 0x04, 0x00, 0x09, 0x00, 0x01, + 0x00, 0x0D, 0x00, 0x01, 0x00, 0x06, 0x00, 0x01, + 0x00, 0x3C, 0x00, 0x01, 0x00, 0x98, 0x00, 0x08, + 0x00, 0x99, 0x00, 0x08, + }, + gopacket.CaptureInfo{ + // 2016-11-30T02:08:55.403349+06:00 + Timestamp: time.Unix(1480450135, 403349000), + CaptureLength: 158, + Length: 158, + }, + }, + // + // Frame 2 + { + []byte{ + // + // Ethernet, off: 0, len: 14 + 0x00, 0x50, 0x56, 0x84, 0x63, 0xC9, 0x84, 0x78, + 0xAC, 0x00, 0x0B, 0x0A, 0x08, 0x00, + // + // IPv4, off: 14, len: 20 + 0x45, 0x00, 0x00, 0x44, 0x00, 0x00, 0x00, 0x00, + 0xFD, 0x11, 0x47, 0x50, 0xCC, 0x2A, 0x6E, 0x1E, + 0xCC, 0x2A, 0x6F, 0xE5, + // + // UDP, off: 34, len: 8 + 0x81, 0x44, 0x27, 0x0B, 0x00, 0x30, 0x00, 0x00, + // + // Payload, off: 42, len: 40 + 0x00, 0x0A, 0x00, 0x28, 0x58, 0x3D, 0xE0, 0x57, + 0x00, 0x00, 0x0E, 0xCF, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x03, 0x00, 0x18, 0x01, 0x34, 0x00, 0x03, + 0x00, 0x01, 0x00, 0x05, 0x00, 0x02, 0x00, 0x24, + 0x00, 0x02, 0x00, 0x25, 0x00, 0x02, 0x00, 0x00, + }, + gopacket.CaptureInfo{ + // 2016-11-30T02:08:55.403463+06:00 + Timestamp: time.Unix(1480450135, 403463000), + CaptureLength: 82, + Length: 82, + }, + }, + // + // Frame 3 + { + []byte{ + // + // Ethernet, off: 0, len: 14 + 0x00, 0x50, 0x56, 0x84, 0x63, 0xC9, 0x84, 0x78, + 0xAC, 0x00, 0x0B, 0x0A, 0x08, 0x00, + // + // IPv4, off: 14, len: 20 + 0x45, 0x00, 0x00, 0x7C, 0x00, 0x00, 0x00, 0x00, + 0xFD, 0x11, 0x47, 0x18, 0xCC, 0x2A, 0x6E, 0x1E, + 0xCC, 0x2A, 0x6F, 0xE5, + // + // UDP, off: 34, len: 8 + 0x81, 0x44, 0x27, 0x0B, 0x00, 0x68, 0x00, 0x00, + // + // Payload, off: 42, len: 96 + 0x00, 0x0A, 0x00, 0x60, 0x58, 0x3D, 0xE0, 0x59, + 0x00, 0x00, 0x0E, 0xE4, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x33, 0x00, 0x50, 0x46, 0x01, 0x73, 0x01, + 0x32, 0x00, 0x47, 0x01, 0x00, 0x3D, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x3B, + 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, + 0xCC, 0x2A, 0x6E, 0x65, 0x00, 0x00, 0x03, 0x56, + 0x00, 0x00, 0x05, 0x20, 0x00, 0x00, 0x00, 0x09, + 0xB3, 0xF9, 0x06, 0xEE, 0xB3, 0xFB, 0xAF, 0x3C, + 0xCC, 0x2A, 0x6E, 0xBD, 0x18, 0x18, 0x00, 0x04, + 0x00, 0x00, 0x01, 0x58, 0xB1, 0xB1, 0x38, 0xFF, + 0x00, 0x00, 0x01, 0x58, 0xB1, 0xB3, 0xE1, 0x4D, + }, + gopacket.CaptureInfo{ + // 2016-11-30T02:08:57.588883+06:00 + Timestamp: time.Unix(1480450137, 588883000), + CaptureLength: 138, + Length: 138, + }, + }, +} diff --git a/decoders/netflow/netflow_test.go b/decoders/netflow/netflow_test.go new file mode 100644 index 0000000..6278336 --- /dev/null +++ b/decoders/netflow/netflow_test.go @@ -0,0 +1,288 @@ +package netflow + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFlowMessage_Decode(t *testing.T) { + f := &FlowMessage{} + templates := CreateTemplateSystem() + + // netflow packet + pkt1 := bytes.NewBuffer(netflowTestPackets[1].Data[42:]) + err := f.Decode(pkt1, templates) + assert.NoError(t, err) + + assert.Equal(t, uint16(9), f.PacketNFv9.Version) + assert.Equal(t, uint16(26), f.PacketNFv9.Count) + assert.Equal(t, uint32(2935025008), f.PacketNFv9.SystemUptime) + assert.Equal(t, uint32(1565290402), f.PacketNFv9.UnixSeconds) + assert.Equal(t, uint32(3076483882), f.PacketNFv9.SequenceNumber) + assert.Equal(t, uint32(200), f.PacketNFv9.SourceId) + + assert.Equal(t, 1, len(f.PacketNFv9.TemplateFS)) + assert.Equal(t, 1, len(f.PacketNFv9.DataFS)) + + nfTempFlowset := f.PacketNFv9.TemplateFS[0] + nfDataFlowSet := f.PacketNFv9.DataFS[0] + assert.Equal(t, 4, len(nfTempFlowset.Records)) + assert.Equal(t, 22, len(nfDataFlowSet.Records)) + + // ipfix data template packet + pkt2 := bytes.NewBuffer(ipfixTestPackets[0].Data[42:]) + err = f.Decode(pkt2, templates) + assert.NoError(t, err) + + assert.Equal(t, 1, len(f.PacketIPFIX.TemplateFS)) + ipxTempFlowSet := f.PacketIPFIX.TemplateFS[0] + assert.Equal(t, 1, len(ipxTempFlowSet.Records)) + + // ipfix data records packet + pkt3 := bytes.NewBuffer(ipfixTestPackets[2].Data[42:]) + err = f.Decode(pkt3, templates) + assert.NoError(t, err) + + assert.Equal(t, uint16(10), f.PacketIPFIX.Version) + assert.Equal(t, uint16(96), f.PacketIPFIX.Length) + assert.Equal(t, uint32(1480450137), f.PacketIPFIX.ExportTime) + assert.Equal(t, uint32(3812), f.PacketIPFIX.SequenceNumber) + assert.Equal(t, uint32(0), f.PacketIPFIX.ObservationDomainId) + assert.Equal(t, 1, len(f.PacketIPFIX.DataFS)) + + // unknown netflow/ipfix version + pkt4 := make([]byte, len(ipfixTestPackets[2].Data[42:])) + copy(pkt4, ipfixTestPackets[2].Data[42:]) + pkt4[1] = 5 + + err = f.Decode(bytes.NewBuffer(pkt4), templates) + assert.Error(t, err) + + // uncomplete ipfix packet + pkt5 := bytes.NewBuffer(ipfixTestPackets[2].Data[42:50]) + err = f.Decode(pkt5, templates) + assert.Error(t, err) +} + +func TestFlowMessage_DecodeIPFIXPacket(t *testing.T) { + f := &FlowMessage{ + PacketIPFIX: IPFIXPacket{Length: 116, ObservationDomainId: 0}, + buf: &bytes.Buffer{}, + } + templates := CreateTemplateSystem() + + // ipfix data template packet + pkt1 := bytes.NewBuffer(ipfixTestPackets[0].Data[58:]) + err := f.DecodeIPFIXPacket(pkt1, templates) + assert.NoError(t, err) + + tempFlowSet := f.PacketIPFIX.TemplateFS[0] + + assert.Equal(t, uint16(2), tempFlowSet.Id) + assert.Equal(t, uint16(100), tempFlowSet.Length) + assert.Equal(t, 23, len(tempFlowSet.Records[0].Fields)) + + // ipfix data records packet + f.PacketIPFIX.Length = 96 + + pkt2 := bytes.NewBuffer(ipfixTestPackets[2].Data[58:]) + err = f.DecodeIPFIXPacket(pkt2, templates) + assert.NoError(t, err) + + dataFS := f.PacketIPFIX.DataFS[0] + assert.Equal(t, uint16(307), dataFS.Id) + assert.Equal(t, uint16(80), dataFS.Length) + assert.Equal(t, 1, len(dataFS.Records)) + + // packet without template and empty NetflowTemplateSystem + pkt3 := bytes.NewBuffer(ipfixTestPackets[2].Data[58:]) + err = f.DecodeIPFIXPacket(pkt3, CreateTemplateSystem()) + assert.Error(t, err) + + // packet with wrong flowSet header id + pkt4 := make([]byte, len(ipfixTestPackets[2].Data[58:])) + copy(pkt4, ipfixTestPackets[2].Data[58:]) + pkt4[1] = 30 + err = f.DecodeIPFIXPacket(bytes.NewBuffer(pkt4), templates) + assert.Error(t, err) + + // packet with wrong flowSet length + pkt5 := make([]byte, len(ipfixTestPackets[2].Data[58:])) + copy(pkt5, ipfixTestPackets[2].Data[58:]) + pkt5[3] = 1 + err = f.DecodeIPFIXPacket(bytes.NewBuffer(pkt5), templates) + assert.Error(t, err) +} + +func TestFlowMessage_DecodeNFv9Packet(t *testing.T) { + templates := CreateTemplateSystem() + f := &FlowMessage{ + PacketNFv9: NFv9Packet{Count: 26, SourceId: 200}, + buf: &bytes.Buffer{}, + } + + // packet with template and data flowSets + pkt1 := bytes.NewBuffer(netflowTestPackets[1].Data[62:]) + err := f.DecodeNFv9Packet(pkt1, templates) + assert.NoError(t, err) + + // packet without template and empty NetFlowTemplateSystem + pkt2 := bytes.NewBuffer(netflowTestPackets[1].Data[206:]) + err = f.DecodeNFv9Packet(pkt2, CreateTemplateSystem()) + assert.Error(t, err) + + // incomplete packet + pkt3 := bytes.NewBuffer(netflowTestPackets[1].Data[62:100]) + err = f.DecodeNFv9Packet(pkt3, templates) + assert.Error(t, err) + + // packet with wrong flowSet header id + pkt4 := make([]byte, len(netflowTestPackets[1].Data[62:])) + copy(pkt4, netflowTestPackets[1].Data[62:]) + pkt4[1] = 30 + err = f.DecodeNFv9Packet(bytes.NewBuffer(pkt4), templates) + assert.Error(t, err) + + // packet with wrong flowSet length + pkt5 := make([]byte, len(netflowTestPackets[1].Data[62:])) + copy(pkt5, netflowTestPackets[1].Data[62:]) + pkt5[3] = 1 + err = f.DecodeNFv9Packet(bytes.NewBuffer(pkt5), templates) + assert.Error(t, err) +} + +func TestDecodeTemplateSet(t *testing.T) { + fs := TemplateFlowSet{} + + // packet without template flowset + pkt0 := bytes.NewBuffer(netflowTestPackets[0].Data[42:]) + err := DecodeTemplateSet(pkt0, &fs) + assert.Error(t, err) + + // template flowSet packet + pkt1 := bytes.NewBuffer(netflowTestPackets[1].Data[66:206]) + err = DecodeTemplateSet(pkt1, &fs) + assert.NoError(t, err) + assert.Equal(t, 4, len(fs.Records)) + + // first template record + record := fs.Records[0] + assert.Equal(t, uint16(259), record.TemplateId) + assert.Equal(t, uint16(9), record.FieldCount) + assert.Equal(t, 9, len(record.Fields)) + + // last template record + record = fs.Records[3] + assert.Equal(t, uint16(256), record.TemplateId) + assert.Equal(t, uint16(12), record.FieldCount) + assert.Equal(t, 12, len(record.Fields)) + + // zero field count + pkt2 := make([]byte, len(netflowTestPackets[1].Data[66:206])) + copy(pkt2, netflowTestPackets[1].Data[66:206]) + pkt2[3] = 0 + + err = DecodeTemplateSet(bytes.NewBuffer(pkt2), &fs) + assert.Error(t, err) + assert.Equal(t, &ErrorDecodingNetFlow{msg: "error decoding TemplateSet: zero count."}, err) +} + +func TestDecodeIPFIXOptionsTemplateSet(t *testing.T) { + ts := IPFIXOptionsTemplateFlowSet{} + + // ipfix options template packet + pkt1 := bytes.NewBuffer(ipfixTestPackets[1].Data[62:]) + + err := DecodeIPFIXOptionsTemplateSet(pkt1, &ts) + assert.NoError(t, err) + assert.Equal(t, 1, len(ts.Records)) + + record := ts.Records[0] + assert.Equal(t, uint16(308), record.TemplateId) + assert.Equal(t, uint16(3), record.FieldCount) + assert.Equal(t, uint16(1), record.ScopeFieldCount) + + // negative length + pkt2 := make([]byte, len(ipfixTestPackets[1].Data[62:])) + copy(pkt2, ipfixTestPackets[1].Data[62:]) + pkt2[3] = 0 + + err = DecodeIPFIXOptionsTemplateSet(bytes.NewBuffer(pkt2), &ts) + assert.Error(t, err) + + // incomplete packet + pkt3 := bytes.NewBuffer(ipfixTestPackets[1].Data[62:70]) + err = DecodeIPFIXOptionsTemplateSet(pkt3, &ts) + assert.Error(t, err) +} + +func TestDecodeDataSet(t *testing.T) { + fs := DataFlowSet{} + + // type and length of a single value in a netflow Data Record + nfListFields := []Field{{8, 4}, {225, 4}, {12, 4}, {226, 4}, {7, 2}, {227, 2}, {11, 2}, {228, 2}, {234, 4}, {4, 1}, {230, 1}, {323, 8}} + + // type and length of a single value in ipfix Flow Data Record + ipxListFields := []Field{{8, 4}, {12, 4}, {5, 1}, {4, 1}, {7, 2}, {11, 2}, {32, 2}, {10, 4}, {16, 4}, {17, 4}, {18, 4}, {14, 4}, {1, 4}, {2, 4}, {22, 4}, {21, 4}, {15, 4}, {9, 1}, {13, 1}, {6, 1}, {60, 1}, {152, 8}, {153, 8}} + + // netflow data flowset + nfData := bytes.NewBuffer(netflowTestPackets[1].Data[210:]) + err := DecodeDataSet(nfData, nfListFields, &fs) + assert.NoError(t, err) + assert.Equal(t, 22, len(fs.Records)) + + // ipfix data flowset + ipxData := bytes.NewBuffer(ipfixTestPackets[2].Data[62:]) + err = DecodeDataSet(ipxData, ipxListFields, &fs) + assert.NoError(t, err) + assert.Equal(t, 1, len(fs.Records)) + + flow := fs.Records[0] + assert.Equal(t, 23, len(flow.Values)) +} + +func TestDecodeDataSetUsingFields(t *testing.T) { + // type and length of a single value in a netflow Data Record + nfListFields := []Field{{8, 4}, {225, 4}, {12, 4}, {226, 4}, {7, 2}, {227, 2}, {11, 2}, {228, 2}, {234, 4}, {4, 1}, {230, 1}, {323, 8}} + + // type and length of a single value in ipfix Flow Data Record + ipxListFields := []Field{{8, 4}, {12, 4}, {5, 1}, {4, 1}, {7, 2}, {11, 2}, {32, 2}, {10, 4}, {16, 4}, {17, 4}, {18, 4}, {14, 4}, {1, 4}, {2, 4}, {22, 4}, {21, 4}, {15, 4}, {9, 1}, {13, 1}, {6, 1}, {60, 1}, {152, 8}, {153, 8}} + + dr := DataRecord{} + + // netflow data flow record + data1 := bytes.NewBuffer(netflowTestPackets[1].Data[210:248]) + err := DecodeDataRecordFields(data1, nfListFields, &dr.Values) + assert.NoError(t, err) + assert.Equal(t, 12, len(dr.Values)) + + flow := dr.Values[0] // first record + assert.Equal(t, uint16(8), flow.Type) + assert.Equal(t, []byte{0x0a, 0xe5, 0x40, 0xdb}, flow.Value) + + flow = dr.Values[11] // last record + assert.Equal(t, uint16(323), flow.Type) + assert.Equal(t, []byte{0x00, 0x00, 0x01, 0x6c, 0x72, 0x94, 0x12, 0x69}, flow.Value) + + // ipfix data flow record + data2 := bytes.NewBuffer(ipfixTestPackets[2].Data[62:]) + err = DecodeDataRecordFields(data2, ipxListFields, &dr.Values) + assert.NoError(t, err) + assert.Equal(t, 23, len(dr.Values)) + + flow = dr.Values[0] // first record + + assert.Equal(t, uint16(8), flow.Type) + assert.Equal(t, []byte{0x46, 0x01, 0x73, 0x01}, flow.Value) + + flow = dr.Values[22] // last record + assert.Equal(t, uint16(153), flow.Type) + assert.Equal(t, []byte{0x00, 0x00, 0x01, 0x58, 0xb1, 0xb3, 0xe1, 0x4d}, flow.Value) + + // incomplete data for template + data3 := bytes.NewBuffer(netflowTestPackets[1].Data[210:247]) + err = DecodeDataRecordFields(data3, nfListFields, &dr.Values) + assert.Error(t, err) +} diff --git a/decoders/netflow/netflow_test_packets.go b/decoders/netflow/netflow_test_packets.go new file mode 100644 index 0000000..ce2c3ae --- /dev/null +++ b/decoders/netflow/netflow_test_packets.go @@ -0,0 +1,358 @@ +package netflow + +// +// Code generated automatically. DO NOT EDIT. +// Generated on 2020-05-27 14:26:01.493859524 +0600 +06 m=+0.007254396 +// +import "time" +import "github.com/google/gopacket" + +var netflowTestPackets = []struct { + Data []byte + gopacket.CaptureInfo +}{ + // + // Frame 1 + { + []byte{ + // + // Ethernet, off: 0, len: 14 + 0xF8, 0x75, 0x88, 0x1A, 0x3C, 0x78, 0x7C, 0xAD, + 0x74, 0xE6, 0x78, 0x19, 0x08, 0x00, + // + // IPv4, off: 14, len: 20 + 0x45, 0x00, 0x00, 0x5C, 0x5D, 0x0C, 0x00, 0x00, + 0xFF, 0x11, 0x31, 0x74, 0xAC, 0x17, 0x6A, 0xFD, + 0xAC, 0x17, 0x69, 0xE4, + // + // UDP, off: 34, len: 8 + 0xF7, 0xEF, 0x27, 0x11, 0x00, 0x48, 0x46, 0x64, + // + // Payload, off: 42, len: 64 + 0x00, 0x09, 0x00, 0x01, 0xAE, 0xF0, 0xED, 0x6F, + 0x5D, 0x4C, 0x6F, 0xA2, 0xB7, 0x5F, 0x6B, 0x29, + 0x00, 0x00, 0x00, 0xC8, 0x01, 0x00, 0x00, 0x2C, + 0x0A, 0xF2, 0xFB, 0xD2, 0x59, 0x25, 0x1A, 0xFB, + 0xAD, 0xC2, 0xDE, 0x5F, 0xAD, 0xC2, 0xDE, 0x5F, + 0xC2, 0x76, 0xF9, 0x8E, 0x01, 0xBB, 0x01, 0xBB, + 0x00, 0x00, 0x00, 0x00, 0x06, 0x02, 0x00, 0x00, + 0x01, 0x6C, 0x72, 0x94, 0x12, 0x69, 0x00, 0x00, + }, + gopacket.CaptureInfo{ + // 2019-08-09T00:53:22.407972+06:00 + Timestamp: time.Unix(1565290402, 407972000), + CaptureLength: 106, + Length: 106, + }, + }, + // + // Frame 2 + { + []byte{ + // + // Ethernet, off: 0, len: 14 + 0xF8, 0x75, 0x88, 0x1A, 0x3C, 0x78, 0x7C, 0xAD, + 0x74, 0xE6, 0x78, 0x19, 0x08, 0x00, + // + // IPv4, off: 14, len: 20 + 0x45, 0x00, 0x04, 0x08, 0x5D, 0x0D, 0x00, 0x00, + 0xFF, 0x11, 0x2D, 0xC7, 0xAC, 0x17, 0x6A, 0xFD, + 0xAC, 0x17, 0x69, 0xE4, + // + // UDP, off: 34, len: 8 + 0xF7, 0xEF, 0x27, 0x11, 0x03, 0xF4, 0xA2, 0x65, + // + // Payload, off: 42, len: 1004 + 0x00, 0x09, 0x00, 0x1A, 0xAE, 0xF0, 0xED, 0x70, + 0x5D, 0x4C, 0x6F, 0xA2, 0xB7, 0x5F, 0x6B, 0x2A, + 0x00, 0x00, 0x00, 0xC8, 0x00, 0x00, 0x00, 0x90, + 0x01, 0x03, 0x00, 0x09, 0x00, 0x08, 0x00, 0x04, + 0x00, 0xE1, 0x00, 0x04, 0x00, 0xEA, 0x00, 0x04, + 0x00, 0x04, 0x00, 0x01, 0x00, 0xE6, 0x00, 0x01, + 0x01, 0x43, 0x00, 0x08, 0x01, 0x69, 0x00, 0x02, + 0x01, 0x6B, 0x00, 0x02, 0x01, 0x6C, 0x00, 0x02, + 0x01, 0x02, 0x00, 0x02, 0x01, 0x1B, 0x00, 0x04, + 0x00, 0xE6, 0x00, 0x01, 0x01, 0x01, 0x00, 0x08, + 0x00, 0x08, 0x00, 0x04, 0x00, 0xE1, 0x00, 0x04, + 0x00, 0x07, 0x00, 0x02, 0x00, 0xE3, 0x00, 0x02, + 0x00, 0xEA, 0x00, 0x04, 0x00, 0x04, 0x00, 0x01, + 0x00, 0xE6, 0x00, 0x01, 0x01, 0x43, 0x00, 0x08, + 0x01, 0x00, 0x00, 0x0C, 0x00, 0x08, 0x00, 0x04, + 0x00, 0xE1, 0x00, 0x04, 0x00, 0x0C, 0x00, 0x04, + 0x00, 0xE2, 0x00, 0x04, 0x00, 0x07, 0x00, 0x02, + 0x00, 0xE3, 0x00, 0x02, 0x00, 0x0B, 0x00, 0x02, + 0x00, 0xE4, 0x00, 0x02, 0x00, 0xEA, 0x00, 0x04, + 0x00, 0x04, 0x00, 0x01, 0x00, 0xE6, 0x00, 0x01, + 0x01, 0x43, 0x00, 0x08, 0x01, 0x00, 0x03, 0x48, + 0x0A, 0xE5, 0x40, 0xDB, 0x59, 0x25, 0x19, 0x40, + 0xC0, 0xA8, 0xFE, 0x5C, 0xC0, 0xA8, 0xFE, 0x5C, + 0xD3, 0x11, 0x1A, 0x0C, 0x01, 0xBD, 0x01, 0xBD, + 0x00, 0x00, 0x00, 0x00, 0x06, 0x01, 0x00, 0x00, + 0x01, 0x6C, 0x72, 0x94, 0x12, 0x69, 0x0A, 0xDB, + 0xC0, 0x59, 0x92, 0x00, 0x3F, 0xC0, 0xA9, 0x2F, + 0x4B, 0xB6, 0xA9, 0x2F, 0x4B, 0xB6, 0xAD, 0x06, + 0x69, 0x59, 0x42, 0x68, 0x42, 0x68, 0x00, 0x00, + 0x00, 0x00, 0x11, 0x01, 0x00, 0x00, 0x01, 0x6C, + 0x72, 0x94, 0x12, 0x69, 0x0A, 0xE5, 0x40, 0xDB, + 0x59, 0x25, 0x19, 0x40, 0xC0, 0xA8, 0xFE, 0x5E, + 0xC0, 0xA8, 0xFE, 0x5E, 0xD3, 0x12, 0x1A, 0x0D, + 0x01, 0xBD, 0x01, 0xBD, 0x00, 0x00, 0x00, 0x00, + 0x06, 0x01, 0x00, 0x00, 0x01, 0x6C, 0x72, 0x94, + 0x12, 0x69, 0x0A, 0xE1, 0x69, 0xB6, 0x59, 0x25, + 0x18, 0x69, 0xD1, 0x55, 0xE9, 0x5F, 0xD1, 0x55, + 0xE9, 0x5F, 0xE1, 0xA0, 0x98, 0xC9, 0x01, 0xBB, + 0x01, 0xBB, 0x00, 0x00, 0x00, 0x00, 0x06, 0x02, + 0x00, 0x00, 0x01, 0x6C, 0x72, 0x94, 0x12, 0x69, + 0x0A, 0xD4, 0x94, 0xFA, 0x59, 0x25, 0xDC, 0x94, + 0x4D, 0x4A, 0x46, 0x2E, 0x4D, 0x4A, 0x46, 0x2E, + 0xFF, 0xE5, 0x2A, 0xE6, 0x00, 0x50, 0x00, 0x50, + 0x00, 0x00, 0x00, 0x00, 0x06, 0x01, 0x00, 0x00, + 0x01, 0x6C, 0x72, 0x94, 0x12, 0x69, 0x0A, 0xE5, + 0x40, 0xDB, 0x59, 0x25, 0x19, 0x40, 0xC0, 0xA8, + 0xFE, 0x5D, 0xC0, 0xA8, 0xFE, 0x5D, 0xD3, 0x13, + 0x1A, 0x0E, 0x01, 0xBD, 0x01, 0xBD, 0x00, 0x00, + 0x00, 0x00, 0x06, 0x01, 0x00, 0x00, 0x01, 0x6C, + 0x72, 0x94, 0x12, 0x69, 0x0A, 0xF2, 0x6F, 0x15, + 0x59, 0x25, 0x1A, 0x6F, 0x9D, 0xF0, 0x02, 0x20, + 0x9D, 0xF0, 0x02, 0x20, 0xBF, 0xBD, 0xEE, 0x10, + 0x01, 0xBB, 0x01, 0xBB, 0x00, 0x00, 0x00, 0x00, + 0x06, 0x02, 0x00, 0x00, 0x01, 0x6C, 0x72, 0x94, + 0x12, 0x69, 0x0A, 0xDB, 0x65, 0xE9, 0x92, 0x00, + 0x3F, 0x65, 0xB0, 0xDE, 0xBB, 0xCD, 0xB0, 0xDE, + 0xBB, 0xCD, 0xCE, 0xEB, 0x99, 0xC6, 0x01, 0xBB, + 0x01, 0xBB, 0x00, 0x00, 0x00, 0x00, 0x06, 0x01, + 0x00, 0x00, 0x01, 0x6C, 0x72, 0x94, 0x12, 0x69, + 0x0A, 0xDB, 0x43, 0xA7, 0x92, 0x00, 0x3F, 0x43, + 0x40, 0xE9, 0xA4, 0x5F, 0x40, 0xE9, 0xA4, 0x5F, + 0xF9, 0xCE, 0x30, 0xE3, 0x01, 0xBB, 0x01, 0xBB, + 0x00, 0x00, 0x00, 0x00, 0x06, 0x01, 0x00, 0x00, + 0x01, 0x6C, 0x72, 0x94, 0x12, 0x69, 0x0A, 0xE5, + 0x40, 0xDB, 0x59, 0x25, 0x19, 0x40, 0xC0, 0xA8, + 0xFE, 0x5F, 0xC0, 0xA8, 0xFE, 0x5F, 0xD3, 0x14, + 0x1A, 0x0F, 0x01, 0xBD, 0x01, 0xBD, 0x00, 0x00, + 0x00, 0x00, 0x06, 0x01, 0x00, 0x00, 0x01, 0x6C, + 0x72, 0x94, 0x12, 0x69, 0x0A, 0xE1, 0xB0, 0x76, + 0x59, 0x25, 0x18, 0xB0, 0x5C, 0x64, 0xFE, 0x39, + 0x5C, 0x64, 0xFE, 0x39, 0x8B, 0x32, 0x04, 0x01, + 0x27, 0x1F, 0x27, 0x1F, 0x00, 0x00, 0x00, 0x00, + 0x11, 0x01, 0x00, 0x00, 0x01, 0x6C, 0x72, 0x94, + 0x12, 0x69, 0x0A, 0xD3, 0xD3, 0xFA, 0x92, 0x00, + 0x3F, 0xD3, 0x95, 0x9A, 0xA7, 0x32, 0x95, 0x9A, + 0xA7, 0x32, 0xD5, 0xED, 0x46, 0x1C, 0x14, 0x66, + 0x14, 0x66, 0x00, 0x00, 0x00, 0x00, 0x06, 0x02, + 0x00, 0x00, 0x01, 0x6C, 0x72, 0x94, 0x12, 0x69, + 0x0A, 0xDB, 0x11, 0x38, 0x92, 0x00, 0x3F, 0x11, + 0x6C, 0xB1, 0x0E, 0x9A, 0x6C, 0xB1, 0x0E, 0x9A, + 0x8A, 0x7F, 0x0A, 0x52, 0x01, 0xBB, 0x01, 0xBB, + 0x00, 0x00, 0x00, 0x00, 0x06, 0x01, 0x00, 0x00, + 0x01, 0x6C, 0x72, 0x94, 0x12, 0x69, 0x0A, 0xD4, + 0xB2, 0x7C, 0x59, 0x25, 0xDC, 0xB2, 0x58, 0xDD, + 0x4A, 0x3C, 0x58, 0xDD, 0x4A, 0x3C, 0xD7, 0x85, + 0x05, 0x49, 0x00, 0x50, 0x00, 0x50, 0x00, 0x00, + 0x00, 0x00, 0x06, 0x01, 0x00, 0x00, 0x01, 0x6C, + 0x72, 0x94, 0x12, 0x69, 0x0A, 0xD3, 0x37, 0x41, + 0x92, 0x00, 0x3F, 0x37, 0xA1, 0x75, 0x47, 0x59, + 0xA1, 0x75, 0x47, 0x59, 0xBC, 0x80, 0xDE, 0x0C, + 0x00, 0x50, 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, + 0x06, 0x01, 0x00, 0x00, 0x01, 0x6C, 0x72, 0x94, + 0x12, 0x69, 0x0A, 0xD4, 0x85, 0x27, 0x59, 0x25, + 0xDC, 0x85, 0xD2, 0x48, 0x91, 0x2C, 0xD2, 0x48, + 0x91, 0x2C, 0xC0, 0x65, 0x04, 0x0F, 0x00, 0x7B, + 0x00, 0x7B, 0x00, 0x00, 0x00, 0x00, 0x11, 0x01, + 0x00, 0x00, 0x01, 0x6C, 0x72, 0x94, 0x12, 0x69, + 0x0A, 0xD4, 0x6D, 0xD8, 0x59, 0x25, 0xDC, 0x6D, + 0x1F, 0x0D, 0x47, 0x22, 0x1F, 0x0D, 0x47, 0x22, + 0x9F, 0xC3, 0x78, 0x72, 0x01, 0xBB, 0x01, 0xBB, + 0x00, 0x00, 0x00, 0x00, 0x06, 0x02, 0x00, 0x00, + 0x01, 0x6C, 0x72, 0x94, 0x12, 0x69, 0x0A, 0xE5, + 0x1F, 0xB6, 0x59, 0x25, 0x19, 0x1F, 0x95, 0x9A, + 0xA7, 0x32, 0x95, 0x9A, 0xA7, 0x32, 0xCB, 0x35, + 0x8E, 0x81, 0x01, 0xBB, 0x01, 0xBB, 0x00, 0x00, + 0x00, 0x00, 0x06, 0x01, 0x00, 0x00, 0x01, 0x6C, + 0x72, 0x94, 0x12, 0x69, 0x0A, 0xE5, 0x21, 0xEF, + 0x59, 0x25, 0x19, 0x21, 0xAD, 0xC2, 0x49, 0x9B, + 0xAD, 0xC2, 0x49, 0x9B, 0xBE, 0xF5, 0x04, 0x4B, + 0x01, 0xBB, 0x01, 0xBB, 0x00, 0x00, 0x00, 0x00, + 0x11, 0x01, 0x00, 0x00, 0x01, 0x6C, 0x72, 0x94, + 0x12, 0x69, 0x0A, 0xD4, 0xB2, 0x7C, 0x59, 0x25, + 0xDC, 0xB2, 0x58, 0xDD, 0x4A, 0x3C, 0x58, 0xDD, + 0x4A, 0x3C, 0xC0, 0xCE, 0x05, 0x4B, 0x01, 0xBB, + 0x01, 0xBB, 0x00, 0x00, 0x00, 0x00, 0x06, 0x01, + 0x00, 0x00, 0x01, 0x6C, 0x72, 0x94, 0x12, 0x69, + 0x0A, 0xE1, 0x89, 0xA8, 0x59, 0x25, 0x18, 0x89, + 0x92, 0xFF, 0xC5, 0x43, 0x92, 0xFF, 0xC5, 0x43, + 0xD7, 0x3E, 0x2F, 0x4E, 0x01, 0xBB, 0x01, 0xBB, + 0x00, 0x00, 0x00, 0x00, 0x06, 0x02, 0x00, 0x00, + 0x01, 0x6C, 0x72, 0x94, 0x12, 0x69, 0x0A, 0xF2, + 0x62, 0xF1, 0x59, 0x25, 0x1A, 0x62, 0x1F, 0x0D, + 0x41, 0x22, 0x1F, 0x0D, 0x41, 0x22, 0xB5, 0x92, + 0xB9, 0xC7, 0x01, 0xBB, 0x01, 0xBB, 0x00, 0x00, + 0x00, 0x00, 0x06, 0x01, 0x00, 0x00, 0x01, 0x6C, + 0x72, 0x94, 0x12, 0x69, + }, + gopacket.CaptureInfo{ + // 2019-08-09T00:53:22.408533+06:00 + Timestamp: time.Unix(1565290402, 408533000), + CaptureLength: 1046, + Length: 1046, + }, + }, + // + // Frame 3 + { + []byte{ + // + // Ethernet, off: 0, len: 14 + 0xF8, 0x75, 0x88, 0x1A, 0x3C, 0x78, 0x7C, 0xAD, + 0x74, 0xE6, 0x78, 0x19, 0x08, 0x00, + // + // IPv4, off: 14, len: 20 + 0x45, 0x00, 0x04, 0x38, 0x5D, 0x0F, 0x00, 0x00, + 0xFF, 0x11, 0x2D, 0x95, 0xAC, 0x17, 0x6A, 0xFD, + 0xAC, 0x17, 0x69, 0xE4, + // + // UDP, off: 34, len: 8 + 0xF7, 0xEF, 0x27, 0x11, 0x04, 0x24, 0xE5, 0x80, + // + // Payload, off: 42, len: 1052 + 0x00, 0x09, 0x00, 0x1B, 0xAE, 0xF0, 0xED, 0x72, + 0x5D, 0x4C, 0x6F, 0xA2, 0xB7, 0x5F, 0x6B, 0x2C, + 0x00, 0x00, 0x00, 0xC8, 0x01, 0x00, 0x04, 0x08, + 0x0A, 0xE1, 0x31, 0x34, 0x59, 0x25, 0x18, 0x31, + 0x6C, 0xB1, 0x0E, 0x5E, 0x6C, 0xB1, 0x0E, 0x5E, + 0xDF, 0x1C, 0x04, 0x16, 0x01, 0xBB, 0x01, 0xBB, + 0x00, 0x00, 0x00, 0x00, 0x11, 0x01, 0x00, 0x00, + 0x01, 0x6C, 0x72, 0x94, 0x12, 0x6A, 0x0A, 0xF2, + 0x19, 0xBB, 0x59, 0x25, 0x1A, 0x19, 0xAD, 0xC2, + 0xDC, 0x5F, 0xAD, 0xC2, 0xDC, 0x5F, 0x9A, 0x51, + 0x23, 0xA5, 0x01, 0xBB, 0x01, 0xBB, 0x00, 0x00, + 0x00, 0x00, 0x06, 0x01, 0x00, 0x00, 0x01, 0x6C, + 0x72, 0x94, 0x12, 0x6A, 0x0A, 0xF3, 0xF1, 0x22, + 0x59, 0x25, 0x1B, 0xF1, 0xB0, 0xDE, 0xBB, 0x64, + 0xB0, 0xDE, 0xBB, 0x64, 0x84, 0x70, 0x6D, 0x80, + 0x01, 0xBB, 0x01, 0xBB, 0x00, 0x00, 0x00, 0x00, + 0x06, 0x02, 0x00, 0x00, 0x01, 0x6C, 0x72, 0x94, + 0x12, 0x6A, 0x0A, 0xDB, 0xEF, 0xD5, 0x92, 0x00, + 0x3F, 0xEF, 0xB0, 0x22, 0x97, 0x75, 0xB0, 0x22, + 0x97, 0x75, 0xC9, 0x3E, 0xB7, 0xC8, 0x01, 0xBB, + 0x01, 0xBB, 0x00, 0x00, 0x00, 0x00, 0x06, 0x02, + 0x00, 0x00, 0x01, 0x6C, 0x72, 0x94, 0x12, 0x6A, + 0x0A, 0xDB, 0x47, 0xEA, 0x92, 0x00, 0x3F, 0x47, + 0x4D, 0x58, 0x15, 0xCF, 0x4D, 0x58, 0x15, 0xCF, + 0x86, 0x21, 0x8B, 0x3F, 0x01, 0xBB, 0x01, 0xBB, + 0x00, 0x00, 0x00, 0x00, 0x06, 0x01, 0x00, 0x00, + 0x01, 0x6C, 0x72, 0x94, 0x12, 0x6A, 0x0A, 0xDD, + 0xC6, 0x0B, 0x92, 0x00, 0x3F, 0xC6, 0xD1, 0x55, + 0xE9, 0x61, 0xD1, 0x55, 0xE9, 0x61, 0x93, 0x0E, + 0x8B, 0x40, 0x01, 0xBB, 0x01, 0xBB, 0x00, 0x00, + 0x00, 0x00, 0x06, 0x01, 0x00, 0x00, 0x01, 0x6C, + 0x72, 0x94, 0x12, 0x6A, 0x0A, 0xE0, 0x51, 0xAD, + 0x59, 0x25, 0xDE, 0x51, 0xAD, 0xC2, 0xDC, 0x5F, + 0xAD, 0xC2, 0xDC, 0x5F, 0xDC, 0x05, 0x95, 0xC2, + 0x01, 0xBB, 0x01, 0xBB, 0x00, 0x00, 0x00, 0x00, + 0x06, 0x02, 0x00, 0x00, 0x01, 0x6C, 0x72, 0x94, + 0x12, 0x6B, 0x0A, 0xDD, 0xFE, 0x74, 0x92, 0x00, + 0x3F, 0xFE, 0x58, 0xDD, 0x4A, 0x3C, 0x58, 0xDD, + 0x4A, 0x3C, 0xA2, 0xB4, 0x2F, 0x42, 0x01, 0xBB, + 0x01, 0xBB, 0x00, 0x00, 0x00, 0x00, 0x06, 0x01, + 0x00, 0x00, 0x01, 0x6C, 0x72, 0x94, 0x12, 0x6B, + 0x0A, 0xD3, 0x2A, 0x40, 0x92, 0x00, 0x3F, 0x2A, + 0xB0, 0xDE, 0xBB, 0x51, 0xB0, 0xDE, 0xBB, 0x51, + 0xA8, 0xA2, 0xCD, 0xE2, 0x01, 0xBB, 0x01, 0xBB, + 0x00, 0x00, 0x00, 0x00, 0x06, 0x02, 0x00, 0x00, + 0x01, 0x6C, 0x72, 0x94, 0x12, 0x6B, 0x0A, 0xD4, + 0x3C, 0x0D, 0x59, 0x25, 0xDC, 0x3C, 0x1F, 0x0D, + 0x46, 0x22, 0x1F, 0x0D, 0x46, 0x22, 0xE6, 0xBE, + 0x6F, 0x21, 0x01, 0xBB, 0x01, 0xBB, 0x00, 0x00, + 0x00, 0x00, 0x06, 0x01, 0x00, 0x00, 0x01, 0x6C, + 0x72, 0x94, 0x12, 0x6B, 0x0A, 0xF2, 0x60, 0x9C, + 0x59, 0x25, 0x1A, 0x60, 0xAD, 0xC2, 0x49, 0x61, + 0xAD, 0xC2, 0x49, 0x61, 0xDC, 0xA0, 0x8D, 0xE8, + 0x01, 0xBB, 0x01, 0xBB, 0x00, 0x00, 0x00, 0x00, + 0x06, 0x01, 0x00, 0x00, 0x01, 0x6C, 0x72, 0x94, + 0x12, 0x6B, 0x0A, 0xF3, 0x07, 0x93, 0x59, 0x25, + 0x1B, 0x07, 0x11, 0x39, 0x92, 0x8A, 0x11, 0x39, + 0x92, 0x8A, 0xCC, 0x5D, 0x04, 0x06, 0x14, 0x67, + 0x14, 0x67, 0x00, 0x00, 0x00, 0x00, 0x06, 0x01, + 0x00, 0x00, 0x01, 0x6C, 0x72, 0x94, 0x12, 0x6B, + 0x0A, 0xF2, 0x5E, 0xB6, 0x59, 0x25, 0x1A, 0x5E, + 0x5E, 0x64, 0xB4, 0xC5, 0x5E, 0x64, 0xB4, 0xC5, + 0xC9, 0x25, 0xD6, 0x81, 0x01, 0xBB, 0x01, 0xBB, + 0x00, 0x00, 0x00, 0x00, 0x06, 0x01, 0x00, 0x00, + 0x01, 0x6C, 0x72, 0x94, 0x12, 0x6B, 0x0A, 0xF2, + 0xFD, 0xBC, 0x59, 0x25, 0x1A, 0xFD, 0x40, 0xE9, + 0xA2, 0x8A, 0x40, 0xE9, 0xA2, 0x8A, 0x98, 0x9A, + 0xB2, 0xF0, 0x01, 0xBB, 0x01, 0xBB, 0x00, 0x00, + 0x00, 0x00, 0x06, 0x01, 0x00, 0x00, 0x01, 0x6C, + 0x72, 0x94, 0x12, 0x6B, 0x0A, 0xD4, 0x61, 0xCB, + 0x59, 0x25, 0xDC, 0x61, 0xB0, 0xDE, 0xBE, 0x2A, + 0xB0, 0xDE, 0xBE, 0x2A, 0xB7, 0x90, 0xD9, 0x8A, + 0x01, 0xBB, 0x01, 0xBB, 0x00, 0x00, 0x00, 0x00, + 0x06, 0x02, 0x00, 0x00, 0x01, 0x6C, 0x72, 0x94, + 0x12, 0x6B, 0x0A, 0xE1, 0x19, 0x4A, 0x59, 0x25, + 0x18, 0x19, 0x7C, 0xCA, 0x8A, 0x0B, 0x7C, 0xCA, + 0x8A, 0x0B, 0xE5, 0x46, 0x04, 0x05, 0x4A, 0x38, + 0x4A, 0x38, 0x00, 0x00, 0x00, 0x00, 0x11, 0x02, + 0x00, 0x00, 0x01, 0x6C, 0x72, 0x94, 0x12, 0x6B, + 0x0A, 0xDD, 0xA0, 0xB4, 0x92, 0x00, 0x3F, 0xA0, + 0x40, 0xE9, 0xA1, 0x5E, 0x40, 0xE9, 0xA1, 0x5E, + 0xA5, 0x4F, 0xD4, 0x5D, 0x01, 0xBB, 0x01, 0xBB, + 0x00, 0x00, 0x00, 0x00, 0x06, 0x01, 0x00, 0x00, + 0x01, 0x6C, 0x72, 0x94, 0x12, 0x6B, 0x0A, 0xF3, + 0x0C, 0x36, 0x59, 0x25, 0x1B, 0x0C, 0x34, 0x4C, + 0xCC, 0x49, 0x34, 0x4C, 0xCC, 0x49, 0xE8, 0x72, + 0x24, 0x87, 0x01, 0xBB, 0x01, 0xBB, 0x00, 0x00, + 0x00, 0x00, 0x06, 0x02, 0x00, 0x00, 0x01, 0x6C, + 0x72, 0x94, 0x12, 0x6B, 0x0A, 0xE1, 0x94, 0xA3, + 0x59, 0x25, 0x18, 0x94, 0x08, 0x08, 0x08, 0x08, + 0x08, 0x08, 0x08, 0x08, 0xE9, 0xA5, 0x3E, 0x19, + 0x00, 0x35, 0x00, 0x35, 0x00, 0x00, 0x00, 0x00, + 0x11, 0x01, 0x00, 0x00, 0x01, 0x6C, 0x72, 0x94, + 0x12, 0x6B, 0x0A, 0xDD, 0x9A, 0xA9, 0x92, 0x00, + 0x3F, 0x9A, 0x4D, 0x58, 0x37, 0x32, 0x4D, 0x58, + 0x37, 0x32, 0xDB, 0x21, 0x15, 0x00, 0x01, 0xBB, + 0x01, 0xBB, 0x00, 0x00, 0x00, 0x00, 0x06, 0x02, + 0x00, 0x00, 0x01, 0x6C, 0x72, 0x94, 0x12, 0x6B, + 0x0A, 0xDD, 0x4F, 0xDF, 0x92, 0x00, 0x3F, 0x4F, + 0xCD, 0xC4, 0x06, 0x4A, 0xCD, 0xC4, 0x06, 0x4A, + 0xD0, 0x1C, 0x8F, 0xB1, 0x69, 0x99, 0x69, 0x99, + 0x00, 0x00, 0x00, 0x00, 0x11, 0x01, 0x00, 0x00, + 0x01, 0x6C, 0x72, 0x94, 0x12, 0x6B, 0x0A, 0xD3, + 0xB0, 0x31, 0x92, 0x00, 0x3F, 0xB0, 0x25, 0x63, + 0x3E, 0x21, 0x25, 0x63, 0x3E, 0x21, 0xD5, 0xE7, + 0xD6, 0x0A, 0x01, 0xBB, 0x01, 0xBB, 0x00, 0x00, + 0x00, 0x00, 0x06, 0x02, 0x00, 0x00, 0x01, 0x6C, + 0x72, 0x94, 0x12, 0x6B, 0x0A, 0xDB, 0x10, 0x97, + 0x92, 0x00, 0x3F, 0x10, 0xB0, 0xDE, 0xBB, 0x4F, + 0xB0, 0xDE, 0xBB, 0x4F, 0xB5, 0xE9, 0x04, 0x02, + 0x01, 0xBB, 0x01, 0xBB, 0x00, 0x00, 0x00, 0x00, + 0x11, 0x01, 0x00, 0x00, 0x01, 0x6C, 0x72, 0x94, + 0x12, 0x6B, 0x0A, 0xDD, 0x58, 0xD4, 0x92, 0x00, + 0x3F, 0x58, 0x9D, 0xF0, 0xC2, 0x0B, 0x9D, 0xF0, + 0xC2, 0x0B, 0xFA, 0x12, 0xF0, 0x64, 0x01, 0xBB, + 0x01, 0xBB, 0x00, 0x00, 0x00, 0x00, 0x06, 0x01, + 0x00, 0x00, 0x01, 0x6C, 0x72, 0x94, 0x12, 0x6B, + 0x0A, 0xE0, 0x27, 0xD2, 0x59, 0x25, 0xDE, 0x27, + 0x40, 0xE9, 0xA4, 0x5F, 0x40, 0xE9, 0xA4, 0x5F, + 0x85, 0x7A, 0xB7, 0x3A, 0x01, 0xBB, 0x01, 0xBB, + 0x00, 0x00, 0x00, 0x00, 0x06, 0x01, 0x00, 0x00, + 0x01, 0x6C, 0x72, 0x94, 0x12, 0x6B, 0x0A, 0xE1, + 0x1F, 0x47, 0x59, 0x25, 0x18, 0x1F, 0x4D, 0x57, + 0x65, 0x39, 0x4D, 0x57, 0x65, 0x39, 0xFA, 0xCB, + 0x6D, 0x58, 0xCC, 0xA0, 0xCC, 0xA0, 0x00, 0x00, + 0x00, 0x00, 0x06, 0x02, 0x00, 0x00, 0x01, 0x6C, + 0x72, 0x94, 0x12, 0x6B, 0x0A, 0xD4, 0x6E, 0xD9, + 0x59, 0x25, 0xDC, 0x6E, 0x11, 0xFD, 0x26, 0x7D, + 0x11, 0xFD, 0x26, 0x7D, 0xE8, 0xFB, 0x3A, 0x21, + 0x00, 0x7B, 0x00, 0x7B, 0x00, 0x00, 0x00, 0x00, + 0x11, 0x01, 0x00, 0x00, 0x01, 0x6C, 0x72, 0x94, + 0x12, 0x6B, 0x00, 0x00, + }, + gopacket.CaptureInfo{ + // 2019-08-09T00:53:22.410626+06:00 + Timestamp: time.Unix(1565290402, 410626000), + CaptureLength: 1094, + Length: 1094, + }, + }, +} From 35ec5eb5cce711f1a6b968898b4f51045d99d8ab Mon Sep 17 00:00:00 2001 From: Batyrkhan Koshenov Date: Mon, 31 Oct 2022 13:19:43 +0600 Subject: [PATCH 04/10] decoders/utils.go: add ReadUint16/32 funcs to avoid allocs when reading from buffer Signed-off-by: Batyrkhan Koshenov --- decoders/utils/utils.go | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/decoders/utils/utils.go b/decoders/utils/utils.go index a36e3b2..b22ccc4 100644 --- a/decoders/utils/utils.go +++ b/decoders/utils/utils.go @@ -1,6 +1,7 @@ package utils import ( + "bytes" "encoding/binary" "io" ) @@ -14,3 +15,41 @@ func BinaryDecoder(payload io.Reader, dests ...interface{}) error { } return nil } + +// ReadUint16FromBuffer reads Uint16 value from buffer and returns +// boolean flag telling if it was a success. +// +// Value is treated as big endian. +func ReadUint16FromBuffer(b *bytes.Buffer, x *uint16) bool { + var buf [2]byte + + for i := range buf { + bt, err := b.ReadByte() + if err != nil { + return false + } + buf[i] = bt + } + + *x = binary.BigEndian.Uint16(buf[:]) + return true +} + +// ReadUint32FromBuffer reads Uint32 value from buffer and returns +// boolean flag telling if it was a success. +// +// Value is treated as big endian. +func ReadUint32FromBuffer(b *bytes.Buffer, x *uint32) bool { + var buf [4]byte + + for i := range buf { + bt, err := b.ReadByte() + if err != nil { + return false + } + buf[i] = bt + } + + *x = binary.BigEndian.Uint32(buf[:]) + return true +} From 2fcef7c2da269f4971bdfa9ca41e2244914c3f6f Mon Sep 17 00:00:00 2001 From: Batyrkhan Koshenov Date: Mon, 31 Oct 2022 13:22:20 +0600 Subject: [PATCH 05/10] mod: add github.com/stretchr/testify, github.com/google/gopacket, github.com/klauspost/compress Signed-off-by: Batyrkhan Koshenov --- go.mod | 8 ++++-- go.sum | 87 ++++++++++++++++++++++++++++++++++++++++++++++------------ 2 files changed, 76 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index cae068e..6c35237 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,14 @@ module github.com/cloudflare/goflow/v3 go 1.12 require ( - github.com/Shopify/sarama v1.22.0 + github.com/Shopify/sarama v1.27.0 github.com/golang/protobuf v1.3.1 + github.com/google/gopacket v1.1.18 + github.com/klauspost/compress v1.11.0 // indirect github.com/libp2p/go-reuseport v0.0.1 github.com/prometheus/client_golang v0.9.2 github.com/sirupsen/logrus v1.4.1 - github.com/stretchr/testify v1.3.0 + github.com/stretchr/testify v1.8.0 + golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc // indirect + golang.org/x/sys v0.0.0-20200820212457-1fb795427249 // indirect ) diff --git a/go.sum b/go.sum index c9f5615..b4d56e6 100644 --- a/go.sum +++ b/go.sum @@ -1,36 +1,58 @@ -github.com/DataDog/zstd v1.3.5 h1:DtpNbljikUepEPD16hD4LvIcmhnhdLTiW/5pHgbmp14= -github.com/DataDog/zstd v1.3.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/Shopify/sarama v1.22.0 h1:rtiODsvY4jW6nUV6n3K+0gx/8WlAwVt+Ixt6RIvpYyo= -github.com/Shopify/sarama v1.22.0/go.mod h1:lm3THZ8reqBDBQKQyb5HB3sY1lKp3grEbQ81aWSgPp4= +github.com/Shopify/sarama v1.27.0 h1:tqo2zmyzPf1+gwTTwhI6W+EXDw4PVSczynpHKFtVAmo= +github.com/Shopify/sarama v1.27.0/go.mod h1:aCdj6ymI8uyPEux1JJ9gcaDT6cinjGhNCAhs54taSUo= github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/eapache/go-resiliency v1.1.0 h1:1NtRmCAqadE2FN4ZcN6g90TP3uk8cg9rn9eNK2197aU= -github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q= +github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/frankban/quicktest v1.10.0 h1:Gfh+GAJZOAoKZsIZeZbdn2JF10kN1XHNvjsvQK8gVkE= +github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1 h1:/exdXoGamhu5ONeUJH0deniYLWYvQwW66yvlfiiKTu0= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gopacket v1.1.18 h1:lum7VRA9kdlvBi7/v2p7/zcbkduHaCH/SVVyurs7OpY= +github.com/google/gopacket v1.1.18/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= +github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8= +github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= +github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.11.0 h1:wJbzvpYMVGG9iTI9VxpnNZfd4DzMPoCWze3GgSqz8yg= +github.com/klauspost/compress v1.11.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/libp2p/go-reuseport v0.0.1 h1:7PhkfH73VXfPJYKQ6JwS5I/eVcoyYi9IMNGc6FWpFLw= github.com/libp2p/go-reuseport v0.0.1/go.mod h1:jn6RmB1ufnQwl0Q1f+YxAj8isJgDCQzaaxIFYDhcYEA= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/pierrec/lz4 v0.0.0-20190327172049-315a67e90e41 h1:GeinFsrjWz97fAxVUEd748aV0cYL+I6k44gFJTCVvpU= -github.com/pierrec/lz4 v0.0.0-20190327172049-315a67e90e41/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI= +github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740= @@ -41,27 +63,58 @@ github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jO github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a h1:9ZKAASQSHhDYGoxY8uLVpewe1GDZ2vu2Tr/vTdVAkFQ= -github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ= +github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20200528225125-3c3fba18258b/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc h1:zK/HqS5bZxDptfPJNq8v7vJfXtkU7r9TLIoSr1bXaP4= +golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e h1:nFYrTHrdrAOpShe27kaFHjsqYSEQ0KWqdWLu3xuZJts= -golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200820212457-1fb795427249 h1:tKP05IMsVLZ4VeeCEFmrIUmxAAx6UD8IBdPtYlYNa8g= +golang.org/x/sys v0.0.0-20200820212457-1fb795427249/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/jcmturner/aescts.v1 v1.0.1 h1:cVVZBK2b1zY26haWB4vbBiZrfFQnfbTVrE3xZq6hrEw= +gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= +gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN5hSM= +gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= +gopkg.in/jcmturner/goidentity.v3 v3.0.0 h1:1duIyWiTaYvVx3YX2CYtpJbUFd7/UuPYCfgXtQ3VTbI= +gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= +gopkg.in/jcmturner/gokrb5.v7 v7.5.0 h1:a9tsXlIDD9SKxotJMK3niV7rPZAJeX2aD/0yg3qlIrg= +gopkg.in/jcmturner/gokrb5.v7 v7.5.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= +gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU= +gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200601152816-913338de1bd2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 3fb99bb80d2fe88908390a7beab1f4a301778bc5 Mon Sep 17 00:00:00 2001 From: Batyrkhan Koshenov Date: Mon, 31 Oct 2022 13:24:38 +0600 Subject: [PATCH 06/10] producer, utils: refactor old funcs for new FlowSets field type Signed-off-by: Batyrkhan Koshenov --- decoders/netflow/netflow.go | 2 +- producer/producer_nf.go | 50 +----- producer/producer_nflegacy.go | 2 +- producer/producer_sf.go | 11 +- producer/producer_test.go | 18 ++- utils/netflow.go | 295 +++++++++++++++++----------------- 6 files changed, 175 insertions(+), 203 deletions(-) diff --git a/decoders/netflow/netflow.go b/decoders/netflow/netflow.go index bd025ad..ecbbdb4 100644 --- a/decoders/netflow/netflow.go +++ b/decoders/netflow/netflow.go @@ -638,4 +638,4 @@ func (f *FlowMessage) DecodeIPFIXPacket(payload *bytes.Buffer, templates NetFlow f.PacketIPFIX.OptionsDataFS = f.PacketIPFIX.OptionsDataFS[:ipxOptsDataFSidx] return nil -} \ No newline at end of file +} diff --git a/producer/producer_nf.go b/producer/producer_nf.go index 53da126..4369bbe 100644 --- a/producer/producer_nf.go +++ b/producer/producer_nf.go @@ -34,7 +34,7 @@ func CreateSamplingSystem() SamplingRateSystem { func (s *basicSamplingRateSystem) AddSamplingRate(version uint16, obsDomainId uint32, samplingRate uint32) { s.samplinglock.Lock() _, exists := s.sampling[version] - if exists != true { + if !exists { s.sampling[version] = make(map[uint32]uint32) } s.sampling[version][obsDomainId] = samplingRate @@ -119,7 +119,7 @@ func DecodeUNumber(b []byte, out interface{}) error { iter++ } } else { - return errors.New(fmt.Sprintf("Non-regular number of bytes for a number: %v", l)) + return fmt.Errorf("non-regular number of bytes for a number: %v", l) } } switch t := out.(type) { @@ -132,7 +132,7 @@ func DecodeUNumber(b []byte, out interface{}) error { case *uint64: *t = o default: - return errors.New("The parameter is not a pointer to a byte/uint16/uint32/uint64 structure") + return errors.New("the parameter is not a pointer to a byte/uint16/uint32/uint64 structure") } return nil } @@ -150,13 +150,9 @@ func ConvertNetFlowDataSet(version uint16, baseTime uint32, uptime uint32, recor for i := range record { df := record[i] - v, ok := df.Value.([]byte) - if !ok { - continue - } + v := df.Value switch df.Type { - // Statistics case netflow.NFV9_FIELD_IN_BYTES: DecodeUNumber(v, &(flowMessage.Bytes)) @@ -382,43 +378,11 @@ func SearchNetFlowOptionDataSets(dataFlowSet []netflow.OptionsDataFlowSet) (uint } func SplitNetFlowSets(packetNFv9 netflow.NFv9Packet) ([]netflow.DataFlowSet, []netflow.TemplateFlowSet, []netflow.NFv9OptionsTemplateFlowSet, []netflow.OptionsDataFlowSet) { - dataFlowSet := make([]netflow.DataFlowSet, 0) - templatesFlowSet := make([]netflow.TemplateFlowSet, 0) - optionsTemplatesFlowSet := make([]netflow.NFv9OptionsTemplateFlowSet, 0) - optionsDataFlowSet := make([]netflow.OptionsDataFlowSet, 0) - for _, flowSet := range packetNFv9.FlowSets { - switch flowSet.(type) { - case netflow.TemplateFlowSet: - templatesFlowSet = append(templatesFlowSet, flowSet.(netflow.TemplateFlowSet)) - case netflow.NFv9OptionsTemplateFlowSet: - optionsTemplatesFlowSet = append(optionsTemplatesFlowSet, flowSet.(netflow.NFv9OptionsTemplateFlowSet)) - case netflow.DataFlowSet: - dataFlowSet = append(dataFlowSet, flowSet.(netflow.DataFlowSet)) - case netflow.OptionsDataFlowSet: - optionsDataFlowSet = append(optionsDataFlowSet, flowSet.(netflow.OptionsDataFlowSet)) - } - } - return dataFlowSet, templatesFlowSet, optionsTemplatesFlowSet, optionsDataFlowSet + return packetNFv9.DataFS, packetNFv9.TemplateFS, packetNFv9.NFv9OptionsTemplateFS, packetNFv9.OptionsDataFS } func SplitIPFIXSets(packetIPFIX netflow.IPFIXPacket) ([]netflow.DataFlowSet, []netflow.TemplateFlowSet, []netflow.IPFIXOptionsTemplateFlowSet, []netflow.OptionsDataFlowSet) { - dataFlowSet := make([]netflow.DataFlowSet, 0) - templatesFlowSet := make([]netflow.TemplateFlowSet, 0) - optionsTemplatesFlowSet := make([]netflow.IPFIXOptionsTemplateFlowSet, 0) - optionsDataFlowSet := make([]netflow.OptionsDataFlowSet, 0) - for _, flowSet := range packetIPFIX.FlowSets { - switch flowSet.(type) { - case netflow.TemplateFlowSet: - templatesFlowSet = append(templatesFlowSet, flowSet.(netflow.TemplateFlowSet)) - case netflow.IPFIXOptionsTemplateFlowSet: - optionsTemplatesFlowSet = append(optionsTemplatesFlowSet, flowSet.(netflow.IPFIXOptionsTemplateFlowSet)) - case netflow.DataFlowSet: - dataFlowSet = append(dataFlowSet, flowSet.(netflow.DataFlowSet)) - case netflow.OptionsDataFlowSet: - optionsDataFlowSet = append(optionsDataFlowSet, flowSet.(netflow.OptionsDataFlowSet)) - } - } - return dataFlowSet, templatesFlowSet, optionsTemplatesFlowSet, optionsDataFlowSet + return packetIPFIX.DataFS, packetIPFIX.TemplateFS, packetIPFIX.IPFIXOptionsTemplateFS, packetIPFIX.OptionsDataFS } // Convert a NetFlow datastructure to a FlowMessage protobuf @@ -474,7 +438,7 @@ func ProcessMessageNetFlow(msgDec interface{}, samplingRateSys SamplingRateSyste fmsg.SamplingRate = uint64(samplingRate) } default: - return flowMessageSet, errors.New("Bad NetFlow/IPFIX version") + return flowMessageSet, errors.New("bad NetFlow/IPFIX version") } return flowMessageSet, nil diff --git a/producer/producer_nflegacy.go b/producer/producer_nflegacy.go index 990da0e..3dd1a94 100644 --- a/producer/producer_nflegacy.go +++ b/producer/producer_nflegacy.go @@ -74,6 +74,6 @@ func ProcessMessageNetFlowLegacy(msgDec interface{}) ([]*flowmessage.FlowMessage return flowMessageSet, nil default: - return []*flowmessage.FlowMessage{}, errors.New("Bad NetFlow v5 version") + return []*flowmessage.FlowMessage{}, errors.New("bad NetFlow v5 version") } } diff --git a/producer/producer_sf.go b/producer/producer_sf.go index 9217c67..47cfb33 100644 --- a/producer/producer_sf.go +++ b/producer/producer_sf.go @@ -319,9 +319,11 @@ func ParseSampledHeaderConfig(flowMessage *flowmessage.FlowMessage, sampledHeade return nil } +/* func SearchSFlowSamples(samples []interface{}) []*flowmessage.FlowMessage { return SearchSFlowSamples(samples) } +*/ func SearchSFlowSamplesConfig(samples []interface{}, config *SFlowProducerConfig, agent net.IP) []*flowmessage.FlowMessage { flowMessageSet := make([]*flowmessage.FlowMessage, 0) @@ -345,9 +347,10 @@ func SearchSFlowSamplesConfig(samples []interface{}, config *SFlowProducerConfig flowMessage.OutIf = flowSample.OutputIfValue } - ipNh := net.IP{} - ipSrc := net.IP{} - ipDst := net.IP{} + var ( + ipNh, ipSrc, ipDst net.IP + ) + flowMessage.Packets = 1 for _, record := range records { switch recordData := record.Data.(type) { @@ -425,6 +428,6 @@ func ProcessMessageSFlowConfig(msgDec interface{}, config *SFlowProducerConfig) return flowMessageSet, nil default: - return []*flowmessage.FlowMessage{}, errors.New("Bad sFlow version") + return []*flowmessage.FlowMessage{}, errors.New("bad sFlow version") } } diff --git a/producer/producer_test.go b/producer/producer_test.go index fcefb3e..c2269f0 100644 --- a/producer/producer_test.go +++ b/producer/producer_test.go @@ -10,30 +10,32 @@ import ( func TestProcessMessageNetFlow(t *testing.T) { records := []netflow.DataRecord{ - netflow.DataRecord{ + { Values: []netflow.DataField{ - netflow.DataField{ + { Type: netflow.NFV9_FIELD_IPV4_SRC_ADDR, Value: []byte{10, 0, 0, 1}, }, }, }, } - dfs := []interface{}{ - netflow.DataFlowSet{ + + dfs := []netflow.DataFlowSet{ + { Records: records, }, } pktnf9 := netflow.NFv9Packet{ - FlowSets: dfs, + FlowSets: netflow.FlowSets{DataFS: dfs}, } + testsr := &SingleSamplingRateSystem{1} _, err := ProcessMessageNetFlow(pktnf9, testsr) assert.Nil(t, err) pktipfix := netflow.IPFIXPacket{ - FlowSets: dfs, + FlowSets: netflow.FlowSets{DataFS: dfs}, } _, err = ProcessMessageNetFlow(pktipfix, testsr) assert.Nil(t, err) @@ -58,7 +60,7 @@ func TestProcessMessageSFlow(t *testing.T) { sflow.FlowSample{ SamplingRate: 1, Records: []sflow.FlowRecord{ - sflow.FlowRecord{ + { Data: sh, }, }, @@ -66,7 +68,7 @@ func TestProcessMessageSFlow(t *testing.T) { sflow.ExpandedFlowSample{ SamplingRate: 1, Records: []sflow.FlowRecord{ - sflow.FlowRecord{ + { Data: sh, }, }, diff --git a/utils/netflow.go b/utils/netflow.go index e1c2b11..9a98fb1 100644 --- a/utils/netflow.go +++ b/utils/netflow.go @@ -96,7 +96,10 @@ func (s *StateNetFlow) DecodeFlow(msg interface{}) error { } timeTrackStart := time.Now() - msgDec, err := netflow.DecodeMessage(buf, templates) + + fm := netflow.FlowMessage{} + err := fm.Decode(buf, templates) + if err != nil { switch err.(type) { case *netflow.ErrorVersion: @@ -133,8 +136,7 @@ func (s *StateNetFlow) DecodeFlow(msg interface{}) error { flowMessageSet := make([]*flowmessage.FlowMessage, 0) - switch msgDecConv := msgDec.(type) { - case netflow.NFv9Packet: + if fm.Version == 9 { NetFlowStats.With( prometheus.Labels{ "router": key, @@ -142,77 +144,78 @@ func (s *StateNetFlow) DecodeFlow(msg interface{}) error { }). Inc() - for _, fs := range msgDecConv.FlowSets { - switch fsConv := fs.(type) { - case netflow.TemplateFlowSet: - NetFlowSetStatsSum.With( - prometheus.Labels{ - "router": key, - "version": "9", - "type": "TemplateFlowSet", - }). - Inc() - - NetFlowSetRecordsStatsSum.With( - prometheus.Labels{ - "router": key, - "version": "9", - "type": "OptionsTemplateFlowSet", - }). - Add(float64(len(fsConv.Records))) - - case netflow.NFv9OptionsTemplateFlowSet: - NetFlowSetStatsSum.With( - prometheus.Labels{ - "router": key, - "version": "9", - "type": "OptionsTemplateFlowSet", - }). - Inc() - - NetFlowSetRecordsStatsSum.With( - prometheus.Labels{ - "router": key, - "version": "9", - "type": "OptionsTemplateFlowSet", - }). - Add(float64(len(fsConv.Records))) - - case netflow.OptionsDataFlowSet: - NetFlowSetStatsSum.With( - prometheus.Labels{ - "router": key, - "version": "9", - "type": "OptionsDataFlowSet", - }). - Inc() - - NetFlowSetRecordsStatsSum.With( - prometheus.Labels{ - "router": key, - "version": "9", - "type": "OptionsDataFlowSet", - }). - Add(float64(len(fsConv.Records))) - case netflow.DataFlowSet: - NetFlowSetStatsSum.With( - prometheus.Labels{ - "router": key, - "version": "9", - "type": "DataFlowSet", - }). - Inc() - - NetFlowSetRecordsStatsSum.With( - prometheus.Labels{ - "router": key, - "version": "9", - "type": "DataFlowSet", - }). - Add(float64(len(fsConv.Records))) - } + for _, fs := range fm.PacketNFv9.TemplateFS { + NetFlowSetStatsSum.With( + prometheus.Labels{ + "router": key, + "version": "9", + "type": "TemplateFlowSet", + }). + Inc() + + NetFlowSetRecordsStatsSum.With( + prometheus.Labels{ + "router": key, + "version": "9", + "type": "OptionsTemplateFlowSet", + }). + Add(float64(len(fs.Records))) + } + + for _, fs := range fm.PacketNFv9.NFv9OptionsTemplateFS { + NetFlowSetStatsSum.With( + prometheus.Labels{ + "router": key, + "version": "9", + "type": "OptionsTemplateFlowSet", + }). + Inc() + + NetFlowSetRecordsStatsSum.With( + prometheus.Labels{ + "router": key, + "version": "9", + "type": "OptionsTemplateFlowSet", + }). + Add(float64(len(fs.Records))) + } + + for _, fs := range fm.PacketNFv9.OptionsDataFS { + NetFlowSetStatsSum.With( + prometheus.Labels{ + "router": key, + "version": "9", + "type": "OptionsDataFlowSet", + }). + Inc() + + NetFlowSetRecordsStatsSum.With( + prometheus.Labels{ + "router": key, + "version": "9", + "type": "OptionsDataFlowSet", + }). + Add(float64(len(fs.Records))) + } + + for _, fs := range fm.PacketNFv9.DataFS { + NetFlowSetStatsSum.With( + prometheus.Labels{ + "router": key, + "version": "9", + "type": "DataFlowSet", + }). + Inc() + + NetFlowSetRecordsStatsSum.With( + prometheus.Labels{ + "router": key, + "version": "9", + "type": "DataFlowSet", + }). + Add(float64(len(fs.Records))) } - flowMessageSet, err = producer.ProcessMessageNetFlow(msgDecConv, sampling) + flowMessageSet, err = producer.ProcessMessageNetFlow(fm.PacketNFv9, sampling) for _, fmsg := range flowMessageSet { fmsg.TimeReceived = ts @@ -225,7 +228,7 @@ func (s *StateNetFlow) DecodeFlow(msg interface{}) error { }). Observe(float64(timeDiff)) } - case netflow.IPFIXPacket: + } else if fm.Version == 10 { NetFlowStats.With( prometheus.Labels{ "router": key, @@ -233,79 +236,79 @@ func (s *StateNetFlow) DecodeFlow(msg interface{}) error { }). Inc() - for _, fs := range msgDecConv.FlowSets { - switch fsConv := fs.(type) { - case netflow.TemplateFlowSet: - NetFlowSetStatsSum.With( - prometheus.Labels{ - "router": key, - "version": "10", - "type": "TemplateFlowSet", - }). - Inc() - - NetFlowSetRecordsStatsSum.With( - prometheus.Labels{ - "router": key, - "version": "10", - "type": "TemplateFlowSet", - }). - Add(float64(len(fsConv.Records))) - - case netflow.IPFIXOptionsTemplateFlowSet: - NetFlowSetStatsSum.With( - prometheus.Labels{ - "router": key, - "version": "10", - "type": "OptionsTemplateFlowSet", - }). - Inc() - - NetFlowSetRecordsStatsSum.With( - prometheus.Labels{ - "router": key, - "version": "10", - "type": "OptionsTemplateFlowSet", - }). - Add(float64(len(fsConv.Records))) - - case netflow.OptionsDataFlowSet: - - NetFlowSetStatsSum.With( - prometheus.Labels{ - "router": key, - "version": "10", - "type": "OptionsDataFlowSet", - }). - Inc() - - NetFlowSetRecordsStatsSum.With( - prometheus.Labels{ - "router": key, - "version": "10", - "type": "OptionsDataFlowSet", - }). - Add(float64(len(fsConv.Records))) - - case netflow.DataFlowSet: - NetFlowSetStatsSum.With( - prometheus.Labels{ - "router": key, - "version": "10", - "type": "DataFlowSet", - }). - Inc() - - NetFlowSetRecordsStatsSum.With( - prometheus.Labels{ - "router": key, - "version": "10", - "type": "DataFlowSet", - }). - Add(float64(len(fsConv.Records))) - } + for _, fs := range fm.PacketIPFIX.TemplateFS { + NetFlowSetStatsSum.With( + prometheus.Labels{ + "router": key, + "version": "10", + "type": "TemplateFlowSet", + }). + Inc() + + NetFlowSetRecordsStatsSum.With( + prometheus.Labels{ + "router": key, + "version": "10", + "type": "TemplateFlowSet", + }). + Add(float64(len(fs.Records))) + } + + for _, fs := range fm.PacketIPFIX.IPFIXOptionsTemplateFS { + NetFlowSetStatsSum.With( + prometheus.Labels{ + "router": key, + "version": "10", + "type": "OptionsTemplateFlowSet", + }). + Inc() + + NetFlowSetRecordsStatsSum.With( + prometheus.Labels{ + "router": key, + "version": "10", + "type": "OptionsTemplateFlowSet", + }). + Add(float64(len(fs.Records))) } - flowMessageSet, err = producer.ProcessMessageNetFlow(msgDecConv, sampling) + + for _, fs := range fm.PacketIPFIX.OptionsDataFS { + NetFlowSetStatsSum.With( + prometheus.Labels{ + "router": key, + "version": "10", + "type": "OptionsDataFlowSet", + }). + Inc() + + NetFlowSetRecordsStatsSum.With( + prometheus.Labels{ + "router": key, + "version": "10", + "type": "OptionsDataFlowSet", + }). + Add(float64(len(fs.Records))) + } + + for _, fs := range fm.PacketIPFIX.DataFS { + NetFlowSetStatsSum.With( + prometheus.Labels{ + "router": key, + "version": "10", + "type": "DataFlowSet", + }). + Inc() + + NetFlowSetRecordsStatsSum.With( + prometheus.Labels{ + "router": key, + "version": "10", + "type": "DataFlowSet", + }). + Add(float64(len(fs.Records))) + } + + flowMessageSet, err = producer.ProcessMessageNetFlow(fm.PacketIPFIX, sampling) for _, fmsg := range flowMessageSet { fmsg.TimeReceived = ts From b99f957037d7bf22570bb82eaf4465f2632f3061 Mon Sep 17 00:00:00 2001 From: Batyrkhan Koshenov Date: Fri, 2 Dec 2022 15:15:50 +0600 Subject: [PATCH 07/10] decoders/utils: change byte reading method Signed-off-by: Batyrkhan Koshenov --- decoders/utils/utils.go | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/decoders/utils/utils.go b/decoders/utils/utils.go index b22ccc4..4c39bc9 100644 --- a/decoders/utils/utils.go +++ b/decoders/utils/utils.go @@ -23,12 +23,8 @@ func BinaryDecoder(payload io.Reader, dests ...interface{}) error { func ReadUint16FromBuffer(b *bytes.Buffer, x *uint16) bool { var buf [2]byte - for i := range buf { - bt, err := b.ReadByte() - if err != nil { - return false - } - buf[i] = bt + if n, _ := b.Read(buf[:]); n < len(buf) { + return false } *x = binary.BigEndian.Uint16(buf[:]) @@ -42,12 +38,8 @@ func ReadUint16FromBuffer(b *bytes.Buffer, x *uint16) bool { func ReadUint32FromBuffer(b *bytes.Buffer, x *uint32) bool { var buf [4]byte - for i := range buf { - bt, err := b.ReadByte() - if err != nil { - return false - } - buf[i] = bt + if n, _ := b.Read(buf[:]); n < len(buf) { + return false } *x = binary.BigEndian.Uint32(buf[:]) From ea4692f440c455369c6918a2ffd4cb779e6ad5e0 Mon Sep 17 00:00:00 2001 From: Batyrkhan Koshenov Date: Tue, 6 Dec 2022 16:10:04 +0600 Subject: [PATCH 08/10] remove func init() to escape import conflict Signed-off-by: Batyrkhan Koshenov --- pb/flow.pb.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pb/flow.pb.go b/pb/flow.pb.go index 9c1a286..fdcd8cf 100644 --- a/pb/flow.pb.go +++ b/pb/flow.pb.go @@ -608,12 +608,15 @@ func (m *FlowMessage) GetPPPAddressControl() uint32 { return 0 } +// Commented due to avoid "duplicate enum registered" error while importing both old and revised versions of parser. +/* func init() { proto.RegisterEnum("flowprotob.FlowMessage_FlowType", FlowMessage_FlowType_name, FlowMessage_FlowType_value) proto.RegisterType((*FlowMessage)(nil), "flowprotob.FlowMessage") } func init() { proto.RegisterFile("pb/flow.proto", fileDescriptor_0beab9b6746e934c) } +*/ var fileDescriptor_0beab9b6746e934c = []byte{ // 943 bytes of a gzipped FileDescriptorProto From 56d241ce147036f06d4e405925780de7c33b5fb9 Mon Sep 17 00:00:00 2001 From: Batyrkhan Koshenov Date: Tue, 6 Dec 2022 16:10:28 +0600 Subject: [PATCH 09/10] gitignore: initial Signed-off-by: Batyrkhan Koshenov --- .gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a725465 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +vendor/ \ No newline at end of file From 1b906773735ddaecb994123d1c13622425daa2ed Mon Sep 17 00:00:00 2001 From: "b.koshenov" Date: Tue, 31 Oct 2023 18:39:36 +0600 Subject: [PATCH 10/10] decoders/netflow: add variable-length fields parsing support Signed-off-by: b.koshenov --- decoders/netflow/netflow.go | 61 ++++++++++++++++++++++++-------- decoders/netflow/netflow_test.go | 8 ++--- decoders/netflow/packet.go | 26 ++++++++++++++ 3 files changed, 77 insertions(+), 18 deletions(-) diff --git a/decoders/netflow/netflow.go b/decoders/netflow/netflow.go index ecbbdb4..f397f20 100644 --- a/decoders/netflow/netflow.go +++ b/decoders/netflow/netflow.go @@ -2,6 +2,7 @@ package netflow import ( "bytes" + "encoding/binary" "fmt" "io" "sync" @@ -164,18 +165,28 @@ func GetTemplateSize(template []Field) int { // DecodeDataRecordFields decodes the fields(type and value) of DataRecord. func DecodeDataRecordFields(payload *bytes.Buffer, listFields []Field, record *[]DataField) error { - size := GetTemplateSize(listFields) - if payload.Len() < size { - return fmt.Errorf("decode dataset: there are fewer than %d bytes in the buffer", size) - } - *record = (*record)[:cap(*record)] if len(*record) < len(listFields) { *record = append(*record, make([]DataField, len(listFields)-len(*record))...) } + var ok bool for i, templateField := range listFields { - value := payload.Next(int(templateField.Length)) + + l := int(templateField.Length) + + if templateField.IsVariableLength() { + if l, ok = getVariableLength(payload); !ok { + return fmt.Errorf("decode DataRecordFields: invalid variable-length: %d", l) + } + } + + // XXX: Retaining a slice returned by Next() may be unsafe according to + // method's documentation as it may be invalidated by future Read call. + value := payload.Next(l) + if len(value) < l { + return fmt.Errorf("decode dataset: there are fewer than %d bytes in the buffer", l) + } (*record)[i].Type = templateField.Type (*record)[i].Value = value @@ -184,6 +195,31 @@ func DecodeDataRecordFields(payload *bytes.Buffer, listFields []Field, record *[ return nil } +func getVariableLength(payload *bytes.Buffer) (int, bool) { + b, err := payload.ReadByte() + if err != nil { + return 0, false + } + + // RFC 7011 Sec. 7.Variable-Length Information Element. + if b != 0xff { + return int(b), true + } + + // RFC 7011 Sec. 7.Variable-Length Information Element: + // The length may also be encoded into 3 octets before the Information + // Element, allowing the length of the Information Element to be greater + // than or equal to 255 octets. In this case, the first octet of the Length + // field MUST be 255, and the length is carried in the second and third + // octets. + length := payload.Next(2) + if len(length) < 2 { + return 0, false + } + + return int(binary.BigEndian.Uint16(length)), true +} + type ErrorTemplateNotFound struct { version uint16 obsDomainId uint32 @@ -281,24 +317,19 @@ func DecodeOptionsDataSet(payload *bytes.Buffer, fs *OptionsDataFlowSet, listFie func DecodeDataSet(payload *bytes.Buffer, listFields []Field, flowSet *DataFlowSet) error { flowSet.Records = flowSet.Records[:cap(flowSet.Records)] - listFieldsSize := GetTemplateSize(listFields) - i := 0 - for payload.Len() >= listFieldsSize { - payloadLim := bytes.NewBuffer(payload.Next(listFieldsSize)) - + for payload.Len() > 0 { if i >= len(flowSet.Records) { flowSet.Records = append(flowSet.Records, DataRecord{}) } datafields := &flowSet.Records[i].Values - if err := DecodeDataRecordFields(payloadLim, listFields, datafields); err != nil { + if err := DecodeDataRecordFields(payload, listFields, datafields); err != nil { return err } i++ } - flowSet.Records = flowSet.Records[:i] return nil } @@ -533,7 +564,9 @@ func (f *FlowMessage) DecodeIPFIXPacket(payload *bytes.Buffer, templates NetFlow ) for i := 0; i < int(f.PacketIPFIX.Length) && payload.Len() > 0; i++ { - f.fsheader.ReadFrom(payload) + if ok := f.fsheader.ReadFrom(payload); !ok { + return NewErrorDecodingNetFlow("error decoding packet: invalid FlowSet header.") + } nextrelpos := int(f.fsheader.Length) - flowSetHeaderSize if nextrelpos < 0 { diff --git a/decoders/netflow/netflow_test.go b/decoders/netflow/netflow_test.go index 6278336..8f5c5a2 100644 --- a/decoders/netflow/netflow_test.go +++ b/decoders/netflow/netflow_test.go @@ -222,10 +222,10 @@ func TestDecodeDataSet(t *testing.T) { fs := DataFlowSet{} // type and length of a single value in a netflow Data Record - nfListFields := []Field{{8, 4}, {225, 4}, {12, 4}, {226, 4}, {7, 2}, {227, 2}, {11, 2}, {228, 2}, {234, 4}, {4, 1}, {230, 1}, {323, 8}} + nfListFields := []Field{{8, 4, 0}, {225, 4, 0}, {12, 4, 0}, {226, 4, 0}, {7, 2, 0}, {227, 2, 0}, {11, 2, 0}, {228, 2, 0}, {234, 4, 0}, {4, 1, 0}, {230, 1, 0}, {323, 8, 0}} // type and length of a single value in ipfix Flow Data Record - ipxListFields := []Field{{8, 4}, {12, 4}, {5, 1}, {4, 1}, {7, 2}, {11, 2}, {32, 2}, {10, 4}, {16, 4}, {17, 4}, {18, 4}, {14, 4}, {1, 4}, {2, 4}, {22, 4}, {21, 4}, {15, 4}, {9, 1}, {13, 1}, {6, 1}, {60, 1}, {152, 8}, {153, 8}} + ipxListFields := []Field{{8, 4, 0}, {12, 4, 0}, {5, 1, 0}, {4, 1, 0}, {7, 2, 0}, {11, 2, 0}, {32, 2, 0}, {10, 4, 0}, {16, 4, 0}, {17, 4, 0}, {18, 4, 0}, {14, 4, 0}, {1, 4, 0}, {2, 4, 0}, {22, 4, 0}, {21, 4, 0}, {15, 4, 0}, {9, 1, 0}, {13, 1, 0}, {6, 1, 0}, {60, 1, 0}, {152, 8, 0}, {153, 8, 0}} // netflow data flowset nfData := bytes.NewBuffer(netflowTestPackets[1].Data[210:]) @@ -245,10 +245,10 @@ func TestDecodeDataSet(t *testing.T) { func TestDecodeDataSetUsingFields(t *testing.T) { // type and length of a single value in a netflow Data Record - nfListFields := []Field{{8, 4}, {225, 4}, {12, 4}, {226, 4}, {7, 2}, {227, 2}, {11, 2}, {228, 2}, {234, 4}, {4, 1}, {230, 1}, {323, 8}} + nfListFields := []Field{{8, 4, 0}, {225, 4, 0}, {12, 4, 0}, {226, 4, 0}, {7, 2, 0}, {227, 2, 0}, {11, 2, 0}, {228, 2, 0}, {234, 4, 0}, {4, 1, 0}, {230, 1, 0}, {323, 8, 0}} // type and length of a single value in ipfix Flow Data Record - ipxListFields := []Field{{8, 4}, {12, 4}, {5, 1}, {4, 1}, {7, 2}, {11, 2}, {32, 2}, {10, 4}, {16, 4}, {17, 4}, {18, 4}, {14, 4}, {1, 4}, {2, 4}, {22, 4}, {21, 4}, {15, 4}, {9, 1}, {13, 1}, {6, 1}, {60, 1}, {152, 8}, {153, 8}} + ipxListFields := []Field{{8, 4, 0}, {12, 4, 0}, {5, 1, 0}, {4, 1, 0}, {7, 2, 0}, {11, 2, 0}, {32, 2, 0}, {10, 4, 0}, {16, 4, 0}, {17, 4, 0}, {18, 4, 0}, {14, 4, 0}, {1, 4, 0}, {2, 4, 0}, {22, 4, 0}, {21, 4, 0}, {15, 4, 0}, {9, 1, 0}, {13, 1, 0}, {6, 1, 0}, {60, 1, 0}, {152, 8, 0}, {153, 8, 0}} dr := DataRecord{} diff --git a/decoders/netflow/packet.go b/decoders/netflow/packet.go index 9c9945f..81af788 100644 --- a/decoders/netflow/packet.go +++ b/decoders/netflow/packet.go @@ -114,6 +114,24 @@ type Field struct { // The length (in bytes) of the field. Length uint16 + + // IANA private enterprise number (PEN) of the authority defining the + // Information Element identifier in this Template Record. + // + // Note: it's valid only for v10(IPFIX). + EnterpriseNumber uint32 +} + +// IsVariableLength reports whether the Field is variable-length Information +// Element (RFC 7011 sec. 3.2. Field Specifier Format). +func (f *Field) IsVariableLength() bool { + return f.Length == 0xffff +} + +// ContainsEnterpriseNumber reports whether the Field is an enterprise-specific +// Information Element (RFC 7011 sec. 3.2. Field Specifier Format). +func (f *Field) ContainsEnterpriseNumber() bool { + return f.Type&0x8000 != 0 } // DataField contains the type and record value itself. @@ -161,9 +179,17 @@ func (f *Field) ReadFrom(b *bytes.Buffer) bool { if ok := utils.ReadUint16FromBuffer(b, &f.Type); !ok { return false } + if ok := utils.ReadUint16FromBuffer(b, &f.Length); !ok { return false } + + if f.ContainsEnterpriseNumber() { + if ok := utils.ReadUint32FromBuffer(b, &f.EnterpriseNumber); !ok { + return false + } + } + return true }