Feature/add call event parser

.gitignore

@@ -0,0 +1,2 @@
+running
+processed/

README.md

@@ -0,0 +1,44 @@
+# Call Event Parser
+
+## Introduction
+
+This application processes sets of call data in a specific CSV format. The data is read, validated and then uploaded to a database. Once a file has been successfully processed it moved to a process directory. Any records that can not be processed form files are logged to the syslog.
+
+This project is designed to be run as a back ground task.
+
+## Config
+
+The project is a CLI application and requires 9 arguments to be run. The variables that can be configured are:
+
+- Database Username
+- Database Password
+- Database Host
+- Database Port
+- Database Name
+- Database Table
+- Process directory path (Where files will be moved when processed)
+- Upload directory path (Directory where incoming cvs files are located)
+- File running path (Path to a temport file that stops this application running in more than one process at a time)
+
+
+## Build and Run
+
+To build this project run the following in the root of the project
+
+`go install`
+
+from you $GOPATH/bin directory run the application passingin the arguments
+
+`./call-event-parser username password host port db_name db_table "./processed/" "./uploaded" "/tmp/call_event_parse_running"`
+
+
+### Things to note
+
+The processed directory argument must include a '/' at the end. This is a bug that needs to be resolved. Make sure the application as read/write access to the `File running path` location.
+
+
+## Tests
+
+This project contains test which can be run from the project root using 
+
+`go test ./...`

call-event-file.go

@@ -0,0 +1,136 @@
+package main
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+)
+
+const (
+	DATE_TIME_FORMAT        = "2006-02-01 15:04:05"
+	COL_EVENT_DATETIME      = 0
+	COL_EVENT_ACTION        = 1
+	COL_CALL_REF            = 2
+	COL_EVENT_VAL           = 3
+	COL_EVENT_CURRENCY_CODE = 4
+)
+
+type CallEventFile struct {
+	filenamePath    string
+	validData       [][]string
+	recordErrors    []string
+	numberOfRecords int
+}
+
+func (f *CallEventFile) GetFilename() string {
+	parts := strings.Split(f.filenamePath, "/")
+
+	return parts[len(parts)-1:][0]
+}
+
+func (f *CallEventFile) RecordErrors() []string {
+	return f.recordErrors
+}
+
+func (f *CallEventFile) ValidData() [][]string {
+	return f.validData
+}
+
+func CreateCallEventFileFromRaw(rawData [][]string, filepath string) CallEventFile {
+	data := removeHeader(rawData)
+	validRecords := [][]string{}
+	errorRecords := []string{}
+
+	for i, row := range data {
+		// COL_EVENT_DATETIME
+		eventDateTimeValue := row[COL_EVENT_DATETIME]
+
+		if eventDateTimeValue == "" {
+			errorRecords = append(
+				errorRecords,
+				fmt.Sprintf("File: %#v. Record: %#v requires 'eventDatetime' value", filepath, i+1),
+			)
+			continue
+		}
+
+		_, err := time.Parse(DATE_TIME_FORMAT, eventDateTimeValue)
+		if err != nil {
+			errorRecords = append(
+				errorRecords,
+				fmt.Sprintf("File: %#v. Record: %#v date format must be yyyy-mm-dd hh:mm:ss", filepath, i+1),
+			)
+			continue
+		}
+
+		// COL_EVENT_ACTION
+		if row[COL_EVENT_ACTION] == "" {
+			errorRecords = append(
+				errorRecords,
+				fmt.Sprintf("File: %#v. Record: %#v requires 'eventAction' value", filepath, i+1),
+			)
+			continue
+		}
+
+		if l := len(row[COL_EVENT_ACTION]); l == 0 || l > 20 {
+			errorRecords = append(
+				errorRecords,
+				fmt.Sprintf("File: %#v. Record: %#v requires 'eventAction' to be between 1 - 20 in length", filepath, i+1),
+			)
+			continue
+		}
+
+		// COL_CALL_REF
+		if row[COL_CALL_REF] == "" {
+			errorRecords = append(
+				errorRecords,
+				fmt.Sprintf("File: %#v. Record: %#v requires 'callRef' value", filepath, i+1),
+			)
+			continue
+		}
+
+		if _, err := strconv.ParseInt(row[COL_CALL_REF], 10, 64); err != nil {
+			errorRecords = append(
+				errorRecords,
+				fmt.Sprintf("File: %#v. Record: %#v requires 'callRef' to be a valid integer", filepath, i+1),
+			)
+			continue
+		}
+
+		// COL_EVENT_VAL
+		if row[COL_EVENT_VAL] == "" {
+			row[COL_EVENT_VAL] = "0.00"
+		}
+
+		eventValue, err := strconv.ParseFloat(row[COL_EVENT_VAL], 64)
+
+		if err != nil {
+			errorRecords = append(
+				errorRecords,
+				fmt.Sprintf("File: %#v. Record: %#v requires 'eventValue' to be a valid float", filepath, i+1),
+			)
+			continue
+		}
+
+		// COL_EVENT_CURRENCY_CODE
+		if row[COL_EVENT_CURRENCY_CODE] == "" && eventValue > 0.0 {
+			errorRecords = append(
+				errorRecords,
+				fmt.Sprintf("File: %#v. Record: %#v requires 'eventCurrencyCode' if event value is more than 0.0", filepath, i+1),
+			)
+			continue
+		}
+
+		validRecords = append(validRecords, row)
+	}
+
+	return CallEventFile{filepath, validRecords, errorRecords, len(data)}
+}
+
+func removeHeader(rawCsv [][]string) [][]string {
+	if len(rawCsv) > 0 {
+		return rawCsv[1:]
+	}
+
+	return rawCsv
+}

call-event-file_test.go

@@ -0,0 +1,137 @@
+package main_test
+
+import (
+	"strings"
+	"testing"
+
+	parser "github.com/kaschula/call-event-parser"
+	. "github.com/stretchr/testify/assert"
+)
+
+func TestItReturnsAnErrorWhenNoEventDateTimeGiven(t *testing.T) {
+	rawCsv := [][]string{
+		[]string{"eventDatetime", "eventAction", "callRef", "eventValue", "eventCurrencyCode"},
+		[]string{""},
+	}
+
+	callFile := parser.CreateCallEventFileFromRaw(rawCsv, "path/to/file")
+
+	Equal(t, 0, len(callFile.ValidData()))
+	True(t, strings.Contains(callFile.RecordErrors()[0], "requires 'eventDatetime' value"))
+}
+
+func TestItReturnsAnErrorWhenDateFormatIsIncorrect(t *testing.T) {
+	rawCsv := [][]string{
+		[]string{"eventDatetime", "eventAction", "callRef", "eventValue", "eventCurrencyCode"},
+		[]string{"2012-01-02 12-01-30"},
+	}
+
+	callFile := parser.CreateCallEventFileFromRaw(rawCsv, "path/to/file")
+
+	Equal(t, 0, len(callFile.ValidData()))
+	True(t, strings.Contains(callFile.RecordErrors()[0], "date format must be yyyy-mm-dd hh:mm:ss"))
+}
+
+func TestItReturnsAnErrorWhenEventActionIsMissing(t *testing.T) {
+	rawCsv := [][]string{
+		[]string{"eventDatetime", "eventAction", "callRef", "eventValue", "eventCurrencyCode"},
+		[]string{"2012-01-02 12:01:30", ""},
+	}
+
+	callFile := parser.CreateCallEventFileFromRaw(rawCsv, "path/to/file")
+
+	Equal(t, 0, len(callFile.ValidData()))
+	True(t, strings.Contains(callFile.RecordErrors()[0], "requires 'eventAction' value"))
+}
+
+func TestItReturnsAnErrorWhenEventActionIsMoreThan20CharactersLong(t *testing.T) {
+	rawCsv := [][]string{
+		[]string{"eventDatetime", "eventAction", "callRef", "eventValue", "eventCurrencyCode"},
+		[]string{"2012-01-02 12:01:30", "abcdefghijklmnopqrstu"},
+	}
+
+	callFile := parser.CreateCallEventFileFromRaw(rawCsv, "path/to/file")
+
+	Equal(t, 0, len(callFile.ValidData()))
+	True(t, strings.Contains(callFile.RecordErrors()[0], "requires 'eventAction' to be between 1 - 20 in length"))
+}
+
+func TestItReturnsAnErrorWhenCallRefIsMissing(t *testing.T) {
+	rawCsv := [][]string{
+		[]string{"eventDatetime", "eventAction", "callRef", "eventValue", "eventCurrencyCode"},
+		[]string{"2012-01-02 12:01:30", "sale", ""},
+	}
+
+	callFile := parser.CreateCallEventFileFromRaw(rawCsv, "path/to/file")
+
+	Equal(t, 0, len(callFile.ValidData()))
+	True(t, strings.Contains(callFile.RecordErrors()[0], "requires 'callRef' value"))
+}
+
+func TestItReturnsAnErrorWhenCallRefIsNotAValidInteger(t *testing.T) {
+	rawCsv := [][]string{
+		[]string{"eventDatetime", "eventAction", "callRef", "eventValue", "eventCurrencyCode"},
+		[]string{"2012-01-02 12:01:30", "sale", "notAnInt"},
+	}
+
+	callFile := parser.CreateCallEventFileFromRaw(rawCsv, "path/to/file")
+
+	Equal(t, 0, len(callFile.ValidData()))
+	True(t, strings.Contains(callFile.RecordErrors()[0], "requires 'callRef' to be a valid integer"))
+}
+
+func TestItReturnsAnErrorWhenEventValueIsNoteAValidFloat(t *testing.T) {
+	rawCsv := [][]string{
+		[]string{"eventDatetime", "eventAction", "callRef", "eventValue", "eventCurrencyCode"},
+		[]string{"2012-01-02 12:01:30", "sale", "1234", "2.1a"},
+	}
+
+	callFile := parser.CreateCallEventFileFromRaw(rawCsv, "path/to/file")
+
+	Equal(t, 0, len(callFile.ValidData()))
+	True(t, strings.Contains(callFile.RecordErrors()[0], "requires 'eventValue' to be a valid float"))
+}
+
+func TestItReturnsAnErrorWhenEventCurrencyCodeIsNotSetAndEventValueIsMoreThanZero(t *testing.T) {
+	rawCsv := [][]string{
+		[]string{"eventDatetime", "eventAction", "callRef", "eventValue", "eventCurrencyCode"},
+		[]string{"2012-01-02 12:01:30", "sale", "1234", "1.0", ""},
+	}
+
+	callFile := parser.CreateCallEventFileFromRaw(rawCsv, "path/to/file")
+
+	Equal(t, 0, len(callFile.ValidData()))
+	True(t, strings.Contains(callFile.RecordErrors()[0], "requires 'eventCurrencyCode' if event value is more than 0.0"))
+}
+
+func TestItDoesNotReturnAnErrorWhenEventValueIsZeroAndNoCurrencyCodeSet(t *testing.T) {
+	rawCsv := [][]string{
+		[]string{"eventDatetime", "eventAction", "callRef", "eventValue", "eventCurrencyCode"},
+		[]string{"2012-01-02 12:01:30", "sale", "1234", "0.0", ""},
+	}
+
+	callFile := parser.CreateCallEventFileFromRaw(rawCsv, "path/to/file")
+
+	Equal(t, 0, len(callFile.RecordErrors()))
+	Equal(t, 1, len(callFile.ValidData()))
+}
+
+func TestItReturnsItSetsTheEventValueToZeroFloatIfEmpty(t *testing.T) {
+	rawCsv := [][]string{
+		[]string{"eventDatetime", "eventAction", "callRef", "eventValue", "eventCurrencyCode"},
+		[]string{"2012-01-02 12:01:30", "sale", "1234", "", "GBP"},
+	}
+
+	callFile := parser.CreateCallEventFileFromRaw(rawCsv, "path/to/file")
+
+	Equal(t, 1, len(callFile.ValidData()))
+	Equal(t, "0.00", callFile.ValidData()[0][parser.COL_EVENT_VAL])
+}
+
+func TestItReturnsTheFileName(t *testing.T) {
+	rawCsv := [][]string{}
+
+	callFile := parser.CreateCallEventFileFromRaw(rawCsv, "path/to/file.txt")
+
+	Equal(t, "file.txt", callFile.GetFilename())
+}