Skip to content

Commit 4b9c10e

Browse files
committed
Add argument in parseCsvWithHeader function
1 parent 1eb027c commit 4b9c10e

6 files changed

+37
-16
lines changed

builtins.go

+27-12
Original file line numberDiff line numberDiff line change
@@ -1516,6 +1516,7 @@ func builtinParseYAML(i *interpreter, str value) (value, error) {
15161516
func builtinParseCSVWithHeader(i *interpreter, arguments []value) (value, error) {
15171517
strv := arguments[0]
15181518
dv := arguments[1]
1519+
odhv := arguments[2]
15191520

15201521
sval, err := i.getString(strv)
15211522
if err != nil {
@@ -1536,6 +1537,15 @@ func builtinParseCSVWithHeader(i *interpreter, arguments []value) (value, error)
15361537
d = rune(ds[0]) // conversion to rune
15371538
}
15381539

1540+
odh := true // default value for overwrite_duplicate_headers
1541+
if odhv.getType() != nullType {
1542+
odhval, err := i.getBoolean(odhv)
1543+
if err != nil {
1544+
return nil, err
1545+
}
1546+
odh = odhval.value
1547+
}
1548+
15391549
json := make([]interface{}, 0)
15401550
var keys []string
15411551

@@ -1552,14 +1562,19 @@ func builtinParseCSVWithHeader(i *interpreter, arguments []value) (value, error)
15521562
}
15531563

15541564
if row == 0 { // consider first row as header
1555-
// detect and handle duplicate headers
1556-
keyCount := map[string]int{}
1557-
for _, k := range record {
1558-
keyCount[k]++
1559-
if c := keyCount[k]; c > 1 {
1560-
keys = append(keys, fmt.Sprintf("%s__%d", k, c-1))
1561-
} else {
1562-
keys = append(keys, k)
1565+
if odh {
1566+
// Overwrite duplicate headers
1567+
keys = record
1568+
} else {
1569+
// detect and handle duplicate headers
1570+
keyCount := map[string]int{}
1571+
for _, k := range record {
1572+
keyCount[k]++
1573+
if c := keyCount[k]; c > 1 {
1574+
keys = append(keys, fmt.Sprintf("%s__%d", k, c-1))
1575+
} else {
1576+
keys = append(keys, k)
1577+
}
15631578
}
15641579
}
15651580
} else {
@@ -2262,12 +2277,12 @@ func builtinAvg(i *interpreter, arrv value) (value, error) {
22622277
if err != nil {
22632278
return nil, err
22642279
}
2265-
2280+
22662281
len := float64(arr.length())
22672282
if len == 0 {
22682283
return nil, i.Error("Cannot calculate average of an empty array.")
22692284
}
2270-
2285+
22712286
sumValue, err := builtinSum(i, arrv)
22722287
if err != nil {
22732288
return nil, err
@@ -2277,7 +2292,7 @@ func builtinAvg(i *interpreter, arrv value) (value, error) {
22772292
return nil, err
22782293
}
22792294

2280-
avg := sum.value/len
2295+
avg := sum.value / len
22812296
return makeValueNumber(avg), nil
22822297
}
22832298

@@ -2685,7 +2700,7 @@ var funcBuiltins = buildBuiltinMap([]builtin{
26852700
&unaryBuiltin{name: "parseInt", function: builtinParseInt, params: ast.Identifiers{"str"}},
26862701
&unaryBuiltin{name: "parseJson", function: builtinParseJSON, params: ast.Identifiers{"str"}},
26872702
&unaryBuiltin{name: "parseYaml", function: builtinParseYAML, params: ast.Identifiers{"str"}},
2688-
&generalBuiltin{name: "parseCsvWithHeader", function: builtinParseCSVWithHeader, params: []generalBuiltinParameter{{name: "str"}, {name: "delimiter", defaultValue: &nullValue}}},
2703+
&generalBuiltin{name: "parseCsvWithHeader", function: builtinParseCSVWithHeader, params: []generalBuiltinParameter{{name: "str"}, {name: "delimiter", defaultValue: &nullValue}, {name: "overwrite_duplicate_headers", defaultValue: &nullValue}}},
26892704
&generalBuiltin{name: "manifestCsv", function: builtinManifestCsv, params: []generalBuiltinParameter{{name: "json"}, {name: "headers", defaultValue: &nullValue}}},
26902705
&generalBuiltin{name: "manifestJsonEx", function: builtinManifestJSONEx, params: []generalBuiltinParameter{{name: "value"}, {name: "indent"},
26912706
{name: "newline", defaultValue: &valueFlatString{value: []rune("\n")}},

linter/internal/types/stdlib.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ func prepareStdlib(g *typeGraph) {
111111
"parseHex": g.newSimpleFuncType(numberType, "str"),
112112
"parseJson": g.newSimpleFuncType(jsonType, "str"),
113113
"parseYaml": g.newSimpleFuncType(jsonType, "str"),
114-
"parseCsvWithHeader": g.newFuncType(jsonType, []ast.Parameter{required("str"), optional("delimiter")}),
114+
"parseCsvWithHeader": g.newFuncType(jsonType, []ast.Parameter{required("str"), optional("delimiter"), optional("overwrite_duplicate_headers")}),
115115
"encodeUTF8": g.newSimpleFuncType(numberArrayType, "str"),
116116
"decodeUTF8": g.newSimpleFuncType(stringType, "arr"),
117117

@@ -154,7 +154,7 @@ func prepareStdlib(g *typeGraph) {
154154
"minArray": g.newFuncType(anyArrayType, []ast.Parameter{required("arr"), optional("keyF")}),
155155
"maxArray": g.newFuncType(anyArrayType, []ast.Parameter{required("arr"), optional("keyF")}),
156156
"contains": g.newSimpleFuncType(boolType, "arr", "elem"),
157-
"avg": g.newSimpleFuncType(numberType, "arr"),
157+
"avg": g.newSimpleFuncType(numberType, "arr"),
158158
"all": g.newSimpleFuncType(boolArrayType, "arr"),
159159
"any": g.newSimpleFuncType(boolArrayType, "arr"),
160160
"remove": g.newSimpleFuncType(anyArrayType, "arr", "elem"),
+1-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
[
22
{
3-
"head1": "val1",
4-
"head1__1": "val2"
3+
"head1": "val2"
54
}
65
]
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
[
2+
{
3+
"head1": "val1",
4+
"head1__1": "val2"
5+
}
6+
]
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
std.parseCsvWithHeader("head1,head1\nval1,val2", overwrite_duplicate_headers = false)

testdata/builtinParseCsvWithHeader4.linter.golden

Whitespace-only changes.

0 commit comments

Comments
 (0)