complete fixed-length fileformat and wire into omni extension v21 (#119)

Author: jf-tech

extensions/omniv21/fileformat/fixedlength/format.go

@@ -129,9 +129,8 @@ func (f *fixedLengthFileFormat) validateColumns(cols []*columnDecl) error {
 
 func (f *fixedLengthFileFormat) CreateFormatReader(
 	name string, r io.Reader, runtime interface{}) (fileformat.FormatReader, error) {
-	// TODO
-	_ = runtime.(*fixedLengthFormatRuntime)
-	return nil, nil
+	rt := runtime.(*fixedLengthFormatRuntime)
+	return NewReader(name, r, rt.Decl, rt.XPath)
 }
 
 func (f *fixedLengthFileFormat) FmtErr(format string, args ...interface{}) error {

extensions/omniv21/fileformat/fixedlength/format_test.go

@@ -1,16 +1,19 @@
 package fixedlength
 
 import (
+	"io"
 	"strings"
 	"testing"
 
 	"github.com/bradleyjkemp/cupaloy"
 	"github.com/jf-tech/go-corelib/jsons"
 	"github.com/jf-tech/go-corelib/strs"
+	"github.com/jf-tech/go-corelib/testlib"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/jf-tech/omniparser/errs"
 	"github.com/jf-tech/omniparser/extensions/omniv21/transform"
+	"github.com/jf-tech/omniparser/idr"
 )
 
 func TestValidateSchema(t *testing.T) {
@@ -304,10 +307,29 @@ func TestValidateSchema(t *testing.T) {
 }
 
 func TestCreateFormatReader(t *testing.T) {
-	_, err := NewFixedLengthFileFormat("test").CreateFormatReader(
-		"test-input",
-		strings.NewReader("TODO"),
-		&fixedLengthFormatRuntime{})
+	r, err := NewFixedLengthFileFormat("test").CreateFormatReader(
+		"test",
+		strings.NewReader("abcd\n1234\n"),
+		&fixedLengthFormatRuntime{
+			Decl: &fileDecl{
+				Envelopes: []*envelopeDecl{
+					{
+						Name:   strs.StrPtr("env1"),
+						ByRows: testlib.IntPtr(2),
+						Columns: []*columnDecl{
+							{Name: "letters", StartPos: 1, Length: 3, LinePattern: strs.StrPtr("^[a-z]")},
+							{Name: "numerics", StartPos: 1, Length: 3, LinePattern: strs.StrPtr("^[0-9]")},
+						},
+					},
+				},
+			},
+		})
+	assert.NoError(t, err)
+	n, err := r.Read()
 	assert.NoError(t, err)
-	// TODO
+	assert.Equal(t, `{"letters":"abc","numerics":"123"}`, idr.JSONify2(n))
+	r.Release(n)
+	n, err = r.Read()
+	assert.Equal(t, io.EOF, err)
+	assert.Nil(t, n)
 }

extensions/omniv21/fileformat/fixedlength/reader.go

@@ -2,8 +2,10 @@ package fixedlength
 
 import (
 	"bufio"
+	"errors"
 	"fmt"
 	"io"
+	"strings"
 
 	"github.com/antchfx/xpath"
 	"github.com/jf-tech/go-corelib/caches"
@@ -170,6 +172,42 @@ readEnvelope:
 	return node, err
 }
 
+func (r *reader) Release(n *idr.Node) {
+	if r.target == n {
+		r.target = nil
+	}
+	idr.RemoveAndReleaseTree(n)
+}
+
+func (r *reader) IsContinuableError(err error) bool {
+	return !IsErrInvalidEnvelope(err) && err != io.EOF
+}
+
+func (r *reader) FmtErr(format string, args ...interface{}) error {
+	return errors.New(r.fmtErrStr(format, args...))
+}
+
 func (r *reader) fmtErrStr(format string, args ...interface{}) string {
 	return fmt.Sprintf("input '%s' line %d: %s", r.inputName, r.line, fmt.Sprintf(format, args...))
 }
+
+// NewReader creates an FormatReader for fixed-length file format.
+func NewReader(inputName string, r io.Reader, decl *fileDecl, xpathStr string) (*reader, error) {
+	var expr *xpath.Expr
+	var err error
+	xpathStr = strings.TrimSpace(xpathStr)
+	if xpathStr != "" && xpathStr != "." {
+		expr, err = caches.GetXPathExpr(xpathStr)
+		if err != nil {
+			return nil, fmt.Errorf("invalid xpath '%s', err: %s", xpathStr, err.Error())
+		}
+	}
+	return &reader{
+		inputName: inputName,
+		r:         bufio.NewReader(r),
+		decl:      decl,
+		xpath:     expr,
+		root:      idr.CreateNode(idr.DocumentNode, "#root"),
+		line:      1,
+	}, nil
+}

extensions/omniv21/fileformat/fixedlength/reader_test.go

@@ -2,6 +2,7 @@ package fixedlength
 
 import (
 	"bufio"
+	"encoding/json"
 	"errors"
 	"io"
 	"strings"
@@ -455,3 +456,38 @@ func TestRead_ByHeaderFooter(t *testing.T) {
 	assert.Equal(t, io.EOF, err)
 	assert.Nil(t, n)
 }
+
+func TestRelease(t *testing.T) {
+	var decl fileDecl
+	err := json.Unmarshal([]byte(`
+		{
+			"envelopes": [
+				{ "name": "env1", "columns": [ { "name": "col1", "start_pos": 1, "length": 3 } ] }
+			]
+		}`), &decl)
+	assert.NoError(t, err)
+	r, err := NewReader("test", strings.NewReader("abcd\n12\n#$%^\n"), &decl, ".[starts-with(., '1')]")
+	assert.NoError(t, err)
+	n, err := r.Read()
+	assert.NoError(t, err)
+	assert.Equal(t, `{"col1":"12"}`, idr.JSONify2(n))
+	assert.Equal(t, `{"env1":{"col1":"12"}}`, idr.JSONify2(r.root))
+	assert.True(t, n == r.target)
+	r.Release(n)
+	assert.Nil(t, r.target)
+	assert.Equal(t, `{}`, idr.JSONify2(r.root))
+}
+
+func TestIsContinuableError(t *testing.T) {
+	r := &reader{}
+	assert.True(t, r.IsContinuableError(r.FmtErr("some error")))
+	assert.False(t, r.IsContinuableError(ErrInvalidEnvelope("invalid envelope")))
+	assert.False(t, r.IsContinuableError(io.EOF))
+}
+
+func TestNewReader(t *testing.T) {
+	r, err := NewReader("test", nil, nil, "[invalid")
+	assert.Error(t, err)
+	assert.Equal(t, `invalid xpath '[invalid', err: expression must evaluate to a node-set`, err.Error())
+	assert.Nil(t, r)
+}

extensions/omniv21/schemahandler.go

@@ -8,6 +8,7 @@ import (
 	"github.com/jf-tech/omniparser/extensions/omniv21/fileformat"
 	"github.com/jf-tech/omniparser/extensions/omniv21/fileformat/csv"
 	"github.com/jf-tech/omniparser/extensions/omniv21/fileformat/edi"
+	"github.com/jf-tech/omniparser/extensions/omniv21/fileformat/fixedlength"
 	"github.com/jf-tech/omniparser/extensions/omniv21/fileformat/json"
 	"github.com/jf-tech/omniparser/extensions/omniv21/fileformat/xml"
 	"github.com/jf-tech/omniparser/extensions/omniv21/transform"
@@ -84,10 +85,10 @@ func customParseFuncs(ctx *schemahandler.CreateCtx) transform.CustomParseFuncs {
 func fileFormats(ctx *schemahandler.CreateCtx) []fileformat.FileFormat {
 	formats := []fileformat.FileFormat{
 		csv.NewCSVFileFormat(ctx.Name),
+		edi.NewEDIFileFormat(ctx.Name),
+		fixedlength.NewFixedLengthFileFormat(ctx.Name),
 		json.NewJSONFileFormat(ctx.Name),
 		xml.NewXMLFileFormat(ctx.Name),
-		edi.NewEDIFileFormat(ctx.Name),
-		// TODO more built-in file formats to come.
 	}
 	if ctx.CreateParams == nil {
 		return formats