Skip to content

Commit

Permalink
read and check ofx 2.x file header
Browse files Browse the repository at this point in the history
  • Loading branch information
@mayswind
mayswind committed Oct 31, 2024
1 parent d174e99 commit ac29f0b
Show file tree
Hide file tree
Showing 4 changed files with 651 additions and 465 deletions.
108 changes: 93 additions & 15 deletions pkg/converters/ofx/ofx_data_reader.go
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,25 @@
package ofx

import (
"bufio"
"bytes"
"encoding/xml"
"regexp"
"strings"

"golang.org/x/net/html/charset"

"github.com/mayswind/ezbookkeeping/pkg/core"
"github.com/mayswind/ezbookkeeping/pkg/errs"
"github.com/mayswind/ezbookkeeping/pkg/log"
)

var ofx2HeaderPattern = regexp.MustCompile("<\\?OFX( +[A-Z]+=\"[^=]*\")* *\\?>")
var ofx2HeaderAttributePattern = regexp.MustCompile(" +([A-Z]+)=\"([^=]*)\"")

// ofxFileReader defines the structure of open financial exchange (ofx) file reader
type ofxFileReader struct {
fileHeader *ofxFileHeader
xmlDecoder *xml.Decoder
}

Expand All @@ -25,27 +33,97 @@ func (r *ofxFileReader) read(ctx core.Context) (*ofxFile, error) {
return nil, err
}

file.FileHeader = r.fileHeader

return file, nil
}

func createNewOFXFileReader(data []byte) (*ofxFileReader, error) {
if len(data) > 5 && data[0] == 0x3C && data[1] == 0x3F && data[2] == 0x78 && data[3] == 0x6D && data[4] == 0x6C { // ofx 2.x starts with <?xml
xmlDecoder := xml.NewDecoder(bytes.NewReader(data))
xmlDecoder.CharsetReader = charset.NewReaderLabel

return &ofxFileReader{
xmlDecoder: xmlDecoder,
}, nil
} else if len(data) > 13 && string(data[0:13]) == "OFXHEADER:100" { // ofx 1.x starts with OFXHEADER:100
func createNewOFXFileReader(ctx core.Context, data []byte) (*ofxFileReader, error) {
if len(data) > 5 && string(data[0:5]) == "<?xml" { // ofx 2.x starts with <?xml
return createNewOFX2FileReader(ctx, data, true)
} else if len(data) > 10 && string(data[0:10]) == "OFXHEADER:" { // ofx 1.x starts with OFXHEADER:

} else if len(data) > 5 && string(data[0:5]) == "<OFX>" { // no ofx header
xmlDecoder := xml.NewDecoder(bytes.NewReader(data))
xmlDecoder.CharsetReader = charset.NewReaderLabel

return &ofxFileReader{
xmlDecoder: xmlDecoder,
}, nil
return createNewOFX2FileReader(ctx, data, false)
}

return nil, errs.ErrInvalidOFXFile
}

func createNewOFX2FileReader(ctx core.Context, data []byte, withHeader bool) (*ofxFileReader, error) {
var fileHeader *ofxFileHeader = nil
var err error

if withHeader {
fileHeader, err = readOFX2FileHeader(ctx, data)

if err != nil {
return nil, err
}

if fileHeader.OFXDeclarationVersion != ofxVersion2 {
log.Errorf(ctx, "[ofx_data_reader.createNewOFX2FileReader] cannot parse ofx 2.x file header, because declaration version is \"%s\"", fileHeader.OFXDeclarationVersion)
return nil, errs.ErrInvalidOFXFile
}
}

xmlDecoder := xml.NewDecoder(bytes.NewReader(data))
xmlDecoder.CharsetReader = charset.NewReaderLabel

return &ofxFileReader{
fileHeader: fileHeader,
xmlDecoder: xmlDecoder,
}, nil
}

func readOFX2FileHeader(ctx core.Context, data []byte) (fileHeader *ofxFileHeader, err error) {
reader := bytes.NewReader(data)
scanner := bufio.NewScanner(reader)
fileHeader = &ofxFileHeader{}
headerLine := ""

for scanner.Scan() {
line := scanner.Text()

ofxHeaderStartIndex := strings.Index(line, "<?OFX ")

if ofxHeaderStartIndex >= 0 {
headerLine = ofx2HeaderPattern.FindString(line)
break
}
}

if headerLine == "" {
log.Errorf(ctx, "[ofx_data_reader.readOFX2FileHeader] cannot find ofx 2.x file header")
return nil, errs.ErrInvalidOFXFile
}

headerAttributes := ofx2HeaderAttributePattern.FindAllStringSubmatch(headerLine, -1)

for _, attributeItems := range headerAttributes {
if len(attributeItems) != 3 {
log.Warnf(ctx, "[ofx_data_reader.readOFX2FileHeader] cannot parse line in ofx 2.x file header, because item is \"%s\"", attributeItems)
continue
}

name := attributeItems[1]
value := attributeItems[2]

if name == "OFXHEADER" {
fileHeader.OFXDeclarationVersion = oFXDeclarationVersion(value)
} else if name == "VERSION" {
fileHeader.OFXDataVersion = value
} else if name == "SECURITY" {
fileHeader.Security = value
} else if name == "OLDFILEUID" {
fileHeader.OldFileUid = value
} else if name == "NEWFILEUID" {
fileHeader.NewFileUid = value
} else {
log.Warnf(ctx, "[ofx_data_reader.readOFX2FileHeader] cannot parse unknown header line in ofx 2.x file header, because item is \"%s\"", attributeItems)
continue
}
}

return fileHeader, nil
}
188 changes: 148 additions & 40 deletions pkg/converters/ofx/ofx_data_reader_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,38 +6,100 @@ import (
"github.com/stretchr/testify/assert"

"github.com/mayswind/ezbookkeeping/pkg/core"
"github.com/mayswind/ezbookkeeping/pkg/errs"
)

func TestCreateNewOFXFileReader_OFX2(t *testing.T) {
context := core.NewNullContext()
reader, err := createNewOFXFileReader([]byte(
"<?xml version=\"1.0\" encoding=\"US-ASCII\"?>" +
"<?OFX OFXHEADER=\"200\" VERSION=\"211\" SECURITY=\"NONE\" OLDFILEUID=\"NONE\" NEWFILEUID=\"NONE\"?>" +
"<OFX>" +
" <BANKMSGSRSV1>" +
" <STMTTRNRS>" +
" <STMTRS>" +
" <CURDEF>CNY</CURDEF>" +
" <BANKACCTFROM>" +
" <ACCTID>123</ACCTID>" +
" </BANKACCTFROM>" +
" <BANKTRANLIST>" +
" <STMTTRN>" +
" <TRNTYPE>DEP</TRNTYPE>" +
" <DTPOSTED>20240901012345.000[+8:CST]</DTPOSTED>" +
" <TRNAMT>123.45</TRNAMT>" +
" </STMTTRN>" +
" </BANKTRANLIST>" +
" </STMTRS>" +
" </STMTTRNRS>" +
" </BANKMSGSRSV1>" +
reader, err := createNewOFXFileReader(context, []byte(
"<?xml version=\"1.0\" encoding=\"US-ASCII\"?>\n"+
"<?OFX OFXHEADER=\"200\" VERSION=\"211\" SECURITY=\"NONE\" OLDFILEUID=\"NONE\" NEWFILEUID=\"NONE\"?>\n"+
"<OFX>\n"+
" <BANKMSGSRSV1>\n"+
" <STMTTRNRS>\n"+
" <STMTRS>\n"+
" <CURDEF>CNY</CURDEF>\n"+
" <BANKACCTFROM>\n"+
" <ACCTID>123</ACCTID>\n"+
" </BANKACCTFROM>\n"+
" <BANKTRANLIST>\n"+
" <STMTTRN>\n"+
" <TRNTYPE>DEP</TRNTYPE>\n"+
" <DTPOSTED>20240901012345.000[+8:CST]</DTPOSTED>\n"+
" <TRNAMT>123.45</TRNAMT>\n"+
" </STMTTRN>\n"+
" </BANKTRANLIST>\n"+
" </STMTRS>\n"+
" </STMTTRNRS>\n"+
" </BANKMSGSRSV1>\n"+
"</OFX>"))

assert.Nil(t, err)

ofxFile, err := reader.read(context)
assert.Nil(t, err)
assert.NotNil(t, ofxFile)

assert.NotNil(t, ofxFile.FileHeader)
assert.Equal(t, ofxVersion2, ofxFile.FileHeader.OFXDeclarationVersion)
assert.Equal(t, "211", ofxFile.FileHeader.OFXDataVersion)
assert.Equal(t, "NONE", ofxFile.FileHeader.Security)
assert.Equal(t, "NONE", ofxFile.FileHeader.OldFileUid)
assert.Equal(t, "NONE", ofxFile.FileHeader.NewFileUid)

assert.NotNil(t, ofxFile.BankMessageResponseV1)
assert.NotNil(t, ofxFile.BankMessageResponseV1.StatementTransactionResponse)
assert.NotNil(t, ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse)

assert.Equal(t, "CNY", ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse.DefaultCurrency)

assert.NotNil(t, ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse.AccountFrom)
assert.Equal(t, "123", ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse.AccountFrom.AccountId)

assert.Equal(t, 1, len(ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse.TransactionList.StatementTransactions))
assert.Equal(t, ofxDepositTransaction, ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse.TransactionList.StatementTransactions[0].TransactionType)
assert.Equal(t, "20240901012345.000[+8:CST]", ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse.TransactionList.StatementTransactions[0].PostedDate)
assert.Equal(t, "123.45", ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse.TransactionList.StatementTransactions[0].Amount)
}

func TestCreateNewOFXFileReader_OFX2WithoutBreakLine(t *testing.T) {
context := core.NewNullContext()
reader, err := createNewOFXFileReader(context, []byte(
"<?xml version=\"1.0\" encoding=\"US-ASCII\"?>"+
"<?OFX OFXHEADER=\"200\" VERSION=\"211\" SECURITY=\"NONE\" OLDFILEUID=\"NONE\" NEWFILEUID=\"NONE\"?>"+
"<OFX>"+
" <BANKMSGSRSV1>"+
" <STMTTRNRS>"+
" <STMTRS>"+
" <CURDEF>CNY</CURDEF>"+
" <BANKACCTFROM>"+
" <ACCTID>123</ACCTID>"+
" </BANKACCTFROM>"+
" <BANKTRANLIST>"+
" <STMTTRN>"+
" <TRNTYPE>DEP</TRNTYPE>"+
" <DTPOSTED>20240901012345.000[+8:CST]</DTPOSTED>"+
" <TRNAMT>123.45</TRNAMT>"+
" </STMTTRN>"+
" </BANKTRANLIST>"+
" </STMTRS>"+
" </STMTTRNRS>"+
" </BANKMSGSRSV1>"+
"</OFX>"))

assert.Nil(t, err)

ofxFile, err := reader.read(context)
assert.Nil(t, err)
assert.NotNil(t, ofxFile)

assert.NotNil(t, ofxFile.FileHeader)
assert.Equal(t, ofxVersion2, ofxFile.FileHeader.OFXDeclarationVersion)
assert.Equal(t, "211", ofxFile.FileHeader.OFXDataVersion)
assert.Equal(t, "NONE", ofxFile.FileHeader.Security)
assert.Equal(t, "NONE", ofxFile.FileHeader.OldFileUid)
assert.Equal(t, "NONE", ofxFile.FileHeader.NewFileUid)

assert.NotNil(t, ofxFile.BankMessageResponseV1)
assert.NotNil(t, ofxFile.BankMessageResponseV1.StatementTransactionResponse)
assert.NotNil(t, ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse)
Expand All @@ -53,34 +115,80 @@ func TestCreateNewOFXFileReader_OFX2(t *testing.T) {
assert.Equal(t, "123.45", ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse.TransactionList.StatementTransactions[0].Amount)
}

func TestCreateNewOFXFileReader_OFX2WithoutOFXHeader(t *testing.T) {
context := core.NewNullContext()
_, err := createNewOFXFileReader(context, []byte(
"<?xml version=\"1.0\" encoding=\"US-ASCII\"?>"+
"<OFX>"+
"</OFX>"))

assert.EqualError(t, err, errs.ErrInvalidOFXFile.Message)
}

func TestCreateNewOFXFileReader_OFX2WithInvalidHeaderVersion(t *testing.T) {
context := core.NewNullContext()
_, err := createNewOFXFileReader(context, []byte(
"<?xml version=\"1.0\" encoding=\"US-ASCII\"?>"+
"<?OFX OFXHEADER=\"100\" VERSION=\"211\" SECURITY=\"NONE\" OLDFILEUID=\"NONE\" NEWFILEUID=\"NONE\"?>"+
"<OFX>"+
"</OFX>"))

assert.EqualError(t, err, errs.ErrInvalidOFXFile.Message)
}

func TestCreateNewOFXFileReader_OFX2WithInvalidHeader(t *testing.T) {
context := core.NewNullContext()
_, err := createNewOFXFileReader(context, []byte(
"<?xml version=\"1.0\" encoding=\"US-ASCII\"?>"+
"<?OFX?>"+
"<OFX>"+
"</OFX>"))

_, err = createNewOFXFileReader(context, []byte(
"<?xml version=\"1.0\" encoding=\"US-ASCII\"?>"+
"<?OFX OFXHEADER=100?>"+
"<OFX>"+
"</OFX>"))

assert.EqualError(t, err, errs.ErrInvalidOFXFile.Message)
_, err = createNewOFXFileReader(context, []byte(
"<?xml version=\"1.0\" encoding=\"US-ASCII\"?>"+
"<?OFX OFXHEADER=\"100\" VERSION=\"211\" SECURITY=\"NONE\" OLDFILEUID=\"NONE\" NEWFILEUID=\"NONE\" test=\"\"?>"+
"<OFX>"+
"</OFX>"))
assert.EqualError(t, err, errs.ErrInvalidOFXFile.Message)
}

func TestCreateNewOFXFileReader_OFX2WithoutAnyHeader(t *testing.T) {
context := core.NewNullContext()
reader, err := createNewOFXFileReader([]byte(
"<OFX>" +
" <BANKMSGSRSV1>" +
" <STMTTRNRS>" +
" <STMTRS>" +
" <CURDEF>CNY</CURDEF>" +
" <BANKACCTFROM>" +
" <ACCTID>123</ACCTID>" +
" </BANKACCTFROM>" +
" <BANKTRANLIST>" +
" <STMTTRN>" +
" <TRNTYPE>DEP</TRNTYPE>" +
" <DTPOSTED>20240901012345.000[+8:CST]</DTPOSTED>" +
" <TRNAMT>123.45</TRNAMT>" +
" </STMTTRN>" +
" </BANKTRANLIST>" +
" </STMTRS>" +
" </STMTTRNRS>" +
" </BANKMSGSRSV1>" +
reader, err := createNewOFXFileReader(context, []byte(
"<OFX>\n"+
" <BANKMSGSRSV1>\n"+
" <STMTTRNRS>\n"+
" <STMTRS>\n"+
" <CURDEF>CNY</CURDEF>\n"+
" <BANKACCTFROM>\n"+
" <ACCTID>123</ACCTID>\n"+
" </BANKACCTFROM>\n"+
" <BANKTRANLIST>\n"+
" <STMTTRN>\n"+
" <TRNTYPE>DEP</TRNTYPE>\n"+
" <DTPOSTED>20240901012345.000[+8:CST]</DTPOSTED>\n"+
" <TRNAMT>123.45</TRNAMT>\n"+
" </STMTTRN>\n"+
" </BANKTRANLIST>\n"+
" </STMTRS>\n"+
" </STMTTRNRS>\n"+
" </BANKMSGSRSV1>\n"+
"</OFX>"))

assert.Nil(t, err)

ofxFile, err := reader.read(context)
assert.Nil(t, err)
assert.NotNil(t, ofxFile)
assert.Nil(t, ofxFile.FileHeader)

assert.NotNil(t, ofxFile.BankMessageResponseV1)
assert.NotNil(t, ofxFile.BankMessageResponseV1.StatementTransactionResponse)
assert.NotNil(t, ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse)
Expand Down
2 changes: 1 addition & 1 deletion pkg/converters/ofx/ofx_transaction_data_file_importer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ var (

// ParseImportedData returns the imported data by parsing the open financial exchange (ofx) file transaction data
func (c *ofxTransactionDataImporter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) {
ofxDataReader, err := createNewOFXFileReader(data)
ofxDataReader, err := createNewOFXFileReader(ctx, data)

if err != nil {
return nil, nil, nil, nil, nil, nil, err
Expand Down
Loading

0 comments on commit ac29f0b

Please sign in to comment.