Merge pull request #6 from d--j/header

Header Unfolding & SetAddressList formatting

integration/README.md

@@ -69,10 +69,10 @@ You need docker since the test are run inside a docker container.
 
 Add a Makefile
 ```makefile
-GO_MILTER_DIR := $(shell go list -f '{{.Dir}}' github.com/d--j/go-milter)
+GO_MILTER_INTEGRATION_DIR := $(shell cd integration && go list -f '{{.Dir}}' github.com/d--j/go-milter/integration)
 
 integration:
-	docker build -q --progress=plain -t go-milter-integration "$(GO_MILTER_DIR)/integration/docker" && \
+	docker build -q -t go-milter-integration "$(GO_MILTER_INTEGRATION_DIR)/docker" && \
 	docker run --rm -w /usr/src/root/integration -v $(PWD):/usr/src/root go-milter-integration \
 	go run github.com/d--j/go-milter/integration/runner -filter '.*' ./tests
 

integration/testcase.go

@@ -729,8 +729,14 @@ func DiffOutput(expected, got *Output) (string, bool) {
 	return b.String(), ok
 }
 
+var unfoldRegex = regexp.MustCompile(`\r?\n\s*`)
+
+func unfold(in []byte) []byte {
+	return unfoldRegex.ReplaceAll(in, []byte(" "))
+}
+
 // CompareOutputSendmail is a relaxed compare function that does only check
-// that the header values are all there – the order does not matter.
+// that the header values are all there – the order and folding do not matter.
 func CompareOutputSendmail(expected, got *Output) bool {
 	if expected == nil && got == nil {
 		return true
@@ -751,11 +757,35 @@ func CompareOutputSendmail(expected, got *Output) bool {
 		return false
 	}
 	if expected.Header != nil {
-		expectedLines := bytes.Split(expected.Header, []byte{'\r', '\n'})
-		gotLines := bytes.Split(got.Header, []byte{'\r', '\n'})
-		if len(expectedLines) != len(gotLines) {
+		r, err := mail.CreateReader(bytes.NewReader(expected.Header))
+		if err != nil {
 			return false
 		}
+		exFields := r.Header.Fields()
+		r, err = mail.CreateReader(bytes.NewReader(got.Header))
+		if err != nil {
+			return false
+		}
+		gotFields := r.Header.Fields()
+		if exFields.Len() != gotFields.Len() {
+			return false
+		}
+		expectedLines := make([][]byte, 0, exFields.Len())
+		for exFields.Next() {
+			b, err := exFields.Raw()
+			if err != nil {
+				return false
+			}
+			expectedLines = append(expectedLines, unfold(b[:len(b)-2]))
+		}
+		gotLines := make([][]byte, 0, gotFields.Len())
+		for gotFields.Next() {
+			b, err := gotFields.Raw()
+			if err != nil {
+				return false
+			}
+			gotLines = append(gotLines, unfold(b[:len(b)-2]))
+		}
 	outer:
 		for _, e := range expectedLines {
 			for _, g := range gotLines {

integration/tests/header/change-to.testcase

@@ -0,0 +1,18 @@
+FROM <[email protected]>
+HEADER
+From: <>
+To: <[email protected]>
+Subject: test
+Date: Fri, 10 Mar 2023 23:29:35 +0000 (UTC)
+Message-ID: <[email protected]>
+.
+DECISION ACCEPT
+HEADER
+Received: placeholder
+From: <>
+To: <[email protected]>,
+ <[email protected]>
+Subject: test
+Date: Fri, 10 Mar 2023 23:29:35 +0000 (UTC)
+Message-ID: <[email protected]>
+.

integration/tests/header/test.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/d--j/go-milter/integration"
 	"github.com/d--j/go-milter/mailfilter"
+	"github.com/emersion/go-message/mail"
 )
 
 func main() {
@@ -57,6 +58,13 @@ func main() {
 			}
 			trx.Headers().Add("X-ADD1", "Test")
 			trx.Headers().Add("X-ADD2", "Test")
+		case "[email protected]":
+			addr, err := trx.Headers().AddressList("To")
+			if err != nil {
+				return nil, err
+			}
+			addr = append(addr, &mail.Address{Address: "[email protected]"})
+			trx.Headers().SetAddressList("To", addr)
 		default:
 			return mailfilter.CustomErrorResponse(500, "unknown mail from"), nil
 		}

internal/header/header.go

@@ -6,13 +6,28 @@ import (
 	"io"
 	netmail "net/mail"
 	"net/textproto"
+	"regexp"
 	"strings"
 	"time"
 
 	"github.com/d--j/go-milter/mailfilter/header"
 	"github.com/emersion/go-message/mail"
 )
 
+var unfoldRegex = regexp.MustCompile(`\r?\n\s*`)
+
+func unfold(lines string) string {
+	return unfoldRegex.ReplaceAllString(lines, " ")
+}
+
+func formatAddressList(l []*mail.Address) string {
+	formatted := make([]string, len(l))
+	for i, a := range l {
+		formatted[i] = a.String()
+	}
+	return strings.Join(formatted, ",\r\n ")
+}
+
 type Field struct {
 	Index        int
 	CanonicalKey string
@@ -27,6 +42,10 @@ func (f *Field) Value() string {
 	return string(f.Raw[len(f.CanonicalKey)+1:])
 }
 
+func (f *Field) UnfoldedValue() string {
+	return unfold(string(f.Raw[len(f.CanonicalKey)+1:]))
+}
+
 func (f *Field) Deleted() bool {
 	return len(f.Raw) <= len(f.CanonicalKey)+1
 }
@@ -94,14 +113,24 @@ func (h *Header) Value(key string) string {
 	return ""
 }
 
+func (h *Header) UnfoldedValue(key string) string {
+	canonicalKey := textproto.CanonicalMIMEHeaderKey(key)
+	for _, f := range h.fields {
+		if f.CanonicalKey == canonicalKey {
+			return f.UnfoldedValue()
+		}
+	}
+	return ""
+}
+
 func (h *Header) Text(key string) (string, error) {
 	if h.helper == nil {
 		h.helper = newHelper()
 	}
 	canonicalKey := textproto.CanonicalMIMEHeaderKey(key)
 	for _, f := range h.fields {
 		if f.CanonicalKey == canonicalKey {
-			h.helper.Set(helperKey, f.Value())
+			h.helper.Set(helperKey, f.UnfoldedValue())
 			return h.helper.Text(helperKey)
 		}
 	}
@@ -115,7 +144,7 @@ func (h *Header) AddressList(key string) ([]*mail.Address, error) {
 	canonicalKey := textproto.CanonicalMIMEHeaderKey(key)
 	for _, f := range h.fields {
 		if f.CanonicalKey == canonicalKey {
-			h.helper.Set(helperKey, f.Value())
+			h.helper.Set(helperKey, f.UnfoldedValue())
 			return h.helper.AddressList(helperKey)
 		}
 	}
@@ -148,11 +177,7 @@ func (h *Header) SetText(key string, value string) {
 }
 
 func (h *Header) SetAddressList(key string, addresses []*mail.Address) {
-	if h.helper == nil {
-		h.helper = newHelper()
-	}
-	h.helper.SetAddressList(helperKey, addresses)
-	h.Set(key, h.helper.Get(helperKey))
+	h.Set(key, formatAddressList(addresses))
 }
 
 func (h *Header) Subject() (string, error) {
@@ -251,13 +276,17 @@ func (f *Fields) Value() string {
 	return f.h.fields[f.index()].Value()
 }
 
+func (f *Fields) UnfoldedValue() string {
+	return f.h.fields[f.index()].UnfoldedValue()
+}
+
 func (f *Fields) Text() (string, error) {
-	f.helper.Set(helperKey, f.Value())
+	f.helper.Set(helperKey, f.UnfoldedValue())
 	return f.helper.Text(helperKey)
 }
 
 func (f *Fields) AddressList() ([]*mail.Address, error) {
-	f.helper.Set(helperKey, f.Value())
+	f.helper.Set(helperKey, f.UnfoldedValue())
 	return f.helper.AddressList(helperKey)
 }
 
@@ -284,8 +313,7 @@ func (f *Fields) SetText(value string) {
 }
 
 func (f *Fields) addressList(value []*mail.Address) string {
-	f.helper.SetAddressList(helperKey, value)
-	return f.helper.Get(helperKey)
+	return formatAddressList(value)
 }
 
 func (f *Fields) SetAddressList(value []*mail.Address) {

internal/header/header_test.go

@@ -35,6 +35,30 @@ func testHeader() *Header {
 	}}
 }
 
+func Test_unfold(t *testing.T) {
+	tests := []struct {
+		lines string
+		want  string
+	}{
+		{"one", "one"},
+		{"one\ntwo", "one two"},
+		{"one\n two", "one two"},
+		{"one\n\ttwo", "one two"},
+		{"one\r\ntwo", "one two"},
+		{"one\r\n\ttwo", "one two"},
+		{"one\r\n two", "one two"},
+		{"one\r\n  two", "one two"},
+		{"one\r\n \t two", "one two"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.lines, func(t *testing.T) {
+			if got := unfold(tt.lines); got != tt.want {
+				t.Errorf("unfold() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
 var root, nobody = mail.Address{
 	Name:    "",
 	Address: "root@localhost",
@@ -100,7 +124,7 @@ func TestHeaderFields_Del(t *testing.T) {
 	}
 }
 
-func TestHeaderFields_Get(t *testing.T) {
+func TestHeaderFields_Value(t *testing.T) {
 	type fields struct {
 		cursor int
 		h      *Header
@@ -129,6 +153,35 @@ func TestHeaderFields_Get(t *testing.T) {
 	}
 }
 
+func TestHeaderFields_UnfoldedValue(t *testing.T) {
+	type fields struct {
+		cursor int
+		h      *Header
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		want   string
+	}{
+		{"From", fields{0, testHeader()}, " <root@localhost>"},
+		{"To", fields{1, testHeader()}, "  <root@localhost>, <nobody@localhost>"},
+		{"Subject", fields{2, testHeader()}, " =?UTF-8?Q?=F0=9F=9F=A2?="},
+		{"Date", fields{3, testHeader()}, "\tWed, 01 Mar 2023 15:47:33 +0100"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			f := &Fields{
+				cursor: tt.fields.cursor,
+				h:      tt.fields.h,
+				helper: newHelper(),
+			}
+			if got := f.UnfoldedValue(); got != tt.want {
+				t.Errorf("Value() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
 func TestHeaderFields_GetAddressList(t *testing.T) {
 	type fields struct {
 		cursor int
@@ -201,8 +254,8 @@ func TestHeaderFields_GetText(t *testing.T) {
 
 func outputFields(fields []*Field) string {
 	h := Header{fields: fields}
-	bytes, _ := io.ReadAll(h.Reader())
-	return string(bytes)
+	b, _ := io.ReadAll(h.Reader())
+	return string(b)
 }
 
 func TestHeaderFields_InsertAfter(t *testing.T) {
@@ -513,7 +566,7 @@ func TestHeaderFields_SetAddressList(t *testing.T) {
 		want   *Field
 	}{
 		{"One", fields{0, testHeader()}, args{[]*mail.Address{&nobody}}, &Field{0, "From", []byte("From: <nobody@localhost>")}},
-		{"Two", fields{1, testHeader()}, args{[]*mail.Address{&nobody, &root}}, &Field{1, "To", []byte("To: <nobody@localhost>, <root@localhost>")}},
+		{"Two", fields{1, testHeader()}, args{[]*mail.Address{&nobody, &root}}, &Field{1, "To", []byte("To: <nobody@localhost>,\r\n <root@localhost>")}},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -525,7 +578,7 @@ func TestHeaderFields_SetAddressList(t *testing.T) {
 			f.SetAddressList(tt.args.value)
 			got := f.h.fields[f.index()]
 			if !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("SetAddressList() = %v, want %v", got, tt.want)
+				t.Errorf("SetAddressList() = %q, want %q", got, tt.want)
 			}
 		})
 	}
@@ -692,7 +745,7 @@ func TestHeader_Fields(t *testing.T) {
 	}
 }
 
-func TestHeader_Get(t *testing.T) {
+func TestHeader_Value(t *testing.T) {
 	type args struct {
 		key string
 	}
@@ -717,6 +770,31 @@ func TestHeader_Get(t *testing.T) {
 	}
 }
 
+func TestHeader_UnfoldedValue(t *testing.T) {
+	type args struct {
+		key string
+	}
+	tests := []struct {
+		name   string
+		fields []*Field
+		args   args
+		want   string
+	}{
+		{"works", testHeader().fields, args{"fRoM"}, " <root@localhost>"},
+		{"not found", testHeader().fields, args{"not-there"}, ""},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			h := &Header{
+				fields: tt.fields,
+			}
+			if got := h.UnfoldedValue(tt.args.key); got != tt.want {
+				t.Errorf("Value() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
 func TestHeader_GetAddressList(t *testing.T) {
 	type args struct {
 		key string