memory cleanup

adapters/fetchers/jshttp/jshttp.go

@@ -16,7 +16,13 @@ func New(headless, disableImages bool, rotator scrapemate.ProxyRotator) (scrapem
 
 	const poolSize = 10
 
+	pw, err := playwright.Run()
+	if err != nil {
+		return nil, err
+	}
+
 	ans := jsFetch{
+		pw:            pw,
 		headless:      headless,
 		disableImages: disableImages,
 		pool:          make(chan *browser, poolSize),
@@ -27,6 +33,7 @@ func New(headless, disableImages bool, rotator scrapemate.ProxyRotator) (scrapem
 }
 
 type jsFetch struct {
+	pw            *playwright.Playwright
 	headless      bool
 	disableImages bool
 	pool          chan *browser
@@ -40,7 +47,7 @@ func (o *jsFetch) GetBrowser(ctx context.Context) (*browser, error) {
 	case ans := <-o.pool:
 		return ans, nil
 	default:
-		ans, err := newBrowser(o.headless, o.disableImages, o.rotator)
+		ans, err := newBrowser(o.pw, o.headless, o.disableImages, o.rotator)
 		if err != nil {
 			return nil, err
 		}
@@ -49,6 +56,18 @@ func (o *jsFetch) GetBrowser(ctx context.Context) (*browser, error) {
 	}
 }
 
+func (o *jsFetch) Close() error {
+	close(o.pool)
+
+	for b := range o.pool {
+		b.Close()
+	}
+
+	_ = o.pw.Stop()
+
+	return nil
+}
+
 func (o *jsFetch) PutBrowser(ctx context.Context, b *browser) {
 	select {
 	case <-ctx.Done():
@@ -100,23 +119,16 @@ func (o *jsFetch) Fetch(ctx context.Context, job scrapemate.IJob) scrapemate.Res
 }
 
 type browser struct {
-	pw      *playwright.Playwright
 	browser playwright.Browser
 	ctx     playwright.BrowserContext
 }
 
 func (o *browser) Close() {
 	_ = o.ctx.Close()
 	_ = o.browser.Close()
-	_ = o.pw.Stop()
 }
 
-func newBrowser(headless, disableImages bool, rotator scrapemate.ProxyRotator) (*browser, error) {
-	pw, err := playwright.Run()
-	if err != nil {
-		return nil, err
-	}
-
+func newBrowser(pw *playwright.Playwright, headless, disableImages bool, rotator scrapemate.ProxyRotator) (*browser, error) {
 	opts := playwright.BrowserTypeLaunchOptions{
 		Headless: playwright.Bool(headless),
 		Args: []string{
@@ -173,7 +185,6 @@ func newBrowser(headless, disableImages bool, rotator scrapemate.ProxyRotator) (
 	}
 
 	ans := browser{
-		pw:      pw,
 		browser: br,
 		ctx:     bctx,
 	}

adapters/fetchers/nethttp/nethttp.go

@@ -27,6 +27,10 @@ type httpFetch struct {
 	netClient HTTPClient
 }
 
+func (o *httpFetch) Close() error {
+	return nil
+}
+
 func (o *httpFetch) Fetch(ctx context.Context, job scrapemate.IJob) scrapemate.Response {
 	u := job.GetFullURL()
 	reqBody := getBuffer()

examples/books-to-scrape-simple/main.go

@@ -156,8 +156,11 @@ func run() error {
 		}
 	}()
 
+	defer mate.Close()
+
 	err = mate.Start()
 	<-resultsDone
+
 	return err
 }
 

go.mod

@@ -192,6 +192,7 @@ require (
 	go-simpler.org/sloglint v0.7.2 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/automaxprocs v1.5.3 // indirect
+	go.uber.org/mock v0.5.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.24.0 // indirect
 	golang.org/x/crypto v0.28.0 // indirect

go.sum

@@ -603,6 +603,8 @@ go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8=
 go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0=
+go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU=
+go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=