diff --git a/cmd/slackdump/internal/archive/archive.go b/cmd/slackdump/internal/archive/archive.go index 3dc915ea..91d25eff 100644 --- a/cmd/slackdump/internal/archive/archive.go +++ b/cmd/slackdump/internal/archive/archive.go @@ -8,6 +8,7 @@ import ( "time" "github.com/rusq/fsadapter" + "github.com/rusq/slackdump/v3/cmd/slackdump/internal/bootstrap" "github.com/rusq/slackdump/v3/cmd/slackdump/internal/cfg" "github.com/rusq/slackdump/v3/cmd/slackdump/internal/golang/base" @@ -35,9 +36,7 @@ func init() { CmdArchive.Wizard = archiveWizard } -var ( - errNoOutput = errors.New("output directory is required") -) +var errNoOutput = errors.New("output directory is required") func RunArchive(ctx context.Context, cmd *base.Command, args []string) error { start := time.Now() @@ -80,7 +79,13 @@ func RunArchive(ctx context.Context, cmd *base.Command, args []string) error { defer stop() // we are using the same file subprocessor as the mattermost export. subproc := fileproc.NewExport(fileproc.STmattermost, dl) - ctrl := control.New(cd, stream, control.WithLogger(lg), control.WithFiler(subproc)) + ctrl := control.New( + cd, + stream, + control.WithLogger(lg), + control.WithFiler(subproc), + control.WithFlags(control.Flags{MemberOnly: cfg.MemberOnly}), + ) if err := ctrl.Run(ctx, list); err != nil { base.SetExitStatus(base.SApplicationError) return err diff --git a/cmd/slackdump/internal/archive/archive_wizard.go b/cmd/slackdump/internal/archive/archive_wizard.go index a486e843..6c5c3b70 100644 --- a/cmd/slackdump/internal/archive/archive_wizard.go +++ b/cmd/slackdump/internal/archive/archive_wizard.go @@ -3,9 +3,11 @@ package archive import ( "context" + "github.com/rusq/slackdump/v3/cmd/slackdump/internal/cfg" "github.com/rusq/slackdump/v3/cmd/slackdump/internal/golang/base" "github.com/rusq/slackdump/v3/cmd/slackdump/internal/ui/cfgui" "github.com/rusq/slackdump/v3/cmd/slackdump/internal/ui/dumpui" + "github.com/rusq/slackdump/v3/cmd/slackdump/internal/ui/updaters" "github.com/rusq/slackdump/v3/internal/structures" ) @@ -33,6 +35,13 @@ func configuration() cfgui.Configuration { Name: "Optional parameters", Params: []cfgui.Parameter{ cfgui.ChannelIDs(&entryList, false), + { + Name: "Member Only", + Value: cfgui.Checkbox(cfg.MemberOnly), + Description: "Export only channels, which current user belongs to", + Inline: true, + Updater: updaters.NewBool(&cfg.MemberOnly), + }, }, }, } diff --git a/cmd/slackdump/internal/archive/search.go b/cmd/slackdump/internal/archive/search.go index ceb1956c..a0efc3a4 100644 --- a/cmd/slackdump/internal/archive/search.go +++ b/cmd/slackdump/internal/archive/search.go @@ -32,7 +32,7 @@ var CmdSearch = &base.Command{ }, } -const flagMask = cfg.OmitUserCacheFlag | cfg.OmitCacheDir | cfg.OmitTimeframeFlag +const flagMask = cfg.OmitUserCacheFlag | cfg.OmitCacheDir | cfg.OmitTimeframeFlag | cfg.OmitMemberOnlyFlag var cmdSearchMessages = &base.Command{ UsageLine: "slackdump search messages [flags] query terms", diff --git a/cmd/slackdump/internal/cfg/cfg.go b/cmd/slackdump/internal/cfg/cfg.go index c0115e41..57033b3f 100644 --- a/cmd/slackdump/internal/cfg/cfg.go +++ b/cmd/slackdump/internal/cfg/cfg.go @@ -40,8 +40,8 @@ var ( LegacyBrowser bool ForceEnterprise bool + MemberOnly bool DownloadFiles bool - NoChunkCache bool // Oldest is the default timestamp of the oldest message to fetch, that is // used by the dump and export commands. @@ -54,6 +54,7 @@ var ( LocalCacheDir string UserCacheRetention time.Duration NoUserCache bool + NoChunkCache bool Log *slog.Logger = slog.Default() // LoadSecrets is a flag that indicates whether to load secrets from the @@ -86,6 +87,7 @@ const ( OmitUserCacheFlag OmitTimeframeFlag OmitChunkCacheFlag + OmitMemberOnlyFlag OmitAll = OmitConfigFlag | OmitDownloadFlag | @@ -95,11 +97,11 @@ const ( OmitAuthFlags | OmitUserCacheFlag | OmitTimeframeFlag | - OmitChunkCacheFlag + OmitChunkCacheFlag | + OmitMemberOnlyFlag ) // SetBaseFlags sets base flags -// TODO: tests. func SetBaseFlags(fs *flag.FlagSet, mask FlagMask) { fs.StringVar(&TraceFile, "trace", os.Getenv("TRACE_FILE"), "trace `filename`") fs.StringVar(&LogFile, "log", os.Getenv("LOG_FILE"), "log `file`, if not specified, messages are printed to STDERR") @@ -136,7 +138,7 @@ func SetBaseFlags(fs *flag.FlagSet, mask FlagMask) { LocalCacheDir = CacheDir() } if mask&OmitWorkspaceFlag == 0 { - fs.StringVar(&Workspace, "workspace", osenv.Value("SLACK_WORKSPACE", ""), "Slack workspace to use") // TODO: load from configuration. + fs.StringVar(&Workspace, "workspace", osenv.Value("SLACK_WORKSPACE", ""), "Slack workspace to use") } if mask&OmitUserCacheFlag == 0 { fs.BoolVar(&NoUserCache, "no-user-cache", false, "disable user cache (file cache)") @@ -151,4 +153,7 @@ func SetBaseFlags(fs *flag.FlagSet, mask FlagMask) { fs.Var(&Oldest, "time-from", "timestamp of the oldest message to fetch (UTC timezone)") fs.Var(&Latest, "time-to", "timestamp of the newest message to fetch (UTC timezone)") } + if mask&OmitMemberOnlyFlag == 0 { + fs.BoolVar(&MemberOnly, "member-only", false, "export only channels, which the current user belongs to (if no channels are specified)") + } } diff --git a/cmd/slackdump/internal/diag/info.go b/cmd/slackdump/internal/diag/info.go index f2c2c64c..d0d38e6a 100644 --- a/cmd/slackdump/internal/diag/info.go +++ b/cmd/slackdump/internal/diag/info.go @@ -6,15 +6,18 @@ import ( "io" "os" + "github.com/rusq/slackdump/v3/cmd/slackdump/internal/cfg" "github.com/rusq/slackdump/v3/cmd/slackdump/internal/diag/info" "github.com/rusq/slackdump/v3/cmd/slackdump/internal/golang/base" ) // cmdInfo is the information command. var cmdInfo = &base.Command{ - UsageLine: "slackdump tools info", - Short: "show information about Slackdump environment", - Run: runInfo, + UsageLine: "slackdump tools info", + Short: "show information about Slackdump environment", + Run: runInfo, + FlagMask: cfg.OmitAll, + PrintFlags: true, Long: `# Info Command diff --git a/cmd/slackdump/internal/diag/info/auth.go b/cmd/slackdump/internal/diag/info/auth.go index b5814877..7e03092b 100644 --- a/cmd/slackdump/internal/diag/info/auth.go +++ b/cmd/slackdump/internal/diag/info/auth.go @@ -8,9 +8,9 @@ import ( "os" "path/filepath" "strings" - "text/tabwriter" "github.com/rusq/encio" + "github.com/rusq/slackdump/v3/auth" "github.com/rusq/slackdump/v3/cmd/slackdump/internal/cfg" "github.com/rusq/slackdump/v3/internal/cache" @@ -43,6 +43,7 @@ func CollectAuth(ctx context.Context, w io.Writer) error { if err != nil { return fmt.Errorf("cache error: %w", err) } + fmt.Fprintf(w, "TOKEN=%s\n", prov.Token) if err := dumpCookiesMozilla(ctx, w, prov.Cookies()); err != nil { return err } @@ -51,12 +52,8 @@ func CollectAuth(ctx context.Context, w io.Writer) error { // dumpCookiesMozilla dumps cookies in Mozilla format. func dumpCookiesMozilla(_ context.Context, w io.Writer, cookies []*http.Cookie) error { - tw := tabwriter.NewWriter(w, 0, 8, 0, '\t', 0) - defer tw.Flush() - fmt.Fprintf(tw, "# name@domain\tvalue_len\tflag\tpath\tsecure\texpiration\n") for _, c := range cookies { - fmt.Fprintf(tw, "%s\t%9d\t%s\t%s\t%s\t%d\n", - c.Name+"@"+c.Domain, len(c.Value), "TRUE", c.Path, strings.ToUpper(fmt.Sprintf("%v", c.Secure)), c.Expires.Unix()) + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%d\t%s\t%s\n", c.Domain, "TRUE", c.Path, strings.ToUpper(fmt.Sprintf("%v", c.Secure)), c.Expires.Unix(), c.Name, c.Value) } return nil } diff --git a/cmd/slackdump/internal/diag/uninstall.go b/cmd/slackdump/internal/diag/uninstall.go index 6df4f90e..b4c08e17 100644 --- a/cmd/slackdump/internal/diag/uninstall.go +++ b/cmd/slackdump/internal/diag/uninstall.go @@ -224,7 +224,7 @@ func (p *uninstOptions) configuration() cfgui.Configuration { Description: "Do not perform the uninstallation, just show what would be done", Updater: updaters.NewBool(&p.dry), }, - // TODO: delete slackdump from user cache options. + // TODO: add an option to delete slackdump from user cache options. }, }, } diff --git a/cmd/slackdump/internal/dump/dump.go b/cmd/slackdump/internal/dump/dump.go index fd855a8c..6313ca01 100644 --- a/cmd/slackdump/internal/dump/dump.go +++ b/cmd/slackdump/internal/dump/dump.go @@ -13,6 +13,7 @@ import ( "time" "github.com/rusq/fsadapter" + "github.com/rusq/slackdump/v3" "github.com/rusq/slackdump/v3/cmd/slackdump/internal/bootstrap" "github.com/rusq/slackdump/v3/cmd/slackdump/internal/cfg" @@ -37,6 +38,7 @@ var CmdDump = &base.Command{ Long: dumpMd, RequireAuth: true, PrintFlags: true, + FlagMask: cfg.OmitMemberOnlyFlag, } func init() { @@ -175,7 +177,9 @@ func dump(ctx context.Context, sess *slackdump.Session, fsa fsadapter.FS, p dump var sdl fileproc.Downloader if p.downloadFiles { dl := downloader.New(sess.Client(), fsa, downloader.WithLogger(lg)) - dl.Start(ctx) + if err := dl.Start(ctx); err != nil { + return err + } defer dl.Stop() sdl = dl } else { @@ -225,7 +229,6 @@ func dump(ctx context.Context, sess *slackdump.Session, fsa fsadapter.FS, p dump return sr.Err } if sr.IsLast { - //TODO: this is unbeautiful. lg.InfoContext(ctx, "dumped", "sr", sr.String()) } return nil diff --git a/cmd/slackdump/internal/export/export.go b/cmd/slackdump/internal/export/export.go index 7591abf4..8e25261e 100644 --- a/cmd/slackdump/internal/export/export.go +++ b/cmd/slackdump/internal/export/export.go @@ -8,6 +8,7 @@ import ( "time" "github.com/rusq/fsadapter" + "github.com/rusq/slackdump/v3/cmd/slackdump/internal/bootstrap" "github.com/rusq/slackdump/v3/cmd/slackdump/internal/cfg" @@ -30,7 +31,6 @@ var CmdExport = &base.Command{ type exportFlags struct { ExportStorageType fileproc.StorageType - MemberOnly bool ExportToken string } @@ -42,9 +42,7 @@ var ( ) func init() { - // TODO: move TimeValue somewhere more appropriate once v1 is sunset. CmdExport.Flag.Var(&options.ExportStorageType, "type", "export file storage type") - CmdExport.Flag.BoolVar(&options.MemberOnly, "member-only", false, "export only channels, which current user belongs to") CmdExport.Flag.StringVar(&options.ExportToken, "export-token", "", "file export token to append to each of the file URLs") CmdExport.Run = runExport diff --git a/cmd/slackdump/internal/export/v3.go b/cmd/slackdump/internal/export/v3.go index ed8c8699..5b133d43 100644 --- a/cmd/slackdump/internal/export/v3.go +++ b/cmd/slackdump/internal/export/v3.go @@ -22,8 +22,6 @@ import ( "github.com/rusq/slackdump/v3/stream" ) -// TODO: check if the features is on par with Export v2. - // export runs the export v3. func export(ctx context.Context, sess *slackdump.Session, fsa fsadapter.FS, list *structures.EntityList, params exportFlags) error { lg := cfg.Log @@ -73,7 +71,7 @@ func export(ctx context.Context, sess *slackdump.Session, fsa fsadapter.FS, list ) flags := control.Flags{ - MemberOnly: params.MemberOnly, + MemberOnly: cfg.MemberOnly, } ctr := control.New( chunkdir, diff --git a/cmd/slackdump/internal/export/wizard.go b/cmd/slackdump/internal/export/wizard.go index fcf6e7f3..6c146d42 100644 --- a/cmd/slackdump/internal/export/wizard.go +++ b/cmd/slackdump/internal/export/wizard.go @@ -4,6 +4,8 @@ import ( "context" "github.com/charmbracelet/huh" + + "github.com/rusq/slackdump/v3/cmd/slackdump/internal/cfg" "github.com/rusq/slackdump/v3/cmd/slackdump/internal/golang/base" "github.com/rusq/slackdump/v3/cmd/slackdump/internal/ui/cfgui" "github.com/rusq/slackdump/v3/cmd/slackdump/internal/ui/dumpui" @@ -51,10 +53,10 @@ func (fl *exportFlags) configuration() cfgui.Configuration { }, { Name: "Member Only", - Value: cfgui.Checkbox(fl.MemberOnly), + Value: cfgui.Checkbox(cfg.MemberOnly), Description: "Export only channels, which current user belongs to", Inline: true, - Updater: updaters.NewBool(&fl.MemberOnly), + Updater: updaters.NewBool(&cfg.MemberOnly), }, { Name: "Export Token", diff --git a/cmd/slackdump/internal/list/channels.go b/cmd/slackdump/internal/list/channels.go index d10257c1..3f89283a 100644 --- a/cmd/slackdump/internal/list/channels.go +++ b/cmd/slackdump/internal/list/channels.go @@ -7,6 +7,7 @@ import ( "time" "github.com/rusq/slack" + "github.com/rusq/slackdump/v3" "github.com/rusq/slackdump/v3/cmd/slackdump/internal/bootstrap" "github.com/rusq/slackdump/v3/cmd/slackdump/internal/cfg" @@ -19,7 +20,7 @@ var CmdListChannels = &base.Command{ Run: runListChannels, UsageLine: "slackdump list channels [flags] [filename]", PrintFlags: true, - FlagMask: cfg.OmitDownloadFlag, + FlagMask: flagMask, Short: "list workspace channels", Long: fmt.Sprintf(` # List Channels Command @@ -75,7 +76,7 @@ func runListChannels(ctx context.Context, cmd *base.Command, args []string) erro return err } - var l = &channels{ + l := &channels{ opts: chanFlags, common: commonFlags, } diff --git a/cmd/slackdump/internal/list/common.go b/cmd/slackdump/internal/list/common.go index eafa7a44..187b36f5 100644 --- a/cmd/slackdump/internal/list/common.go +++ b/cmd/slackdump/internal/list/common.go @@ -8,6 +8,7 @@ import ( "os" "github.com/rusq/slack" + "github.com/rusq/slackdump/v3" "github.com/rusq/slackdump/v3/cmd/slackdump/internal/cfg" "github.com/rusq/slackdump/v3/cmd/slackdump/internal/golang/base" @@ -16,6 +17,8 @@ import ( "github.com/rusq/slackdump/v3/types" ) +const flagMask = cfg.OmitDownloadFlag | cfg.OmitMemberOnlyFlag + // CmdList is the list command. The logic is in the subcommands. var CmdList = &base.Command{ UsageLine: "slackdump list", diff --git a/cmd/slackdump/internal/list/users.go b/cmd/slackdump/internal/list/users.go index d72b1082..4a98e232 100644 --- a/cmd/slackdump/internal/list/users.go +++ b/cmd/slackdump/internal/list/users.go @@ -8,6 +8,7 @@ import ( "time" "github.com/rusq/slack" + "github.com/rusq/slackdump/v3" "github.com/rusq/slackdump/v3/cmd/slackdump/internal/bootstrap" "github.com/rusq/slackdump/v3/cmd/slackdump/internal/cfg" @@ -21,7 +22,7 @@ var CmdListUsers = &base.Command{ Run: runListUsers, UsageLine: "slackdump list users [flags] [filename]", PrintFlags: true, - FlagMask: cfg.OmitDownloadFlag, + FlagMask: flagMask, Short: "list workspace users", RequireAuth: true, Long: fmt.Sprintf(` @@ -46,7 +47,7 @@ func runListUsers(ctx context.Context, cmd *base.Command, args []string) error { return err } - var l = &users{ + l := &users{ common: commonFlags, } @@ -81,7 +82,6 @@ func (u *users) Retrieve(ctx context.Context, sess *slackdump.Session, m *cache. } //go:generate mockgen -source=users.go -destination=mocks_test.go -package=list userGetter,userCacher - type userGetter interface { GetUsers(ctx context.Context) (types.Users, error) } diff --git a/cmd/slackdump/internal/workspace/list.go b/cmd/slackdump/internal/workspace/list.go index 1b9729dc..f8b9336e 100644 --- a/cmd/slackdump/internal/workspace/list.go +++ b/cmd/slackdump/internal/workspace/list.go @@ -55,12 +55,12 @@ func runList(ctx context.Context, cmd *base.Command, args []string) error { fmtFn = printAll } - return list(m, fmtFn) + return list(ctx, m, fmtFn) } -type formatFunc func(io.Writer, manager, string, []string) error +type formatFunc func(context.Context, io.Writer, manager, string, []string) error -func list(m manager, formatter formatFunc) error { +func list(ctx context.Context, m manager, formatter formatFunc) error { entries, err := m.List() if err != nil { if errors.Is(err, cache.ErrNoWorkspaces) { @@ -84,10 +84,10 @@ func list(m manager, formatter formatFunc) error { } - return formatter(os.Stdout, m, current, entries) + return formatter(ctx, os.Stdout, m, current, entries) } -func printDefault(w io.Writer, m manager, current string, wsps []string) error { +func printDefault(_ context.Context, w io.Writer, m manager, current string, wsps []string) error { ew := &errWriter{w: w} fmt.Fprintf(ew, "Workspaces in %q:\n\n", cfg.CacheDir()) for _, row := range simpleList(m, current, wsps) { @@ -97,8 +97,8 @@ func printDefault(w io.Writer, m manager, current string, wsps []string) error { return ew.Err() } -func printAll(w io.Writer, m manager, current string, wsps []string) error { - ctx, task := trace.NewTask(context.TODO(), "printAll") +func printAll(ctx context.Context, w io.Writer, m manager, current string, wsps []string) error { + ctx, task := trace.NewTask(ctx, "printAll") defer task.End() ew := &errWriter{w: w} @@ -116,7 +116,7 @@ func printAll(w io.Writer, m manager, current string, wsps []string) error { return ew.Err() } -func printBare(w io.Writer, _ manager, current string, workspaces []string) error { +func printBare(_ context.Context, w io.Writer, _ manager, current string, workspaces []string) error { ew := &errWriter{w: w} for _, name := range workspaces { if current == name { diff --git a/cmd/slackdump/internal/workspace/list_test.go b/cmd/slackdump/internal/workspace/list_test.go index 2d3f6282..7b1ae27e 100644 --- a/cmd/slackdump/internal/workspace/list_test.go +++ b/cmd/slackdump/internal/workspace/list_test.go @@ -2,6 +2,7 @@ package workspace import ( "bytes" + "context" "errors" "fmt" "io" @@ -45,7 +46,7 @@ func Test_printBare(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { w := &bytes.Buffer{} - if err := printBare(w, nil, tt.args.current, tt.args.workspaces); (err != nil) != tt.wantErr { + if err := printBare(context.Background(), w, nil, tt.args.current, tt.args.workspaces); (err != nil) != tt.wantErr { t.Errorf("printBare() error = %v, wantErr %v", err, tt.wantErr) return } @@ -133,14 +134,14 @@ func Test_list(t *testing.T) { ctrl := gomock.NewController(t) mm := NewMockmanager(ctrl) tt.expectFn(mm) - if err := list(mm, tt.args.formatter); (err != nil) != tt.wantErr { + if err := list(context.Background(), mm, tt.args.formatter); (err != nil) != tt.wantErr { t.Errorf("list() error = %v, wantErr %v", err, tt.wantErr) } }) } } -func testFmt(w io.Writer, m manager, current string, wsps []string) error { +func testFmt(_ context.Context, w io.Writer, m manager, current string, wsps []string) error { for _, wsp := range wsps { if wsp == current { fmt.Fprint(w, ">") diff --git a/downloader/downloader.go b/downloader/downloader.go index 2ec36b39..94c4dda0 100644 --- a/downloader/downloader.go +++ b/downloader/downloader.go @@ -28,8 +28,9 @@ const ( ) var ( - ErrNoFS = errors.New("fs adapter not initialised") - ErrNotStarted = errors.New("downloader not started") + ErrNoFS = errors.New("fs adapter not initialised") + ErrNotStarted = errors.New("downloader not started") + ErrAlreadyStarted = errors.New("downloader already started") ) // Downloader is the file downloader interface. It exists primarily for mocking @@ -141,37 +142,37 @@ type Request struct { // Start starts an async file downloader. If the downloader is already // started, it does nothing. -func (c *Client) Start(ctx context.Context) { +func (c *Client) Start(ctx context.Context) error { c.mu.Lock() defer c.mu.Unlock() - slog.Debug("starting downloader") - if c.started.Load() { - return + return ErrAlreadyStarted } - c.requests = make(chan Request, c.chanBufSz) - c.wg = c.startWorkers(ctx, c.requests) + c.lg.Debug("starting downloader") + c.startWorkers(ctx) c.started.Store(true) + return nil } // startWorkers starts download workers. It returns a sync.WaitGroup. If the // req channel is closed, workers will stop, and wg.Wait() completes. -func (c *Client) startWorkers(ctx context.Context, req <-chan Request) *sync.WaitGroup { +func (c *Client) startWorkers(ctx context.Context) { if c.workers == 0 { c.workers = defNumWorkers } - seen := fltSeen(req) - var wg sync.WaitGroup + c.requests = make(chan Request, c.chanBufSz) + c.wg = new(sync.WaitGroup) + + seen := fltSeen(c.requests) // create workers for i := range c.workers { - wg.Add(1) + c.wg.Add(1) go func(workerNum int) { c.worker(ctx, seen) - wg.Done() + c.wg.Done() c.lg.DebugContext(ctx, "download worker terminated", "worker", workerNum) }(i) } - return &wg } // fltSeen filters the files from filesC to ensure that no duplicates @@ -316,7 +317,10 @@ func (c *Client) Download(fullpath string, url string) error { func (c *Client) AsyncDownloader(ctx context.Context, queueC <-chan Request) (<-chan struct{}, error) { done := make(chan struct{}) - c.Start(ctx) + if err := c.Start(ctx); err != nil { + close(done) + return done, err + } go func() { defer close(done) for r := range queueC { diff --git a/downloader/downloader_test.go b/downloader/downloader_test.go index 1ae52859..2065134f 100644 --- a/downloader/downloader_test.go +++ b/downloader/downloader_test.go @@ -1,13 +1,15 @@ package downloader import ( + "context" "log/slog" "sync" "testing" "time" - "github.com/rusq/slackdump/v3/internal/fixtures" "github.com/stretchr/testify/assert" + + "github.com/rusq/slackdump/v3/internal/fixtures" ) func init() { @@ -62,7 +64,6 @@ func Test_fltSeen(t *testing.T) { var benchInput = makeFileReqQ(100_000) func BenchmarkFltSeen(b *testing.B) { - inputC := make(chan Request) go func() { defer close(inputC) @@ -131,6 +132,49 @@ func TestClient_Download(t *testing.T) { case r := <-requests: assert.Equal(t, Request{Fullpath: "x/file", URL: "http://example.com"}, r, "expected request to be sent") } + }) +} + +func TestClient_startWorkers(t *testing.T) { + t.Parallel() + t.Run("starts workers", func(t *testing.T) { + t.Parallel() + c := &Client{ + requests: make(chan Request), + wg: new(sync.WaitGroup), + options: options{lg: slog.Default(), workers: 3}, + } + defer close(c.requests) + c.startWorkers(context.Background()) + assert.Equal(t, 3, c.options.workers) + assert.NotNil(t, c.wg) + }) + t.Run("no workers specified", func(t *testing.T) { + t.Parallel() + c := &Client{ + requests: make(chan Request), + wg: new(sync.WaitGroup), + options: options{lg: slog.Default()}, + } + defer close(c.requests) + c.startWorkers(context.Background()) + assert.Equal(t, defNumWorkers, c.options.workers) + assert.NotNil(t, c.wg) + }) +} +func TestStart(t *testing.T) { + t.Parallel() + t.Run("already started", func(t *testing.T) { + c := &Client{} + c.lg = slog.Default() + c.started.Store(true) + assert.Error(t, c.Start(context.Background()), "expected error") + }) + t.Run("not started", func(t *testing.T) { + c := &Client{} + c.lg = slog.Default() + assert.NoError(t, c.Start(context.Background()), "expected no error") + assert.True(t, c.started.Load(), "expected started to be true") }) } diff --git a/internal/chunk/control/control.go b/internal/chunk/control/control.go index ca1b38ad..cf981957 100644 --- a/internal/chunk/control/control.go +++ b/internal/chunk/control/control.go @@ -110,6 +110,10 @@ func (e Error) Error() string { return fmt.Sprintf("controller error in subroutine %s on stage %s: %v", e.Subroutine, e.Stage, e.Err) } +func (e Error) Unwrap() error { + return e.Err +} + func (c *Controller) Run(ctx context.Context, list *structures.EntityList) error { ctx, task := trace.NewTask(ctx, "Controller.Run") defer task.End() diff --git a/internal/chunk/obfuscate/obfuscate_test.go b/internal/chunk/obfuscate/obfuscate_test.go index b88d92c1..148a9f7a 100644 --- a/internal/chunk/obfuscate/obfuscate_test.go +++ b/internal/chunk/obfuscate/obfuscate_test.go @@ -12,9 +12,10 @@ import ( "testing" "github.com/rusq/slack" + "github.com/stretchr/testify/assert" + "github.com/rusq/slackdump/v3/internal/chunk" "github.com/rusq/slackdump/v3/internal/fixtures" - "github.com/stretchr/testify/assert" ) const testSeed = 0 diff --git a/internal/chunk/transform/fileproc/fileproc.go b/internal/chunk/transform/fileproc/fileproc.go index 3dcdfdb0..370b607e 100644 --- a/internal/chunk/transform/fileproc/fileproc.go +++ b/internal/chunk/transform/fileproc/fileproc.go @@ -129,7 +129,10 @@ func NewDownloader(ctx context.Context, gEnabled bool, cl FileGetter, fsa fsadap return NoopDownloader{}, func() {} } else { dl := downloader.New(cl, fsa, downloader.WithLogger(lg)) - dl.Start(ctx) + if err := dl.Start(ctx); err != nil { + lg.Error("failed to start downloader", "error", err) + return NoopDownloader{}, func() {} + } return dl, dl.Stop } } diff --git a/internal/edge/edge_test.go b/internal/edge/edge_test.go index b222a730..7ff1bf07 100644 --- a/internal/edge/edge_test.go +++ b/internal/edge/edge_test.go @@ -11,14 +11,16 @@ import ( "testing" "github.com/joho/godotenv" - "github.com/rusq/slackdump/v3/auth" "github.com/stretchr/testify/assert" + + "github.com/rusq/slackdump/v3/auth" ) var _ = godotenv.Load() var ( - testToken = os.Getenv("EDGE_TOKEN)") + // preferrably guest workspace token. + testToken = os.Getenv("EDGE_TOKEN") testCookie = os.Getenv("EDGE_COOKIE") ) diff --git a/internal/osext/osext.go b/internal/osext/osext.go index 2b8ebfca..a1cde33a 100644 --- a/internal/osext/osext.go +++ b/internal/osext/osext.go @@ -16,6 +16,10 @@ func (e *Error) Error() string { return fmt.Sprintf("%s: %s", e.Err, e.File) } +func (e *Error) Unwrap() error { + return e.Err +} + // Namer is an interface that allows us to get the name of the file. type Namer interface { // Name should return the name of the file. *os.File implements this diff --git a/slackdump.1 b/slackdump.1 index f5e36e58..a4ccf5c6 100644 --- a/slackdump.1 +++ b/slackdump.1 @@ -11,6 +11,7 @@ or .Dq Ar yes ), .. + .Dd $Mdocdate$ .Dt SLACKDUMP 1 .Os @@ -36,9 +37,9 @@ Includes all message replies or, in other words — threads. Files are dumped along with messages they belong to. .It Em emojis Emojis are dumped along with their index which contains their names and aliases -as a JSON file. +as a JSON file, in the "slow" mode -- includes the user ID of the uploader. .It Em users -Users include full profile information without custom fields. +Users includes full profile information without custom fields. .It Em channels Channels, that are visible to the current authenticated user. This includes: .Bl -dash -compact @@ -51,14 +52,14 @@ private group conversations; direct messages (private conversations between two users). .El .It Em search -Searches for messages and/or files in the workspace. +Searches and saves messages and/or files from the workspace. .El .Pp If no command is given, on a dumb terminal, the .Cm help command is assumed. On an interactive terminal a list of options will -be presented, allowing the user to enter an interactive mode, display -help, or exit. +be presented, allowing the user to enter an interactive mode (wizard), +display help, or exit. .Sh COMMANDS The following commands are supported (listed in alphabetical order): .Bl -tag -width workspace @@ -102,6 +103,10 @@ to run these commands to help with debugging. See TOOLS section for more information. .It Cm version Display version information. +.It Cm view +Allows to view the exported data (archive, dump and export) in the browser. +.It Cm wiz +Starts up an interactive mode. .It Cm workspace Manage stored credentials for authenticated workspaces, or authenticates in a new workspace. @@ -112,7 +117,7 @@ This section lists all available flags, availability of which depends on the command. The flags are listed in alphabetical order. .Bl -tag -width -base dir .It Fl api-config Ar path -Use the specified API limits configuration file (see the +Use the specified API limits configuration TOML file (see the .Cm config command). .It Fl base Ar path @@ -150,6 +155,10 @@ Enables or disables JSON log format. The default is disabled. .It Fl token Ar token Specifies the token to use for the authentication. This flag is only used with the manual authentication methods. +.It Fl member-only +Specify this flag to export only conversations (channels) that the current user +is part of. Works only if the list of channels/threads is not explicitly +specified. .It Fl no-user-cache Disables caching of users for the subcommands of the .Cm list