diff --git a/cmd/alp/cmd/command.go b/cmd/alp/cmd/command.go new file mode 100644 index 0000000..af22b1d --- /dev/null +++ b/cmd/alp/cmd/command.go @@ -0,0 +1,93 @@ +package cmd + +import ( + "github.com/spf13/cobra" +) + +type Command struct { + // alp + rootCmd *cobra.Command + + // alp diff + diffCmd *cobra.Command + + // alp count + countCmd *cobra.Command + + // alp json + jsonCmd *cobra.Command + // alp json diff + jsonDiffCmd *cobra.Command + + // alp ltsv + ltsvCmd *cobra.Command + // alp ltsv diff + ltsvDiffCmd *cobra.Command + + // alp regexp + regexpCmd *cobra.Command + // alp regexp diff + regexpDiffCmd *cobra.Command + + // alp pcap + pcapCmd *cobra.Command + // alp pcap diff + pcapDiffCmd *cobra.Command + + flags *flags +} + +func NewCommand(version string) *Command { + command := &Command{} + command.flags = newFlags() + + command.rootCmd = newRootCmd(version) + + command.flags.defineGlobalOptions(command.rootCmd) + + // alp ltsv + command.ltsvCmd = newLTSVCmd(command.flags) + command.rootCmd.AddCommand(command.ltsvCmd) + // alp ltsv diff + command.ltsvDiffCmd = newLTSVDiffCmd(command.flags) + command.ltsvCmd.AddCommand(command.ltsvDiffCmd) + + // alp json + command.jsonCmd = newJSONCmd(command.flags) + command.rootCmd.AddCommand(command.jsonCmd) + // alp json diff + command.jsonDiffCmd = newJsonDiffCmd(command.flags) + command.jsonCmd.AddCommand(command.jsonDiffCmd) + + // alp regexp + command.regexpCmd = newRegexpCmd(command.flags) + command.rootCmd.AddCommand(command.regexpCmd) + // alp regexp diff + command.regexpDiffCmd = newRegexpDiffCmd(command.flags) + command.regexpCmd.AddCommand(command.regexpDiffCmd) + + // alp pcap + command.pcapCmd = newPcapCmd(command.flags) + command.rootCmd.AddCommand(command.pcapCmd) + // alp pcap diff + command.pcapDiffCmd = newPcapDiffCmd(command.flags) + command.pcapCmd.AddCommand(command.pcapDiffCmd) + + // alp diff + command.diffCmd = newDiffCmd(command.flags) + command.rootCmd.AddCommand(command.diffCmd) + + // alp count + command.countCmd = newCountCmd(command.flags) + command.rootCmd.AddCommand(command.countCmd) + + return command +} + +func (c *Command) Execute() error { + return c.rootCmd.Execute() +} + +func (c *Command) setArgs(args []string) { + c.rootCmd.SetArgs(args) +} diff --git a/cmd/alp/cmd/common_test.go b/cmd/alp/cmd/common_test.go index f87a116..39137e8 100644 --- a/cmd/alp/cmd/common_test.go +++ b/cmd/alp/cmd/common_test.go @@ -99,10 +99,10 @@ func TestCommonFlags(t *testing.T) { for _, tt := range tests { t.Run(strings.Join(tt.args, " "), func(t *testing.T) { - rootCmd := NewRootCmd("test") - rootCmd.SetArgs(tt.args) + command := NewCommand("test") + command.setArgs(tt.args) - err := rootCmd.Execute() + err := command.Execute() if err != nil { t.Fatal(err) } diff --git a/cmd/alp/cmd/count.go b/cmd/alp/cmd/count.go index 4d737d9..a98d4da 100644 --- a/cmd/alp/cmd/count.go +++ b/cmd/alp/cmd/count.go @@ -10,13 +10,13 @@ import ( "github.com/tkuchiki/alp/parsers" ) -func NewCountCmd(commandFlags *flags) *cobra.Command { +func newCountCmd(flags *flags) *cobra.Command { var countCmd = &cobra.Command{ Use: "count", Short: "Count by log entries", Long: `Count by log entries`, RunE: func(cmd *cobra.Command, args []string) error { - opts, err := commandFlags.createCountOptions(cmd) + opts, err := flags.createCountOptions(cmd) if err != nil { return err } @@ -75,7 +75,7 @@ func NewCountCmd(commandFlags *flags) *cobra.Command { }, } - commandFlags.defineCountOptions(countCmd) + flags.defineCountOptions(countCmd) // TODO: Remove these after implementing `alp (json|ltsv|regex) count`. countCmd.PersistentFlags().StringP("pattern", "", options.DefaultPatternOption, "Regular expressions pattern matching the log") diff --git a/cmd/alp/cmd/count_test.go b/cmd/alp/cmd/count_test.go index 4216e03..74058cb 100644 --- a/cmd/alp/cmd/count_test.go +++ b/cmd/alp/cmd/count_test.go @@ -13,10 +13,10 @@ func TestCountCmd(t *testing.T) { "--keys", "ua", } - rootCmd := NewRootCmd("test") - rootCmd.SetArgs(args) + command := NewCommand("test") + command.setArgs(args) - err := rootCmd.Execute() + err := command.Execute() if err != nil { t.Fatal(err) } diff --git a/cmd/alp/cmd/diff.go b/cmd/alp/cmd/diff.go index 4968352..52bd0bb 100644 --- a/cmd/alp/cmd/diff.go +++ b/cmd/alp/cmd/diff.go @@ -4,17 +4,19 @@ import ( "os" "github.com/spf13/cobra" + "github.com/tkuchiki/alp/parsers" + "github.com/tkuchiki/alp/profiler" "github.com/tkuchiki/alp/stats" ) -func NewDiffCmd(commandFlags *flags) *cobra.Command { +func newDiffCmd(flags *flags) *cobra.Command { diffCmd := &cobra.Command{ Use: "diff ", Args: cobra.ExactArgs(2), Short: "Show the difference between the two profile results", Long: `Show the difference between the two profile results`, RunE: func(cmd *cobra.Command, args []string) error { - opts, err := commandFlags.createDiffOptions(cmd) + opts, err := flags.createDiffOptions(cmd) if err != nil { return err } @@ -30,7 +32,7 @@ func NewDiffCmd(commandFlags *flags) *cobra.Command { } sts.SetOptions(opts) - sts.SetSortOptions(commandFlags.sortOptions) + sts.SetSortOptions(flags.sortOptions) printOptions := stats.NewPrintOptions(opts.NoHeaders, opts.ShowFooters, opts.DecodeUri, opts.PaginationLimit) printer := stats.NewPrinter(os.Stdout, opts.Output, opts.Format, opts.Percentiles, printOptions) @@ -57,7 +59,7 @@ func NewDiffCmd(commandFlags *flags) *cobra.Command { } toSts.SetOptions(opts) - toSts.SetSortOptions(commandFlags.sortOptions) + toSts.SetSortOptions(flags.sortOptions) tof, err := os.Open(to) if err != nil { @@ -77,7 +79,7 @@ func NewDiffCmd(commandFlags *flags) *cobra.Command { }, } - commandFlags.defineDiffOptions(diffCmd) + flags.defineDiffOptions(diffCmd) diffCmd.Flags().SortFlags = false diffCmd.PersistentFlags().SortFlags = false @@ -85,3 +87,32 @@ func NewDiffCmd(commandFlags *flags) *cobra.Command { return diffCmd } + +func newDiffSubCmd() *cobra.Command { + return &cobra.Command{ + Use: "diff [] ", + Args: cobra.RangeArgs(1, 2), + Short: "Show the difference between the two access logs", + Long: `Show the difference between the two access logs`, + } +} + +func getFromTo(load string, args []string) (string, string) { + if load != "" { + return "", args[0] + } + + return args[0], args[1] +} + +func runDiff(sortOptions *stats.SortOptions, + fromProf *profiler.Profiler, fromParser parsers.Parser, + toProf *profiler.Profiler, toParser parsers.Parser) error { + + fromSts, err := fromProf.Profile(sortOptions, fromParser) + if err != nil { + return err + } + + return toProf.Run(sortOptions, toParser, fromSts) +} diff --git a/cmd/alp/cmd/diff_test.go b/cmd/alp/cmd/diff_test.go index 19f4491..bfbbcba 100644 --- a/cmd/alp/cmd/diff_test.go +++ b/cmd/alp/cmd/diff_test.go @@ -9,10 +9,10 @@ func TestDiffCmd(t *testing.T) { from, to, } - rootCmd := NewRootCmd("test") - rootCmd.SetArgs(args) + command := NewCommand("test") + command.setArgs(args) - err := rootCmd.Execute() + err := command.Execute() if err != nil { t.Fatal(err) } diff --git a/cmd/alp/cmd/flags.go b/cmd/alp/cmd/flags.go index b2e3dcc..6db220c 100644 --- a/cmd/alp/cmd/flags.go +++ b/cmd/alp/cmd/flags.go @@ -350,6 +350,32 @@ func (f *flags) defineDiffOptions(cmd *cobra.Command) { f.definePage(cmd) } +func (f *flags) defineDiffSubCommandOptions(cmd *cobra.Command) { + // overwrite and hidden => remove flag + cmd.LocalFlags().String(flagFile, "", "") + cmd.LocalFlags().MarkHidden(flagFile) + + f.defineDump(cmd) + f.defineLoad(cmd) + f.defineFormat(cmd) + f.defineSort(cmd) + f.defineReverse(cmd) + f.defineNoHeaders(cmd) + f.defineShowFooters(cmd) + f.defineLimit(cmd) + f.defineOutput(cmd) + f.defineQueryString(cmd) + f.defineQueryStringIgnoreValues(cmd) + f.defineLocation(cmd) + f.defineDecodeUri(cmd) + f.defineMatchingGroups(cmd) + f.defineFilters(cmd) + f.definePositionFile(cmd) + f.defineNoSavePositionFile(cmd) + f.definePercentiles(cmd) + f.definePage(cmd) +} + func (f *flags) bindFlags(cmd *cobra.Command) { viper.BindPFlag("file", cmd.PersistentFlags().Lookup(flagFile)) viper.BindPFlag("dump", cmd.PersistentFlags().Lookup(flagDump)) @@ -614,7 +640,6 @@ func (f *flags) setOptions(cmd *cobra.Command, opts *options.Options, flags []st func (f *flags) setProfileOptions(cmd *cobra.Command, opts *options.Options) (*options.Options, error) { _flags := []string{ - flagConfig, flagFile, flagDump, flagLoad, @@ -836,6 +861,32 @@ func (f *flags) setDiffOptions(cmd *cobra.Command, opts *options.Options) (*opti return f.setOptions(cmd, opts, _flags) } +func (f *flags) setDiffSubCommandOptions(cmd *cobra.Command, opts *options.Options) (*options.Options, error) { + _flags := []string{ + flagDump, + flagLoad, + flagFormat, + flagSort, + flagReverse, + flagNoHeaders, + flagShowFooters, + flagLimit, + flagOutput, + flagQueryString, + flagQueryStringIgnoreValues, + flagLocation, + flagDecodeUri, + flagMatchingGroups, + flagFilters, + flagPositionFile, + flagNoSavePositionFile, + flagPercentiles, + flagPage, + } + + return f.setOptions(cmd, opts, _flags) +} + func (f *flags) createJSONOptions(cmd *cobra.Command) (*options.Options, error) { if f.config != "" { f.bindFlags(cmd) @@ -850,6 +901,20 @@ func (f *flags) createJSONOptions(cmd *cobra.Command) (*options.Options, error) return f.setJSONOptions(cmd, opts) } +func (f *flags) createJSONDiffOptions(cmd *cobra.Command) (*options.Options, error) { + if f.config != "" { + f.bindFlags(cmd) + return f.createOptionsFromConfig(cmd) + } + + opts, err := f.setDiffSubCommandOptions(cmd, options.NewOptions()) + if err != nil { + return nil, err + } + + return f.setJSONOptions(cmd, opts) +} + func (f *flags) createLTSVOptions(cmd *cobra.Command) (*options.Options, error) { if f.config != "" { f.bindFlags(cmd) @@ -864,6 +929,20 @@ func (f *flags) createLTSVOptions(cmd *cobra.Command) (*options.Options, error) return f.setLTSVOptions(cmd, opts) } +func (f *flags) createLTSVDiffOptions(cmd *cobra.Command) (*options.Options, error) { + if f.config != "" { + f.bindFlags(cmd) + return f.createOptionsFromConfig(cmd) + } + + opts, err := f.setDiffSubCommandOptions(cmd, options.NewOptions()) + if err != nil { + return nil, err + } + + return f.setLTSVOptions(cmd, opts) +} + func (f *flags) createRegexpOptions(cmd *cobra.Command) (*options.Options, error) { if f.config != "" { f.bindFlags(cmd) @@ -878,6 +957,20 @@ func (f *flags) createRegexpOptions(cmd *cobra.Command) (*options.Options, error return f.setRegexpOptions(cmd, opts) } +func (f *flags) createRegexpDiffOptions(cmd *cobra.Command) (*options.Options, error) { + if f.config != "" { + f.bindFlags(cmd) + return f.createOptionsFromConfig(cmd) + } + + opts, err := f.setDiffSubCommandOptions(cmd, options.NewOptions()) + if err != nil { + return nil, err + } + + return f.setRegexpOptions(cmd, opts) +} + func (f *flags) createPcapOptions(cmd *cobra.Command) (*options.Options, error) { if f.config != "" { f.bindFlags(cmd) @@ -892,6 +985,20 @@ func (f *flags) createPcapOptions(cmd *cobra.Command) (*options.Options, error) return f.setPcapOptions(cmd, opts) } +func (f *flags) createPcapDiffOptions(cmd *cobra.Command) (*options.Options, error) { + if f.config != "" { + f.bindFlags(cmd) + return f.createOptionsFromConfig(cmd) + } + + opts, err := f.setDiffSubCommandOptions(cmd, options.NewOptions()) + if err != nil { + return nil, err + } + + return f.setPcapOptions(cmd, opts) +} + func (f *flags) createCountOptions(cmd *cobra.Command) (*options.Options, error) { if f.config != "" { f.bindFlags(cmd) diff --git a/cmd/alp/cmd/flags_test.go b/cmd/alp/cmd/flags_test.go index d9e8ad1..1b3f68a 100644 --- a/cmd/alp/cmd/flags_test.go +++ b/cmd/alp/cmd/flags_test.go @@ -13,28 +13,20 @@ import ( func Test_createOptionsFromConfig(t *testing.T) { viper.Reset() - rootCmd := NewRootCmd("test") - flags := newFlags() - - flags.defineProfileOptions(rootCmd) - flags.defineJSONOptions(rootCmd) - flags.defineLTSVOptions(rootCmd) - flags.defineRegexpOptions(rootCmd) - flags.definePcapOptions(rootCmd) - flags.defineCountKeys(rootCmd) + command := NewCommand("test") tempDir := t.TempDir() sort := "max" dummyOpts := testutil.DummyOptions(sort) var err error - flags.config, err = testutil.CreateTempDirAndFile(tempDir, "test_create_options_from_config_config", testutil.DummyConfigFile(sort, dummyOpts)) + command.flags.config, err = testutil.CreateTempDirAndFile(tempDir, "test_create_options_from_config_config", testutil.DummyConfigFile(sort, dummyOpts)) if err != nil { t.Fatal(err) } var opts *options.Options - opts, err = flags.createOptionsFromConfig(rootCmd) + opts, err = command.flags.createOptionsFromConfig(command.rootCmd) if err != nil { t.Fatal(err) } @@ -45,15 +37,7 @@ func Test_createOptionsFromConfig(t *testing.T) { } func Test_createOptionsFromConfig_overwrite(t *testing.T) { - rootCmd := NewRootCmd("test") - flags := newFlags() - - flags.defineProfileOptions(rootCmd) - flags.defineJSONOptions(rootCmd) - flags.defineLTSVOptions(rootCmd) - flags.defineRegexpOptions(rootCmd) - flags.definePcapOptions(rootCmd) - flags.defineCountKeys(rootCmd) + command := NewCommand("test") tempDir := t.TempDir() sort := "max" @@ -62,7 +46,7 @@ func Test_createOptionsFromConfig_overwrite(t *testing.T) { overwrittenOpts := testutil.DummyOverwrittenOptions(overwrittenSort) var err error - flags.config, err = testutil.CreateTempDirAndFile(tempDir, "test_create_options_from_config_overwrite_config", testutil.DummyConfigFile(sort, overwrittenOpts)) + command.flags.config, err = testutil.CreateTempDirAndFile(tempDir, "test_create_options_from_config_overwrite_config", testutil.DummyConfigFile(sort, overwrittenOpts)) if err != nil { t.Fatal(err) } @@ -124,7 +108,7 @@ func Test_createOptionsFromConfig_overwrite(t *testing.T) { viper.Set("count.keys", overwrittenOpts.Count.Keys) var opts *options.Options - opts, err = flags.createOptionsFromConfig(rootCmd) + opts, err = command.flags.createOptionsFromConfig(command.rootCmd) if err != nil { t.Fatal(err) } diff --git a/cmd/alp/cmd/json.go b/cmd/alp/cmd/json.go index 2e34940..880a499 100644 --- a/cmd/alp/cmd/json.go +++ b/cmd/alp/cmd/json.go @@ -3,43 +3,45 @@ package cmd import ( "os" + "github.com/spf13/cobra" + "github.com/tkuchiki/alp/options" "github.com/tkuchiki/alp/parsers" "github.com/tkuchiki/alp/profiler" - - "github.com/spf13/cobra" ) -func NewJSONCmd(commandFlags *flags) *cobra.Command { +func newJSONCmd(flags *flags) *cobra.Command { var jsonCmd = &cobra.Command{ Use: "json", Short: "Profile the logs for JSON", Long: `Profile the logs for JSON`, RunE: func(cmd *cobra.Command, args []string) error { - opts, err := commandFlags.createJSONOptions(cmd) + opts, err := flags.createJSONOptions(cmd) if err != nil { return err } prof := profiler.NewProfiler(os.Stdout, os.Stderr, opts) + if err = prof.ValidatePrinter(); err != nil { + return err + } + f, err := prof.Open(opts.File) if err != nil { return err } defer f.Close() - keys := parsers.NewJSONKeys(opts.JSON.UriKey, opts.JSON.MethodKey, opts.JSON.TimeKey, - opts.JSON.ResponseTimeKey, opts.JSON.RequestTimeKey, opts.JSON.BodyBytesKey, opts.JSON.StatusKey) - parser := parsers.NewJSONParser(f, keys, opts.QueryString, opts.QueryStringIgnoreValues) + parser := newJsonParser(opts, f) - err = prof.Run(commandFlags.sortOptions, parser) + err = prof.Run(flags.sortOptions, parser, nil) return err }, } - commandFlags.defineProfileOptions(jsonCmd) - commandFlags.defineJSONOptions(jsonCmd) + flags.defineProfileOptions(jsonCmd) + flags.defineJSONOptions(jsonCmd) jsonCmd.Flags().SortFlags = false jsonCmd.PersistentFlags().SortFlags = false @@ -47,3 +49,61 @@ func NewJSONCmd(commandFlags *flags) *cobra.Command { return jsonCmd } + +func newJsonParser(opts *options.Options, f *os.File) parsers.Parser { + keys := parsers.NewJSONKeys(opts.JSON.UriKey, opts.JSON.MethodKey, opts.JSON.TimeKey, + opts.JSON.ResponseTimeKey, opts.JSON.RequestTimeKey, opts.JSON.BodyBytesKey, opts.JSON.StatusKey) + + return parsers.NewJSONParser(f, keys, opts.QueryString, opts.QueryStringIgnoreValues) +} + +func newJsonDiffCmd(flags *flags) *cobra.Command { + jsonDiffCmd := newDiffSubCmd() + jsonDiffCmd.RunE = func(cmd *cobra.Command, args []string) error { + opts, err := flags.createJSONDiffOptions(cmd) + if err != nil { + return err + } + + from, to := getFromTo(opts.Load, args) + + fromProf := profiler.NewProfiler(os.Stdout, os.Stderr, opts) + + if err = fromProf.ValidatePrinter(); err != nil { + return err + } + + fromf, err := fromProf.Open(from) + if err != nil { + return err + } + defer fromf.Close() + + fromParser := newJsonParser(opts, fromf) + + toProf := profiler.NewProfiler(os.Stdout, os.Stderr, opts) + toProf.DisableLoad() + + tof, err := toProf.Open(to) + if err != nil { + return err + } + defer tof.Close() + + toParser := newJsonParser(opts, tof) + + return runDiff(flags.sortOptions, + fromProf, fromParser, + toProf, toParser, + ) + } + + flags.defineDiffSubCommandOptions(jsonDiffCmd) + flags.defineJSONOptions(jsonDiffCmd) + + jsonDiffCmd.Flags().SortFlags = false + jsonDiffCmd.PersistentFlags().SortFlags = false + jsonDiffCmd.InheritedFlags().SortFlags = false + + return jsonDiffCmd +} diff --git a/cmd/alp/cmd/json_test.go b/cmd/alp/cmd/json_test.go index f630f3d..59888d6 100644 --- a/cmd/alp/cmd/json_test.go +++ b/cmd/alp/cmd/json_test.go @@ -35,11 +35,73 @@ func TestJSONCmd(t *testing.T) { "--status-key", keys.Status, } - rootCmd := NewRootCmd("test") - rootCmd.SetArgs(args) + command := NewCommand("test") + command.setArgs(args) - err = rootCmd.Execute() + err = command.Execute() if err != nil { t.Fatal(err) } } + +func TestJSONDiffCmd(t *testing.T) { + keys := testutil.LogKeys{ + Uri: "u", + Method: "m", + Time: "t", + ResponseTime: "r", + RequestTime: "r2", + BodyBytes: "b", + Status: "s", + } + + jsonLog := testutil.JsonLog(keys) + + tempDir := t.TempDir() + + tempFromFile, err := testutil.CreateTempDirAndFile(tempDir, "test_json_diff_cmd_temp_from_file", jsonLog) + if err != nil { + t.Fatal(err) + } + + tempToFile, err := testutil.CreateTempDirAndFile(tempDir, "test_json_diff_cmd_temp_to_file", jsonLog) + if err != nil { + t.Fatal(err) + } + + tempDump, err := testutil.CreateTempDirAndFile(tempDir, "test_json_diff_cmd_temp_dump", "") + if err != nil { + t.Fatal(err) + } + + t.Run("alp json diff ", func(t *testing.T) { + args := []string{"json", "diff", + tempFromFile, + tempToFile, + "--dump", tempDump, + } + + command := NewCommand("test") + command.setArgs(args) + + err = command.Execute() + if err != nil { + t.Fatal(err) + } + }) + + t.Run("alp json diff --load ", func(t *testing.T) { + args := []string{"json", "diff", + "--load", tempDump, + tempToFile, + } + + command := NewCommand("test") + command.setArgs(args) + + err = command.Execute() + if err != nil { + t.Fatal(err) + } + }) +} diff --git a/cmd/alp/cmd/ltsv.go b/cmd/alp/cmd/ltsv.go index d6afe32..02385ff 100644 --- a/cmd/alp/cmd/ltsv.go +++ b/cmd/alp/cmd/ltsv.go @@ -3,44 +3,45 @@ package cmd import ( "os" + "github.com/spf13/cobra" + "github.com/tkuchiki/alp/options" "github.com/tkuchiki/alp/parsers" "github.com/tkuchiki/alp/profiler" - - "github.com/spf13/cobra" ) -func NewLTSVCmd(commandFlags *flags) *cobra.Command { +func newLTSVCmd(flags *flags) *cobra.Command { var ltsvCmd = &cobra.Command{ Use: "ltsv", Short: "Profile the logs for LTSV", Long: `Profile the logs for LTSV`, RunE: func(cmd *cobra.Command, args []string) error { - opts, err := commandFlags.createLTSVOptions(cmd) + opts, err := flags.createLTSVOptions(cmd) if err != nil { return err } prof := profiler.NewProfiler(os.Stdout, os.Stderr, opts) + if err = prof.ValidatePrinter(); err != nil { + return err + } + f, err := prof.Open(opts.File) if err != nil { return err } defer f.Close() - label := parsers.NewLTSVLabel(opts.LTSV.UriLabel, opts.LTSV.MethodLabel, opts.LTSV.TimeLabel, - opts.LTSV.ApptimeLabel, opts.LTSV.ReqtimeLabel, opts.LTSV.SizeLabel, opts.LTSV.StatusLabel, - ) - parser := parsers.NewLTSVParser(f, label, opts.QueryString, opts.QueryStringIgnoreValues) + parser := newLTSVParser(opts, f) - err = prof.Run(commandFlags.sortOptions, parser) + err = prof.Run(flags.sortOptions, parser, nil) return err }, } - commandFlags.defineProfileOptions(ltsvCmd) - commandFlags.defineLTSVOptions(ltsvCmd) + flags.defineProfileOptions(ltsvCmd) + flags.defineLTSVOptions(ltsvCmd) ltsvCmd.Flags().SortFlags = false ltsvCmd.PersistentFlags().SortFlags = false @@ -48,3 +49,62 @@ func NewLTSVCmd(commandFlags *flags) *cobra.Command { return ltsvCmd } + +func newLTSVParser(opts *options.Options, f *os.File) parsers.Parser { + label := parsers.NewLTSVLabel(opts.LTSV.UriLabel, opts.LTSV.MethodLabel, opts.LTSV.TimeLabel, + opts.LTSV.ApptimeLabel, opts.LTSV.ReqtimeLabel, opts.LTSV.SizeLabel, opts.LTSV.StatusLabel, + ) + + return parsers.NewLTSVParser(f, label, opts.QueryString, opts.QueryStringIgnoreValues) +} + +func newLTSVDiffCmd(flags *flags) *cobra.Command { + ltsvDiffCmd := newDiffSubCmd() + ltsvDiffCmd.RunE = func(cmd *cobra.Command, args []string) error { + opts, err := flags.createLTSVDiffOptions(cmd) + if err != nil { + return err + } + + from, to := getFromTo(opts.Load, args) + + fromProf := profiler.NewProfiler(os.Stdout, os.Stderr, opts) + + if err = fromProf.ValidatePrinter(); err != nil { + return err + } + + fromf, err := fromProf.Open(from) + if err != nil { + return err + } + defer fromf.Close() + + fromParser := newLTSVParser(opts, fromf) + + toProf := profiler.NewProfiler(os.Stdout, os.Stderr, opts) + toProf.DisableLoad() + + tof, err := toProf.Open(to) + if err != nil { + return err + } + defer tof.Close() + + toParser := newLTSVParser(opts, tof) + + return runDiff(flags.sortOptions, + fromProf, fromParser, + toProf, toParser, + ) + } + + flags.defineDiffSubCommandOptions(ltsvDiffCmd) + flags.defineLTSVOptions(ltsvDiffCmd) + + ltsvDiffCmd.Flags().SortFlags = false + ltsvDiffCmd.PersistentFlags().SortFlags = false + ltsvDiffCmd.InheritedFlags().SortFlags = false + + return ltsvDiffCmd +} diff --git a/cmd/alp/cmd/ltsv_test.go b/cmd/alp/cmd/ltsv_test.go index 8745627..2b84499 100644 --- a/cmd/alp/cmd/ltsv_test.go +++ b/cmd/alp/cmd/ltsv_test.go @@ -35,11 +35,73 @@ func TestLTSVCmd(t *testing.T) { "--status-label", keys.Status, } - rootCmd := NewRootCmd("test") - rootCmd.SetArgs(args) + command := NewCommand("test") + command.setArgs(args) - err = rootCmd.Execute() + err = command.Execute() if err != nil { t.Fatal(err) } } + +func TestLTSVDiffCmd(t *testing.T) { + keys := testutil.LogKeys{ + Uri: "u", + Method: "m", + Time: "t", + ResponseTime: "r", + RequestTime: "r2", + BodyBytes: "b", + Status: "s", + } + + ltsvLog := testutil.LTSVLog(keys) + + tempDir := t.TempDir() + + tempFromFile, err := testutil.CreateTempDirAndFile(tempDir, "test_ltsv_diff_cmd_temp_from_file", ltsvLog) + if err != nil { + t.Fatal(err) + } + + tempToFile, err := testutil.CreateTempDirAndFile(tempDir, "test_ltsv_diff_cmd_temp_to_file", ltsvLog) + if err != nil { + t.Fatal(err) + } + + tempDump, err := testutil.CreateTempDirAndFile(tempDir, "test_ltsv_diff_cmd_temp_dump", "") + if err != nil { + t.Fatal(err) + } + + t.Run("alp ltsv diff ", func(t *testing.T) { + args := []string{"ltsv", "diff", + tempFromFile, + tempToFile, + "--dump", tempDump, + } + + command := NewCommand("test") + command.setArgs(args) + + err = command.Execute() + if err != nil { + t.Fatal(err) + } + }) + + t.Run("alp ltsv diff --load ", func(t *testing.T) { + args := []string{"ltsv", "diff", + "--load", tempDump, + tempToFile, + } + + command := NewCommand("test") + command.setArgs(args) + + err = command.Execute() + if err != nil { + t.Fatal(err) + } + }) +} diff --git a/cmd/alp/cmd/pcap.go b/cmd/alp/cmd/pcap.go index 57905f7..a365e9b 100644 --- a/cmd/alp/cmd/pcap.go +++ b/cmd/alp/cmd/pcap.go @@ -3,44 +3,48 @@ package cmd import ( "os" + "github.com/spf13/cobra" + "github.com/tkuchiki/alp/options" "github.com/tkuchiki/alp/parsers" "github.com/tkuchiki/alp/profiler" - - "github.com/spf13/cobra" ) -func NewPcapCmd(commandFlags *flags) *cobra.Command { +func newPcapCmd(flags *flags) *cobra.Command { var pcapCmd = &cobra.Command{ Use: "pcap", Short: "Profile the HTTP requests for captured packets", Long: `Profile the HTTP requests for captured packets`, RunE: func(cmd *cobra.Command, args []string) error { - opts, err := commandFlags.createPcapOptions(cmd) + opts, err := flags.createPcapOptions(cmd) if err != nil { return err } prof := profiler.NewProfiler(os.Stdout, os.Stderr, opts) + if err = prof.ValidatePrinter(); err != nil { + return err + } + f, err := prof.Open(opts.File) if err != nil { return err } defer f.Close() - parser, err := parsers.NewPcapParser(f, opts.Pcap.ServerIPs, opts.Pcap.ServerPort, opts.QueryString, opts.QueryStringIgnoreValues) + parser, err := newPcapParser(opts, f) if err != nil { return err } - err = prof.Run(commandFlags.sortOptions, parser) + err = prof.Run(flags.sortOptions, parser, nil) return err }, } - commandFlags.defineProfileOptions(pcapCmd) - commandFlags.definePcapOptions(pcapCmd) + flags.defineProfileOptions(pcapCmd) + flags.definePcapOptions(pcapCmd) pcapCmd.Flags().SortFlags = false pcapCmd.PersistentFlags().SortFlags = false @@ -48,3 +52,67 @@ func NewPcapCmd(commandFlags *flags) *cobra.Command { return pcapCmd } + +func newPcapParser(opts *options.Options, f *os.File) (parsers.Parser, error) { + return parsers.NewPcapParser(f, opts.Pcap.ServerIPs, opts.Pcap.ServerPort, opts.QueryString, opts.QueryStringIgnoreValues) +} + +func newPcapDiffCmd(flags *flags) *cobra.Command { + pcapDiffCmd := newDiffSubCmd() + pcapDiffCmd.RunE = func(cmd *cobra.Command, args []string) error { + opts, err := flags.createPcapDiffOptions(cmd) + if err != nil { + return err + } + + from, to := getFromTo(opts.Load, args) + + fromProf := profiler.NewProfiler(os.Stdout, os.Stderr, opts) + + if err = fromProf.ValidatePrinter(); err != nil { + return err + } + + fromf, err := fromProf.Open(from) + if err != nil { + return err + } + defer fromf.Close() + + var fromParser parsers.Parser + if opts.Load == "" { + fromParser, err = newPcapParser(opts, fromf) + if err != nil { + return err + } + } + + toProf := profiler.NewProfiler(os.Stdout, os.Stderr, opts) + toProf.DisableLoad() + + tof, err := toProf.Open(to) + if err != nil { + return err + } + defer tof.Close() + + toParser, err := newPcapParser(opts, tof) + if err != nil { + return err + } + + return runDiff(flags.sortOptions, + fromProf, fromParser, + toProf, toParser, + ) + } + + flags.defineDiffSubCommandOptions(pcapDiffCmd) + flags.definePcapOptions(pcapDiffCmd) + + pcapDiffCmd.Flags().SortFlags = false + pcapDiffCmd.PersistentFlags().SortFlags = false + pcapDiffCmd.InheritedFlags().SortFlags = false + + return pcapDiffCmd +} diff --git a/cmd/alp/cmd/pcap_test.go b/cmd/alp/cmd/pcap_test.go index 5eaffe1..3c251c2 100644 --- a/cmd/alp/cmd/pcap_test.go +++ b/cmd/alp/cmd/pcap_test.go @@ -3,6 +3,8 @@ package cmd import ( "testing" + "github.com/tkuchiki/alp/internal/testutil" + "github.com/tkuchiki/alp/options" ) @@ -16,11 +18,58 @@ func TestPcapCmd(t *testing.T) { "--pcap-server-port", pcapServerPort, } - rootCmd := NewRootCmd("test") - rootCmd.SetArgs(args) + command := NewCommand("test") + command.setArgs(args) + + err := command.Execute() + if err != nil { + t.Fatal(err) + } +} + +func TestPcapDiffCmd(t *testing.T) { + pcapFile := "../../../example/logs/http.cap" + pcapServerPort := "18080" + + tempDir := t.TempDir() - err := rootCmd.Execute() + tempDump, err := testutil.CreateTempDirAndFile(tempDir, "test_pcap_diff_cmd_temp_dump", "") if err != nil { t.Fatal(err) } + + t.Run("alp pcap diff ", func(t *testing.T) { + args := []string{"pcap", "diff", + pcapFile, + pcapFile, + "--dump", tempDump, + "--pcap-server-ip", options.DefaultPcapServerIPsOption[0], + "--pcap-server-port", pcapServerPort, + } + + command := NewCommand("test") + command.setArgs(args) + + err = command.Execute() + if err != nil { + t.Fatal(err) + } + }) + + t.Run("alp pcap diff --load ", func(t *testing.T) { + args := []string{"pcap", "diff", + "--load", tempDump, + pcapFile, + "--pcap-server-ip", options.DefaultPcapServerIPsOption[0], + "--pcap-server-port", pcapServerPort, + } + + command := NewCommand("test") + command.setArgs(args) + + err = command.Execute() + if err != nil { + t.Fatal(err) + } + }) } diff --git a/cmd/alp/cmd/regexp.go b/cmd/alp/cmd/regexp.go index b86e999..9d2297e 100644 --- a/cmd/alp/cmd/regexp.go +++ b/cmd/alp/cmd/regexp.go @@ -3,46 +3,48 @@ package cmd import ( "os" + "github.com/spf13/cobra" + "github.com/tkuchiki/alp/options" "github.com/tkuchiki/alp/parsers" "github.com/tkuchiki/alp/profiler" - - "github.com/spf13/cobra" ) -func NewRegexpCmd(commandFlags *flags) *cobra.Command { +func newRegexpCmd(flags *flags) *cobra.Command { var regexpCmd = &cobra.Command{ Use: "regexp", Short: "Profile the logs that match a regular expression", Long: `Profile the logs that match a regular expression`, RunE: func(cmd *cobra.Command, args []string) error { - opts, err := commandFlags.createRegexpOptions(cmd) + opts, err := flags.createRegexpOptions(cmd) if err != nil { return err } prof := profiler.NewProfiler(os.Stdout, os.Stderr, opts) + if err = prof.ValidatePrinter(); err != nil { + return err + } + f, err := prof.Open(opts.File) if err != nil { return err } defer f.Close() - names := parsers.NewSubexpNames(opts.Regexp.UriSubexp, opts.Regexp.MethodSubexp, opts.Regexp.TimeSubexp, - opts.Regexp.ResponseTimeSubexp, opts.Regexp.RequestTimeSubexp, opts.Regexp.BodyBytesSubexp, opts.Regexp.StatusSubexp) - parser, err := parsers.NewRegexpParser(f, opts.Regexp.Pattern, names, opts.QueryString, opts.QueryStringIgnoreValues) + parser, err := newRegexpParser(opts, f) if err != nil { return err } - err = prof.Run(commandFlags.sortOptions, parser) + err = prof.Run(flags.sortOptions, parser, nil) return err }, } - commandFlags.defineProfileOptions(regexpCmd) - commandFlags.defineRegexpOptions(regexpCmd) + flags.defineProfileOptions(regexpCmd) + flags.defineRegexpOptions(regexpCmd) regexpCmd.Flags().SortFlags = false regexpCmd.PersistentFlags().SortFlags = false @@ -50,3 +52,66 @@ func NewRegexpCmd(commandFlags *flags) *cobra.Command { return regexpCmd } + +func newRegexpParser(opts *options.Options, f *os.File) (parsers.Parser, error) { + names := parsers.NewSubexpNames(opts.Regexp.UriSubexp, opts.Regexp.MethodSubexp, opts.Regexp.TimeSubexp, + opts.Regexp.ResponseTimeSubexp, opts.Regexp.RequestTimeSubexp, opts.Regexp.BodyBytesSubexp, opts.Regexp.StatusSubexp) + return parsers.NewRegexpParser(f, opts.Regexp.Pattern, names, opts.QueryString, opts.QueryStringIgnoreValues) +} + +func newRegexpDiffCmd(flags *flags) *cobra.Command { + regexpDiffCmd := newDiffSubCmd() + regexpDiffCmd.RunE = func(cmd *cobra.Command, args []string) error { + opts, err := flags.createRegexpDiffOptions(cmd) + if err != nil { + return err + } + + from, to := getFromTo(opts.Load, args) + + fromProf := profiler.NewProfiler(os.Stdout, os.Stderr, opts) + + if err = fromProf.ValidatePrinter(); err != nil { + return err + } + + fromf, err := fromProf.Open(from) + if err != nil { + return err + } + defer fromf.Close() + + fromParser, err := newRegexpParser(opts, fromf) + if err != nil { + return err + } + + toProf := profiler.NewProfiler(os.Stdout, os.Stderr, opts) + toProf.DisableLoad() + + tof, err := toProf.Open(to) + if err != nil { + return err + } + defer tof.Close() + + toParser, err := newRegexpParser(opts, tof) + if err != nil { + return err + } + + return runDiff(flags.sortOptions, + fromProf, fromParser, + toProf, toParser, + ) + } + + flags.defineDiffSubCommandOptions(regexpDiffCmd) + flags.defineRegexpOptions(regexpDiffCmd) + + regexpDiffCmd.Flags().SortFlags = false + regexpDiffCmd.PersistentFlags().SortFlags = false + regexpDiffCmd.InheritedFlags().SortFlags = false + + return regexpDiffCmd +} diff --git a/cmd/alp/cmd/regexp_test.go b/cmd/alp/cmd/regexp_test.go index 1a5b837..268ac62 100644 --- a/cmd/alp/cmd/regexp_test.go +++ b/cmd/alp/cmd/regexp_test.go @@ -36,11 +36,63 @@ func TestRegexpCmd(t *testing.T) { "--status-subexp", keys.Status, } - rootCmd := NewRootCmd("test") - rootCmd.SetArgs(args) + command := NewCommand("test") + command.setArgs(args) - err = rootCmd.Execute() + err = command.Execute() if err != nil { t.Fatal(err) } } + +func TestRegexpDiffCmd(t *testing.T) { + regexpLog := testutil.RegexpLog() + + tempDir := t.TempDir() + + tempFromFile, err := testutil.CreateTempDirAndFile(tempDir, "test_regexp_diff_cmd_temp_from_file", regexpLog) + if err != nil { + t.Fatal(err) + } + + tempToFile, err := testutil.CreateTempDirAndFile(tempDir, "test_regexp_diff_cmd_temp_to_file", regexpLog) + if err != nil { + t.Fatal(err) + } + + tempDump, err := testutil.CreateTempDirAndFile(tempDir, "test_regexp_diff_cmd_temp_dump", "") + if err != nil { + t.Fatal(err) + } + + t.Run("alp regexp diff ", func(t *testing.T) { + args := []string{"regexp", "diff", + tempFromFile, + tempToFile, + "--dump", tempDump, + } + + command := NewCommand("test") + command.setArgs(args) + + err = command.Execute() + if err != nil { + t.Fatal(err) + } + }) + + t.Run("alp regexp diff --load ", func(t *testing.T) { + args := []string{"regexp", "diff", + "--load", tempDump, + tempToFile, + } + + command := NewCommand("test") + command.setArgs(args) + + err = command.Execute() + if err != nil { + t.Fatal(err) + } + }) +} diff --git a/cmd/alp/cmd/root.go b/cmd/alp/cmd/root.go index a2aaa0b..7ba4a6d 100644 --- a/cmd/alp/cmd/root.go +++ b/cmd/alp/cmd/root.go @@ -6,7 +6,7 @@ import ( "github.com/spf13/cobra" ) -func NewRootCmd(version string) *cobra.Command { +func newRootCmd(version string) *cobra.Command { var rootCmd = &cobra.Command{ Use: "alp", Short: "Access Log Profiler", @@ -23,22 +23,11 @@ func NewRootCmd(version string) *cobra.Command { }, } - commandFlags := newFlags() - - commandFlags.defineGlobalOptions(rootCmd) - - rootCmd.AddCommand(NewLTSVCmd(commandFlags)) - rootCmd.AddCommand(NewJSONCmd(commandFlags)) - rootCmd.AddCommand(NewRegexpCmd(commandFlags)) - rootCmd.AddCommand(NewPcapCmd(commandFlags)) - rootCmd.AddCommand(NewDiffCmd(commandFlags)) - rootCmd.AddCommand(NewCountCmd(commandFlags)) rootCmd.SetVersionTemplate(fmt.Sprintln(version)) - return rootCmd -} + rootCmd.Flags().SortFlags = false + rootCmd.PersistentFlags().SortFlags = false + rootCmd.InheritedFlags().SortFlags = false -func Execute(version string) error { - rootCmd := NewRootCmd(version) - return rootCmd.Execute() + return rootCmd } diff --git a/cmd/alp/cmd/root_test.go b/cmd/alp/cmd/root_test.go index 2cc7ddd..50ae311 100644 --- a/cmd/alp/cmd/root_test.go +++ b/cmd/alp/cmd/root_test.go @@ -3,9 +3,9 @@ package cmd import "testing" func TestNewRootCmd(t *testing.T) { - rootCmd := NewRootCmd("test") + command := NewCommand("test") - err := rootCmd.Execute() + err := command.Execute() if err != nil { t.Fatal(err) } diff --git a/cmd/alp/main.go b/cmd/alp/main.go index 6f7b13b..9322e5b 100644 --- a/cmd/alp/main.go +++ b/cmd/alp/main.go @@ -9,7 +9,8 @@ import ( var version string func main() { - if err := cmd.Execute(version); err != nil { + command := cmd.NewCommand(version) + if err := command.Execute(); err != nil { log.Fatal(err) } } diff --git a/profiler/profiler.go b/profiler/profiler.go index 5b070fb..c385393 100644 --- a/profiler/profiler.go +++ b/profiler/profiler.go @@ -6,27 +6,33 @@ import ( "io" "os" - "github.com/tkuchiki/alp/helpers" - "github.com/tkuchiki/alp/errors" + "github.com/tkuchiki/alp/helpers" "github.com/tkuchiki/alp/options" "github.com/tkuchiki/alp/parsers" "github.com/tkuchiki/alp/stats" ) type Profiler struct { - options *options.Options - outWriter io.Writer - errWriter io.Writer - inReader *os.File + options *options.Options + outWriter io.Writer + errWriter io.Writer + inReader *os.File + printer *stats.Printer + loadEnabled bool } func NewProfiler(outw, errw io.Writer, opts *options.Options) *Profiler { + printOptions := stats.NewPrintOptions(opts.NoHeaders, opts.ShowFooters, opts.DecodeUri, opts.PaginationLimit) + printer := stats.NewPrinter(outw, opts.Output, opts.Format, opts.Percentiles, printOptions) + return &Profiler{ - options: opts, - outWriter: outw, - errWriter: errw, - inReader: os.Stdin, + options: opts, + outWriter: outw, + errWriter: errw, + inReader: os.Stdin, + printer: printer, + loadEnabled: true, } } @@ -61,43 +67,66 @@ func (p *Profiler) ReadPosFile(f *os.File) (int, error) { return helpers.StringToInt(string(pos)) } -func (p *Profiler) Run(sortOptions *stats.SortOptions, parser parsers.Parser) error { - sts := stats.NewHTTPStats(true, false, false) +func (p *Profiler) SetPrinter(printer *stats.Printer) { + p.printer = printer +} +func (p *Profiler) ValidatePrinter() error { + return p.printer.Validate() +} + +func (p *Profiler) Options() *options.Options { + return p.options +} + +func (p *Profiler) DisableLoad() { + p.loadEnabled = false +} + +func (p *Profiler) Load(sortOptions *stats.SortOptions) (*stats.HTTPStats, error) { + sts := stats.NewHTTPStats(true, false, false) err := sts.InitFilter(p.options) if err != nil { - return err + return nil, err } sts.SetOptions(p.options) sts.SetSortOptions(sortOptions) - printOptions := stats.NewPrintOptions(p.options.NoHeaders, p.options.ShowFooters, p.options.DecodeUri, p.options.PaginationLimit) - printer := stats.NewPrinter(p.outWriter, p.options.Output, p.options.Format, p.options.Percentiles, printOptions) - if err = printer.Validate(); err != nil { - return err + lf, err := os.Open(p.options.Load) + if err != nil { + return nil, err } + err = sts.LoadStats(lf) + if err != nil { + return nil, err + } + defer lf.Close() - if p.options.Load != "" { - lf, err := os.Open(p.options.Load) - if err != nil { - return err - } - err = sts.LoadStats(lf) - if err != nil { - return err - } - defer lf.Close() + sts.SortWithOptions() - sts.SortWithOptions() - printer.Print(sts, nil) - return nil + return sts, nil +} + +func (p *Profiler) profile(sortOptions *stats.SortOptions, parser parsers.Parser) (*stats.HTTPStats, error) { + sts := stats.NewHTTPStats(true, false, false) + + if p.options.Load != "" && p.loadEnabled { + return p.Load(sortOptions) } + err := sts.InitFilter(p.options) + if err != nil { + return nil, err + } + + sts.SetOptions(p.options) + sts.SetSortOptions(sortOptions) + if len(p.options.MatchingGroups) > 0 { err = sts.SetURIMatchingGroups(p.options.MatchingGroups) if err != nil { - return err + return nil, err } } @@ -105,18 +134,18 @@ func (p *Profiler) Run(sortOptions *stats.SortOptions, parser parsers.Parser) er if p.options.PosFile != "" { posfile, err = p.OpenPosFile(p.options.PosFile) if err != nil { - return err + return nil, err } defer posfile.Close() pos, err := p.ReadPosFile(posfile) if err != nil && err != io.EOF { - return err + return nil, err } err = parser.Seek(pos) if err != nil { - return err + return nil, err } parser.SetReadBytes(pos) @@ -132,13 +161,13 @@ Loop: continue Loop } - return err + return nil, err } var b bool b, err = sts.DoFilter(s) if err != nil { - return err + return nil, err } if !b { @@ -148,29 +177,66 @@ Loop: sts.Set(s.Uri, s.Method, s.Status, s.ResponseTime, s.BodyBytes, 0) if sts.CountUris() > p.options.Limit { - return fmt.Errorf("Too many URI's (%d or less)", p.options.Limit) + return nil, fmt.Errorf("Too many URI's (%d or less)", p.options.Limit) } } - if p.options.Dump != "" { - df, err := os.OpenFile(p.options.Dump, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) - err = sts.DumpStats(df) + if !p.options.NoSavePos && p.options.PosFile != "" { + posfile.Seek(0, 0) + _, err = posfile.Write([]byte(fmt.Sprint(parser.ReadBytes()))) if err != nil { - return err + return nil, err } - defer df.Close() } - if !p.options.NoSavePos && p.options.PosFile != "" { - posfile.Seek(0, 0) - _, err = posfile.Write([]byte(fmt.Sprint(parser.ReadBytes()))) + return sts, nil +} + +func (p *Profiler) Profile(sortOptions *stats.SortOptions, parser parsers.Parser) (*stats.HTTPStats, error) { + sts, err := p.profile(sortOptions, parser) + if err != nil { + return nil, err + } + + sts.SortWithOptions() + + return sts, nil +} + +func (p *Profiler) Run(sortOptions *stats.SortOptions, parser parsers.Parser, from *stats.HTTPStats) error { + sts, err := p.profile(sortOptions, parser) + if err != nil { + return err + } + + if p.options.Load != "" { + if from == nil { + p.printer.Print(sts, nil) + } else { + // diff + p.printer.Print(from, sts) + } + + return nil + } + + if p.options.Dump != "" { + df, err := os.OpenFile(p.options.Dump, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) + err = sts.DumpStats(df) if err != nil { return err } + defer df.Close() } sts.SortWithOptions() - printer.Print(sts, nil) + + if from == nil { + p.printer.Print(sts, nil) + } else { + // diff + p.printer.Print(from, sts) + } return nil }