diff --git a/docs/Dockerfile b/docs/Dockerfile index 4f174137d..a555064fa 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -3,7 +3,7 @@ FROM ghcr.io/gofr-dev/website:latest WORKDIR /app COPY docs/quick-start /app/src/app/docs/quick-start -COPY docs/public /app/public +COPY docs/public/ /app/public COPY docs/advanced-guide /app/src/app/docs/advanced-guide COPY docs/references /app/src/app/docs/references COPY docs/page.md /app/src/app/docs diff --git a/docs/navigation.js b/docs/navigation.js index b218e568d..4c502fd7e 100644 --- a/docs/navigation.js +++ b/docs/navigation.js @@ -1,55 +1,162 @@ export const navigation = [ { title: 'Quick Start Guide', + desc: "Get started with GoFR through our Quick Start Guide. Learn to build scalable applications with easy-to-follow instructions on server setup, database connections, configuration management, and more. Boost your productivity and streamline your development process.", links: [ - { title: 'Hello Server', href: '/docs/quick-start/introduction' }, - { title: 'Configuration', href: '/docs/quick-start/configuration' }, - { title: 'Connecting Redis', href: '/docs/quick-start/connecting-redis' }, - { title: 'Connecting MySQL', href: '/docs/quick-start/connecting-mysql' }, - { title: 'Observability', href: '/docs/quick-start/observability' }, - { title: 'Adding REST Handlers', href: '/docs/quick-start/add-rest-handlers' }, + { + title: 'Hello Server', + href: '/docs/quick-start/introduction' , + desc: "Getting started with how to write a server using GoFR with basic examples and explanations. Boost your productivity with efficient coding practices and learn to build scalable applications quickly."}, + { + title: 'Configuration', + href: '/docs/quick-start/configuration', + desc: "Set up environment variables, manage settings, and streamline your development process." + }, + { + title: 'Connecting Redis', + href: '/docs/quick-start/connecting-redis', + desc: "Discover how to connect your GoFR application to Redis for fast in-memory data storage." + }, + { + title: 'Connecting MySQL', + href: '/docs/quick-start/connecting-mysql', + desc: "Step-by-step guide on integrating MySQL with your GoFR application. With managed database connections and new methods for increasing your productivity." + }, + { + title: 'Observability', + href: '/docs/quick-start/observability', + desc: "Inbuilt logging, tracing, and metrics to enhance reliability and performance." + }, + { + title: 'Adding REST Handlers', + href: '/docs/quick-start/add-rest-handlers', + desc: "Fastest way to create CRUD APIs by just providing the entity." + } ], }, { title: 'Advanced Guide', links: [ - { title: "Scheduling Cron Jobs", href: "/docs/advanced-guide/using-cron"}, - { title: 'Overriding Default', href: '/docs/advanced-guide/overriding-default' }, - { title: 'Remote Log Level Change', href: '/docs/advanced-guide/remote-log-level-change' }, - { title: 'Publishing Custom Metrics', href: '/docs/advanced-guide/publishing-custom-metrics' }, - { title: 'Custom Spans in Tracing', href: '/docs/advanced-guide/custom-spans-in-tracing' }, - { title: 'Adding Custom Middleware',href: '/docs/advanced-guide/middlewares'}, - { title: 'HTTP Communication', href: '/docs/advanced-guide/http-communication' }, - { title: 'HTTP Authentication', href: '/docs/advanced-guide/http-authentication' }, - { title: 'Circuit Breaker Support', href: '/docs/advanced-guide/circuit-breaker' }, - { title: 'Monitoring Service Health', href: '/docs/advanced-guide/monitoring-service-health' }, - { title: 'Handling Data Migrations', href: '/docs/advanced-guide/handling-data-migrations' }, - { title: 'Writing gRPC Server', href: '/docs/advanced-guide/grpc' }, - { title: 'Using Pub/Sub', href: '/docs/advanced-guide/using-publisher-subscriber' }, - { title: 'Injecting Databases', href: '/docs/advanced-guide/injecting-databases-drivers' }, - { title: 'Key Value Store', href: '/docs/advanced-guide/key-value-store' }, - { title: 'Dealing with SQL', href: '/docs/advanced-guide/dealing-with-sql' }, - { title: 'Automatic SwaggerUI Rendering', href: '/docs/advanced-guide/swagger-documentation' }, - { title: 'Error Handling',href: '/docs/advanced-guide/gofr-errors'}, - { title: 'Handling File', href: '/docs/advanced-guide/handling-file'}, - { title: 'WebSockets', href: '/docs/advanced-guide/websocket' }, + { + title: "Scheduling Cron Jobs", + href: "/docs/advanced-guide/using-cron", + desc: "Learn how to schedule and manage cron jobs in your application for automated tasks and background processes with GoFr's CRON job management." + }, + { + title: 'Overriding Default', + href: '/docs/advanced-guide/overriding-default', + desc: "Understand how to override default configurations and behaviors in GoFR to tailor framework to your specific needs." + }, + { + title: 'Remote Log Level Change', + href: '/docs/advanced-guide/remote-log-level-change', + desc: "Discover how to dynamically change log levels remotely, enabling you to adjust logging verbosity without redeploying your application." + }, + { + title: 'Publishing Custom Metrics', + href: '/docs/advanced-guide/publishing-custom-metrics', + desc: "Explore methods for publishing custom metrics to monitor your application's performance and gain valuable insights." + }, + { + title: 'Custom Spans in Tracing', + href: '/docs/advanced-guide/custom-spans-in-tracing', + desc: "Learn to create custom spans for tracing to enhance observability and analyze the performance of your services." + }, + { + title: 'Adding Custom Middleware', + href: '/docs/advanced-guide/middlewares', + desc: "Learn how to add custom middleware to your GoFR application for enhanced functionality and request processing." + }, + { + title: 'HTTP Communication', + href: '/docs/advanced-guide/http-communication', + desc: "Get familiar with making HTTP requests and handling responses within your GoFR application to facilitate seamless communication." + }, + { + title: 'HTTP Authentication', + href: '/docs/advanced-guide/http-authentication', + desc: "Implement various HTTP authentication methods to secure your GoFR application and protect sensitive endpoints." + }, + { + title: 'Circuit Breaker Support', + href: '/docs/advanced-guide/circuit-breaker', + desc: "Understand how to implement circuit breaker patterns to enhance the resilience of your services against failures." + }, + { + title: 'Monitoring Service Health', + href: '/docs/advanced-guide/monitoring-service-health', + desc: "Learn to monitor the health of your services effectively, ensuring optimal performance and quick issue resolution." + }, + { + title: 'Handling Data Migrations', + href: '/docs/advanced-guide/handling-data-migrations', + desc: "Explore strategies for managing data migrations within your GoFR application to ensure smooth transitions and data integrity." + }, + { + title: 'Writing gRPC Server', + href: '/docs/advanced-guide/grpc', + desc: "Step-by-step guide on writing a gRPC server in GoFR to facilitate efficient communication between services." + }, + { + title: 'Using Pub/Sub', + href: '/docs/advanced-guide/using-publisher-subscriber', + desc: "Discover how to gofr seamlessly allows to integrate different Pub/Sub systems in your application for effective messaging and event-driven architectures." + }, + { + title: 'Injecting Databases', + href: '/docs/advanced-guide/injecting-databases-drivers', + desc: "Learn how to inject database drivers into your GoFR application for seamless data management and operations." + }, + { + title: 'Key Value Store', + href: '/docs/advanced-guide/key-value-store', + desc: "Explore how to implement and manage a key-value store in your GoFR application for fast and efficient data retrieval." + }, + { + title: 'Dealing with SQL', + href: '/docs/advanced-guide/dealing-with-sql', + desc: "Get insights into best practices for working with SQL databases in GoFR, including query optimization and error handling." + }, + { + title: 'Automatic SwaggerUI Rendering', + href: '/docs/advanced-guide/swagger-documentation', + desc: "Learn how to automatically render SwaggerUI documentation for your GoFR APIs, improving discoverability and usability." + }, + { + title: 'Error Handling', + href: '/docs/advanced-guide/gofr-errors', + desc: "Understand error handling mechanisms in GoFR to ensure robust applications and improved user experience." + }, + { + title: 'Handling File', + href: '/docs/advanced-guide/handling-file', + desc: "Explore how GoFR enables efficient file handling by abstracting remote and local filestore providers in your Go application. Learn to manage file uploads, downloads, and storage seamlessly, enhancing your application's capability to work with diverse data sources." + }, + { + title: 'WebSockets', + href: '/docs/advanced-guide/websocket', + desc: "Explore how gofr eases the process of WebSocket communication in your Golang application for real-time data exchange." + } ], }, { title: 'References', links: [ - { title: 'Context', href: '/docs/references/context' }, - { title: 'Configs', href: '/docs/references/configs' }, - {tile: 'Testing', href:'/docs/references/testing'}, - // { title: 'HTTP Service', href: '/docs/references/http-service' }, - // { title: 'Files', href: '/docs/references/files' }, - // { title: 'Datastore', href: '/docs/references/datastore' }, - // { title: 'PubSub', href: '/docs/references/pubsub' }, - // { title: 'Metrics', href: '/docs/references/metrics' }, - // { title: 'Traces', href: '/docs/references/traces' }, - // { title: 'Logs', href: '/docs/references/logs' }, - // { title: 'Errors', href: '/docs/references/errors' }, - // { title: 'Swaggger', href: '/docs/references/swagger' }, + { + title: 'Context', + href: '/docs/references/context', + desc: "Discover the GoFR context, an injected object that simplifies request-specific data handling for HTTP, gRPC, and Pub/Sub calls. Learn how it extends Go's context, providing easy access to dependencies like databases, loggers, and HTTP clients. Explore features for reading HTTP requests, binding data, and accessing query and path parameters efficiently, all while reducing application complexity." + }, + { + title: 'Configs', + href: '/docs/references/configs', + desc: "Learn how to manage configuration settings in your GoFR applications, including default values for environment variables. This section provides a comprehensive list of all available configurations to streamline your setup." + }, + { + title: 'Testing', + href: '/docs/references/testing', + desc: "GoFR provides a centralized collection of mocks to facilitate writing effective unit tests. Explore testing strategies and tools for GoFR applications, ensuring your code is robust, reliable, and maintainable." + } ], }, -] +] \ No newline at end of file diff --git a/pkg/gofr/datasource/cassandra/cassandra.go b/pkg/gofr/datasource/cassandra/cassandra.go index 802e9e8f3..76d1fa93f 100644 --- a/pkg/gofr/datasource/cassandra/cassandra.go +++ b/pkg/gofr/datasource/cassandra/cassandra.go @@ -122,7 +122,7 @@ func (c *Client) NewBatch(name string, batchType int) error { func (c *Client) QueryWithCtx(ctx context.Context, dest any, stmt string, values ...any) error { _, span := c.addTrace(ctx, "query", stmt) - defer c.sendOperationStats(&QueryLog{Query: stmt, Keyspace: c.config.Keyspace}, time.Now(), "query", span) + defer c.sendOperationStats(&QueryLog{Operation: "QueryWithCtx", Query: stmt, Keyspace: c.config.Keyspace}, time.Now(), "query", span) rvo := reflect.ValueOf(dest) if rvo.Kind() != reflect.Ptr { @@ -171,7 +171,7 @@ func (c *Client) QueryWithCtx(ctx context.Context, dest any, stmt string, values func (c *Client) ExecWithCtx(ctx context.Context, stmt string, values ...any) error { _, span := c.addTrace(ctx, "exec", stmt) - defer c.sendOperationStats(&QueryLog{Query: stmt, Keyspace: c.config.Keyspace}, time.Now(), "exec", span) + defer c.sendOperationStats(&QueryLog{Operation: "ExecWithCtx", Query: stmt, Keyspace: c.config.Keyspace}, time.Now(), "exec", span) return c.cassandra.session.query(stmt, values...).exec() } @@ -185,7 +185,7 @@ func (c *Client) ExecCASWithCtx(ctx context.Context, dest any, stmt string, valu _, span := c.addTrace(ctx, "exec-cas", stmt) - defer c.sendOperationStats(&QueryLog{Query: stmt, Keyspace: c.config.Keyspace}, time.Now(), "exec-cas", span) + defer c.sendOperationStats(&QueryLog{Operation: "ExecCASWithCtx", Query: stmt, Keyspace: c.config.Keyspace}, time.Now(), "exec-cas", span) rvo := reflect.ValueOf(dest) if rvo.Kind() != reflect.Ptr { diff --git a/pkg/gofr/datasource/cassandra/cassandra_batch.go b/pkg/gofr/datasource/cassandra/cassandra_batch.go index 8f75feca6..f05e1747c 100644 --- a/pkg/gofr/datasource/cassandra/cassandra_batch.go +++ b/pkg/gofr/datasource/cassandra/cassandra_batch.go @@ -20,7 +20,7 @@ func (c *Client) ExecuteBatchCAS(name string, dest ...any) (bool, error) { func (c *Client) BatchQueryWithCtx(ctx context.Context, name, stmt string, values ...any) error { _, span := c.addTrace(ctx, "batch-query", stmt) - defer c.sendOperationStats(&QueryLog{Query: stmt, Keyspace: c.config.Keyspace}, time.Now(), "batch-query", span) + defer c.sendOperationStats(&QueryLog{Operation: "BatchQueryWithCtx", Query: stmt, Keyspace: c.config.Keyspace}, time.Now(), "batch-query", span) b, ok := c.cassandra.batches[name] if !ok { @@ -35,7 +35,7 @@ func (c *Client) BatchQueryWithCtx(ctx context.Context, name, stmt string, value func (c *Client) ExecuteBatchWithCtx(ctx context.Context, name string) error { _, span := c.addTrace(ctx, "execute-batch", "batch") - defer c.sendOperationStats(&QueryLog{Query: "batch", Keyspace: c.config.Keyspace}, time.Now(), "execute-batch", + defer c.sendOperationStats(&QueryLog{Operation: "ExecuteBatchWithCtx", Query: "batch", Keyspace: c.config.Keyspace}, time.Now(), "execute-batch", span) b, ok := c.cassandra.batches[name] @@ -49,7 +49,7 @@ func (c *Client) ExecuteBatchWithCtx(ctx context.Context, name string) error { func (c *Client) ExecuteBatchCASWithCtx(ctx context.Context, name string, dest ...any) (bool, error) { _, span := c.addTrace(ctx, "execute-batch-cas", "batch") - defer c.sendOperationStats(&QueryLog{Query: "batch", Keyspace: c.config.Keyspace}, time.Now(), "execute-batch-cas", + defer c.sendOperationStats(&QueryLog{Operation: "ExecuteBatchCASWithCtx", Query: "batch", Keyspace: c.config.Keyspace}, time.Now(), "execute-batch-cas", span) b, ok := c.cassandra.batches[name] diff --git a/pkg/gofr/datasource/cassandra/cassandra_batch_test.go b/pkg/gofr/datasource/cassandra/cassandra_batch_test.go index 565e55168..4c8cb67cf 100644 --- a/pkg/gofr/datasource/cassandra/cassandra_batch_test.go +++ b/pkg/gofr/datasource/cassandra/cassandra_batch_test.go @@ -1,3 +1,6 @@ +//go:build exclude +// +build exclude + package cassandra import ( diff --git a/pkg/gofr/datasource/cassandra/cassandra_test.go b/pkg/gofr/datasource/cassandra/cassandra_test.go index a8f452e80..16c87a8ba 100644 --- a/pkg/gofr/datasource/cassandra/cassandra_test.go +++ b/pkg/gofr/datasource/cassandra/cassandra_test.go @@ -1,3 +1,6 @@ +//go:build exclude +// +build exclude + package cassandra import ( diff --git a/pkg/gofr/datasource/cassandra/logger.go b/pkg/gofr/datasource/cassandra/logger.go index 709a239db..1c90a5e55 100644 --- a/pkg/gofr/datasource/cassandra/logger.go +++ b/pkg/gofr/datasource/cassandra/logger.go @@ -17,14 +17,15 @@ type Logger interface { } type QueryLog struct { - Query string `json:"query"` - Duration int64 `json:"duration"` - Keyspace string `json:"keyspace,omitempty"` + Operation string `json:"operation"` + Query string `json:"query"` + Duration int64 `json:"duration"` + Keyspace string `json:"keyspace,omitempty"` } func (ql *QueryLog) PrettyPrint(writer io.Writer) { - fmt.Fprintf(writer, "\u001B[38;5;8m%-32s \u001B[38;5;206m%-6s\u001B[0m %8d\u001B[38;5;8mµs\u001B[0m %s\n", - clean(ql.Query), "CASS", ql.Duration, clean(ql.Keyspace)) + fmt.Fprintf(writer, "\u001B[38;5;8m%-32s \u001B[38;5;206m%-6s\u001B[0m %8d\u001B[38;5;8mµs\u001B[0m %s \u001B[38;5;8m%-32s\u001B[0m\n", + clean(ql.Operation), "CASS", ql.Duration, clean(ql.Keyspace), clean(ql.Query)) } // clean takes a string query as input and performs two operations to clean it up: diff --git a/pkg/gofr/datasource/cassandra/mock_interfaces.go b/pkg/gofr/datasource/cassandra/mock_interfaces.go index ad671a7f0..f3649ec97 100644 --- a/pkg/gofr/datasource/cassandra/mock_interfaces.go +++ b/pkg/gofr/datasource/cassandra/mock_interfaces.go @@ -1,3 +1,6 @@ +//go:build exclude +// +build exclude + // Code generated by MockGen. DO NOT EDIT. // Source: interfaces.go // @@ -6,7 +9,6 @@ // mockgen -source=interfaces.go -destination=mock_interfaces.go -package=cassandra // -// Package cassandra is a generated GoMock package. package cassandra import ( diff --git a/pkg/gofr/datasource/cassandra/mock_logger.go b/pkg/gofr/datasource/cassandra/mock_logger.go index 167a52927..5d2a61ee0 100644 --- a/pkg/gofr/datasource/cassandra/mock_logger.go +++ b/pkg/gofr/datasource/cassandra/mock_logger.go @@ -1,3 +1,6 @@ +//go:build exclude +// +build exclude + // Code generated by MockGen. DO NOT EDIT. // Source: logger.go // @@ -6,7 +9,6 @@ // mockgen -source=logger.go -destination=mock_logger_old.go -package=cassandra // -// Package cassandra is a generated GoMock package. package cassandra import ( diff --git a/pkg/gofr/datasource/cassandra/mock_metrics.go b/pkg/gofr/datasource/cassandra/mock_metrics.go index fa606a516..67795769d 100644 --- a/pkg/gofr/datasource/cassandra/mock_metrics.go +++ b/pkg/gofr/datasource/cassandra/mock_metrics.go @@ -1,3 +1,6 @@ +//go:build exclude +// +build exclude + // Code generated by MockGen. DO NOT EDIT. // Source: metrics.go // @@ -6,7 +9,6 @@ // mockgen -source=metrics.go -destination=mock_metrics.go -package=cassandra // -// Package cassandra is a generated GoMock package. package cassandra import ( diff --git a/pkg/gofr/datasource/file/ftp/file_test.go b/pkg/gofr/datasource/file/ftp/file_test.go index bab312d2c..2d67bcc52 100644 --- a/pkg/gofr/datasource/file/ftp/file_test.go +++ b/pkg/gofr/datasource/file/ftp/file_test.go @@ -1,3 +1,6 @@ +//go:build exclude +// +build exclude + package ftp import ( diff --git a/pkg/gofr/datasource/file/ftp/fs_test.go b/pkg/gofr/datasource/file/ftp/fs_test.go index 8a5012baa..41456bbd4 100644 --- a/pkg/gofr/datasource/file/ftp/fs_test.go +++ b/pkg/gofr/datasource/file/ftp/fs_test.go @@ -1,3 +1,6 @@ +//go:build exclude +// +build exclude + package ftp import ( diff --git a/pkg/gofr/datasource/file/ftp/mock_interface.go b/pkg/gofr/datasource/file/ftp/mock_interface.go index 865759d1d..e45287af0 100644 --- a/pkg/gofr/datasource/file/ftp/mock_interface.go +++ b/pkg/gofr/datasource/file/ftp/mock_interface.go @@ -1,3 +1,6 @@ +//go:build exclude +// +build exclude + // Code generated by MockGen. DO NOT EDIT. // Source: interface.go // diff --git a/pkg/gofr/datasource/file/sftp/go.mod b/pkg/gofr/datasource/file/sftp/go.mod index 784769eaa..8c6c99a07 100644 --- a/pkg/gofr/datasource/file/sftp/go.mod +++ b/pkg/gofr/datasource/file/sftp/go.mod @@ -18,6 +18,6 @@ require ( github.com/kr/fs v0.1.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/sys v0.25.0 // indirect + golang.org/x/sys v0.26.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect -) \ No newline at end of file +) diff --git a/pkg/gofr/datasource/file/sftp/go.sum b/pkg/gofr/datasource/file/sftp/go.sum index 6f14fcab7..23a4e010a 100644 --- a/pkg/gofr/datasource/file/sftp/go.sum +++ b/pkg/gofr/datasource/file/sftp/go.sum @@ -42,6 +42,7 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/pkg/gofr/datasource/pubsub/eventhub/eventhub.go b/pkg/gofr/datasource/pubsub/eventhub/eventhub.go index 136d2da57..ce172f038 100644 --- a/pkg/gofr/datasource/pubsub/eventhub/eventhub.go +++ b/pkg/gofr/datasource/pubsub/eventhub/eventhub.go @@ -29,9 +29,9 @@ type Config struct { StorageServiceURL string StorageContainerName string EventhubName string - // if not provided it will read from the $Default consumergroup. + // if not provided, it will read from the $Default consumergroup. ConsumerGroup string - // following configs are for advance setup of the eventhub. + // the following configs are for advance setup of the eventhub. StorageOptions *container.ClientOptions BlobStoreOptions *checkpoints.BlobStoreOptions ConsumerOptions *azeventhubs.ConsumerClientOptions @@ -41,9 +41,9 @@ type Config struct { type Client struct { producer *azeventhubs.ProducerClient consumer *azeventhubs.ConsumerClient - // we are using processor such that to keep consuming the events from all the different partitions. + // we are using a processor such that to keep consuming the events from all the different partitions. processor *azeventhubs.Processor - // checkpoint is being called while committing the event received from the event. + // a checkpoint is being called while committing the event received from the event. checkPoint *checkpoints.BlobStore // processorCtx is being stored such that to gracefully shutting down the application. processorCtx context.CancelFunc @@ -220,31 +220,33 @@ func (c *Client) Subscribe(ctx context.Context, topic string) (*pubsub.Message, start := time.Now() - msg, err = c.processEvents(ctx, partitionClient) - switch err { - case errNoMsgReceived: - // if no message is received, we don't achieve anything by returning error rather check in a different partition. - // this logic may change if we remove the timeout while receiving message, but waiting on just one partition - //might lead to miss data, so spawning one go-routine or having a worker pool can be an option to do this operation faster. - break + select { + case <-ctx.Done(): + return nil, nil default: - return nil, err + msg, err = c.processEvents(ctx, partitionClient) + if errors.Is(err, errNoMsgReceived) { + // If no message is received, we don't achieve anything by returning error rather check in a different partition. + // This logic may change if we remove the timeout while receiving a message. However, waiting on just one partition + // might lead to missing data, so spawning one go-routine or having a worker pool can be an option to do this operation faster. + continue + } + + end := time.Since(start) + + c.logger.Debug(&Log{ + Mode: "SUB", + MessageValue: strings.Join(strings.Fields(string(msg.Value)), " "), + Topic: topic, + Host: fmt.Sprint(c.cfg.EventhubName + ":" + c.cfg.ConsumerGroup + ":" + partitionClient.PartitionID()), + PubSubBackend: "EVHUB", + Time: end.Microseconds(), + }) + + c.metrics.IncrementCounter(ctx, "app_pubsub_subscribe_success_count", "topic", topic, "subscription_name", partitionClient.PartitionID()) + + return msg, nil } - - end := time.Since(start) - - c.logger.Debug(&Log{ - Mode: "SUB", - MessageValue: strings.Join(strings.Fields(string(msg.Value)), " "), - Topic: topic, - Host: fmt.Sprint(c.cfg.EventhubName + ":" + c.cfg.ConsumerGroup + ":" + partitionClient.PartitionID()), - PubSubBackend: "EVHUB", - Time: end.Microseconds(), - }) - - c.metrics.IncrementCounter(ctx, "app_pubsub_subscribe_success_count", "topic", topic, "subscription_name", partitionClient.PartitionID()) - - return msg, nil } return nil, nil diff --git a/pkg/gofr/datasource/pubsub/google/google.go b/pkg/gofr/datasource/pubsub/google/google.go index 122116767..1b815d71f 100644 --- a/pkg/gofr/datasource/pubsub/google/google.go +++ b/pkg/gofr/datasource/pubsub/google/google.go @@ -169,12 +169,15 @@ func (g *googleClient) Subscribe(ctx context.Context, topic string) (*pubsub.Mes } }() - m := <-receiveChan - - g.metrics.IncrementCounter(ctx, "app_pubsub_subscribe_success_count", "topic", topic, "subscription_name", - g.Config.SubscriptionName) - - return m, nil + select { + case m := <-receiveChan: + g.metrics.IncrementCounter(ctx, "app_pubsub_subscribe_success_count", "topic", topic, "subscription_name", + g.Config.SubscriptionName) + + return m, nil + case <-ctx.Done(): + return nil, nil + } } func (g *googleClient) getTopic(ctx context.Context, topic string) (*gcPubSub.Topic, error) { diff --git a/pkg/gofr/datasource/pubsub/mqtt/mqtt.go b/pkg/gofr/datasource/pubsub/mqtt/mqtt.go index 0ab089c33..0d2bd29a6 100644 --- a/pkg/gofr/datasource/pubsub/mqtt/mqtt.go +++ b/pkg/gofr/datasource/pubsub/mqtt/mqtt.go @@ -172,11 +172,15 @@ func (m *MQTT) Subscribe(ctx context.Context, topic string) (*pubsub.Message, er } m.mu.Unlock() + select { // blocks if there are no messages in the channel - msg := <-subs.msgs - m.metrics.IncrementCounter(msg.Context(), "app_pubsub_subscribe_success_count", "topic", msg.Topic) + case msg := <-subs.msgs: + m.metrics.IncrementCounter(msg.Context(), "app_pubsub_subscribe_success_count", "topic", msg.Topic) - return msg, nil + return msg, nil + case <-ctx.Done(): + return nil, nil + } } func (m *MQTT) createMqttHandler(_ context.Context, topic string, msgs chan *pubsub.Message) mqtt.MessageHandler { diff --git a/pkg/gofr/datasource/solr/go.mod b/pkg/gofr/datasource/solr/go.mod index d3f4bfa79..611d4a4cc 100644 --- a/pkg/gofr/datasource/solr/go.mod +++ b/pkg/gofr/datasource/solr/go.mod @@ -4,13 +4,17 @@ go 1.22 require ( github.com/stretchr/testify v1.9.0 - go.opentelemetry.io/otel/trace v1.30.0 + go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.56.0 + go.opentelemetry.io/otel v1.31.0 + go.opentelemetry.io/otel/trace v1.31.0 go.uber.org/mock v0.4.0 ) require ( github.com/davecgh/go-spew v1.1.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - go.opentelemetry.io/otel v1.30.0 // indirect + go.opentelemetry.io/otel/metric v1.31.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/pkg/gofr/datasource/solr/go.sum b/pkg/gofr/datasource/solr/go.sum index fe0856a8c..baa9879d5 100644 --- a/pkg/gofr/datasource/solr/go.sum +++ b/pkg/gofr/datasource/solr/go.sum @@ -1,15 +1,28 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts= -go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc= -go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc= -go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.56.0 h1:4BZHA+B1wXEQoGNHxW8mURaLhcdGwvRnmhGbm+odRbc= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.56.0/go.mod h1:3qi2EEwMgB4xnKgPLqsDP3j9qxnHDZeHsnAxfjQqTko= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 h1:UP6IpuHFkUgOQL9FFQFrZ+5LiwhhYRbi7VZSIx6Nj5s= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0/go.mod h1:qxuZLtbq5QDtdeSHsS7bcf6EH6uO6jUAgk764zd3rhM= +go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= +go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= +go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= +go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= +go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= +go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= diff --git a/pkg/gofr/datasource/solr/solr.go b/pkg/gofr/datasource/solr/solr.go index ba0ca367e..68f53c4e5 100644 --- a/pkg/gofr/datasource/solr/solr.go +++ b/pkg/gofr/datasource/solr/solr.go @@ -7,8 +7,12 @@ import ( "fmt" "io" "net/http" + "net/http/httptrace" "time" + "go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace" + + "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) @@ -18,11 +22,11 @@ type Config struct { } type Client struct { - url string - + url string logger Logger metrics Metrics tracer trace.Tracer + client *http.Client } // New initializes Solr driver with the provided configuration. @@ -35,6 +39,7 @@ type Client struct { func New(conf Config) *Client { s := &Client{} s.url = "http://" + conf.Host + ":" + conf.Port + "/solr" + s.client = &http.Client{} return s } @@ -78,87 +83,114 @@ func (c *Client) HealthCheck(ctx context.Context) (any, error) { // This can be used for making any queries to SOLR func (c *Client) Search(ctx context.Context, collection string, params map[string]any) (any, error) { url := c.url + "/" + collection + "/select" + startTime := time.Now() - defer c.sendOperationStats(&QueryLog{Type: "Search", Url: url}, time.Now()) + resp, err, span := c.call(ctx, http.MethodGet, url, params, nil) - return call(ctx, http.MethodGet, url, params, nil) + defer c.sendOperationStats(ctx, &QueryLog{Type: "Search", Url: url}, startTime, "search", span) + + return resp, err } // Create makes documents in the specified collection. params can be used to send parameters like commit=true func (c *Client) Create(ctx context.Context, collection string, document *bytes.Buffer, params map[string]any) (any, error) { - url := c.url + collection + "/update" + url := c.url + "/" + collection + "/update" + startTime := time.Now() - defer c.sendOperationStats(&QueryLog{Type: "Create", Url: url}, time.Now()) + resp, err, span := c.call(ctx, http.MethodPost, url, params, document) - return call(ctx, http.MethodPost, url, params, document) + defer c.sendOperationStats(ctx, &QueryLog{Type: "Create", Url: url}, startTime, "create", span) + + return resp, err } // Update updates documents in the specified collection. params can be used to send parameters like commit=true func (c *Client) Update(ctx context.Context, collection string, document *bytes.Buffer, params map[string]any) (any, error) { url := c.url + collection + "/update" + startTime := time.Now() + + resp, err, span := c.call(ctx, http.MethodPost, url, params, document) - defer c.sendOperationStats(&QueryLog{Type: "Update", Url: url}, time.Now()) + defer c.sendOperationStats(ctx, &QueryLog{Type: "Update", Url: url}, startTime, "update", span) - return call(ctx, http.MethodPost, url, params, document) + return resp, err } // Delete deletes documents in the specified collection. params can be used to send parameters like commit=true func (c *Client) Delete(ctx context.Context, collection string, document *bytes.Buffer, params map[string]any) (any, error) { url := c.url + collection + "/update" + startTime := time.Now() + + resp, err, span := c.call(ctx, http.MethodPost, url, params, document) - defer c.sendOperationStats(&QueryLog{Type: "Delete", Url: url}, time.Now()) + defer c.sendOperationStats(ctx, &QueryLog{Type: "Delete", Url: url}, startTime, "delete", span) - return call(ctx, http.MethodPost, url, params, document) + return resp, err } // ListFields retrieves all the fields in the schema for the specified collection. // params can be used to send query parameters like wt, fl, includeDynamic etc. func (c *Client) ListFields(ctx context.Context, collection string, params map[string]any) (any, error) { url := c.url + collection + "/schema/fields" + startTime := time.Now() - defer c.sendOperationStats(&QueryLog{Type: "ListFields", Url: url}, time.Now()) + resp, err, span := c.call(ctx, http.MethodGet, url, params, nil) - return call(ctx, http.MethodGet, url, params, nil) + defer c.sendOperationStats(ctx, &QueryLog{Type: "ListFields", Url: url}, startTime, "list-fields", span) + + return resp, err } // Retrieve retrieves the entire schema that includes all the fields,field types,dynamic rules and copy field rules. // params can be used to specify the format of response func (c *Client) Retrieve(ctx context.Context, collection string, params map[string]any) (any, error) { url := c.url + collection + "/schema" + startTime := time.Now() + + resp, err, span := c.call(ctx, http.MethodGet, url, params, nil) - defer c.sendOperationStats(&QueryLog{Type: "Retrieve", Url: url}, time.Now()) + defer c.sendOperationStats(ctx, &QueryLog{Type: "Retrieve", Url: url}, startTime, "retrieve", span) - return call(ctx, http.MethodGet, url, params, nil) + return resp, err } // AddField adds Field in the schema for the specified collection func (c *Client) AddField(ctx context.Context, collection string, document *bytes.Buffer) (any, error) { url := c.url + collection + "/schema" + startTime := time.Now() - defer c.sendOperationStats(&QueryLog{Type: "AddField", Url: url}, time.Now()) + resp, err, span := c.call(ctx, http.MethodPost, url, nil, document) - return call(ctx, http.MethodPost, url, nil, document) + defer c.sendOperationStats(ctx, &QueryLog{Type: "AddField", Url: url}, startTime, "add-field", span) + + return resp, err } // UpdateField updates the field definitions in the schema for the specified collection func (c *Client) UpdateField(ctx context.Context, collection string, document *bytes.Buffer) (any, error) { url := c.url + collection + "/schema" + startTime := time.Now() + + resp, err, span := c.call(ctx, http.MethodPost, url, nil, document) - defer c.sendOperationStats(&QueryLog{Type: "UpdateField", Url: url}, time.Now()) + defer c.sendOperationStats(ctx, &QueryLog{Type: "UpdateField", Url: url}, startTime, "update-field", span) - return call(ctx, http.MethodPost, url, nil, document) + return resp, err } // DeleteField deletes the field definitions in the schema for the specified collection func (c *Client) DeleteField(ctx context.Context, collection string, document *bytes.Buffer) (any, error) { url := c.url + collection + "/schema" + startTime := time.Now() - defer c.sendOperationStats(&QueryLog{Type: "DeleteField", Url: url}, time.Now()) + resp, err, span := c.call(ctx, http.MethodPost, url, nil, document) - return call(ctx, http.MethodPost, url, nil, document) + defer c.sendOperationStats(ctx, &QueryLog{Type: "DeleteField", Url: url}, startTime, "delete-field", span) + + return resp, err } // Response stores the response from SOLR @@ -168,8 +200,53 @@ type Response struct { } // call forms the http request and makes a call to solr and populates the solr response -func call(ctx context.Context, method, url string, params map[string]any, body io.Reader) (any, error) { - req, err := http.NewRequest(method, url, body) +func (c *Client) call(ctx context.Context, method, url string, params map[string]any, body io.Reader) (any, error, trace.Span) { + var span trace.Span + + if c.tracer != nil { + ctx, span = c.tracer.Start(ctx, fmt.Sprintf("Solr %s", method), + trace.WithAttributes( + attribute.String("solr.url", url), + ), + ) + } + + ctx = httptrace.WithClientTrace(ctx, otelhttptrace.NewClientTrace(ctx)) + + req, err := c.createRequest(ctx, method, url, params, body) + if err != nil { + return nil, err, nil + } + + resp, err := c.client.Do(req) + if err != nil { + return nil, err, nil + } + defer resp.Body.Close() + + var respBody any + + b, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err, nil + } + + err = json.Unmarshal(b, &respBody) + if err != nil { + return nil, err, nil + } + + if span != nil { + span.SetAttributes( + attribute.Int("http.status_code", resp.StatusCode), + ) + } + + return Response{resp.StatusCode, respBody}, nil, span +} + +func (c *Client) createRequest(ctx context.Context, method, url string, params map[string]any, body io.Reader) (*http.Request, error) { + req, err := http.NewRequestWithContext(ctx, method, url, body) if err != nil { return nil, err } @@ -193,36 +270,23 @@ func call(ctx context.Context, method, url string, params map[string]any, body i req.URL.RawQuery = q.Encode() - client := &http.Client{} - - resp, err := client.Do(req.WithContext(ctx)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - var respBody any - - b, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - - err = json.Unmarshal(b, &respBody) - if err != nil { - return nil, err - } - - return Response{resp.StatusCode, respBody}, nil + return req, nil } -func (c *Client) sendOperationStats(ql *QueryLog, startTime time.Time) { +func (c *Client) sendOperationStats(ctx context.Context, ql *QueryLog, startTime time.Time, method string, span trace.Span) { duration := time.Since(startTime).Milliseconds() ql.Duration = duration c.logger.Debug(ql) - c.metrics.RecordHistogram(context.Background(), "app_solr_stats", float64(duration), + c.metrics.RecordHistogram(ctx, "app_solr_stats", float64(duration), "type", ql.Type) + + if span != nil { + defer span.End() + span.SetAttributes( + attribute.String("solr.type", ql.Type), + attribute.Int64(fmt.Sprintf("solr.%v.duration", method), duration)) + } } diff --git a/pkg/gofr/datasource/solr/solr_test.go b/pkg/gofr/datasource/solr/solr_test.go index 3a4469867..28fcfb287 100644 --- a/pkg/gofr/datasource/solr/solr_test.go +++ b/pkg/gofr/datasource/solr/solr_test.go @@ -13,7 +13,9 @@ import ( ) func Test_InvalidRequest(t *testing.T) { - _, err := call(context.Background(), "GET", ":/localhost:", nil, nil) + client := New(Config{}) + + _, err, _ := client.call(context.Background(), "GET", ":/localhost:", nil, nil) require.Error(t, err, "TEST Failed.\n") } @@ -24,7 +26,9 @@ func Test_InvalidJSONBody(t *testing.T) { })) defer ts.Close() - _, err := call(context.Background(), "GET", ts.URL, nil, nil) + client := New(Config{}) + + _, err, _ := client.call(context.Background(), "GET", ts.URL, nil, nil) require.Error(t, err, "TEST Failed.\n") } @@ -35,7 +39,9 @@ func Test_ErrorResponse(t *testing.T) { })) ts.Close() - _, err := call(context.Background(), "GET", ts.URL, nil, nil) + client := New(Config{}) + + _, err, _ := client.call(context.Background(), "GET", ts.URL, nil, nil) require.Error(t, err, "TEST Failed.\n") } diff --git a/pkg/gofr/external_db.go b/pkg/gofr/external_db.go index 3bf71d717..0c7d40853 100644 --- a/pkg/gofr/external_db.go +++ b/pkg/gofr/external_db.go @@ -102,6 +102,10 @@ func (a *App) AddSolr(db container.SolrProvider) { db.UseLogger(a.Logger()) db.UseMetrics(a.Metrics()) + tracer := otel.GetTracerProvider().Tracer("gofr-solr") + + db.UseTracer(tracer) + db.Connect() a.container.Solr = db diff --git a/pkg/gofr/version/version.go b/pkg/gofr/version/version.go index a71219c32..56fcb12ae 100644 --- a/pkg/gofr/version/version.go +++ b/pkg/gofr/version/version.go @@ -1,3 +1,3 @@ package version -const Framework = "v1.24.1" +const Framework = "v1.24.2"