Add AdaptiveRangeLimit constant for adaptive range limits

connectors/memory/memory.go

@@ -65,6 +65,8 @@ type partitionRange struct {
 	end          int
 }
 
+const defaultRangeLimit = 200
+
 // remove deletes the values referenced by the partitionRange. Since this function modifies
 // the data stored in the in-memory connector, a write lock must be held when calling
 // this function.
@@ -492,6 +494,11 @@ func (c *Connector) Range(_ context.Context, ei *dosa.EntityInfo, columnConditio
 			partitionRange.start += offset
 		}
 	}
+
+	if limit == dosa.AdaptiveRangeLimit {
+		limit = defaultRangeLimit
+	}
+
 	slice := partitionRange.values()
 	token = ""
 	if len(slice) > limit {

connectors/memory/memory_test.go

@@ -665,6 +665,13 @@ func TestConnector_Range(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Empty(t, data)
 	assert.Empty(t, token)
+
+	// Test with adaptive limits
+	data, _, _ = sut.Range(context.TODO(), clusteredEi, map[string][]*dosa.Condition{
+		"f1": {{Op: dosa.Eq, Value: dosa.FieldValue("data")}},
+		"c1": {{Op: dosa.Eq, Value: dosa.FieldValue(int64(1))}},
+	}, dosa.All(), "", 200)
+	assert.Len(t, data, idcount)
 }
 
 func TestConnector_TUUIDs(t *testing.T) {

connectors/random/random.go

@@ -31,8 +31,9 @@ import (
 )
 
 const (
-	maxBlobSize   = 32
-	maxStringSize = 64
+	maxBlobSize       = 32
+	maxStringSize     = 64
+	defaultRangeLimit = 200
 )
 
 // Connector is a connector implementation for testing
@@ -149,6 +150,9 @@ func (c *Connector) MultiRemove(ctx context.Context, ei *dosa.EntityInfo, multiV
 
 // Range returns a random set of data, and a random continuation token
 func (c *Connector) Range(ctx context.Context, ei *dosa.EntityInfo, columnConditions map[string][]*dosa.Condition, minimumFields []string, token string, limit int) ([]map[string]dosa.FieldValue, string, error) {
+	if limit == dosa.AdaptiveRangeLimit {
+		limit = defaultRangeLimit
+	}
 	vals := make([]map[string]dosa.FieldValue, limit)
 	for inx := range vals {
 		vals[inx] = Data(ei, minimumFields)

connectors/random/random_test.go

@@ -121,6 +121,12 @@ func TestRandom_Range(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestRandom_RangeAdaptiveLimits(t *testing.T) {
+	vals, _, err := sut.Range(ctx, testInfo, testConditions, minimumFields, "", dosa.AdaptiveRangeLimit)
+	assert.Len(t, vals, 200)
+	assert.NoError(t, err)
+}
+
 func TestRandom_Scan(t *testing.T) {
 	vals, _, err := sut.Scan(ctx, testInfo, minimumFields, "", 32)
 	assert.NotNil(t, vals)

pager.go

@@ -32,7 +32,7 @@ type pager struct {
 }
 
 func addLimitTokenString(w io.Writer, limit int, token string) {
-	if limit > 0 {
+	if limit == AdaptiveRangeLimit || limit > 0 {
 		fmt.Fprintf(w, " limit %d", limit)
 	}
 	if token != "" {

range.go

@@ -30,6 +30,14 @@ import (
 	"github.com/pkg/errors"
 )
 
+const (
+	// AdaptiveRangeLimit is a sentinel value that is used to indicate an intent
+	// to range over data in a partition as fast as possible. The server will
+	// determine an appropriate limit to use to range over the partition as fast
+	// as possible while ensuring the server remains healthy.
+	AdaptiveRangeLimit = -1
+)
+
 // RangeOp is used to specify constraints to Range calls
 type RangeOp struct {
 	pager

range_test.go

@@ -69,6 +69,12 @@ var rangeTestCases = []struct {
 		stringer:  "<empty> limit 10",
 		converted: "<empty> limit 10",
 	},
+	{
+		descript:  "empty with adaptive limit",
+		rop:       NewRangeOp(&AllTypes{}).Limit(AdaptiveRangeLimit),
+		stringer:  "<empty> limit -1",
+		converted: "<empty> limit -1",
+	},
 	{
 		descript:  "empty with token",
 		rop:       NewRangeOp(&AllTypes{}).Offset("toketoketoke"),