Skip to content

Commit

Permalink
Read-only scope metadata access for the CLI (#349)
Browse files Browse the repository at this point in the history
Read-only scope metadata access for the CLI
  • Loading branch information
phliar committed Jul 30, 2018
1 parent c391244 commit d835d64
Show file tree
Hide file tree
Showing 9 changed files with 315 additions and 23 deletions.
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
# Changelog

## v3.2.0 (unreleased)
- add readonly scope metadata access to CLI

## v3.1.1 (2018-07-25)
- Add keyPrefix to the config of redis connector (#344)
- Fixed memory connector upsert bug (#350)
Expand Down
19 changes: 11 additions & 8 deletions cmd/dosa/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@ type exiter func(int)

var exit = os.Exit

type clientProvider func(opts GlobalOptions) (dosa.AdminClient, error)

type adminClientProvider func(opts GlobalOptions) (dosa.AdminClient, error)
type mdClientProvider func(opts GlobalOptions) (dosa.Client, error)
type queryClientProvider func(opts GlobalOptions, scope, prefix, path, structName string) (ShellQueryClient, error)

// these are overridden at build-time w/ the -ldflags -X option
Expand Down Expand Up @@ -88,15 +88,18 @@ dosa manages your schema both in production and development scopes`
c, _ := OptionsParser.AddCommand("version", "display build info", "display build info", &BuildInfo{})

c, _ = OptionsParser.AddCommand("scope", "commands to manage scope", "create, drop, or truncate development scopes", &ScopeOptions{})
_, _ = c.AddCommand("create", "Create scope", "creates a new scope", newScopeCreate(provideYarpcClient))
_, _ = c.AddCommand("drop", "Drop scope", "drops a scope", newScopeDrop(provideYarpcClient))
_, _ = c.AddCommand("truncate", "Truncate scope", "truncates a scope", newScopeTruncate(provideYarpcClient))
_, _ = c.AddCommand("create", "Create scope", "creates a new scope", newScopeCreate(provideAdminClient))
_, _ = c.AddCommand("drop", "Drop scope", "drops a scope", newScopeDrop(provideAdminClient))
_, _ = c.AddCommand("truncate", "Truncate scope", "truncates a scope", newScopeTruncate(provideAdminClient))

_, _ = c.AddCommand("list", "List scopes", "lists scopes", newScopeList(provideMDClient))
_, _ = c.AddCommand("show", "Show scope MD", "show scope metadata", newScopeShow(provideMDClient))

c, _ = OptionsParser.AddCommand("schema", "commands to manage schemas", "check or update schemas", &SchemaOptions{})
_, _ = c.AddCommand("check", "Check schema", "check the schema", newSchemaCheck(provideYarpcClient))
_, _ = c.AddCommand("upsert", "Upsert schema", "insert or update the schema", newSchemaUpsert(provideYarpcClient))
_, _ = c.AddCommand("check", "Check schema", "check the schema", newSchemaCheck(provideAdminClient))
_, _ = c.AddCommand("upsert", "Upsert schema", "insert or update the schema", newSchemaUpsert(provideAdminClient))
_, _ = c.AddCommand("dump", "Dump schema", "display the schema in a given format", &SchemaDump{})
_, _ = c.AddCommand("status", "Check schema status", "Check application status of schema", newSchemaStatus(provideYarpcClient))
_, _ = c.AddCommand("status", "Check schema status", "Check application status of schema", newSchemaStatus(provideAdminClient))

c, _ = OptionsParser.AddCommand("query", "commands to do query", "fetch one or multiple rows", &QueryOptions{})
_, _ = c.AddCommand("read", "Read query", "read a row by primary keys", newQueryRead(provideShellQueryClient))
Expand Down
47 changes: 42 additions & 5 deletions cmd/dosa/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,14 @@ import (

const _defServiceName = "dosa-gateway"

// from YARPC: "must begin with a letter and consist only of dash-delimited
// lower-case ASCII alphanumeric words" -- we do this here because YARPC
// will panic if caller name is invalid.
var validNameRegex = regexp.MustCompile("^[a-z]+([a-z0-9]|[^-]-)*[^-]$")

// Error message for malformed service names.
const errmsg = "callerName %s must begin with a letter and consist only of dash-separated lower-case ASCII alphanumeric words"

type callerFlag string

func (s *callerFlag) setString(value string) {
Expand Down Expand Up @@ -85,12 +91,9 @@ func (t *timeFlag) UnmarshalFlag(value string) error {
return nil
}

func provideYarpcClient(opts GlobalOptions) (dosa.AdminClient, error) {
// from YARPC: "must begin with a letter and consist only of dash-delimited
// lower-case ASCII alphanumeric words" -- we do this here because YARPC
// will panic if caller name is invalid.
func provideAdminClient(opts GlobalOptions) (dosa.AdminClient, error) {
if !validNameRegex.MatchString(string(opts.CallerName)) {
return nil, fmt.Errorf("invalid caller name: %s, must begin with a letter and consist only of dash-delimited lower-case ASCII alphanumeric words", opts.CallerName)
return nil, fmt.Errorf(errmsg, opts.CallerName)
}

ycfg := yarpc.Config{
Expand All @@ -114,6 +117,40 @@ func shutdownAdminClient(client dosa.AdminClient) {
}
}

func provideMDClient(opts GlobalOptions) (c dosa.Client, err error) {
if !validNameRegex.MatchString(string(opts.CallerName)) {
return nil, fmt.Errorf(errmsg, opts.CallerName)
}

cfg := yarpc.ClientConfig{
Scope: "production",
NamePrefix: "dosa_scopes_metadata",
Yarpc: yarpc.Config{
Host: opts.Host,
Port: opts.Port,
CallerName: opts.CallerName.String(),
ServiceName: opts.ServiceName,
},
}

if c, err = cfg.NewClient((*dosa.ScopeMetadata)(nil)); err != nil {
return
}
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
defer cancel()
if err = c.Initialize(ctx); err != nil {
c = nil
}
return

}

func shutdownMDClient(client dosa.Client) {
if client.Shutdown() != nil {
_, _ = fmt.Fprintf(os.Stderr, "Failed to properly shutdown client")
}
}

func provideShellQueryClient(opts GlobalOptions, scope, prefix, path, structName string) (ShellQueryClient, error) {
// from YARPC: "must begin with a letter and consist only of dash-delimited
// lower-case ASCII alphanumeric words" -- we do this here because YARPC
Expand Down
8 changes: 4 additions & 4 deletions cmd/dosa/schema.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ type SchemaCmd struct {
Scope scopeFlag `short:"s" long:"scope" description:"Storage scope for the given operation." required:"true"`
NamePrefix string `short:"n" long:"namePrefix" description:"Name prefix for schema types."`
Prefix string `short:"p" long:"prefix" description:"Name prefix for schema types." hidden:"true"`
provideClient clientProvider
provideClient adminClientProvider
}

func getNamePrefix(namePrefix, prefix string) (string, error) {
Expand Down Expand Up @@ -144,7 +144,7 @@ type SchemaCheck struct {
} `positional-args:"yes"`
}

func newSchemaCheck(provideClient clientProvider) *SchemaCheck {
func newSchemaCheck(provideClient adminClientProvider) *SchemaCheck {
return &SchemaCheck{
SchemaCmd: &SchemaCmd{
provideClient: provideClient,
Expand All @@ -165,7 +165,7 @@ type SchemaUpsert struct {
} `positional-args:"yes"`
}

func newSchemaUpsert(provideClient clientProvider) *SchemaUpsert {
func newSchemaUpsert(provideClient adminClientProvider) *SchemaUpsert {
return &SchemaUpsert{
SchemaCmd: &SchemaCmd{
provideClient: provideClient,
Expand All @@ -184,7 +184,7 @@ type SchemaStatus struct {
Version int32 `long:"version" description:"Specify schema version."`
}

func newSchemaStatus(provideClient clientProvider) *SchemaStatus {
func newSchemaStatus(provideClient adminClientProvider) *SchemaStatus {
return &SchemaStatus{
SchemaCmd: &SchemaCmd{
provideClient: provideClient,
Expand Down
9 changes: 5 additions & 4 deletions cmd/dosa/scope.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,8 @@ type ScopeOptions struct{}

// ScopeCmd are options for all scope commands
type ScopeCmd struct {
provideClient clientProvider
provideClient adminClientProvider
provideMDClient mdClientProvider
}

func (c *ScopeCmd) doScopeOp(name string, f func(dosa.AdminClient, context.Context, string) error, scopes []string) error {
Expand Down Expand Up @@ -73,7 +74,7 @@ type ScopeCreate struct {
} `positional-args:"yes" required:"1"`
}

func newScopeCreate(provideClient clientProvider) *ScopeCreate {
func newScopeCreate(provideClient adminClientProvider) *ScopeCreate {
return &ScopeCreate{
ScopeCmd: &ScopeCmd{
provideClient: provideClient,
Expand Down Expand Up @@ -112,7 +113,7 @@ type ScopeDrop struct {
} `positional-args:"yes" required:"1"`
}

func newScopeDrop(provideClient clientProvider) *ScopeDrop {
func newScopeDrop(provideClient adminClientProvider) *ScopeDrop {
return &ScopeDrop{
ScopeCmd: &ScopeCmd{
provideClient: provideClient,
Expand All @@ -133,7 +134,7 @@ type ScopeTruncate struct {
} `positional-args:"yes" required:"1"`
}

func newScopeTruncate(provideClient clientProvider) *ScopeTruncate {
func newScopeTruncate(provideClient adminClientProvider) *ScopeTruncate {
return &ScopeTruncate{
ScopeCmd: &ScopeCmd{
provideClient: provideClient,
Expand Down
141 changes: 141 additions & 0 deletions cmd/dosa/scopemd.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
// Copyright (c) 2018 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package main

import (
"context"
"fmt"

"github.com/pkg/errors"
"github.com/uber-go/dosa"
)

// ScopeList contains data for executing scope truncate command.
type ScopeList struct {
*ScopeCmd
}

func newScopeList(provideClient mdClientProvider) *ScopeList {
return &ScopeList{
ScopeCmd: &ScopeCmd{
provideMDClient: provideClient,
},
}
}

// Execute executes a scope list command
func (c *ScopeList) Execute(args []string) error {
client, err := c.makeClient()
if err != nil {
return errors.Wrap(err, "could not make client")
}
defer shutdownMDClient(client)

var scopes []string
if scopes, err = c.getScopes(client); err == nil {
return err
}
for _, sp := range scopes {
fmt.Println(sp)
}
return nil
}

func (c *ScopeList) getScopes(client dosa.Client) ([]string, error) {
scopeList := []string{}

var md dosa.ScopeMetadata
scanOp := dosa.NewScanOp(&md).Limit(100).Fields([]string{"Name"})
for {
ctx, cancel := context.WithTimeout(context.Background(), options.Timeout.Duration())
defer cancel()

scopes, token, err := client.ScanEverything(ctx, scanOp)
if err != nil {
if dosa.ErrorIsNotFound(err) {
break
}
fmt.Printf("MD table scan failed (token=%q): %v\n", token, err)
continue
}
scanOp.Offset(token)

for _, e := range scopes {
md := e.(*dosa.ScopeMetadata)
scopeList = append(scopeList, md.Name)
}
}
return scopeList, nil
}

// ScopeShow displays metadata for the specified scopes.
type ScopeShow struct {
*ScopeCmd
Args struct {
Scopes []string `positional-arg-name:"scopes" required:"1"`
} `positional-args:"yes" required:"1"`
}

func newScopeShow(provideClient mdClientProvider) *ScopeShow {
return &ScopeShow{
ScopeCmd: &ScopeCmd{
provideMDClient: provideClient,
},
}
}

// Execute shows MD for the scope(s)
func (c *ScopeShow) Execute(args []string) error {
client, err := c.makeClient()
if err != nil {
return errors.Wrap(err, "could not make client")
}
defer shutdownMDClient(client)

for _, scope := range args {
if md, err := c.getMetadata(client, scope); err != nil {
fmt.Printf("Could not read scope metadata for %q: %v\n", scope, err)
} else {
fmt.Printf("%+v\n", md)
}
}
return nil
}

// getMetadata returns the MD for a scope. Currently prefixes (for prod) are not handled; the intent
// is that a scope may be qualified by a name-prefix, as in "production.vsoffers".
func (c *ScopeShow) getMetadata(client dosa.Client, scope string) (*dosa.ScopeMetadata, error) {
ctx, cancel := context.WithTimeout(context.Background(), options.Timeout.Duration())
defer cancel()

md := &dosa.ScopeMetadata{Name: scope}
if err := client.Read(ctx, dosa.All(), md); err != nil {
return nil, err
}
return md, nil
}

func (c *ScopeCmd) makeClient() (dosa.Client, error) {
if options.ServiceName == "" {
options.ServiceName = _defServiceName
}
return c.provideMDClient(options)
}
Loading

0 comments on commit d835d64

Please sign in to comment.