Skip to content

Commit

Permalink
refactor: remove grw in favor of caching layer
Browse files Browse the repository at this point in the history
Signed-off-by: Nico Braun <[email protected]>
  • Loading branch information
bluebrown committed Jan 20, 2024
1 parent fe9ad3c commit 39e80ff
Show file tree
Hide file tree
Showing 30 changed files with 558 additions and 893 deletions.
27 changes: 7 additions & 20 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -409,13 +409,13 @@ Kobold exposes prometheus metrics on port 8080. The metrics are exposed in the
# HELP kobold_git_fetch number of git fetches
# TYPE kobold_git_fetch counter
kobold_git_fetch{repo="[email protected]:bluebrown/foobar"} 3
kobold_git_fetch{repo="[email protected]:v3/myorg/myproject/kobold-test"} 3
# HELP kobold_git_push number of git pushes
# TYPE kobold_git_push counter
kobold_git_push{repo="[email protected]:bluebrown/foobar"} 1
kobold_git_push{repo="[email protected]:bluebrown/foobar"} 4
# HELP kobold_image_seen number of images seen
# TYPE kobold_image_seen counter
kobold_image_seen{ref="docker.io/bluebrown/busybox"} 3
kobold_image_seen{ref="docker.io/bluebrown/nginx"} 2
kobold_image_seen{ref="library/busybox"} 5
# HELP kobold_msg_recv number of messages received
# TYPE kobold_msg_recv counter
kobold_msg_recv{channel="dockerhub",rejected="false"} 5
Expand All @@ -424,8 +424,8 @@ kobold_msg_recv{channel="dockerhub",rejected="false"} 5
kobold_run_active 0
# HELP kobold_run_status run status (task groups)
# TYPE kobold_run_status counter
kobold_run_status{repo="[email protected]:bluebrown/foobar",status="success"} 2
kobold_run_status{repo="git@github.com:bluebrown/foobar",status="failure"} 1
kobold_run_status{repo="[email protected]:bluebrown/foobar",status="success"} 6
kobold_run_status{repo="git@ssh.dev.azure.com:v3/myorg/myproject/kobold-test",status="success"} 3
```

## Web API
Expand Down Expand Up @@ -498,23 +498,10 @@ items:
- test.azurecr.io/nginx:v1@sha256:993518ca49ede3c4e751fe799837ede16e60bc410452e3922602ebceda9b4c73
```

### Git Read/Writer

The `grw` command is a git read/writer. It reads from a source repository and
emits the resources to stdout. Optionally, by using the `-a` flag, it sets
tracking annotation, to improve the write performance, by preventing multiple
clones.

This example reads from git, pipes to a krm filter, and writes back to git:

```bash
bin/grw -a source '[email protected]:bluebrown/foobar.git@main/manifests' \
| bin/image-ref-updater testdata/events.yaml - \
| bin/grw sink '[email protected]:bluebrown/foobar.git@main/manifests'
```

### ConFix

TODO: fix this program. It is currently broken.

The `confix` command can be used to migrate from the v1 to the v2 config format.
I tries to conver the config on a best effort basis, but careful review is
required. Some features previously supported, are not supported anymore. For
Expand Down
22 changes: 18 additions & 4 deletions api/docs/docs.go
Original file line number Diff line number Diff line change
Expand Up @@ -498,6 +498,20 @@ const docTemplate = `{
}
}
},
"git.PackageURI": {
"type": "object",
"properties": {
"pkg": {
"type": "string"
},
"ref": {
"type": "string"
},
"repo": {
"type": "string"
}
}
},
"store.Channel": {
"type": "object",
"properties": {
Expand Down Expand Up @@ -542,7 +556,7 @@ const docTemplate = `{
"type": "string"
},
"repo_uri": {
"type": "string"
"$ref": "#/definitions/git.PackageURI"
}
}
},
Expand All @@ -569,7 +583,7 @@ const docTemplate = `{
"type": "string"
},
"repo_uri": {
"type": "string"
"$ref": "#/definitions/git.PackageURI"
},
"status": {
"type": "string"
Expand Down Expand Up @@ -617,7 +631,7 @@ const docTemplate = `{
"type": "string"
},
"repo_uri": {
"type": "string"
"$ref": "#/definitions/git.PackageURI"
},
"status": {
"type": "string"
Expand Down Expand Up @@ -653,7 +667,7 @@ const docTemplate = `{
"type": "string"
},
"repo_uri": {
"type": "string"
"$ref": "#/definitions/git.PackageURI"
},
"status": {
"type": "string"
Expand Down
22 changes: 18 additions & 4 deletions api/docs/swagger.json
Original file line number Diff line number Diff line change
Expand Up @@ -487,6 +487,20 @@
}
}
},
"git.PackageURI": {
"type": "object",
"properties": {
"pkg": {
"type": "string"
},
"ref": {
"type": "string"
},
"repo": {
"type": "string"
}
}
},
"store.Channel": {
"type": "object",
"properties": {
Expand Down Expand Up @@ -531,7 +545,7 @@
"type": "string"
},
"repo_uri": {
"type": "string"
"$ref": "#/definitions/git.PackageURI"
}
}
},
Expand All @@ -558,7 +572,7 @@
"type": "string"
},
"repo_uri": {
"type": "string"
"$ref": "#/definitions/git.PackageURI"
},
"status": {
"type": "string"
Expand Down Expand Up @@ -606,7 +620,7 @@
"type": "string"
},
"repo_uri": {
"type": "string"
"$ref": "#/definitions/git.PackageURI"
},
"status": {
"type": "string"
Expand Down Expand Up @@ -642,7 +656,7 @@
"type": "string"
},
"repo_uri": {
"type": "string"
"$ref": "#/definitions/git.PackageURI"
},
"status": {
"type": "string"
Expand Down
17 changes: 13 additions & 4 deletions api/docs/swagger.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,15 @@ definitions:
message:
type: string
type: object
git.PackageURI:
properties:
pkg:
type: string
ref:
type: string
repo:
type: string
type: object
store.Channel:
properties:
decoder_name:
Expand Down Expand Up @@ -35,7 +44,7 @@ definitions:
post_hook_name:
type: string
repo_uri:
type: string
$ref: '#/definitions/git.PackageURI'
type: object
store.PipelineRunListRow:
properties:
Expand All @@ -53,7 +62,7 @@ definitions:
post_hook:
type: string
repo_uri:
type: string
$ref: '#/definitions/git.PackageURI'
status:
type: string
timestamp: {}
Expand Down Expand Up @@ -85,7 +94,7 @@ definitions:
post_hook:
type: string
repo_uri:
type: string
$ref: '#/definitions/git.PackageURI'
status:
type: string
timestamp: {}
Expand All @@ -109,7 +118,7 @@ definitions:
post_hook_name:
type: string
repo_uri:
type: string
$ref: '#/definitions/git.PackageURI'
status:
type: string
task_group_fingerprint:
Expand Down
2 changes: 1 addition & 1 deletion build/.swaggo
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
replace github.com/volatiletech/null/v8.String string
replace kioutil.GitPackageURI string
replace git.GitPackageURI string
90 changes: 45 additions & 45 deletions build/sqlc.yaml
Original file line number Diff line number Diff line change
@@ -1,54 +1,54 @@
version: "2"
sql:
- engine: sqlite
queries: ../sql/*.query.sql
schema: ../sql/*.schema.sql
gen:
go:
package: store
out: ../store/
emit_json_tags: true
emit_empty_slices: true
emit_sql_as_comment: true
- engine: sqlite
queries: ../sql/*.query.sql
schema: ../sql/*.schema.sql
gen:
go:
package: store
out: ../store/
emit_json_tags: true
emit_empty_slices: true
emit_sql_as_comment: true

overrides:
- db_type: text
nullable: true
go_type:
import: github.com/volatiletech/null/v8
package: "null"
type: String
overrides:
- db_type: text
nullable: true
go_type:
import: github.com/volatiletech/null/v8
package: "null"
type: String

- column: "*.repo_uri"
go_type:
import: github.com/bluebrown/kobold/kioutil
package: kioutil
type: GitPackageURI
- column: "*.repo_uri"
go_type:
import: github.com/bluebrown/kobold/git
package: git
type: PackageURI

- column: "*.msgs"
go_type:
import: github.com/bluebrown/kobold/dbutil
package: dbutil
type: SliceText
- column: "*.msgs"
go_type:
import: github.com/bluebrown/kobold/dbutil
package: dbutil
type: SliceText

- column: "*.warnings"
go_type:
import: github.com/bluebrown/kobold/dbutil
package: dbutil
type: SliceText
- column: "*.warnings"
go_type:
import: github.com/bluebrown/kobold/dbutil
package: dbutil
type: SliceText

- column: "*.task_ids"
go_type:
import: github.com/bluebrown/kobold/dbutil
package: dbutil
type: SliceText
- column: "*.task_ids"
go_type:
import: github.com/bluebrown/kobold/dbutil
package: dbutil
type: SliceText

- column: "*.fingerprint"
go_type:
type: string
- column: "*.fingerprint"
go_type:
type: string

- column: "*.channels"
go_type:
import: github.com/bluebrown/kobold/dbutil
package: dbutil
type: JsonArray
- column: "*.channels"
go_type:
import: github.com/bluebrown/kobold/dbutil
package: dbutil
type: JsonArray
8 changes: 4 additions & 4 deletions cmd/cli/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,10 @@ func main() {
func run(ctx context.Context, args []string, env []string, input io.Reader) error {
var (
channel string
handler task.TaskHandler = task.KoboldHandler
set = flag.NewFlagSet("kobold-cli", flag.ExitOnError)
opts = config.Options().Bind(set)
maxprocs = 10
handler task.Handler = task.KoboldHandler
set = flag.NewFlagSet("kobold-cli", flag.ExitOnError)
opts = config.Options().Bind(set)
maxprocs = 10
)

set.StringVar(&channel, "channel", "", "channel to publish msgs to")
Expand Down
52 changes: 0 additions & 52 deletions cmd/grw/main.go

This file was deleted.

Loading

0 comments on commit 39e80ff

Please sign in to comment.