From 2f07ba70e9f13b1cdcd268b57a1e0611ca524508 Mon Sep 17 00:00:00 2001 From: Stefan Majewsky Date: Fri, 15 Sep 2023 14:55:11 +0200 Subject: [PATCH] add full AZ awareness to quota plugin interface This is the second part of the change that was started in the previous commit. This change is much more straight-forward than the one to the capacity plugins, since quota plugins do not report usage per AZ at all right now. We are still evaluating which plugins to move to per-AZ usage reporting and which changes in the backend services will be necessary to enable this. --- internal/collector/scrape.go | 13 +++--- internal/collector/scrape_test.go | 4 +- internal/core/data.go | 34 +++++++++++++++- internal/core/plugin.go | 12 ------ internal/plugins/archer.go | 8 +++- internal/plugins/cinder.go | 13 ++++-- internal/plugins/designate.go | 8 +++- internal/plugins/keppel.go | 4 +- internal/plugins/manila.go | 30 ++++++++------ internal/plugins/neutron.go | 8 +++- internal/plugins/nova.go | 36 +++++++++++------ internal/plugins/swift.go | 10 +++-- internal/test/plugins/quota_autoapproval.go | 4 +- internal/test/plugins/quota_generic.go | 44 +++++++++++++-------- 14 files changed, 149 insertions(+), 79 deletions(-) diff --git a/internal/collector/scrape.go b/internal/collector/scrape.go index e158ac592..3ea47696a 100644 --- a/internal/collector/scrape.go +++ b/internal/collector/scrape.go @@ -220,8 +220,9 @@ func (c *Collector) writeResourceScrapeResult(dbDomain db.Domain, dbProject db.P //this is the callback that ProjectResourceUpdate will use to write the scraped data into the project_resources updateResource := func(res *db.ProjectResource) error { data := resourceData[res.Name] - res.Usage = data.Usage - res.PhysicalUsage = data.PhysicalUsage + usageData := data.UsageData.Sum() + res.Usage = usageData.Usage + res.PhysicalUsage = usageData.PhysicalUsage resInfo := c.Cluster.InfoForResource(srv.Type, res.Name) if !resInfo.NoQuota { @@ -238,17 +239,17 @@ func (c *Collector) writeResourceScrapeResult(dbDomain db.Domain, dbProject db.P res.BackendQuota = &data.Quota } - if len(data.Subresources) == 0 { + if len(usageData.Subresources) == 0 { res.SubresourcesJSON = "" } else { //warn when the backend is inconsistent with itself - if uint64(len(data.Subresources)) != res.Usage { + if uint64(len(usageData.Subresources)) != res.Usage { logg.Info("resource quantity mismatch in project %s, resource %s/%s: usage = %d, but found %d subresources", dbProject.UUID, srv.Type, res.Name, - res.Usage, len(data.Subresources), + res.Usage, len(usageData.Subresources), ) } - bytes, err := json.Marshal(data.Subresources) + bytes, err := json.Marshal(usageData.Subresources) if err != nil { return fmt.Errorf("failed to convert subresources to JSON: %s", err.Error()) } diff --git a/internal/collector/scrape_test.go b/internal/collector/scrape_test.go index 31bb2bce7..c1ed97760 100644 --- a/internal/collector/scrape_test.go +++ b/internal/collector/scrape_test.go @@ -173,7 +173,7 @@ func Test_ScrapeSuccess(t *testing.T) { //change the data that is reported by the plugin s.Clock.StepBy(scrapeInterval) plugin.StaticResourceData["capacity"].Quota = 110 - plugin.StaticResourceData["things"].Usage = 5 + plugin.StaticResourceData["things"].UsageData.Regional.Usage = 5 //Scrape should pick up the changed resource data mustT(t, job.ProcessOne(s.Ctx, withLabel)) mustT(t, job.ProcessOne(s.Ctx, withLabel)) @@ -269,7 +269,7 @@ func Test_ScrapeSuccess(t *testing.T) { //"capacity_portion" (otherwise this resource has been all zeroes this entire //time) s.Clock.StepBy(scrapeInterval) - plugin.StaticResourceData["capacity"].Usage = 20 + plugin.StaticResourceData["capacity"].UsageData.Regional.Usage = 20 mustT(t, job.ProcessOne(s.Ctx, withLabel)) mustT(t, job.ProcessOne(s.Ctx, withLabel)) diff --git a/internal/core/data.go b/internal/core/data.go index 0a2cb6c23..37c078627 100644 --- a/internal/core/data.go +++ b/internal/core/data.go @@ -68,13 +68,45 @@ func (t Topological[D]) Sum() D { type TopologicalData[Self any] interface { // List of permitted types. This is required for type inference, as explained here: // - CapacityData + UsageData | CapacityData // Computes the sum of this structure and `other`. // This is used to implement Topological.Sum(). add(other Self) Self } +// ResourceData contains quota and usage data for a single project resource. +type ResourceData struct { + Quota int64 //negative values indicate infinite quota + UsageData Topological[UsageData] +} + +// UsageData contains usage data for a single project resource. +// It appears in type ResourceData. +type UsageData struct { + Usage uint64 + PhysicalUsage *uint64 //only supported by some plugins + Subresources []any //only if supported by plugin and enabled in config +} + +// add implements the TopologicalData interface. +// +//nolint:unused // looks like a linter bug +func (d UsageData) add(other UsageData) UsageData { + result := UsageData{ + Usage: d.Usage + other.Usage, + Subresources: append(slices.Clone(d.Subresources), other.Subresources...), + } + + //the sum can only have a PhysicalUsage value if both sides have it + if d.PhysicalUsage != nil && other.PhysicalUsage != nil { + physUsage := *d.PhysicalUsage + *other.PhysicalUsage + result.PhysicalUsage = &physUsage + } + + return result +} + // CapacityData contains capacity data for a single project resource. type CapacityData struct { Capacity uint64 diff --git a/internal/core/plugin.go b/internal/core/plugin.go index c40d9e4e8..e42720d36 100644 --- a/internal/core/plugin.go +++ b/internal/core/plugin.go @@ -82,18 +82,6 @@ type DiscoveryPlugin interface { ListProjects(domain KeystoneDomain) ([]KeystoneProject, error) } -// ResourceData contains quota and usage data for a single resource. -// -// The Subresources field may optionally be populated with subresources, if the -// quota plugin providing this ResourceData instance has been instructed to (and -// is able to) scrape subresources for this resource. -type ResourceData struct { - Quota int64 //negative values indicate infinite quota - Usage uint64 - PhysicalUsage *uint64 //only supported by some plugins - Subresources []any -} - // QuotaPlugin is the interface that the quota/usage collector plugins for all // backend services must implement. There can only be one QuotaPlugin for each // backend service. diff --git a/internal/plugins/archer.go b/internal/plugins/archer.go index f68f7f73d..33b64512a 100644 --- a/internal/plugins/archer.go +++ b/internal/plugins/archer.go @@ -113,11 +113,15 @@ func (p *archerPlugin) Scrape(project core.KeystoneProject) (result map[string]c result = map[string]core.ResourceData{ "endpoints": { Quota: archerQuota.Endpoint, - Usage: archerQuota.InUseEndpoint, + UsageData: core.Regional(core.UsageData{ + Usage: archerQuota.InUseEndpoint, + }), }, "services": { Quota: archerQuota.Service, - Usage: archerQuota.InUseService, + UsageData: core.Regional(core.UsageData{ + Usage: archerQuota.InUseService, + }), }, } return result, "", nil diff --git a/internal/plugins/cinder.go b/internal/plugins/cinder.go index ce412e48b..7c00d5876 100644 --- a/internal/plugins/cinder.go +++ b/internal/plugins/cinder.go @@ -118,7 +118,10 @@ func (p *cinderPlugin) makeResourceName(kind, volumeType string) string { return kind + "_" + volumeType } -type quotaSetField core.ResourceData +type quotaSetField struct { + Quota int64 + Usage uint64 +} func (f *quotaSetField) UnmarshalJSON(buf []byte) error { //The `quota_set` field in the os-quota-sets response is mostly @@ -142,9 +145,11 @@ func (f *quotaSetField) UnmarshalJSON(buf []byte) error { func (f quotaSetField) ToResourceData(subresources []any) core.ResourceData { return core.ResourceData{ - Quota: f.Quota, - Usage: f.Usage, - Subresources: subresources, + Quota: f.Quota, + UsageData: core.Regional(core.UsageData{ + Usage: f.Usage, + Subresources: subresources, + }), } } diff --git a/internal/plugins/designate.go b/internal/plugins/designate.go index ffb7c8d4c..f3cef6f76 100644 --- a/internal/plugins/designate.go +++ b/internal/plugins/designate.go @@ -121,11 +121,15 @@ func (p *designatePlugin) Scrape(project core.KeystoneProject) (result map[strin return map[string]core.ResourceData{ "zones": { Quota: quotas.Zones, - Usage: uint64(len(zoneIDs)), + UsageData: core.Regional(core.UsageData{ + Usage: uint64(len(zoneIDs)), + }), }, "recordsets": { Quota: quotas.ZoneRecordsets, - Usage: maxRecordsetsPerZone, + UsageData: core.Regional(core.UsageData{ + Usage: maxRecordsetsPerZone, + }), }, }, "", nil } diff --git a/internal/plugins/keppel.go b/internal/plugins/keppel.go index 76610b0f9..b9a29b026 100644 --- a/internal/plugins/keppel.go +++ b/internal/plugins/keppel.go @@ -92,7 +92,9 @@ func (p *keppelPlugin) Scrape(project core.KeystoneProject) (result map[string]c return map[string]core.ResourceData{ "images": { Quota: quotas.Manifests.Quota, - Usage: quotas.Manifests.Usage, + UsageData: core.Regional(core.UsageData{ + Usage: quotas.Manifests.Usage, + }), }, }, "", nil } diff --git a/internal/plugins/manila.go b/internal/plugins/manila.go index 900a85357..7f2400215 100644 --- a/internal/plugins/manila.go +++ b/internal/plugins/manila.go @@ -231,12 +231,12 @@ func (p *manilaPlugin) Scrape(project core.KeystoneProject) (result map[string]c for idx, shareType := range p.ShareTypes { stName := resolveManilaShareType(shareType, project) if stName == "" { - result[p.makeResourceName("shares", shareType)] = core.ResourceData{Quota: 0, Usage: 0} - result[p.makeResourceName("share_capacity", shareType)] = core.ResourceData{Quota: 0, Usage: 0} - result[p.makeResourceName("share_snapshots", shareType)] = core.ResourceData{Quota: 0, Usage: 0} - result[p.makeResourceName("snapshot_capacity", shareType)] = core.ResourceData{Quota: 0, Usage: 0} + result[p.makeResourceName("shares", shareType)] = core.ResourceData{Quota: 0, UsageData: core.Regional(core.UsageData{})} + result[p.makeResourceName("share_capacity", shareType)] = core.ResourceData{Quota: 0, UsageData: core.Regional(core.UsageData{})} + result[p.makeResourceName("share_snapshots", shareType)] = core.ResourceData{Quota: 0, UsageData: core.Regional(core.UsageData{})} + result[p.makeResourceName("snapshot_capacity", shareType)] = core.ResourceData{Quota: 0, UsageData: core.Regional(core.UsageData{})} if p.PrometheusAPIConfig != nil { - result[p.makeResourceName("snapmirror_capacity", shareType)] = core.ResourceData{Quota: 0, Usage: 0} + result[p.makeResourceName("snapmirror_capacity", shareType)] = core.ResourceData{Quota: 0, UsageData: core.Regional(core.UsageData{})} } continue } @@ -255,8 +255,8 @@ func (p *manilaPlugin) Scrape(project core.KeystoneProject) (result map[string]c sharesData := quotaSets[stName].Shares.ToResourceData(nil) shareCapacityData := quotaSets[stName].Gigabytes.ToResourceData(gigabytesPhysical) if p.hasReplicaQuotas && shareType.ReplicationEnabled { - sharesData.Usage = quotaSets[stName].Replicas.Usage - shareCapacityData.Usage = quotaSets[stName].ReplicaGigabytes.Usage + sharesData.UsageData.Regional.Usage = quotaSets[stName].Replicas.Usage + shareCapacityData.UsageData.Regional.Usage = quotaSets[stName].ReplicaGigabytes.Usage //if share quotas and replica quotas disagree, report quota = -1 to force Limes to reapply the replica quota if quotaSets[stName].Replicas.Quota != sharesData.Quota { logg.Info("found mismatch between share quota (%d) and replica quota (%d) for share type %q in project %s", @@ -431,9 +431,11 @@ type manilaQuotaDetail struct { func (q manilaQuotaDetail) ToResourceData(physicalUsage *uint64) core.ResourceData { return core.ResourceData{ - Quota: q.Quota, - Usage: q.Usage, - PhysicalUsage: physicalUsage, + Quota: q.Quota, + UsageData: core.Regional(core.UsageData{ + Usage: q.Usage, + PhysicalUsage: physicalUsage, + }), } } @@ -536,9 +538,11 @@ func (p *manilaPlugin) collectSnapmirrorUsage(project core.KeystoneProject, shar bytesUsedAsUint64 := roundUpIntoGigabytes(bytesUsed) return core.ResourceData{ - Quota: 0, //NoQuota = true - Usage: roundUpIntoGigabytes(bytesTotal), - PhysicalUsage: &bytesUsedAsUint64, + Quota: 0, //NoQuota = true + UsageData: core.Regional(core.UsageData{ + Usage: roundUpIntoGigabytes(bytesTotal), + PhysicalUsage: &bytesUsedAsUint64, + }), }, nil } diff --git a/internal/plugins/neutron.go b/internal/plugins/neutron.go index be02af132..57dcd9bbb 100644 --- a/internal/plugins/neutron.go +++ b/internal/plugins/neutron.go @@ -365,7 +365,9 @@ func (p *neutronPlugin) scrapeNeutronInto(result map[string]core.ResourceData, p values := quotas.Values[res.NeutronName] result[res.LimesName] = core.ResourceData{ Quota: values.Quota, - Usage: values.Usage, + UsageData: core.Regional(core.UsageData{ + Usage: values.Usage, + }), } } return nil @@ -394,7 +396,9 @@ func (p *neutronPlugin) scrapeOctaviaInto(result map[string]core.ResourceData, p } result[res.LimesName] = core.ResourceData{ Quota: quota, - Usage: usage[res.OctaviaName], + UsageData: core.Regional(core.UsageData{ + Usage: usage[res.OctaviaName], + }), } } return nil diff --git a/internal/plugins/nova.go b/internal/plugins/nova.go index fda85cd8c..2243f7169 100644 --- a/internal/plugins/nova.go +++ b/internal/plugins/nova.go @@ -311,23 +311,33 @@ func (p *novaPlugin) Scrape(project core.KeystoneProject) (result map[string]cor resultPtr := map[string]*core.ResourceData{ "cores": { Quota: limitsData.Limits.Absolute.MaxTotalCores, - Usage: limitsData.Limits.Absolute.TotalCoresUsed, + UsageData: core.Regional(core.UsageData{ + Usage: limitsData.Limits.Absolute.TotalCoresUsed, + }), }, "instances": { Quota: limitsData.Limits.Absolute.MaxTotalInstances, - Usage: limitsData.Limits.Absolute.TotalInstancesUsed, + UsageData: core.Regional(core.UsageData{ + Usage: limitsData.Limits.Absolute.TotalInstancesUsed, + }), }, "ram": { Quota: limitsData.Limits.Absolute.MaxTotalRAMSize, - Usage: limitsData.Limits.Absolute.TotalRAMUsed, + UsageData: core.Regional(core.UsageData{ + Usage: limitsData.Limits.Absolute.TotalRAMUsed, + }), }, "server_groups": { Quota: limitsData.Limits.Absolute.MaxServerGroups, - Usage: limitsData.Limits.Absolute.TotalServerGroupsUsed, + UsageData: core.Regional(core.UsageData{ + Usage: limitsData.Limits.Absolute.TotalServerGroupsUsed, + }), }, "server_group_members": { Quota: limitsData.Limits.Absolute.MaxServerGroupMembers, - Usage: totalServerGroupMembersUsed, + UsageData: core.Regional(core.UsageData{ + Usage: totalServerGroupMembersUsed, + }), }, } @@ -337,7 +347,9 @@ func (p *novaPlugin) Scrape(project core.KeystoneProject) (result map[string]cor if p.SeparateInstanceQuotas.FlavorNameRx.MatchString(flavorName) { resultPtr[p.ftt.LimesResourceNameForFlavor(flavorName)] = &core.ResourceData{ Quota: flavorLimits.MaxTotalInstances, - Usage: flavorLimits.TotalInstancesUsed, + UsageData: core.Regional(core.UsageData{ + Usage: flavorLimits.TotalInstancesUsed, + }), } } } @@ -350,8 +362,8 @@ func (p *novaPlugin) Scrape(project core.KeystoneProject) (result map[string]cor for _, res := range p.resources { if _, exists := resultPtr[res.Name]; !exists { resultPtr[res.Name] = &core.ResourceData{ - Quota: 0, - Usage: 0, + Quota: 0, + UsageData: core.Regional(core.UsageData{}), } } } @@ -432,9 +444,9 @@ func (p *novaPlugin) Scrape(project core.KeystoneProject) (result map[string]cor //do not count baremetal instances into `{cores,instances,ram}_{bigvm,regular}` if _, exists := resultPtr[p.ftt.LimesResourceNameForFlavor(flavorName)]; !exists { - resultPtr["cores_"+class].Usage += flavor.VCPUs - resultPtr["instances_"+class].Usage++ - resultPtr["ram_"+class].Usage += flavor.MemoryMiB + resultPtr["cores_"+class].UsageData.Regional.Usage += flavor.VCPUs + resultPtr["instances_"+class].UsageData.Regional.Usage++ + resultPtr["ram_"+class].UsageData.Regional.Usage += flavor.MemoryMiB } } } @@ -468,7 +480,7 @@ func (p *novaPlugin) Scrape(project core.KeystoneProject) (result map[string]cor if !exists { resource = resultPtr["instances"] } - resource.Subresources = append(resource.Subresources, subResource) + resource.UsageData.Regional.Subresources = append(resource.UsageData.Regional.Subresources, subResource) } return true, nil }) diff --git a/internal/plugins/swift.go b/internal/plugins/swift.go index 84891162b..5dcdce881 100644 --- a/internal/plugins/swift.go +++ b/internal/plugins/swift.go @@ -129,11 +129,11 @@ func (p *swiftPlugin) Scrape(project core.KeystoneProject) (result map[string]co account := p.Account(project.UUID) headers, err := account.Headers() if schwift.Is(err, http.StatusNotFound) || schwift.Is(err, http.StatusGone) { - //Swift account does not exist or was deleted and not yet reaped, but the keystone project exist + //Swift account does not exist or was deleted and not yet reaped, but the keystone project exists return map[string]core.ResourceData{ "capacity": { - Quota: 0, - Usage: 0, + Quota: 0, + UsageData: core.Regional(core.UsageData{Usage: 0}), }, }, "", nil } else if err != nil { @@ -164,8 +164,10 @@ func (p *swiftPlugin) Scrape(project core.KeystoneProject) (result map[string]co } data := core.ResourceData{ - Usage: headers.BytesUsed().Get(), Quota: int64(headers.BytesUsedQuota().Get()), + UsageData: core.Regional(core.UsageData{ + Usage: headers.BytesUsed().Get(), + }), } if !headers.BytesUsedQuota().Exists() { data.Quota = -1 diff --git a/internal/test/plugins/quota_autoapproval.go b/internal/test/plugins/quota_autoapproval.go index abdfc2231..d0a8efd2e 100644 --- a/internal/test/plugins/quota_autoapproval.go +++ b/internal/test/plugins/quota_autoapproval.go @@ -97,8 +97,8 @@ func (p *AutoApprovalQuotaPlugin) CollectMetrics(ch chan<- prometheus.Metric, pr // Scrape implements the core.QuotaPlugin interface. func (p *AutoApprovalQuotaPlugin) Scrape(project core.KeystoneProject) (result map[string]core.ResourceData, serializedMetrics string, err error) { return map[string]core.ResourceData{ - "approve": {Usage: 0, Quota: int64(p.StaticBackendQuota)}, - "noapprove": {Usage: 0, Quota: int64(p.StaticBackendQuota) + 10}, + "approve": {UsageData: core.Regional(core.UsageData{Usage: 0}), Quota: int64(p.StaticBackendQuota)}, + "noapprove": {UsageData: core.Regional(core.UsageData{Usage: 0}), Quota: int64(p.StaticBackendQuota) + 10}, }, "", nil } diff --git a/internal/test/plugins/quota_generic.go b/internal/test/plugins/quota_generic.go index 4c9520fee..fcc48867d 100644 --- a/internal/test/plugins/quota_generic.go +++ b/internal/test/plugins/quota_generic.go @@ -73,8 +73,18 @@ var resources = []limesresources.ResourceInfo{ // Init implements the core.QuotaPlugin interface. func (p *GenericQuotaPlugin) Init(provider *gophercloud.ProviderClient, eo gophercloud.EndpointOpts, scrapeSubresources map[string]bool) error { p.StaticResourceData = map[string]*core.ResourceData{ - "things": {Quota: 42, Usage: 2}, - "capacity": {Quota: 100, Usage: 0}, + "things": { + Quota: 42, + UsageData: core.Regional(core.UsageData{ + Usage: 2, + }), + }, + "capacity": { + Quota: 100, + UsageData: core.Regional(core.UsageData{ + Usage: 0, + }), + }, } p.OverrideQuota = make(map[string]map[string]uint64) return nil @@ -141,16 +151,23 @@ func (p *GenericQuotaPlugin) Scrape(project core.KeystoneProject) (result map[st result = make(map[string]core.ResourceData) for key, val := range p.StaticResourceData { - copyOfVal := *val + copyOfVal := core.ResourceData{ + Quota: val.Quota, + UsageData: core.Regional(core.UsageData{ + Usage: val.UsageData.Regional.Usage, + }), + } //test coverage for PhysicalUsage != Usage if key == "capacity" { - physUsage := val.Usage / 2 - copyOfVal.PhysicalUsage = &physUsage + physUsage := val.UsageData.Regional.Usage / 2 + copyOfVal.UsageData.Regional.PhysicalUsage = &physUsage //derive a resource that does not track quota result["capacity_portion"] = core.ResourceData{ - Usage: val.Usage / 4, + UsageData: core.Regional(core.UsageData{ + Usage: val.UsageData.Regional.Usage / 4, + }), } } @@ -161,32 +178,27 @@ func (p *GenericQuotaPlugin) Scrape(project core.KeystoneProject) (result map[st if exists { for resourceName, quota := range data { result[resourceName] = core.ResourceData{ - Quota: int64(quota), - Usage: result[resourceName].Usage, - PhysicalUsage: result[resourceName].PhysicalUsage, + Quota: int64(quota), + UsageData: result[resourceName].UsageData, } } } //make up some subresources for "things" - thingsUsage := int(result["things"].Usage) + thingsUsage := int(result["things"].UsageData.Regional.Usage) subres := make([]any, thingsUsage) for idx := 0; idx < thingsUsage; idx++ { subres[idx] = map[string]any{ "index": idx, } } - result["things"] = core.ResourceData{ - Quota: result["things"].Quota, - Usage: result["things"].Usage, - Subresources: subres, - } + result["things"].UsageData.Regional.Subresources = subres //make up some serialized metrics (reporting usage as a metric is usually //nonsensical since limes-collect already reports all usages as metrics, but //this is only a testcase anyway) serializedMetrics = fmt.Sprintf(`{"capacity_usage":%d,"things_usage":%d}`, - result["capacity"].Usage, result["things"].Usage) + result["capacity"].UsageData.Regional.Usage, result["things"].UsageData.Regional.Usage) return result, serializedMetrics, nil }