Introduce import source

Signed-off-by: Dominika Zemanovicova <[email protected]>
redhat-developer · Nov 18, 2024 · d7b2de6 · d7b2de6
1 parent 926482a
commit d7b2de6
Show file tree

Hide file tree

Showing 19 changed files with 344 additions and 79 deletions.
diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/.openapi-generator/FILES b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/.openapi-generator/FILES
@@ -17,6 +17,8 @@ Models/Organization.md
 Models/OrganizationList.md
 Models/Repository.md
 Models/RepositoryList.md
+Models/Source.md
+Models/SourceImport.md
 Models/findAllImports_200_response.md
 Models/findAllImports_500_response.md
 Models/ping_200_response.md

diff --git a/...aces/bulk-import/plugins/bulk-import-backend/api-docs/Models/ImportJobListV2.md b/...aces/bulk-import/plugins/bulk-import-backend/api-docs/Models/ImportJobListV2.md
@@ -2,12 +2,12 @@
 
 ## Properties
 
-| Name           | Type                  | Description | Notes                        |
-| -------------- | --------------------- | ----------- | ---------------------------- |
-| **imports**    | [**List**](Import.md) |             | [optional] [default to null] |
-| **errors**     | **List**              |             | [optional] [default to null] |
-| **totalCount** | **Integer**           |             | [optional] [default to null] |
-| **page**       | **Integer**           |             | [optional] [default to null] |
-| **size**       | **Integer**           |             | [optional] [default to null] |
+| Name           | Type                        | Description | Notes                        |
+| -------------- | --------------------------- | ----------- | ---------------------------- |
+| **imports**    | [**List**](SourceImport.md) |             | [optional] [default to null] |
+| **errors**     | **List**                    |             | [optional] [default to null] |
+| **totalCount** | **Integer**                 |             | [optional] [default to null] |
+| **page**       | **Integer**                 |             | [optional] [default to null] |
+| **size**       | **Integer**                 |             | [optional] [default to null] |
 
 [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Models/Source.md b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Models/Source.md
@@ -0,0 +1,8 @@
+# Source
+
+## Properties
+
+| Name | Type | Description | Notes |
+| ---- | ---- | ----------- | ----- |
+
+[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Models/SourceImport.md b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Models/SourceImport.md
@@ -0,0 +1,17 @@
+# SourceImport
+
+## Properties
+
+| Name                  | Type                                  | Description                                                                                | Notes                        |
+| --------------------- | ------------------------------------- | ------------------------------------------------------------------------------------------ | ---------------------------- |
+| **id**                | **String**                            |                                                                                            | [optional] [default to null] |
+| **status**            | [**ImportStatus**](ImportStatus.md)   |                                                                                            | [optional] [default to null] |
+| **catalogEntityName** | **String**                            | Specified entity name in the catalog. Filled only in response for dry-run import requests. | [optional] [default to null] |
+| **lastUpdate**        | **Date**                              |                                                                                            | [optional] [default to null] |
+| **errors**            | **List**                              |                                                                                            | [optional] [default to null] |
+| **approvalTool**      | [**ApprovalTool**](ApprovalTool.md)   |                                                                                            | [optional] [default to null] |
+| **repository**        | [**Repository**](Repository.md)       |                                                                                            | [optional] [default to null] |
+| **github**            | [**Import_github**](Import_github.md) |                                                                                            | [optional] [default to null] |
+| **source**            | [**Source**](Source.md)               |                                                                                            | [optional] [default to null] |
+
+[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
diff --git a/...port/plugins/bulk-import-backend/api-docs/Models/findAllImports_200_response.md b/...port/plugins/bulk-import-backend/api-docs/Models/findAllImports_200_response.md
@@ -2,12 +2,12 @@
 
 ## Properties
 
-| Name           | Type                  | Description | Notes                        |
-| -------------- | --------------------- | ----------- | ---------------------------- |
-| **imports**    | [**List**](Import.md) |             | [optional] [default to null] |
-| **errors**     | **List**              |             | [optional] [default to null] |
-| **totalCount** | **Integer**           |             | [optional] [default to null] |
-| **page**       | **Integer**           |             | [optional] [default to null] |
-| **size**       | **Integer**           |             | [optional] [default to null] |
+| Name           | Type                        | Description | Notes                        |
+| -------------- | --------------------------- | ----------- | ---------------------------- |
+| **imports**    | [**List**](SourceImport.md) |             | [optional] [default to null] |
+| **errors**     | **List**                    |             | [optional] [default to null] |
+| **totalCount** | **Integer**                 |             | [optional] [default to null] |
+| **page**       | **Integer**                 |             | [optional] [default to null] |
+| **size**       | **Integer**                 |             | [optional] [default to null] |
 
 [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
diff --git a/...port/plugins/bulk-import-backend/api-docs/Models/findAllImports_500_response.md b/...port/plugins/bulk-import-backend/api-docs/Models/findAllImports_500_response.md
@@ -2,12 +2,12 @@
 
 ## Properties
 
-| Name           | Type                  | Description | Notes                        |
-| -------------- | --------------------- | ----------- | ---------------------------- |
-| **imports**    | [**List**](Import.md) |             | [optional] [default to null] |
-| **errors**     | **List**              |             | [optional] [default to null] |
-| **totalCount** | **Integer**           |             | [optional] [default to null] |
-| **page**       | **Integer**           |             | [optional] [default to null] |
-| **size**       | **Integer**           |             | [optional] [default to null] |
+| Name           | Type                        | Description | Notes                        |
+| -------------- | --------------------------- | ----------- | ---------------------------- |
+| **imports**    | [**List**](SourceImport.md) |             | [optional] [default to null] |
+| **errors**     | **List**                    |             | [optional] [default to null] |
+| **totalCount** | **Integer**                 |             | [optional] [default to null] |
+| **page**       | **Integer**                 |             | [optional] [default to null] |
+| **size**       | **Integer**                 |             | [optional] [default to null] |
 
 [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/README.md b/workspaces/bulk-import/plugins/bulk-import-backend/api-docs/README.md
@@ -35,6 +35,8 @@ All URIs are relative to _http://localhost:7007/api/bulk-import_
 - [OrganizationList](./Models/OrganizationList.md)
 - [Repository](./Models/Repository.md)
 - [RepositoryList](./Models/RepositoryList.md)
+- [Source](./Models/Source.md)
+- [SourceImport](./Models/SourceImport.md)
 - [findAllImports_200_response](./Models/findAllImports_200_response.md)
 - [findAllImports_500_response](./Models/findAllImports_500_response.md)
 - [ping_200_response](./Models/ping_200_response.md)

diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/catalog/catalogHttpClient.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/catalog/catalogHttpClient.ts
@@ -27,6 +27,7 @@ import fetch from 'node-fetch';
 
 import { getTokenForPlugin, logErrorIfNeeded } from '../helpers';
 import { filterLocations, getCatalogUrl } from './catalogUtils';
+import { CatalogLocation } from './types';
 
 export class CatalogHttpClient {
   private readonly logger: LoggerService;
@@ -78,18 +79,25 @@ export class CatalogHttpClient {
     search?: string,
     pageNumber?: number,
     pageSize?: number,
-  ): Promise<{ targetUrls: string[]; totalCount?: number }> {
+  ): Promise<{
+    uniqueCatalogUrlLocations: Map<string, CatalogLocation>;
+    totalCount?: number;
+  }> {
+    // byId order: config, locations, other
     const byId = await this.listCatalogUrlLocationsById(
       search,
       pageNumber,
       pageSize,
     );
-    const result = new Set<string>();
+    const result = new Map<string, CatalogLocation>();
+
     for (const l of byId.locations) {
-      result.add(l.target);
+      if (!result.has(l.target)) {
+        result.set(l.target, l);
+      }
     }
     return {
-      targetUrls: Array.from(result.values()),
+      uniqueCatalogUrlLocations: result,
       totalCount: byId.totalCount,
     };
   }
@@ -99,7 +107,7 @@ export class CatalogHttpClient {
     pageNumber?: number,
     pageSize?: number,
   ): Promise<{
-    locations: { id?: string; target: string }[];
+    locations: CatalogLocation[];
     totalCount?: number;
   }> {
     const result = await Promise.all([
@@ -121,7 +129,7 @@ export class CatalogHttpClient {
   async listCatalogUrlLocationsByIdFromLocationsEndpoint(
     search?: string,
   ): Promise<{
-    locations: { id?: string; target: string }[];
+    locations: CatalogLocation[];
     totalCount?: number;
   }> {
     const url = `${await this.discovery.getBaseUrl('catalog')}/locations`;
@@ -136,7 +144,7 @@ export class CatalogHttpClient {
       method: 'GET',
     });
     const locations = (await response.json()) as {
-      data: { id: string; target: string; type: string };
+      data: CatalogLocation;
     }[];
     if (!Array.isArray(locations)) {
       return { locations: [] };
@@ -149,14 +157,15 @@ export class CatalogHttpClient {
         return {
           id: location.data?.id,
           target: location.data.target,
-        };
+          source: 'location',
+        } as CatalogLocation;
       });
     const filtered = filterLocations(res, search);
     return { locations: filtered, totalCount: filtered.length };
   }
 
   listCatalogUrlLocationsFromConfig(search?: string): {
-    locations: { id?: string; target: string }[];
+    locations: CatalogLocation[];
     totalCount?: number;
   } {
     const locationConfigs =
@@ -169,9 +178,11 @@ export class CatalogHttpClient {
       )
       .map(location => {
         const target = location.getString('target');
+        const source = 'config';
         return {
           id: `app-config-location--${target}`,
           target,
+          source,
         };
       });
     const filtered = filterLocations(res, search);
@@ -183,7 +194,7 @@ export class CatalogHttpClient {
     _pageNumber?: number,
     _pageSize?: number,
   ): Promise<{
-    locations: { id?: string; target: string }[];
+    locations: CatalogLocation[];
     totalCount?: number;
   }> {
     const result = await this.catalogApi.getEntities(
@@ -210,7 +221,8 @@ export class CatalogHttpClient {
         return {
           id: location.metadata.uid,
           target: location.spec.target!,
-        };
+          source: 'other',
+        } as CatalogLocation;
       });
     const filtered = filterLocations(res, search);
     return { locations: filtered, totalCount: filtered.length };

diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/catalog/catalogUtils.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/catalog/catalogUtils.ts
@@ -18,6 +18,8 @@ import type { Config } from '@backstage/config';
 
 import gitUrlParse from 'git-url-parse';
 
+import { CatalogLocation } from './types';
+
 export function getCatalogFilename(config: Config): string {
   return (
     config.getOptionalString('catalog.import.entityFilename') ??
@@ -41,7 +43,7 @@ export function getCatalogUrl(
 }
 
 export function filterLocations(
-  res: { id: string | undefined; target: string }[],
+  res: CatalogLocation[],
   search: string | undefined,
 ) {
   return search

diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/catalog/types.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/catalog/types.ts
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2024 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export type CatalogLocation = {
+  id?: string;
+  target: string;
+  source: SourceType;
+};
+
+export type SourceType = 'location' | 'config' | 'other';
diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapi.d.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapi.d.ts
@@ -89,7 +89,7 @@ declare namespace Components {
      * Import Job List
      */
     export interface ImportJobListV2 {
-      imports?: /* Import Job */ Import[];
+      imports?: /* Import Job with source it originates from */ SourceImport[];
       errors?: string[];
       totalCount?: number;
       page?: number;
@@ -227,6 +227,53 @@ declare namespace Components {
       pagePerIntegration?: number;
       sizePerIntegration?: number;
     }
+    /**
+     * Import Source
+     */
+    export type Source = 'config' | 'location' | 'other' | null;
+    /**
+     * Import Job with source it originates from
+     */
+    export interface SourceImport {
+      id?: string;
+      status?: /* Import Job status */ ImportStatus;
+      /**
+       * Specified entity name in the catalog. Filled only in response for dry-run import requests.
+       */
+      catalogEntityName?: string;
+      lastUpdate?: string; // date-time
+      errors?: string[];
+      approvalTool?: ApprovalTool;
+      repository?: /* Repository */ Repository;
+      /**
+       * GitHub details. Applicable if approvalTool is git.
+       */
+      github?: {
+        pullRequest?: {
+          /**
+           * URL of the Pull Request
+           */
+          url?: string;
+          /**
+           * Pull Request number
+           */
+          number?: number;
+          /**
+           * title of the Pull Request
+           */
+          title?: string;
+          /**
+           * body of the Pull Request
+           */
+          body?: string;
+          /**
+           * content of the catalog-info.yaml as fetched from the Pull Request.
+           */
+          catalogInfoContent?: string;
+        };
+      };
+      source?: /* Import Source */ Source;
+    }
   }
 }
 declare namespace Paths {
@@ -277,9 +324,8 @@ declare namespace Paths {
       search?: Parameters.Search;
     }
     namespace Responses {
-      export type $200 =
-        /* Import Job */
-        | Components.Schemas.Import[]
+      export type $200 = /* Import Job with source it originates from */
+        | Components.Schemas.SourceImport[]
         | /* Import Job List */ Components.Schemas.ImportJobListV2;
       export type $500 =
         | string
@@ -540,3 +586,5 @@ export type Organization = Components.Schemas.Organization;
 export type OrganizationList = Components.Schemas.OrganizationList;
 export type Repository = Components.Schemas.Repository;
 export type RepositoryList = Components.Schemas.RepositoryList;
+export type Source = Components.Schemas.Source;
+export type SourceImport = Components.Schemas.SourceImport;
diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapidocument.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapidocument.ts
@@ -319,7 +319,7 @@ const OPENAPI = `
                     {
                       "type": "array",
                       "items": {
-                        "$ref": "#/components/schemas/Import"
+                        "$ref": "#/components/schemas/SourceImport"
                       }
                     },
                     {
@@ -750,7 +750,7 @@ const OPENAPI = `
           "imports": {
             "type": "array",
             "items": {
-              "$ref": "#/components/schemas/Import"
+              "$ref": "#/components/schemas/SourceImport"
             }
           },
           "errors": {
@@ -833,6 +833,33 @@ const OPENAPI = `
           }
         }
       },
+      "SourceImport": {
+        "title": "Import Job with source it originates from",
+        "allOf": [
+          {
+            "$ref": "#/components/schemas/Import"
+          },
+          {
+            "type": "object",
+            "properties": {
+              "source": {
+                "$ref": "#/components/schemas/Source"
+              }
+            }
+          }
+        ]
+      },
+      "Source": {
+        "type": "string",
+        "nullable": true,
+        "description": "Import Source",
+        "enum": [
+          "config",
+          "location",
+          "other",
+          null
+        ]
+      },
       "ImportRequest": {
         "title": "Import Job request",
         "type": "object",