feat(bulk-import): add import source

workspaces/bulk-import/.changeset/brown-rabbits-turn.md

@@ -0,0 +1,5 @@
+---
+'@red-hat-developer-hub/backstage-plugin-bulk-import-backend': minor
+---
+
+Introduced a new response key 'source' in the GET /imports endpoint to indicate from which source the import originated from ('config', 'location', 'integration'). In case of duplicates, it returns first source it finds in order 'config', 'location', 'integration'.

workspaces/bulk-import/plugins/bulk-import-backend/api-docs/.openapi-generator/FILES

@@ -17,6 +17,8 @@ Models/Organization.md
 Models/OrganizationList.md
 Models/Repository.md
 Models/RepositoryList.md
+Models/Source.md
+Models/SourceImport.md
 Models/findAllImports_200_response.md
 Models/findAllImports_500_response.md
 Models/ping_200_response.md

workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Models/ImportJobListV2.md

@@ -2,12 +2,12 @@
 
 ## Properties
 
-| Name           | Type                  | Description | Notes                        |
-| -------------- | --------------------- | ----------- | ---------------------------- |
-| **imports**    | [**List**](Import.md) |             | [optional] [default to null] |
-| **errors**     | **List**              |             | [optional] [default to null] |
-| **totalCount** | **Integer**           |             | [optional] [default to null] |
-| **page**       | **Integer**           |             | [optional] [default to null] |
-| **size**       | **Integer**           |             | [optional] [default to null] |
+| Name           | Type                        | Description | Notes                        |
+| -------------- | --------------------------- | ----------- | ---------------------------- |
+| **imports**    | [**List**](SourceImport.md) |             | [optional] [default to null] |
+| **errors**     | **List**                    |             | [optional] [default to null] |
+| **totalCount** | **Integer**                 |             | [optional] [default to null] |
+| **page**       | **Integer**                 |             | [optional] [default to null] |
+| **size**       | **Integer**                 |             | [optional] [default to null] |
 
 [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)

workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Models/Source.md

@@ -0,0 +1,8 @@
+# Source
+
+## Properties
+
+| Name | Type | Description | Notes |
+| ---- | ---- | ----------- | ----- |
+
+[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)

workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Models/SourceImport.md

@@ -0,0 +1,17 @@
+# SourceImport
+
+## Properties
+
+| Name                  | Type                                  | Description                                                                                | Notes                        |
+| --------------------- | ------------------------------------- | ------------------------------------------------------------------------------------------ | ---------------------------- |
+| **id**                | **String**                            |                                                                                            | [optional] [default to null] |
+| **status**            | [**ImportStatus**](ImportStatus.md)   |                                                                                            | [optional] [default to null] |
+| **catalogEntityName** | **String**                            | Specified entity name in the catalog. Filled only in response for dry-run import requests. | [optional] [default to null] |
+| **lastUpdate**        | **Date**                              |                                                                                            | [optional] [default to null] |
+| **errors**            | **List**                              |                                                                                            | [optional] [default to null] |
+| **approvalTool**      | [**ApprovalTool**](ApprovalTool.md)   |                                                                                            | [optional] [default to null] |
+| **repository**        | [**Repository**](Repository.md)       |                                                                                            | [optional] [default to null] |
+| **github**            | [**Import_github**](Import_github.md) |                                                                                            | [optional] [default to null] |
+| **source**            | [**Source**](Source.md)               |                                                                                            | [optional] [default to null] |
+
+[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)

workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Models/findAllImports_200_response.md

@@ -2,12 +2,12 @@
 
 ## Properties
 
-| Name           | Type                  | Description | Notes                        |
-| -------------- | --------------------- | ----------- | ---------------------------- |
-| **imports**    | [**List**](Import.md) |             | [optional] [default to null] |
-| **errors**     | **List**              |             | [optional] [default to null] |
-| **totalCount** | **Integer**           |             | [optional] [default to null] |
-| **page**       | **Integer**           |             | [optional] [default to null] |
-| **size**       | **Integer**           |             | [optional] [default to null] |
+| Name           | Type                        | Description | Notes                        |
+| -------------- | --------------------------- | ----------- | ---------------------------- |
+| **imports**    | [**List**](SourceImport.md) |             | [optional] [default to null] |
+| **errors**     | **List**                    |             | [optional] [default to null] |
+| **totalCount** | **Integer**                 |             | [optional] [default to null] |
+| **page**       | **Integer**                 |             | [optional] [default to null] |
+| **size**       | **Integer**                 |             | [optional] [default to null] |
 
 [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)

workspaces/bulk-import/plugins/bulk-import-backend/api-docs/Models/findAllImports_500_response.md

@@ -2,12 +2,12 @@
 
 ## Properties
 
-| Name           | Type                  | Description | Notes                        |
-| -------------- | --------------------- | ----------- | ---------------------------- |
-| **imports**    | [**List**](Import.md) |             | [optional] [default to null] |
-| **errors**     | **List**              |             | [optional] [default to null] |
-| **totalCount** | **Integer**           |             | [optional] [default to null] |
-| **page**       | **Integer**           |             | [optional] [default to null] |
-| **size**       | **Integer**           |             | [optional] [default to null] |
+| Name           | Type                        | Description | Notes                        |
+| -------------- | --------------------------- | ----------- | ---------------------------- |
+| **imports**    | [**List**](SourceImport.md) |             | [optional] [default to null] |
+| **errors**     | **List**                    |             | [optional] [default to null] |
+| **totalCount** | **Integer**                 |             | [optional] [default to null] |
+| **page**       | **Integer**                 |             | [optional] [default to null] |
+| **size**       | **Integer**                 |             | [optional] [default to null] |
 
 [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)

workspaces/bulk-import/plugins/bulk-import-backend/api-docs/README.md

@@ -35,6 +35,8 @@ All URIs are relative to _http://localhost:7007/api/bulk-import_
 - [OrganizationList](./Models/OrganizationList.md)
 - [Repository](./Models/Repository.md)
 - [RepositoryList](./Models/RepositoryList.md)
+- [Source](./Models/Source.md)
+- [SourceImport](./Models/SourceImport.md)
 - [findAllImports_200_response](./Models/findAllImports_200_response.md)
 - [findAllImports_500_response](./Models/findAllImports_500_response.md)
 - [ping_200_response](./Models/ping_200_response.md)

workspaces/bulk-import/plugins/bulk-import-backend/src/catalog/catalogHttpClient.ts

@@ -27,6 +27,7 @@ import fetch from 'node-fetch';
 
 import { getTokenForPlugin, logErrorIfNeeded } from '../helpers';
 import { filterLocations, getCatalogUrl } from './catalogUtils';
+import { CatalogLocation } from './types';
 
 export class CatalogHttpClient {
   private readonly logger: LoggerService;
@@ -78,18 +79,25 @@ export class CatalogHttpClient {
     search?: string,
     pageNumber?: number,
     pageSize?: number,
-  ): Promise<{ targetUrls: string[]; totalCount?: number }> {
+  ): Promise<{
+    uniqueCatalogUrlLocations: Map<string, CatalogLocation>;
+    totalCount?: number;
+  }> {
+    // byId order: config, locations, other
     const byId = await this.listCatalogUrlLocationsById(
       search,
       pageNumber,
       pageSize,
     );
-    const result = new Set<string>();
+    const result = new Map<string, CatalogLocation>();
+
     for (const l of byId.locations) {
-      result.add(l.target);
+      if (!result.has(l.target)) {
+        result.set(l.target, l);
+      }
     }
     return {
-      targetUrls: Array.from(result.values()),
+      uniqueCatalogUrlLocations: result,
       totalCount: byId.totalCount,
     };
   }
@@ -99,7 +107,7 @@ export class CatalogHttpClient {
     pageNumber?: number,
     pageSize?: number,
   ): Promise<{
-    locations: { id?: string; target: string }[];
+    locations: CatalogLocation[];
     totalCount?: number;
   }> {
     const result = await Promise.all([
@@ -121,7 +129,7 @@ export class CatalogHttpClient {
   async listCatalogUrlLocationsByIdFromLocationsEndpoint(
     search?: string,
   ): Promise<{
-    locations: { id?: string; target: string }[];
+    locations: CatalogLocation[];
     totalCount?: number;
   }> {
     const url = `${await this.discovery.getBaseUrl('catalog')}/locations`;
@@ -149,14 +157,15 @@ export class CatalogHttpClient {
         return {
           id: location.data?.id,
           target: location.data.target,
-        };
+          source: 'location',
+        } as CatalogLocation;
       });
     const filtered = filterLocations(res, search);
     return { locations: filtered, totalCount: filtered.length };
   }
 
   listCatalogUrlLocationsFromConfig(search?: string): {
-    locations: { id?: string; target: string }[];
+    locations: CatalogLocation[];
     totalCount?: number;
   } {
     const locationConfigs =
@@ -172,7 +181,8 @@ export class CatalogHttpClient {
         return {
           id: `app-config-location--${target}`,
           target,
-        };
+          source: 'config',
+        } as CatalogLocation;
       });
     const filtered = filterLocations(res, search);
     return { locations: filtered, totalCount: filtered.length };
@@ -183,7 +193,7 @@ export class CatalogHttpClient {
     _pageNumber?: number,
     _pageSize?: number,
   ): Promise<{
-    locations: { id?: string; target: string }[];
+    locations: CatalogLocation[];
     totalCount?: number;
   }> {
     const result = await this.catalogApi.getEntities(
@@ -210,7 +220,8 @@ export class CatalogHttpClient {
         return {
           id: location.metadata.uid,
           target: location.spec.target!,
-        };
+          source: 'integration',
+        } as CatalogLocation;
       });
     const filtered = filterLocations(res, search);
     return { locations: filtered, totalCount: filtered.length };

workspaces/bulk-import/plugins/bulk-import-backend/src/catalog/catalogUtils.ts

@@ -18,6 +18,8 @@ import type { Config } from '@backstage/config';
 
 import gitUrlParse from 'git-url-parse';
 
+import { CatalogLocation } from './types';
+
 export function getCatalogFilename(config: Config): string {
   return (
     config.getOptionalString('catalog.import.entityFilename') ??
@@ -41,7 +43,7 @@ export function getCatalogUrl(
 }
 
 export function filterLocations(
-  res: { id: string | undefined; target: string }[],
+  res: CatalogLocation[],
   search: string | undefined,
 ) {
   return search

workspaces/bulk-import/plugins/bulk-import-backend/src/catalog/types.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright 2024 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Source } from '../generated/openapi';
+
+export type CatalogLocation = {
+  id?: string;
+  target: string;
+  source: Source;
+};

workspaces/bulk-import/plugins/bulk-import-backend/src/generated/openapi.d.ts

@@ -89,7 +89,7 @@ declare namespace Components {
      * Import Job List
      */
     export interface ImportJobListV2 {
-      imports?: /* Import Job */ Import[];
+      imports?: /* Import Job with source it originates from */ SourceImport[];
       errors?: string[];
       totalCount?: number;
       page?: number;
@@ -227,6 +227,66 @@ declare namespace Components {
       pagePerIntegration?: number;
       sizePerIntegration?: number;
     }
+    /**
+     * Import Source:
+     *   * 'config' - Import from static catalog location configuration in 'app-config'
+     *   * 'location' - Import of user registered entities using locations endpoint
+     *   * 'integration' - Import using a GitHub integration
+     *   * null - Import source is unknown
+     *
+     */
+    export type Source = 'config' | 'location' | 'integration' | null;
+    /**
+     * Import Job with source it originates from
+     */
+    export interface SourceImport {
+      id?: string;
+      status?: /* Import Job status */ ImportStatus;
+      /**
+       * Specified entity name in the catalog. Filled only in response for dry-run import requests.
+       */
+      catalogEntityName?: string;
+      lastUpdate?: string; // date-time
+      errors?: string[];
+      approvalTool?: ApprovalTool;
+      repository?: /* Repository */ Repository;
+      /**
+       * GitHub details. Applicable if approvalTool is git.
+       */
+      github?: {
+        pullRequest?: {
+          /**
+           * URL of the Pull Request
+           */
+          url?: string;
+          /**
+           * Pull Request number
+           */
+          number?: number;
+          /**
+           * title of the Pull Request
+           */
+          title?: string;
+          /**
+           * body of the Pull Request
+           */
+          body?: string;
+          /**
+           * content of the catalog-info.yaml as fetched from the Pull Request.
+           */
+          catalogInfoContent?: string;
+        };
+      };
+      source?: /**
+       * Import Source:
+       *   * 'config' - Import from static catalog location configuration in 'app-config'
+       *   * 'location' - Import of user registered entities using locations endpoint
+       *   * 'integration' - Import using a GitHub integration
+       *   * null - Import source is unknown
+       *
+       */
+      Source;
+    }
   }
 }
 declare namespace Paths {
@@ -278,8 +338,8 @@ declare namespace Paths {
     }
     namespace Responses {
       export type $200 =
-        /* Import Job */
-        | Components.Schemas.Import[]
+        /* Import Job with source it originates from */
+        | Components.Schemas.SourceImport[]
         | /* Import Job List */ Components.Schemas.ImportJobListV2;
       export type $500 =
         | string
@@ -540,3 +600,5 @@ export type Organization = Components.Schemas.Organization;
 export type OrganizationList = Components.Schemas.OrganizationList;
 export type Repository = Components.Schemas.Repository;
 export type RepositoryList = Components.Schemas.RepositoryList;
+export type Source = Components.Schemas.Source;
+export type SourceImport = Components.Schemas.SourceImport;