From 8a1e9155b885a8a8639e4b114f5a51516c3e4ce1 Mon Sep 17 00:00:00 2001 From: Ani Argjiri Date: Wed, 18 Sep 2024 10:28:31 +0200 Subject: [PATCH 01/23] chore: release branch [ci skip] (#116) --- CHANGELOG.md | 10 ++++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ac8988..5dd265b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [3.2.0](https://github.com/linode/apl-tasks/compare/v3.1.0...v3.2.0) (2024-09-18) + + +### Bug Fixes + +* check if setup was a success ([#113](https://github.com/linode/apl-tasks/issues/113)) ([d359ef5](https://github.com/linode/apl-tasks/commit/d359ef56d770205c4230435a374713393a5934b3)) +* group mapping for gitea oidc ([#111](https://github.com/linode/apl-tasks/issues/111)) ([6efe076](https://github.com/linode/apl-tasks/commit/6efe07673e42e3d46cd5b0e673e264c540ea8c75)) +* keycloakstyling ([#114](https://github.com/linode/apl-tasks/issues/114)) ([c355214](https://github.com/linode/apl-tasks/commit/c3552144b94a6e669f66207aad4982f59b2350b8)) +* set keycloak theme ([#115](https://github.com/linode/apl-tasks/issues/115)) ([8e4fc5b](https://github.com/linode/apl-tasks/commit/8e4fc5b9fa93cdf60dbfa75ac7ab840e1002ad84)) + ## [3.1.0](https://github.com/linode/apl-tasks/compare/v3.0.0...v3.1.0) (2024-09-09) diff --git a/package-lock.json b/package-lock.json index 70b408f..a45e7d7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "otomi-tasks", - "version": "3.1.0", + "version": "3.2.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "otomi-tasks", - "version": "3.1.0", + "version": "3.2.0", "license": "Apache-2.0", "dependencies": { "@apidevtools/json-schema-ref-parser": "9.0.6", diff --git a/package.json b/package.json index 90abc0f..b538324 100644 --- a/package.json +++ b/package.json @@ -146,5 +146,5 @@ "tag": true } }, - "version": "3.1.0" + "version": "3.2.0" } From 1647ce6a521f9239c56e5051d3cc9b1b8d7035e8 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 18 Sep 2024 15:04:04 +0200 Subject: [PATCH 02/23] feat: update readme and launch&tasks.json files for keycloak operator --- .vscode/launch.json | 6 ++++-- .vscode/tasks.json | 11 +++++++++++ README.md | 2 +- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 2800ec1..0e8d6ba 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -146,8 +146,10 @@ "console": "integratedTerminal", "envFile": "${workspaceFolder}/.env", "env": { - "NODE_EXTRA_CA_CERTS": "${workspaceFolder}/.env.ca" - } + "NODE_EXTRA_CA_CERTS": "${workspaceFolder}/.env.ca", + "KUBECONFIG": "/path/to/your/kubeconfig.yaml", + }, + "preLaunchTask": "port-forward-keycloak" }, { "type": "node", diff --git a/.vscode/tasks.json b/.vscode/tasks.json index a53e416..7ea8100 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -22,6 +22,17 @@ "reveal": "always", "panel": "new" } + }, + { + "label": "port-forward-keycloak", + "type": "shell", + "command": "export KUBECONFIG=/path/to/your/kubeconfig.yaml && kubectl -n keycloak port-forward svc/keycloak-operator 8084:80", + "problemMatcher": [], + "isBackground": true, + "presentation": { + "reveal": "always", + "panel": "new" + } } ] } diff --git a/README.md b/README.md index e29060a..ddca3d4 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Then start a proxy to the api you wish to target: - gitea: `k -n gitea port-forward svc/gitea-http 8082:3000 &` - harbor: `k -n harbor port-forward svc/harbor-core 8083:80 &` -- keycloak: `k -n keycloak port-forward svc/keycloak-http 8084:80 &` +- keycloak: `k -n keycloak port-forward svc/keycloak-operator 8084:80 &` Or start them all with `bin/start-proxies.sh` From b26edab8461d4e2ef2915152c91a89b1ef62b94c Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 18 Sep 2024 15:23:03 +0200 Subject: [PATCH 03/23] feat: rm unnecessary lines from launch.json and package.json --- .cspell.json | 1 + .vscode/launch.json | 28 +--------------------------- package.json | 4 ---- 3 files changed, 2 insertions(+), 31 deletions(-) diff --git a/.cspell.json b/.cspell.json index 92a14a7..fcdd29d 100644 --- a/.cspell.json +++ b/.cspell.json @@ -10,6 +10,7 @@ "camelcase", "creds", "gitea", + "keycloak", "kubernetes", "oidc", "openid", diff --git a/.vscode/launch.json b/.vscode/launch.json index 0e8d6ba..4b56314 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -113,33 +113,7 @@ { "type": "node", "request": "launch", - "name": "Debug Harbor task", - "runtimeExecutable": "npm", - "runtimeArgs": ["run-script", "tasks:harbor-dev"], - "cwd": "${workspaceRoot}", - "console": "integratedTerminal", - "envFile": "${workspaceFolder}/.env" - // "env": { - // "NODE_EXTRA_CA_CERTS": "${workspaceFolder}/.env.ca" - // } - }, - { - "type": "node", - "request": "launch", - "name": "Debug Keycloak task", - "runtimeExecutable": "npm", - "runtimeArgs": ["run-script", "tasks:keycloak-dev"], - "cwd": "${workspaceRoot}", - "console": "integratedTerminal", - "envFile": "${workspaceFolder}/.env", - "env": { - "NODE_EXTRA_CA_CERTS": "${workspaceFolder}/.env.ca" - } - }, - { - "type": "node", - "request": "launch", - "name": "Debug Keycloak operator", + "name": "Debug keycloak operator", "runtimeExecutable": "npm", "runtimeArgs": ["run-script", "operator:keycloak-dev"], "cwd": "${workspaceRoot}", diff --git a/package.json b/package.json index b538324..168af89 100644 --- a/package.json +++ b/package.json @@ -122,10 +122,6 @@ "tasks:copy-certs-dev": "ts-node-dev ./src/tasks/otomi/copy-certs.ts", "tasks:copy-certs": "node dist/tasks/otomi/copy-certs.js", "tasks:copy-certs-argo": "node dist/tasks/otomi/copy-certs-argo.js", - "tasks:harbor-dev": "NODE_TLS_REJECT_UNAUTHORIZED=0 ts-node-dev ./src/tasks/harbor/harbor.ts", - "tasks:harbor": "node dist/tasks/harbor/harbor.js", - "tasks:keycloak-dev": "NODE_TLS_REJECT_UNAUTHORIZED=0 ts-node-dev ./src/tasks/keycloak/keycloak.ts", - "tasks:keycloak": "node dist/tasks/keycloak/keycloak.js", "tasks:keycloak-users-dev": "NODE_TLS_REJECT_UNAUTHORIZED=0 ts-node-dev ./src/tasks/keycloak/users.ts", "tasks:otomi-chart-dev": "NODE_TLS_REJECT_UNAUTHORIZED=0 ts-node-dev ./src/tasks/otomi/otomi-chart.ts", "tasks:otomi-chart": "node dist/tasks/otomi/otomi-chart.js", From b7f14f74c16e9ec032d179c8632227d7a9dad777 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 19 Sep 2024 22:16:31 +0200 Subject: [PATCH 04/23] feat: add createTeamUser in the keycloak operator --- src/operator/keycloak.ts | 38 +++++++++++++++++++++++++++-- src/tasks/keycloak/config.ts | 25 +++++++++++++++++++ src/tasks/keycloak/realm-factory.ts | 14 +++++++++++ 3 files changed, 75 insertions(+), 2 deletions(-) diff --git a/src/operator/keycloak.ts b/src/operator/keycloak.ts index 8ec00f5..70f1235 100644 --- a/src/operator/keycloak.ts +++ b/src/operator/keycloak.ts @@ -36,9 +36,10 @@ import { createIdProvider, createLoginThemeConfig, createRealm, + createTeamUser, mapTeamsToRoles, } from '../tasks/keycloak/realm-factory' -import { doApiCall, waitTillAvailable } from '../utils' +import { doApiCall } from '../utils' import { cleanEnv, KEYCLOAK_TOKEN_OFFLINE_MAX_TTL_ENABLED, @@ -96,6 +97,7 @@ const env = { REDIRECT_URIS: [] as string[], TEAM_IDS: [] as string[], WAIT_OPTIONS: {}, + USERS: [], } const kc = new KubeConfig() @@ -168,6 +170,7 @@ async function runKeycloakUpdater(key: string) { await keycloakConfigMapChanges().then(async () => { await runKeycloakUpdater('addTeam') }) + if (env.USERS.length > 0) await manageUsers(env.USERS) break } catch (error) { console.debug('Error could not update configMap', error) @@ -206,6 +209,7 @@ export default class MyOperator extends Operator { if (data!.IDP_CLIENT_ID) env.IDP_CLIENT_ID = Buffer.from(data!.IDP_CLIENT_ID, 'base64').toString() if (data!.IDP_CLIENT_SECRET) env.IDP_CLIENT_SECRET = Buffer.from(data!.IDP_CLIENT_SECRET, 'base64').toString() + env.USERS = JSON.parse(Buffer.from(data!.USERS, 'base64').toString()) await runKeycloakUpdater('updateConfig').then(() => { console.log('Updated Config') }) @@ -358,7 +362,7 @@ async function keycloakTeamDeleted() { } async function createKeycloakConnection(): Promise { - await waitTillAvailable(env.KEYCLOAK_HOSTNAME_URL, undefined, env.WAIT_OPTIONS) + // await waitTillAvailable(env.KEYCLOAK_HOSTNAME_URL, undefined, env.WAIT_OPTIONS) const keycloakAddress = env.KEYCLOAK_HOSTNAME_URL const basePath = `${keycloakAddress}/admin/realms` let token: TokenSet @@ -655,6 +659,7 @@ async function internalIdp(api: KeycloakApi, connection: KeycloakConnection) { api.users.realmUsersGet(keycloakRealm, false, userConf.email), )) as UserRepresentation[] const existingUser: UserRepresentation = existingUsersByAdminEmail?.[0] + try { if (existingUser) { await doApiCall(errors, `Updating user ${env.KEYCLOAK_ADMIN}`, async () => @@ -700,3 +705,32 @@ async function manageGroups(connection: KeycloakConnection) { console.error('Error in manageGroups: ', error) } } + +async function createUser(api: any, user: any) { + const { name: username, email, firstName, lastName, teamId } = user + const userConf = createTeamUser(username, email, firstName, lastName, teamId) + const existingUsersByAdminEmail = (await doApiCall([], `Getting users`, () => + api.users.realmUsersGet(keycloakRealm, false, `${email}`), + )) as UserRepresentation[] + const existingUser: UserRepresentation = existingUsersByAdminEmail?.[0] + + try { + if (existingUser) { + console.debug(`User with email ${email} already exists`) + // await doApiCall(errors, `Updating user ${user}`, async () => + // api.users.realmUsersIdPut(keycloakRealm, existingUser.id as string, userConf), + // ) + } else { + await doApiCall(errors, `Creating user ${username}`, () => api.users.realmUsersPost(keycloakRealm, userConf)) + } + } catch (error) { + console.error('Error in internalIDP: ', error) + } +} + +async function manageUsers(users: any[]) { + const connection = await createKeycloakConnection() + const api = setupKeycloakApi(connection) + // Create/Update users in realm 'otomi' + await Promise.all(users.map((user) => createUser(api, user))) +} diff --git a/src/tasks/keycloak/config.ts b/src/tasks/keycloak/config.ts index 6b8ae3b..2f0bfd2 100644 --- a/src/tasks/keycloak/config.ts +++ b/src/tasks/keycloak/config.ts @@ -70,6 +70,31 @@ export const adminUserCfgTpl = (username: string, password: string): Record => ({ + username, + enabled: true, + email, + emailVerified: true, + firstName, + lastName, + realmRoles: ['team'], + groups: [`team-${teamId}`], + credentials: [ + { + type: 'password', + value: `${username}@APL`, + temporary: true, + }, + ], + requiredActions: [], +}) + export const realmCfgTpl = (realm: string): Record => ({ id: realm, realm, diff --git a/src/tasks/keycloak/realm-factory.ts b/src/tasks/keycloak/realm-factory.ts index c86472d..add0b65 100644 --- a/src/tasks/keycloak/realm-factory.ts +++ b/src/tasks/keycloak/realm-factory.ts @@ -24,6 +24,7 @@ import { protocolMappersList, realmCfgTpl, roleTpl, + teamUserCfgTpl, } from './config' export function createClient(redirectUris: string[], webOrigins: string, secret: string): ClientRepresentation { @@ -99,6 +100,19 @@ export function createAdminUser(username: string, password: string): UserReprese const userRepresentation = defaultsDeep(new UserRepresentation(), adminUserCfgTpl(username, password)) return userRepresentation } +export function createTeamUser( + username: string, + email: string, + firstName: string, + lastName: string, + teamId: string, +): UserRepresentation { + const userRepresentation = defaultsDeep( + new UserRepresentation(), + teamUserCfgTpl(username, email, firstName, lastName, teamId), + ) + return userRepresentation +} export function createRealm(realm: string): RealmRepresentation { const realmRepresentation = defaultsDeep(new RealmRepresentation(), realmCfgTpl(realm)) From b9d112e4581ce2e192ea61e20a166b9fa28b8a66 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 25 Sep 2024 10:12:38 +0200 Subject: [PATCH 05/23] feat: update groups --- src/operator/keycloak.ts | 28 ++++++++++-------- src/tasks/keycloak/config.ts | 6 ++-- src/tasks/keycloak/realm-factory.ts | 44 +++++++++++++++++++++-------- src/validators.ts | 6 ++-- 4 files changed, 56 insertions(+), 28 deletions(-) diff --git a/src/operator/keycloak.ts b/src/operator/keycloak.ts index 624df4c..b07982d 100644 --- a/src/operator/keycloak.ts +++ b/src/operator/keycloak.ts @@ -80,8 +80,9 @@ const env = { IDP_OIDC_URL: '', IDP_CLIENT_ID: '', IDP_CLIENT_SECRET: '', - IDP_GROUP_OTOMI_ADMIN: '', IDP_GROUP_TEAM_ADMIN: '', + IDP_GROUP_ALL_TEAMS_ADMIN: '', + IDP_GROUP_PLATFORM_ADMIN: '', IDP_GROUP_MAPPINGS_TEAMS: {} || undefined, IDP_SUB_CLAIM_MAPPER: '', IDP_USERNAME_CLAIM_MAPPER: '', @@ -117,8 +118,8 @@ async function runKeycloakUpdater(key: string) { !env.IDP_OIDC_URL || !env.IDP_CLIENT_ID || !env.IDP_CLIENT_SECRET || - !env.IDP_GROUP_OTOMI_ADMIN || - !env.IDP_GROUP_TEAM_ADMIN || + !env.IDP_GROUP_PLATFORM_ADMIN || + !env.IDP_GROUP_ALL_TEAMS_ADMIN || !env.IDP_GROUP_MAPPINGS_TEAMS || !env.IDP_SUB_CLAIM_MAPPER || !env.IDP_USERNAME_CLAIM_MAPPER @@ -254,8 +255,9 @@ export default class MyOperator extends Operator { if (env.FEAT_EXTERNAL_IDP === 'true') { env.IDP_ALIAS = data!.IDP_ALIAS env.IDP_OIDC_URL = data!.IDP_OIDC_URL - env.IDP_GROUP_OTOMI_ADMIN = data!.IDP_GROUP_OTOMI_ADMIN - env.IDP_GROUP_TEAM_ADMIN = data!.IDP_GROUP_TEAM_ADMIN + env.IDP_GROUP_PLATFORM_ADMIN = data!.IDP_GROUP_PLATFORM_ADMIN + env.IDP_GROUP_ALL_TEAMS_ADMIN = data!.IDP_GROUP_TEAM_ADMIN + env.IDP_GROUP_TEAM_ADMIN = 'xxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx' env.IDP_GROUP_MAPPINGS_TEAMS = Object.keys(data!.IDP_GROUP_MAPPINGS_TEAMS).length === 0 ? JSON.parse(data!.IDP_GROUP_MAPPINGS_TEAMS) @@ -450,7 +452,8 @@ async function keycloakRealmProviderConfigurer(api: KeycloakApi) { env.TEAM_IDS, env.IDP_GROUP_MAPPINGS_TEAMS, env.IDP_GROUP_TEAM_ADMIN, - env.IDP_GROUP_OTOMI_ADMIN, + env.IDP_GROUP_ALL_TEAMS_ADMIN, + env.IDP_GROUP_PLATFORM_ADMIN, env.KEYCLOAK_REALM, ) const existingRealmRoles = ((await doApiCall(errors, `Getting all roles from realm ${keycloakRealm}`, async () => @@ -520,7 +523,8 @@ async function externalIDP(api: KeycloakApi) { const idpMappers = createIdpMappers( env.IDP_ALIAS, env.IDP_GROUP_MAPPINGS_TEAMS, - env.IDP_GROUP_OTOMI_ADMIN, + env.IDP_GROUP_PLATFORM_ADMIN, + env.IDP_GROUP_ALL_TEAMS_ADMIN, env.IDP_GROUP_TEAM_ADMIN, env.IDP_USERNAME_CLAIM_MAPPER, env.IDP_SUB_CLAIM_MAPPER, @@ -612,7 +616,7 @@ async function internalIdp(api: KeycloakApi, connection: KeycloakConnection) { // set realm roles const roles: Array = [] const existingRole = updatedExistingRealmRoles.find( - (el) => el.name === (groupName === 'otomi-admin' ? 'admin' : groupName), + (el) => el.name === (groupName === 'otomi-admin' ? 'platform-admin' : groupName), ) as RoleRepresentation roles.push(existingRole) await doApiCall(errors, `Creating role mapping for group ${groupName}`, async () => @@ -634,10 +638,10 @@ async function internalIdp(api: KeycloakApi, connection: KeycloakConnection) { if (!existingClientRoleMapping) { // let team members see other users const accessRoles: Array = [userViewerRole] - // both otomi-admin and team-admin role will get access to manage users - // so the otomi-admin can login to the 'otomi' realm just like team-admin and see the same - if (groupName === 'team-admin') accessRoles.push(userManagementRole) - if (groupName === 'otomi-admin') accessRoles.push(realmManagementRole) + // both platform-admin and all-teams-admin role will get access to manage users + // so the platform-admin can login to the 'otomi' realm just like all-teams-admin and see the same + if (groupName === 'all-teams-admin') accessRoles.push(userManagementRole) + if (groupName === 'platform-admin') accessRoles.push(realmManagementRole) await doApiCall( errors, `Creating access roles [${accessRoles.map((r) => r.name).join(',')}] mapping for group ${groupName}`, diff --git a/src/tasks/keycloak/config.ts b/src/tasks/keycloak/config.ts index 2f0bfd2..c68ace9 100644 --- a/src/tasks/keycloak/config.ts +++ b/src/tasks/keycloak/config.ts @@ -58,8 +58,8 @@ export const adminUserCfgTpl = (username: string, password: string): Record { - const groupNames: string[] = teamIds.map((id) => `team-${id}`).concat(['otomi-admin', 'team-admin']) + const groupNames: string[] = teamIds + .map((id) => `team-${id}`) + .concat(['platform-admin', 'all-teams-admin', 'team-admin']) const groups = groupNames.map((name) => defaultsDeep(new GroupRepresentation(), { name })) return groups } @@ -44,18 +46,31 @@ export function createGroups(teamIds: string[]): Array { export function createIdpMappers( idpAlias: string, teams: {} | undefined, - adminGroupMapping: string, + platformAdminGroupMapping: string, + allTeamsAdminGroupMapping: string, teamAdminGroupMapping: string, userClaimMapper: string, idpSubClaimMapper: string, ): Array { - // admin idp mapper case - const admin = idpMapperTpl('otomi-admin group to role', idpAlias, 'admin', adminGroupMapping) - const adminMapper = defaultsDeep(new IdentityProviderMapperRepresentation(), admin) + // platform admin idp mapper case + const platformAdmin = idpMapperTpl( + 'platform-admin group to role', + idpAlias, + 'platform-admin', + platformAdminGroupMapping, + ) + const platformAdminMapper = defaultsDeep(new IdentityProviderMapperRepresentation(), platformAdmin) + // all teams admin idp mapper case + const allTeamsAdmin = idpMapperTpl( + 'all-teams-admin group to role', + idpAlias, + 'all-teams-admin', + allTeamsAdminGroupMapping, + ) + const allTeamsAdminMapper = defaultsDeep(new IdentityProviderMapperRepresentation(), allTeamsAdmin) // team admin idp mapper case const teamAdmin = idpMapperTpl('team-admin group to role', idpAlias, 'team-admin', teamAdminGroupMapping) const teamAdminMapper = defaultsDeep(new IdentityProviderMapperRepresentation(), teamAdmin) - // default idp mappers case const defaultIdps = defaultsIdpMapperTpl(idpAlias, userClaimMapper, idpSubClaimMapper) @@ -68,7 +83,11 @@ export function createIdpMappers( const teamMapper = idpMapperTpl(`${team.name} group to role`, idpAlias, team.name, team.groupMapping) return defaultsDeep(new IdentityProviderMapperRepresentation(), teamMapper) }) - return teamMappers.concat(defaultMapper).concat(adminMapper).concat(teamAdminMapper) + return teamMappers + .concat(defaultMapper) + .concat(platformAdminMapper) + .concat(allTeamsAdminMapper) + .concat(teamAdminMapper) } export async function createIdProvider( @@ -131,7 +150,8 @@ export function mapTeamsToRoles( teamIds: string[], idpGroupMappings: {} | undefined, idpGroupTeamAdmin: string, - groupOtomiAdmin: string, + idpGroupAllTeamsAdmin: string, + idpGroupPlatformAdmin: string, realm: string, ): Array { // eslint-disable-next-line no-param-reassign @@ -145,10 +165,12 @@ export function mapTeamsToRoles( }, {}) // create static admin teams const teamAdmin = Object.create({ name: 'team-admin', groupMapping: idpGroupTeamAdmin }) as TeamMapping - const adminTeams = [teamAdmin] + const allTeamsAdmin = Object.create({ name: 'all-teams-admin', groupMapping: idpGroupAllTeamsAdmin }) as TeamMapping + const adminTeams = [teamAdmin, allTeamsAdmin] + const otomiAdmin = Object.create({ - name: 'admin', - groupMapping: groupOtomiAdmin, + name: 'platform-admin', + groupMapping: idpGroupPlatformAdmin, }) as TeamMapping adminTeams.push(otomiAdmin) // iterate through all the teams and map groups diff --git a/src/validators.ts b/src/validators.ts index f163fc6..1d49211 100644 --- a/src/validators.ts +++ b/src/validators.ts @@ -33,7 +33,8 @@ export const IDP_GROUP_MAPPINGS_TEAMS = json({ default: undefined, }) export const IDP_GROUP_TEAM_ADMIN = str({ desc: 'APL team-admin group name' }) -export const IDP_GROUP_OTOMI_ADMIN = str({ desc: 'APL admin group name', default: undefined }) +export const IDP_GROUP_ALL_TEAMS_ADMIN = str({ desc: 'APL all-teams-admin group name' }) +export const IDP_GROUP_PLATFORM_ADMIN = str({ desc: 'APL platform admin group name', default: undefined }) export const IDP_OIDC_URL = str({ desc: "The IDP's OIDC enpoints url", default: undefined }) export const IDP_USERNAME_CLAIM_MAPPER = str({ desc: "The IDP's OIDC claim to username mapper string", @@ -97,7 +98,8 @@ if (!feat.FEAT_EXTERNAL_IDP) { ;[ IDP_ALIAS, IDP_GROUP_TEAM_ADMIN, - IDP_GROUP_OTOMI_ADMIN, + IDP_GROUP_ALL_TEAMS_ADMIN, + IDP_GROUP_PLATFORM_ADMIN, IDP_OIDC_URL, IDP_USERNAME_CLAIM_MAPPER, IDP_SUB_CLAIM_MAPPER, From 21a7ada0890a7682e8e37fc3ba82834e55ed86c1 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 25 Sep 2024 10:58:42 +0200 Subject: [PATCH 06/23] feat: update create user --- src/operator/keycloak.ts | 4 ++-- src/tasks/keycloak/config.ts | 6 ++++-- src/tasks/keycloak/realm-factory.ts | 4 +++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/operator/keycloak.ts b/src/operator/keycloak.ts index b07982d..79490d3 100644 --- a/src/operator/keycloak.ts +++ b/src/operator/keycloak.ts @@ -711,8 +711,8 @@ async function manageGroups(connection: KeycloakConnection) { } async function createUser(api: any, user: any) { - const { name: username, email, firstName, lastName, teamId } = user - const userConf = createTeamUser(username, email, firstName, lastName, teamId) + const { name: username, email, firstName, lastName, isPlatformAdmin, isTeamAdmin, teamId } = user + const userConf = createTeamUser(username, email, firstName, lastName, isPlatformAdmin, isTeamAdmin, teamId) const existingUsersByAdminEmail = (await doApiCall([], `Getting users`, () => api.users.realmUsersGet(keycloakRealm, false, `${email}`), )) as UserRepresentation[] diff --git a/src/tasks/keycloak/config.ts b/src/tasks/keycloak/config.ts index c68ace9..13ccd5b 100644 --- a/src/tasks/keycloak/config.ts +++ b/src/tasks/keycloak/config.ts @@ -75,6 +75,8 @@ export const teamUserCfgTpl = ( email: string, firstName: string, lastName: string, + isPlatformAdmin: boolean, + isTeamAdmin: boolean, teamId: string, ): Record => ({ username, @@ -83,8 +85,8 @@ export const teamUserCfgTpl = ( emailVerified: true, firstName, lastName, - realmRoles: ['teamMember'], - groups: [`team-${teamId}`], + realmRoles: [...(isPlatformAdmin ? ['platformAdmin'] : []), ...(isTeamAdmin ? ['teamAdmin'] : []), 'teamMember'], + groups: [...(isPlatformAdmin ? ['platform-admin'] : []), ...(isTeamAdmin ? ['team-admin'] : []), `team-${teamId}`], credentials: [ { type: 'password', diff --git a/src/tasks/keycloak/realm-factory.ts b/src/tasks/keycloak/realm-factory.ts index e01b2b3..4777db8 100644 --- a/src/tasks/keycloak/realm-factory.ts +++ b/src/tasks/keycloak/realm-factory.ts @@ -124,11 +124,13 @@ export function createTeamUser( email: string, firstName: string, lastName: string, + isPlatformAdmin: boolean, + isTeamAdmin: boolean, teamId: string, ): UserRepresentation { const userRepresentation = defaultsDeep( new UserRepresentation(), - teamUserCfgTpl(username, email, firstName, lastName, teamId), + teamUserCfgTpl(username, email, firstName, lastName, isPlatformAdmin, isTeamAdmin, teamId), ) return userRepresentation } From 86a92341b663d822139ff492a3ed20baa3a3518b Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 25 Sep 2024 13:17:38 +0200 Subject: [PATCH 07/23] feat: update harbor operator --- src/operator/harbor.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/operator/harbor.ts b/src/operator/harbor.ts index c9b07b9..bfbe16b 100644 --- a/src/operator/harbor.ts +++ b/src/operator/harbor.ts @@ -257,7 +257,7 @@ async function setupHarbor() { const config: any = { auth_mode: 'oidc_auth', - oidc_admin_group: 'admin', + oidc_admin_group: 'platform-admin', oidc_client_id: 'otomi', oidc_client_secret: env.oidcClientSecret, oidc_endpoint: env.oidcEndpoint, @@ -309,7 +309,7 @@ async function getBearerToken(): Promise { // unauthenticated, so remove and recreate secret await k8sApi.deleteNamespacedSecret(systemSecretName, systemNamespace) // now, the next call might throw IF: - // - authMode oidc was already turned on and an otomi admin accidentally removed the secret + // - authMode oidc was already turned on and an platform admin accidentally removed the secret // but that is very unlikely, an unresolvable problem and needs a manual db fix robotSecret = await createSystemRobotSecret() } @@ -366,7 +366,7 @@ async function processNamespace(namespace: string) { const projAdminMember: ProjectMember = { roleId: HarborRole.admin, memberGroup: { - groupName: 'team-admin', + groupName: 'all-teams-admin', groupType: HarborGroupType.http, }, } @@ -377,7 +377,7 @@ async function processNamespace(namespace: string) { ) await doApiCall( errors, - `Associating "project-admin" role for "team-admin" with harbor project "${projectName}"`, + `Associating "project-admin" role for "all-teams-admin" with harbor project "${projectName}"`, () => memberApi.createProjectMember(projectId, undefined, undefined, projAdminMember), ) From 3d769bddc6630a7a57f031f54bbbd94c1824dbd6 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 26 Sep 2024 11:41:12 +0200 Subject: [PATCH 08/23] feat: update create user --- src/operator/keycloak.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/operator/keycloak.ts b/src/operator/keycloak.ts index 79490d3..a6aaf4e 100644 --- a/src/operator/keycloak.ts +++ b/src/operator/keycloak.ts @@ -711,7 +711,7 @@ async function manageGroups(connection: KeycloakConnection) { } async function createUser(api: any, user: any) { - const { name: username, email, firstName, lastName, isPlatformAdmin, isTeamAdmin, teamId } = user + const { username, email, firstName, lastName, isPlatformAdmin, isTeamAdmin, teamId } = user const userConf = createTeamUser(username, email, firstName, lastName, isPlatformAdmin, isTeamAdmin, teamId) const existingUsersByAdminEmail = (await doApiCall([], `Getting users`, () => api.users.realmUsersGet(keycloakRealm, false, `${email}`), From e72ac42203566dc6a0f8355467a18850a3acc02f Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 26 Sep 2024 23:23:29 +0200 Subject: [PATCH 09/23] feat: add delete keycloak users function --- src/operator/keycloak.ts | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/src/operator/keycloak.ts b/src/operator/keycloak.ts index a6aaf4e..ddb610c 100644 --- a/src/operator/keycloak.ts +++ b/src/operator/keycloak.ts @@ -710,17 +710,20 @@ async function manageGroups(connection: KeycloakConnection) { } } -async function createUser(api: any, user: any) { +async function createUpdateUser(api: any, user: any) { const { username, email, firstName, lastName, isPlatformAdmin, isTeamAdmin, teamId } = user const userConf = createTeamUser(username, email, firstName, lastName, isPlatformAdmin, isTeamAdmin, teamId) - const existingUsersByAdminEmail = (await doApiCall([], `Getting users`, () => + const existingUsersByUserEmail = (await doApiCall([], `Getting users`, () => api.users.realmUsersGet(keycloakRealm, false, `${email}`), )) as UserRepresentation[] - const existingUser: UserRepresentation = existingUsersByAdminEmail?.[0] + const existingUser: UserRepresentation = existingUsersByUserEmail?.[0] try { if (existingUser) { - console.debug(`User with email ${email} already exists`) + console.debug(`User with email ${email} already exists, updating user`) + await doApiCall(errors, `Updating user ${username}`, async () => + api.users.realmUsersIdPut(keycloakRealm, existingUser.id as string, userConf), + ) } else { await doApiCall(errors, `Creating user ${username}`, () => api.users.realmUsersPost(keycloakRealm, userConf)) } @@ -729,9 +732,32 @@ async function createUser(api: any, user: any) { } } +async function deleteUsers(api: any, users: any[]) { + try { + const { body: keycloakUsers } = await api.users.realmUsersGet(keycloakRealm) + const filteredUsers = keycloakUsers.filter((user) => user.username !== 'otomi-admin') + const usersToDelete = filteredUsers.filter((user) => !users.some((u) => u.email === user.email)) + + await Promise.all( + usersToDelete.map(async (user) => { + try { + await api.users.realmUsersIdDelete(keycloakRealm, user.id) + console.debug(`Deleted user ${user.email}`) + } catch (error) { + console.error(`Error deleting user ${user.email}:`, error) + } + }), + ) + } catch (error) { + console.error('Error fetching users from Keycloak:', error) + } +} + async function manageUsers(users: any[]) { const connection = await createKeycloakConnection() const api = setupKeycloakApi(connection) // Create/Update users in realm 'otomi' - await Promise.all(users.map((user) => createUser(api, user))) + await Promise.all(users.map((user) => createUpdateUser(api, user))) + // Delete users not in users list + await deleteUsers(api, users) } From 1bb4d4e411db14e21c0f281e213461498641b09f Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 27 Sep 2024 12:11:15 +0200 Subject: [PATCH 10/23] test: delete keycloak users --- src/operator/keycloak.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/operator/keycloak.ts b/src/operator/keycloak.ts index ddb610c..14abed1 100644 --- a/src/operator/keycloak.ts +++ b/src/operator/keycloak.ts @@ -171,7 +171,7 @@ async function runKeycloakUpdater(key: string) { await keycloakConfigMapChanges().then(async () => { await runKeycloakUpdater('addTeam') }) - if (env.USERS.length > 0) await manageUsers(env.USERS) + await manageUsers(env.USERS) break } catch (error) { console.debug('Error could not update configMap', error) From 9dae38511555dfd78c298762711109c13bdd087c Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 27 Sep 2024 12:56:36 +0200 Subject: [PATCH 11/23] feat: update create keycloak user --- src/operator/keycloak.ts | 4 ++-- src/tasks/keycloak/config.ts | 8 +++++++- src/tasks/keycloak/realm-factory.ts | 3 ++- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/operator/keycloak.ts b/src/operator/keycloak.ts index 14abed1..be9e742 100644 --- a/src/operator/keycloak.ts +++ b/src/operator/keycloak.ts @@ -711,8 +711,8 @@ async function manageGroups(connection: KeycloakConnection) { } async function createUpdateUser(api: any, user: any) { - const { username, email, firstName, lastName, isPlatformAdmin, isTeamAdmin, teamId } = user - const userConf = createTeamUser(username, email, firstName, lastName, isPlatformAdmin, isTeamAdmin, teamId) + const { username, email, firstName, lastName, isPlatformAdmin, isTeamAdmin, teams, teamId } = user + const userConf = createTeamUser(username, email, firstName, lastName, isPlatformAdmin, isTeamAdmin, teams, teamId) const existingUsersByUserEmail = (await doApiCall([], `Getting users`, () => api.users.realmUsersGet(keycloakRealm, false, `${email}`), )) as UserRepresentation[] diff --git a/src/tasks/keycloak/config.ts b/src/tasks/keycloak/config.ts index 13ccd5b..3b5a0d6 100644 --- a/src/tasks/keycloak/config.ts +++ b/src/tasks/keycloak/config.ts @@ -77,6 +77,7 @@ export const teamUserCfgTpl = ( lastName: string, isPlatformAdmin: boolean, isTeamAdmin: boolean, + teams: string[], teamId: string, ): Record => ({ username, @@ -86,7 +87,12 @@ export const teamUserCfgTpl = ( firstName, lastName, realmRoles: [...(isPlatformAdmin ? ['platformAdmin'] : []), ...(isTeamAdmin ? ['teamAdmin'] : []), 'teamMember'], - groups: [...(isPlatformAdmin ? ['platform-admin'] : []), ...(isTeamAdmin ? ['team-admin'] : []), `team-${teamId}`], + groups: [ + ...(isPlatformAdmin ? ['platform-admin'] : []), + ...(isTeamAdmin ? ['team-admin'] : []), + `team-${teamId}`, + ...(teams.length > 0 ? teams.map((team) => `team-${team}`) : []), + ], credentials: [ { type: 'password', diff --git a/src/tasks/keycloak/realm-factory.ts b/src/tasks/keycloak/realm-factory.ts index 4777db8..f15305e 100644 --- a/src/tasks/keycloak/realm-factory.ts +++ b/src/tasks/keycloak/realm-factory.ts @@ -126,11 +126,12 @@ export function createTeamUser( lastName: string, isPlatformAdmin: boolean, isTeamAdmin: boolean, + teams: string[], teamId: string, ): UserRepresentation { const userRepresentation = defaultsDeep( new UserRepresentation(), - teamUserCfgTpl(username, email, firstName, lastName, isPlatformAdmin, isTeamAdmin, teamId), + teamUserCfgTpl(username, email, firstName, lastName, isPlatformAdmin, isTeamAdmin, teams, teamId), ) return userRepresentation } From b720b98932382c0f36581fee911272133c911890 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 27 Sep 2024 13:37:15 +0200 Subject: [PATCH 12/23] feat: update keycloak user without credentials --- src/operator/keycloak.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/operator/keycloak.ts b/src/operator/keycloak.ts index be9e742..cb710b8 100644 --- a/src/operator/keycloak.ts +++ b/src/operator/keycloak.ts @@ -23,7 +23,7 @@ import { UserRepresentation, UsersApi, } from '@linode/keycloak-client-node' -import { forEach } from 'lodash' +import { forEach, omit } from 'lodash' import { custom, Issuer, TokenSet } from 'openid-client' import { keycloakRealm } from '../tasks/keycloak/config' import { @@ -721,8 +721,9 @@ async function createUpdateUser(api: any, user: any) { try { if (existingUser) { console.debug(`User with email ${email} already exists, updating user`) + const userConfWithoutCredentials = omit(userConf, ['credentials']) await doApiCall(errors, `Updating user ${username}`, async () => - api.users.realmUsersIdPut(keycloakRealm, existingUser.id as string, userConf), + api.users.realmUsersIdPut(keycloakRealm, existingUser.id as string, userConfWithoutCredentials), ) } else { await doApiCall(errors, `Creating user ${username}`, () => api.users.realmUsersPost(keycloakRealm, userConf)) From 008fb62e377dd7ecc0aa50cfa24071a76ca7c702 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 27 Sep 2024 16:12:53 +0200 Subject: [PATCH 13/23] test: update user --- src/operator/keycloak.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/operator/keycloak.ts b/src/operator/keycloak.ts index cb710b8..d4fbeba 100644 --- a/src/operator/keycloak.ts +++ b/src/operator/keycloak.ts @@ -720,10 +720,13 @@ async function createUpdateUser(api: any, user: any) { try { if (existingUser) { + console.log('existingUser', existingUser) console.debug(`User with email ${email} already exists, updating user`) - const userConfWithoutCredentials = omit(userConf, ['credentials']) + const updatedUserConf = existingUser.requiredActions?.includes('UPDATE_PASSWORD') + ? userConf + : omit(userConf, ['credentials']) await doApiCall(errors, `Updating user ${username}`, async () => - api.users.realmUsersIdPut(keycloakRealm, existingUser.id as string, userConfWithoutCredentials), + api.users.realmUsersIdPut(keycloakRealm, existingUser.id as string, updatedUserConf), ) } else { await doApiCall(errors, `Creating user ${username}`, () => api.users.realmUsersPost(keycloakRealm, userConf)) From 7d3f1737b717267d6b5605091421e005eddc1e15 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 27 Sep 2024 16:34:03 +0200 Subject: [PATCH 14/23] feat: update keycloak createUpdateUser --- src/operator/keycloak.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/operator/keycloak.ts b/src/operator/keycloak.ts index d4fbeba..713b816 100644 --- a/src/operator/keycloak.ts +++ b/src/operator/keycloak.ts @@ -720,7 +720,6 @@ async function createUpdateUser(api: any, user: any) { try { if (existingUser) { - console.log('existingUser', existingUser) console.debug(`User with email ${email} already exists, updating user`) const updatedUserConf = existingUser.requiredActions?.includes('UPDATE_PASSWORD') ? userConf From e6e8fc46e0017b631e20470f40cd1a4a82ed023f Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Mon, 30 Sep 2024 16:35:21 +0200 Subject: [PATCH 15/23] feat: update keycloak.ts --- src/operator/keycloak.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/operator/keycloak.ts b/src/operator/keycloak.ts index 713b816..36f97a3 100644 --- a/src/operator/keycloak.ts +++ b/src/operator/keycloak.ts @@ -256,8 +256,8 @@ export default class MyOperator extends Operator { env.IDP_ALIAS = data!.IDP_ALIAS env.IDP_OIDC_URL = data!.IDP_OIDC_URL env.IDP_GROUP_PLATFORM_ADMIN = data!.IDP_GROUP_PLATFORM_ADMIN - env.IDP_GROUP_ALL_TEAMS_ADMIN = data!.IDP_GROUP_TEAM_ADMIN - env.IDP_GROUP_TEAM_ADMIN = 'xxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx' + env.IDP_GROUP_ALL_TEAMS_ADMIN = data!.IDP_GROUP_ALL_TEAMS_ADMIN + env.IDP_GROUP_TEAM_ADMIN = data!.IDP_GROUP_TEAM_ADMIN env.IDP_GROUP_MAPPINGS_TEAMS = Object.keys(data!.IDP_GROUP_MAPPINGS_TEAMS).length === 0 ? JSON.parse(data!.IDP_GROUP_MAPPINGS_TEAMS) @@ -364,7 +364,6 @@ async function keycloakTeamDeleted() { } async function createKeycloakConnection(): Promise { - // await waitTillAvailable(env.KEYCLOAK_HOSTNAME_URL, undefined, env.WAIT_OPTIONS) const keycloakAddress = env.KEYCLOAK_HOSTNAME_URL const basePath = `${keycloakAddress}/admin/realms` let token: TokenSet From a89275da850c1c2cfaf4cc5d3e2dbe5fa9d0e9f3 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 2 Oct 2024 10:05:25 +0200 Subject: [PATCH 16/23] test: users --- src/operator/keycloak.ts | 8 ++++---- src/tasks/keycloak/config.ts | 7 ++----- src/tasks/keycloak/realm-factory.ts | 4 +--- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/src/operator/keycloak.ts b/src/operator/keycloak.ts index 36f97a3..d1a113e 100644 --- a/src/operator/keycloak.ts +++ b/src/operator/keycloak.ts @@ -710,8 +710,8 @@ async function manageGroups(connection: KeycloakConnection) { } async function createUpdateUser(api: any, user: any) { - const { username, email, firstName, lastName, isPlatformAdmin, isTeamAdmin, teams, teamId } = user - const userConf = createTeamUser(username, email, firstName, lastName, isPlatformAdmin, isTeamAdmin, teams, teamId) + const { email, firstName, lastName, isPlatformAdmin, isTeamAdmin, teams } = user + const userConf = createTeamUser(email, firstName, lastName, isPlatformAdmin, isTeamAdmin, teams) const existingUsersByUserEmail = (await doApiCall([], `Getting users`, () => api.users.realmUsersGet(keycloakRealm, false, `${email}`), )) as UserRepresentation[] @@ -723,11 +723,11 @@ async function createUpdateUser(api: any, user: any) { const updatedUserConf = existingUser.requiredActions?.includes('UPDATE_PASSWORD') ? userConf : omit(userConf, ['credentials']) - await doApiCall(errors, `Updating user ${username}`, async () => + await doApiCall(errors, `Updating user ${email}`, async () => api.users.realmUsersIdPut(keycloakRealm, existingUser.id as string, updatedUserConf), ) } else { - await doApiCall(errors, `Creating user ${username}`, () => api.users.realmUsersPost(keycloakRealm, userConf)) + await doApiCall(errors, `Creating user ${email}`, () => api.users.realmUsersPost(keycloakRealm, userConf)) } } catch (error) { console.error('Error in internalIDP: ', error) diff --git a/src/tasks/keycloak/config.ts b/src/tasks/keycloak/config.ts index 3b5a0d6..8d27cae 100644 --- a/src/tasks/keycloak/config.ts +++ b/src/tasks/keycloak/config.ts @@ -71,16 +71,14 @@ export const adminUserCfgTpl = (username: string, password: string): Record => ({ - username, + username: email, enabled: true, email, emailVerified: true, @@ -90,13 +88,12 @@ export const teamUserCfgTpl = ( groups: [ ...(isPlatformAdmin ? ['platform-admin'] : []), ...(isTeamAdmin ? ['team-admin'] : []), - `team-${teamId}`, ...(teams.length > 0 ? teams.map((team) => `team-${team}`) : []), ], credentials: [ { type: 'password', - value: `${username}@APL`, + value: `${email}`, temporary: true, }, ], diff --git a/src/tasks/keycloak/realm-factory.ts b/src/tasks/keycloak/realm-factory.ts index f15305e..34e265b 100644 --- a/src/tasks/keycloak/realm-factory.ts +++ b/src/tasks/keycloak/realm-factory.ts @@ -120,18 +120,16 @@ export function createAdminUser(username: string, password: string): UserReprese return userRepresentation } export function createTeamUser( - username: string, email: string, firstName: string, lastName: string, isPlatformAdmin: boolean, isTeamAdmin: boolean, teams: string[], - teamId: string, ): UserRepresentation { const userRepresentation = defaultsDeep( new UserRepresentation(), - teamUserCfgTpl(username, email, firstName, lastName, isPlatformAdmin, isTeamAdmin, teams, teamId), + teamUserCfgTpl(email, firstName, lastName, isPlatformAdmin, isTeamAdmin, teams), ) return userRepresentation } From efcd26c1a4301bdc39aea5337e93ededde541149 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 2 Oct 2024 15:09:24 +0200 Subject: [PATCH 17/23] feat: keycloak userCreateUpdate --- src/operator/keycloak.ts | 49 ++++++++++++++++++++++++++++- src/tasks/keycloak/config.ts | 8 ++--- src/tasks/keycloak/realm-factory.ts | 4 +-- 3 files changed, 52 insertions(+), 9 deletions(-) diff --git a/src/operator/keycloak.ts b/src/operator/keycloak.ts index d1a113e..d3f8dd5 100644 --- a/src/operator/keycloak.ts +++ b/src/operator/keycloak.ts @@ -64,6 +64,7 @@ interface KeycloakApi { protocols: ProtocolMappersApi realms: RealmsAdminApi users: UsersApi + groups: GroupsApi } // Create realm roles @@ -401,6 +402,7 @@ function setupKeycloakApi(connection: KeycloakConnection) { protocols: new ProtocolMappersApi(basePath), realms: new RealmsAdminApi(basePath), users: new UsersApi(basePath), + groups: new GroupsApi(basePath), } // eslint-disable-next-line no-return-assign,no-param-reassign forEach(api, (a) => (a.accessToken = String(token.access_token))) @@ -709,9 +711,52 @@ async function manageGroups(connection: KeycloakConnection) { } } +async function removeUserGroups(api: any, existingUser: any, teamGroups: string[]) { + try { + const { body: existingUserGroups } = await api.users.realmUsersIdGroupsGet(keycloakRealm, existingUser.id as string) + + await Promise.all( + existingUserGroups.map(async (group) => { + if (!teamGroups.includes(group.name)) { + await api.users.realmUsersIdGroupsGroupIdDelete(keycloakRealm, existingUser.id as string, group.id as string) + } + }), + ) + } catch (error) { + console.error('Error removing user groups:', error) + } +} + +async function addUserGroups(api: any, existingUser: any, teamGroups: string[]) { + try { + const { body: currentKeycloakGroups } = await api.groups.realmGroupsGet(keycloakRealm) + const { body: existingUserGroups } = await api.users.realmUsersIdGroupsGet(keycloakRealm, existingUser.id as string) + + await Promise.all( + teamGroups.map(async (teamGroup) => { + const existingGroup = existingUserGroups.find((el) => el.name === teamGroup) + + if (!existingGroup) { + const groupId = currentKeycloakGroups.find((el) => el.name === teamGroup)?.id + if (groupId) { + await api.users.realmUsersIdGroupsGroupIdPut(keycloakRealm, existingUser.id as string, groupId as string) + } + } + }), + ) + } catch (error) { + console.error('Error adding user groups:', error) + } +} + async function createUpdateUser(api: any, user: any) { const { email, firstName, lastName, isPlatformAdmin, isTeamAdmin, teams } = user - const userConf = createTeamUser(email, firstName, lastName, isPlatformAdmin, isTeamAdmin, teams) + const teamGroups = [ + ...(isPlatformAdmin ? ['platform-admin'] : []), + ...(isTeamAdmin ? ['team-admin'] : []), + ...(teams.length > 0 ? teams.map((team) => `team-${team}`) : []), + ] + const userConf = createTeamUser(email, firstName, lastName, isPlatformAdmin, isTeamAdmin, teamGroups) const existingUsersByUserEmail = (await doApiCall([], `Getting users`, () => api.users.realmUsersGet(keycloakRealm, false, `${email}`), )) as UserRepresentation[] @@ -726,6 +771,8 @@ async function createUpdateUser(api: any, user: any) { await doApiCall(errors, `Updating user ${email}`, async () => api.users.realmUsersIdPut(keycloakRealm, existingUser.id as string, updatedUserConf), ) + await removeUserGroups(api, existingUser, teamGroups) + await addUserGroups(api, existingUser, teamGroups) } else { await doApiCall(errors, `Creating user ${email}`, () => api.users.realmUsersPost(keycloakRealm, userConf)) } diff --git a/src/tasks/keycloak/config.ts b/src/tasks/keycloak/config.ts index 8d27cae..1814ac6 100644 --- a/src/tasks/keycloak/config.ts +++ b/src/tasks/keycloak/config.ts @@ -76,7 +76,7 @@ export const teamUserCfgTpl = ( lastName: string, isPlatformAdmin: boolean, isTeamAdmin: boolean, - teams: string[], + teamGroups: string[], ): Record => ({ username: email, enabled: true, @@ -85,11 +85,7 @@ export const teamUserCfgTpl = ( firstName, lastName, realmRoles: [...(isPlatformAdmin ? ['platformAdmin'] : []), ...(isTeamAdmin ? ['teamAdmin'] : []), 'teamMember'], - groups: [ - ...(isPlatformAdmin ? ['platform-admin'] : []), - ...(isTeamAdmin ? ['team-admin'] : []), - ...(teams.length > 0 ? teams.map((team) => `team-${team}`) : []), - ], + groups: teamGroups, credentials: [ { type: 'password', diff --git a/src/tasks/keycloak/realm-factory.ts b/src/tasks/keycloak/realm-factory.ts index 34e265b..a20df06 100644 --- a/src/tasks/keycloak/realm-factory.ts +++ b/src/tasks/keycloak/realm-factory.ts @@ -125,11 +125,11 @@ export function createTeamUser( lastName: string, isPlatformAdmin: boolean, isTeamAdmin: boolean, - teams: string[], + teamGroups: string[], ): UserRepresentation { const userRepresentation = defaultsDeep( new UserRepresentation(), - teamUserCfgTpl(email, firstName, lastName, isPlatformAdmin, isTeamAdmin, teams), + teamUserCfgTpl(email, firstName, lastName, isPlatformAdmin, isTeamAdmin, teamGroups), ) return userRepresentation } From 5575ba992e3b98e72bf2dd7ca74a2df678649627 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 4 Oct 2024 13:03:31 +0200 Subject: [PATCH 18/23] feat: update user creation --- src/operator/keycloak.ts | 4 ++-- src/tasks/keycloak/config.ts | 3 ++- src/tasks/keycloak/realm-factory.ts | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/operator/keycloak.ts b/src/operator/keycloak.ts index d3f8dd5..29a9bd2 100644 --- a/src/operator/keycloak.ts +++ b/src/operator/keycloak.ts @@ -750,13 +750,13 @@ async function addUserGroups(api: any, existingUser: any, teamGroups: string[]) } async function createUpdateUser(api: any, user: any) { - const { email, firstName, lastName, isPlatformAdmin, isTeamAdmin, teams } = user + const { email, firstName, lastName, isPlatformAdmin, isTeamAdmin, teams, initialPassword } = user const teamGroups = [ ...(isPlatformAdmin ? ['platform-admin'] : []), ...(isTeamAdmin ? ['team-admin'] : []), ...(teams.length > 0 ? teams.map((team) => `team-${team}`) : []), ] - const userConf = createTeamUser(email, firstName, lastName, isPlatformAdmin, isTeamAdmin, teamGroups) + const userConf = createTeamUser(email, firstName, lastName, isPlatformAdmin, isTeamAdmin, teamGroups, initialPassword) const existingUsersByUserEmail = (await doApiCall([], `Getting users`, () => api.users.realmUsersGet(keycloakRealm, false, `${email}`), )) as UserRepresentation[] diff --git a/src/tasks/keycloak/config.ts b/src/tasks/keycloak/config.ts index 1814ac6..d1a4296 100644 --- a/src/tasks/keycloak/config.ts +++ b/src/tasks/keycloak/config.ts @@ -77,6 +77,7 @@ export const teamUserCfgTpl = ( isPlatformAdmin: boolean, isTeamAdmin: boolean, teamGroups: string[], + initialPassword: string, ): Record => ({ username: email, enabled: true, @@ -89,7 +90,7 @@ export const teamUserCfgTpl = ( credentials: [ { type: 'password', - value: `${email}`, + value: `${initialPassword}`, temporary: true, }, ], diff --git a/src/tasks/keycloak/realm-factory.ts b/src/tasks/keycloak/realm-factory.ts index a20df06..9391523 100644 --- a/src/tasks/keycloak/realm-factory.ts +++ b/src/tasks/keycloak/realm-factory.ts @@ -126,10 +126,11 @@ export function createTeamUser( isPlatformAdmin: boolean, isTeamAdmin: boolean, teamGroups: string[], + initialPassword: string, ): UserRepresentation { const userRepresentation = defaultsDeep( new UserRepresentation(), - teamUserCfgTpl(email, firstName, lastName, isPlatformAdmin, isTeamAdmin, teamGroups), + teamUserCfgTpl(email, firstName, lastName, isPlatformAdmin, isTeamAdmin, teamGroups, initialPassword), ) return userRepresentation } From a6bae9e6e2d0f84f65d07a9ddd33e0029dada686 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Mon, 7 Oct 2024 08:57:59 +0200 Subject: [PATCH 19/23] feat: update keycloak manageUsers flow --- src/operator/keycloak.ts | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/operator/keycloak.ts b/src/operator/keycloak.ts index 29a9bd2..5c1a560 100644 --- a/src/operator/keycloak.ts +++ b/src/operator/keycloak.ts @@ -167,12 +167,26 @@ async function runKeycloakUpdater(key: string) { await runKeycloakUpdater('removeTeam') } break + case 'manageUsers': + try { + await manageUsers(env.USERS) + break + } catch (error) { + console.debug('Error could update users', error) + console.debug('Retrying in 30 seconds') + await new Promise((resolve) => setTimeout(resolve, 30000)) + console.log('Retrying to update users') + await runKeycloakUpdater('manageUsers') + } + break case 'updateConfig': try { await keycloakConfigMapChanges().then(async () => { await runKeycloakUpdater('addTeam') }) - await manageUsers(env.USERS) + if (!JSON.parse(env.FEAT_EXTERNAL_IDP)) { + await runKeycloakUpdater('manageUsers') + } break } catch (error) { console.debug('Error could not update configMap', error) From 2479e86c4ed00f4a68b5359b8ef5328dd1fd66d5 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Tue, 8 Oct 2024 16:16:35 +0200 Subject: [PATCH 20/23] feat: update keycloak.ts and add keycloak.test.ts --- src/operator/keycloak.test.ts | 76 +++++++++++++++++++++++++++++++++++ src/operator/keycloak.ts | 74 ++++++++++++++-------------------- 2 files changed, 106 insertions(+), 44 deletions(-) create mode 100644 src/operator/keycloak.test.ts diff --git a/src/operator/keycloak.test.ts b/src/operator/keycloak.test.ts new file mode 100644 index 0000000..2d17260 --- /dev/null +++ b/src/operator/keycloak.test.ts @@ -0,0 +1,76 @@ +import { expect } from 'chai' +import sinon from 'sinon' +import { addUserGroups, removeUserGroups } from './keycloak' + +describe('Keycloak User Group Management', () => { + let api: any + let existingUser: any + let keycloakRealm: string + + beforeEach(() => { + keycloakRealm = 'otomi' + existingUser = { id: 'user-id' } + api = { + users: { + realmUsersIdGroupsGet: sinon.stub(), + realmUsersIdGroupsGroupIdDelete: sinon.stub(), + realmUsersIdGroupsGroupIdPut: sinon.stub(), + }, + groups: { + realmGroupsGet: sinon.stub(), + }, + } + }) + + afterEach(() => { + sinon.restore() + }) + + describe('removeUserGroups', () => { + it('should remove user from groups not in teamGroups', async () => { + const existingUserGroups = [ + { name: 'group1', id: 'group1-id' }, + { name: 'group2', id: 'group2-id' }, + ] + api.users.realmUsersIdGroupsGet.resolves({ body: existingUserGroups }) + + await removeUserGroups(api, existingUser, ['group1']) + + expect(api.users.realmUsersIdGroupsGroupIdDelete.calledWith(keycloakRealm, 'user-id', 'group2-id')).to.be.true + expect(api.users.realmUsersIdGroupsGroupIdDelete.calledWith(keycloakRealm, 'user-id', 'group1-id')).to.be.false + }) + + it('should handle errors gracefully', async () => { + api.users.realmUsersIdGroupsGet.rejects(new Error('API Error')) + + await removeUserGroups(api, existingUser, ['group1']) + + expect(api.users.realmUsersIdGroupsGroupIdDelete.called).to.be.false + }) + }) + + describe('addUserGroups', () => { + it('should add user to groups in teamGroups if not already present', async () => { + const currentKeycloakGroups = [ + { name: 'group1', id: 'group1-id' }, + { name: 'group2', id: 'group2-id' }, + ] + const existingUserGroups = [{ name: 'group1', id: 'group1-id' }] + api.groups.realmGroupsGet.resolves({ body: currentKeycloakGroups }) + api.users.realmUsersIdGroupsGet.resolves({ body: existingUserGroups }) + + await addUserGroups(api, existingUser, ['group1', 'group2']) + + expect(api.users.realmUsersIdGroupsGroupIdPut.calledWith(keycloakRealm, 'user-id', 'group2-id')).to.be.true + expect(api.users.realmUsersIdGroupsGroupIdPut.calledWith(keycloakRealm, 'user-id', 'group1-id')).to.be.false + }) + + it('should handle errors gracefully', async () => { + api.groups.realmGroupsGet.rejects(new Error('API Error')) + + await addUserGroups(api, existingUser, ['group1']) + + expect(api.users.realmUsersIdGroupsGroupIdPut.called).to.be.false + }) + }) +}) diff --git a/src/operator/keycloak.ts b/src/operator/keycloak.ts index 5c1a560..24fe025 100644 --- a/src/operator/keycloak.ts +++ b/src/operator/keycloak.ts @@ -142,59 +142,37 @@ async function runKeycloakUpdater(key: string) { console.info('Missing required keycloak variables for Keycloak setup/reconfiguration') return } + + async function retryOperation(operation: () => Promise, operationName: string) { + try { + await operation() + } catch (error) { + console.debug(`Error could not ${operationName}`, error) + console.debug('Retrying in 30 seconds') + await new Promise((resolve) => setTimeout(resolve, 30000)) + console.log(`Retrying to ${operationName}`) + await runKeycloakUpdater(operationName) + } + } + switch (key) { case 'addTeam': - try { - await keycloakTeamAdded() - break - } catch (error) { - console.debug('Error could not add team', error) - console.debug('Retrying in 30 seconds') - await new Promise((resolve) => setTimeout(resolve, 30000)) - console.log('Retrying to add team') - await runKeycloakUpdater('addTeam') - } + await retryOperation(keycloakTeamAdded, 'add team') break case 'removeTeam': - try { - await keycloakTeamDeleted() - break - } catch (error) { - console.debug('Error could not delete team', error) - console.debug('Retrying in 30 seconds') - await new Promise((resolve) => setTimeout(resolve, 30000)) - console.log('Retrying to delete team') - await runKeycloakUpdater('removeTeam') - } + await retryOperation(keycloakTeamDeleted, 'delete team') break case 'manageUsers': - try { - await manageUsers(env.USERS) - break - } catch (error) { - console.debug('Error could update users', error) - console.debug('Retrying in 30 seconds') - await new Promise((resolve) => setTimeout(resolve, 30000)) - console.log('Retrying to update users') - await runKeycloakUpdater('manageUsers') - } + await retryOperation(() => manageUsers(env.USERS), 'update users') break case 'updateConfig': - try { - await keycloakConfigMapChanges().then(async () => { - await runKeycloakUpdater('addTeam') - }) + await retryOperation(async () => { + await keycloakConfigMapChanges() + await runKeycloakUpdater('addTeam') if (!JSON.parse(env.FEAT_EXTERNAL_IDP)) { await runKeycloakUpdater('manageUsers') } - break - } catch (error) { - console.debug('Error could not update configMap', error) - console.debug('Retrying in 30 seconds') - await new Promise((resolve) => setTimeout(resolve, 30000)) - console.log('Retrying to update configMap') - await runKeycloakUpdater('updateConfig') - } + }, 'update configMap') break default: break @@ -725,7 +703,11 @@ async function manageGroups(connection: KeycloakConnection) { } } -async function removeUserGroups(api: any, existingUser: any, teamGroups: string[]) { +export async function removeUserGroups( + api: { users: UsersApi; groups: GroupsApi }, + existingUser: UserRepresentation, + teamGroups: string[], +): Promise { try { const { body: existingUserGroups } = await api.users.realmUsersIdGroupsGet(keycloakRealm, existingUser.id as string) @@ -741,7 +723,11 @@ async function removeUserGroups(api: any, existingUser: any, teamGroups: string[ } } -async function addUserGroups(api: any, existingUser: any, teamGroups: string[]) { +export async function addUserGroups( + api: { users: UsersApi; groups: GroupsApi }, + existingUser: UserRepresentation, + teamGroups: string[], +): Promise { try { const { body: currentKeycloakGroups } = await api.groups.realmGroupsGet(keycloakRealm) const { body: existingUserGroups } = await api.users.realmUsersIdGroupsGet(keycloakRealm, existingUser.id as string) From 4c3fd7fd8dfeef85ed7dbf31d17369ca117743b3 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Wed, 9 Oct 2024 08:21:18 +0200 Subject: [PATCH 21/23] feat: update manageUsers --- src/operator/keycloak.ts | 13 ++++--------- src/tasks/keycloak/config.ts | 8 +++----- src/tasks/keycloak/realm-factory.ts | 6 ++---- 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/src/operator/keycloak.ts b/src/operator/keycloak.ts index 24fe025..bc7e054 100644 --- a/src/operator/keycloak.ts +++ b/src/operator/keycloak.ts @@ -750,13 +750,8 @@ export async function addUserGroups( } async function createUpdateUser(api: any, user: any) { - const { email, firstName, lastName, isPlatformAdmin, isTeamAdmin, teams, initialPassword } = user - const teamGroups = [ - ...(isPlatformAdmin ? ['platform-admin'] : []), - ...(isTeamAdmin ? ['team-admin'] : []), - ...(teams.length > 0 ? teams.map((team) => `team-${team}`) : []), - ] - const userConf = createTeamUser(email, firstName, lastName, isPlatformAdmin, isTeamAdmin, teamGroups, initialPassword) + const { email, firstName, lastName, groups, initialPassword } = user + const userConf = createTeamUser(email, firstName, lastName, groups, initialPassword) const existingUsersByUserEmail = (await doApiCall([], `Getting users`, () => api.users.realmUsersGet(keycloakRealm, false, `${email}`), )) as UserRepresentation[] @@ -771,8 +766,8 @@ async function createUpdateUser(api: any, user: any) { await doApiCall(errors, `Updating user ${email}`, async () => api.users.realmUsersIdPut(keycloakRealm, existingUser.id as string, updatedUserConf), ) - await removeUserGroups(api, existingUser, teamGroups) - await addUserGroups(api, existingUser, teamGroups) + await removeUserGroups(api, existingUser, groups) + await addUserGroups(api, existingUser, groups) } else { await doApiCall(errors, `Creating user ${email}`, () => api.users.realmUsersPost(keycloakRealm, userConf)) } diff --git a/src/tasks/keycloak/config.ts b/src/tasks/keycloak/config.ts index d1a4296..2724a18 100644 --- a/src/tasks/keycloak/config.ts +++ b/src/tasks/keycloak/config.ts @@ -74,9 +74,7 @@ export const teamUserCfgTpl = ( email: string, firstName: string, lastName: string, - isPlatformAdmin: boolean, - isTeamAdmin: boolean, - teamGroups: string[], + groups: string[], initialPassword: string, ): Record => ({ username: email, @@ -85,8 +83,8 @@ export const teamUserCfgTpl = ( emailVerified: true, firstName, lastName, - realmRoles: [...(isPlatformAdmin ? ['platformAdmin'] : []), ...(isTeamAdmin ? ['teamAdmin'] : []), 'teamMember'], - groups: teamGroups, + realmRoles: ['teamMember'], + groups, credentials: [ { type: 'password', diff --git a/src/tasks/keycloak/realm-factory.ts b/src/tasks/keycloak/realm-factory.ts index 9391523..07f0df3 100644 --- a/src/tasks/keycloak/realm-factory.ts +++ b/src/tasks/keycloak/realm-factory.ts @@ -123,14 +123,12 @@ export function createTeamUser( email: string, firstName: string, lastName: string, - isPlatformAdmin: boolean, - isTeamAdmin: boolean, - teamGroups: string[], + groups: string[], initialPassword: string, ): UserRepresentation { const userRepresentation = defaultsDeep( new UserRepresentation(), - teamUserCfgTpl(email, firstName, lastName, isPlatformAdmin, isTeamAdmin, teamGroups, initialPassword), + teamUserCfgTpl(email, firstName, lastName, groups, initialPassword), ) return userRepresentation } From 13414898ec64b66f9a16b6c5b12cea28bcccb090 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 10 Oct 2024 15:52:41 +0200 Subject: [PATCH 22/23] feat: update existing otomi-admin user groups --- src/operator/keycloak.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/operator/keycloak.ts b/src/operator/keycloak.ts index bc7e054..2003210 100644 --- a/src/operator/keycloak.ts +++ b/src/operator/keycloak.ts @@ -662,6 +662,7 @@ async function internalIdp(api: KeycloakApi, connection: KeycloakConnection) { await doApiCall(errors, `Updating user ${env.KEYCLOAK_ADMIN}`, async () => api.users.realmUsersIdPut(keycloakRealm, existingUser.id as string, userConf), ) + await addUserGroups(api, existingUser, ['platform-admin']) } else { await doApiCall(errors, `Creating user ${env.KEYCLOAK_ADMIN}`, () => api.users.realmUsersPost(keycloakRealm, userConf), From df3f196545d9ccfdedc9ea03f7ddc9aaae41604b Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 10 Oct 2024 16:16:06 +0200 Subject: [PATCH 23/23] fix: update platformAdminUser variable name --- src/operator/keycloak.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/operator/keycloak.ts b/src/operator/keycloak.ts index 2003210..16f3fbd 100644 --- a/src/operator/keycloak.ts +++ b/src/operator/keycloak.ts @@ -655,14 +655,14 @@ async function internalIdp(api: KeycloakApi, connection: KeycloakConnection) { const existingUsersByAdminEmail = (await doApiCall([], `Getting users`, () => api.users.realmUsersGet(keycloakRealm, false, userConf.email), )) as UserRepresentation[] - const existingUser: UserRepresentation = existingUsersByAdminEmail?.[0] + const existingPlatformAdminUser: UserRepresentation = existingUsersByAdminEmail?.[0] try { - if (existingUser) { + if (existingPlatformAdminUser) { await doApiCall(errors, `Updating user ${env.KEYCLOAK_ADMIN}`, async () => - api.users.realmUsersIdPut(keycloakRealm, existingUser.id as string, userConf), + api.users.realmUsersIdPut(keycloakRealm, existingPlatformAdminUser.id as string, userConf), ) - await addUserGroups(api, existingUser, ['platform-admin']) + await addUserGroups(api, existingPlatformAdminUser, ['platform-admin']) } else { await doApiCall(errors, `Creating user ${env.KEYCLOAK_ADMIN}`, () => api.users.realmUsersPost(keycloakRealm, userConf),