From 7cc3f7f89450e5cff977f5d9a0143a821e41d6fe Mon Sep 17 00:00:00 2001 From: Petr Vokac Date: Sat, 4 Apr 2026 17:52:00 +0200 Subject: [PATCH 1/3] OIDC identity synchronization --- atlas/check_map_voms_roles | 66 ++++++++++++++++++++++++++++++++------ atlas/check_voms | 51 +++++++++++++++++++++-------- atlas/check_voms_admin | 7 ++-- 3 files changed, 99 insertions(+), 25 deletions(-) diff --git a/atlas/check_map_voms_roles b/atlas/check_map_voms_roles index 091539a..0a58938 100755 --- a/atlas/check_map_voms_roles +++ b/atlas/check_map_voms_roles @@ -22,7 +22,7 @@ import requests.auth from rucio.client import Client from rucio.common.config import config_get -from rucio.common.exception import Duplicate, IdentityError +from rucio.common.exception import Duplicate, IdentityError, AccountNotFound UNKNOWN = 3 CRITICAL = 2 @@ -46,6 +46,7 @@ def get_access_token(token_url, client_id, client_secret): def get_userid2certs_mapping(scim_url, access_token): ret = {} + userid2email = {} with requests.Session() as session: session.headers = { @@ -62,11 +63,13 @@ def get_userid2certs_mapping(scim_url, access_token): d = response.json() if d['itemsPerPage'] == 0: break for user in d['Resources']: + if not user['active']: continue + userid2email[user['id']] = user['emails'][0]['value'] for certificate in user.get('urn:indigo-dc:scim:schemas:IndigoUser', {}).get('certificates', []): ret.setdefault(user['id'], []).append((certificate['subjectDn'], certificate['issuerDn'], user['emails'][0]['value'])) start += page_size - return ret + return ret, userid2email def get_group2id_mapping(scim_url, access_token): ret = {} @@ -119,10 +122,13 @@ def get_groupid_members(scim_url, groupid, access_token): def get_account_identities(account): ret = [] client = Client() - for identity in client.list_identities(account): - if identity['type'] != 'X509': - continue - ret.append(identity['identity']) + try: + for identity in client.list_identities(account): + if identity['type'] not in ['X509', 'OIDC']: + continue + ret.append(identity['identity']) + except rucio.common.exception.AccountNotFound: + return None return ret @@ -156,7 +162,7 @@ if __name__ == '__main__': CLIENT = Client() access_token = get_access_token(TOKEN_URL, CLIENT_ID, CLIENT_SECRET) - userid2certs = get_userid2certs_mapping(SCIM_URL, access_token) + userid2certs, userid2email = get_userid2certs_mapping(SCIM_URL, access_token) access_token = get_access_token(TOKEN_URL, CLIENT_ID, CLIENT_SECRET) group2id = get_group2id_mapping(SCIM_URL, access_token) @@ -169,8 +175,27 @@ if __name__ == '__main__': dns = [] groupid = group2id["atlas/{}".format(account)] account_identities = get_account_identities(ACCOUNT_MAP[account]) + if account_identities == None: + print("missing rucio account for {}".format(ACCOUNT_MAP[account])) access_token = get_access_token(TOKEN_URL, CLIENT_ID, CLIENT_SECRET) for userid in get_groupid_members(SCIM_URL, groupid, access_token): + if userid not in userid2email: + print("missing or disabled account for userid {}".format(userid)) + continue + dns.append(userid) + if userid not in account_identities: + NBUSERS += 1 + try: + if not TEST: + CLIENT.add_identity(account=ACCOUNT_MAP[account], identity=userid, authtype='OIDC', email=userid2email[userid], default=True) + else: + print("CMD: rucio-admin identity add --account {0} --type OIDC --id '{1}' --email '{2}'".format(ACCOUNT_MAP[account], userid, userid2email[userid])) + print('Identity {0} added to {1}'.format(userid, ACCOUNT_MAP[account])) + except Duplicate: + pass + except Exception as error: + print('ERROR {0} not added to {1}: {2}'.format(userid, ACCOUNT_MAP[account], error)) + STATUS = WARNING if userid not in userid2certs: print("missing user details for userid {}".format(userid)) continue @@ -194,12 +219,13 @@ if __name__ == '__main__': for dn in account_identities: if dn in dns: continue + itype = 'X509' if '=' in dn else 'OIDC' NDUSERS += 1 try: if not TEST: CLIENT.del_identity(account=ACCOUNT_MAP[account], identity=dn, authtype='X509') else: - print("CMD: rucio-admin identity delete --account {0} --type X509 --id '{1}'".format(ACCOUNT_MAP[account], dn)) + print("CMD: rucio-admin identity delete --account {0} --type {1} --id '{2}'".format(ACCOUNT_MAP[account], itype, dn)) print('Identity {0} removed from {1}'.format(dn, ACCOUNT_MAP[account])) except IdentityError: pass @@ -225,8 +251,27 @@ if __name__ == '__main__': dns = [] groupid = group2id["atlas/{}".format(account)] account_identities = get_account_identities(account) + if account_identities == None: + print("missing rucio account for {}".format(account)) access_token = get_access_token(TOKEN_URL, CLIENT_ID, CLIENT_SECRET) for userid in get_groupid_members(SCIM_URL, groupid, access_token): + if userid not in userid2email: + print("missing or disabled account for userid {}".format(userid)) + continue + dns.append(userid) + if userid not in account_identities: + NBUSERS += 1 + try: + if not TEST: + CLIENT.add_identity(account=account, identity=userid, authtype='OIDC', email=userid2email[userid], default=True) + else: + print("CMD: rucio-admin identity add --account {0} --type OIDC --id '{1}' --email '{2}'".format(account, userid, userid2email[userid])) + print('Identity {0} added to {1}'.format(userid, account)) + except Duplicate: + pass + except Exception as error: + print('ERROR {0} not added to {1}: {2}'.format(userid, ACCOUNT_MAP[account], error)) + STATUS = WARNING if userid not in userid2certs: print("missing user details for userid {}".format(userid)) continue @@ -250,12 +295,13 @@ if __name__ == '__main__': for dn in account_identities: if dn in dns: continue + itype = 'X509' if '=' in dn else 'OIDC' NDUSERS += 1 try: if not TEST: - CLIENT.del_identity(account=account, identity=dn, authtype='X509') + CLIENT.del_identity(account=account, identity=dn, authtype=itype) else: - print("CMD: rucio-admin identity delete --account {0} --type X509 --id '{1}'".format(account, dn)) + print("CMD: rucio-admin identity delete --account {0} --type {1} --id '{2}'".format(account, itype, dn)) print('Identity {0} removed from {1}'.format(dn, account)) except IdentityError: pass diff --git a/atlas/check_voms b/atlas/check_voms index 191b7cb..f28de0c 100755 --- a/atlas/check_voms +++ b/atlas/check_voms @@ -54,30 +54,36 @@ LDAP_PAGE_SIZE = 1000 def get_accounts_identities_fast(): from rucio.db.sqla.session import get_session session = get_session() - query = '''select b.identity, a.account, a.email, a.account_type from atlas_rucio.accounts a, atlas_rucio.account_map b where a.account=b.account and identity_type='X509' ''' + query = '''select b.identity, a.account, a.email, a.account_type, b.identity_type from atlas_rucio.accounts a, atlas_rucio.account_map b where a.account=b.account''' dns = {} + account2identities = {} try: result = session.execute(query) - for dn, account, email, atype in result: - dns.setdefault(atype, {})[dn] = (account, email) - return dns + for identity, account, email, atype, itype in result: + account2identities.setdefault(account, []).append((itype, identity)) + if itype not in ['X509', 'OIDC']: + continue + dns.setdefault(atype, {})[identity] = (account, email) + return dns, account2identities except Exception as error: print(error) def get_accounts_identities_slow(): dns = {} + account2identities = {} client = Client() for account in client.list_accounts(): #if account['type'] != 'USER': # continue for identity in client.list_identities(account['account']): - if identity['type'] != 'X509': + account2identities.setdefault(account['account'], []).append((identity['type'], identity['identity'])) + if identity['type'] not in ['X509', 'OIDC']: continue - if identity['identity'] in dns: - print("Duplicate identity for users {} and {}: {}".format(dns[identity['identity']][0], account['account'], identity['identity'])) + if account['type'] == 'USER' and identity['identity'] in dns.get(account['type'], {}): + print("Duplicate identity for users {} and {}: {}".format(dns[account['type']][identity['identity']][0], account['account'], identity['identity'])) dns.setdefault(account['type'], {})[identity['identity']] = (account['account'], account['email']) - return dns + return dns, account2identities get_accounts_identities = get_accounts_identities_slow @@ -189,11 +195,12 @@ def get_users(scim_url, access_token): d = response.json() if d['itemsPerPage'] == 0: break for user in d['Resources']: + if not user['active']: continue certs = [] for certificate in user.get('urn:indigo-dc:scim:schemas:IndigoUser', {}).get('certificates', []): certs.append(certificate['subjectDn']) groups = [x['display'] for x in user.get('groups', [])] - ret.append((user['userName'], user['active'], user['emails'][0]['value'], certs, groups)) + ret.append((user['id'], user['userName'], user['active'], user['emails'][0]['value'], certs, groups)) start += page_size return ret @@ -229,13 +236,13 @@ if __name__ == '__main__': client = Client() accounts = {account['account']: account for account in client.list_accounts()} scopes = [_ for _ in client.list_scopes()] - dns = get_accounts_identities() + dns, account2identities = get_accounts_identities() ldap_accounts = get_ldap_identities() # synchronize accounts, identities and scopes synced_accounts = [] access_token = get_access_token(TOKEN_URL, CLIENT_ID, CLIENT_SECRET) - for nickname, active, email, certs, groups in get_users(SCIM_URL, access_token): + for userid, nickname, active, email, certs, groups in get_users(SCIM_URL, access_token): if 'atlas' not in groups: print("Skipping user {0} because it is not member of ATLAS groups".format(nickname)) continue @@ -296,10 +303,21 @@ if __name__ == '__main__': print('Identity {0} added to {2} account {1}'.format(dn, nickname, atype)) except Duplicate: pass + # c) add OIDC identity + if userid not in dns.get(atype, []): + try: + if not TEST: + client.add_identity(account=nickname, identity=userid, authtype='OIDC', email=email, default=True) + else: + print("CMD: rucio-admin identity add --account {0} --type OIDC --id '{1}' --email '{2}'".format(nickname, userid, email)) + nbusers += 1 + print('Identity {0} added to {2} account {1}'.format(userid, nickname, atype)) + except Duplicate: + pass if atype == 'USER': scope = 'user.' + nickname - elif active and nickname in ldap_accounts and len(certs) > 0: + elif active and nickname in ldap_accounts: # no rucio account exists for this nickname if email.lower() not in [mail.lower() for mail in ldap_accounts[nickname]['mails']]: print('Account %s (%s) without matching email in LDAP does not exist. To create it : rucio-admin account add --type USER --email %s %s' % (nickname, ldap_accounts[nickname]['type'], email, nickname)) @@ -318,6 +336,12 @@ if __name__ == '__main__': print("CMD: rucio-admin identity add --account {0} --type X509 --id '{1}' --email '{2}'".format(nickname, dn, email)) nbusers += 1 print('Identity {0} added to USER account {1}'.format(dn, nickname)) + if not TEST: + client.add_identity(account=nickname, identity=userid, authtype='OIDC', email=email, default=True) + else: + print("CMD: rucio-admin identity add --account {0} --type OIDC --id '{1}' --email '{2}'".format(nickname, userid, email)) + nbusers += 1 + print('Identity {0} added to {2} account {1}'.format(userid, nickname, atype)) scope = 'user.' + nickname except Exception: print('Failed to add new account %s (%s)' % (nickname, ldap_accounts[nickname]['type'])) @@ -343,8 +367,9 @@ if __name__ == '__main__': aname = account['account'] if aname not in synced_accounts: delusers += 1 + identities = account2identities.get(aname, []) if TEST: - print('User account {0} is no longer member VO, details {1}'.format(aname, account)) + print('User account {0} is no longer member VO, details {1}, identities({2}) {3}'.format(aname, account, len(identities), identities)) print('%i users extracted from VOMS, %i users updated, %i users not in VO' % (syncusers, nbusers, delusers)) diff --git a/atlas/check_voms_admin b/atlas/check_voms_admin index a5d03bb..d940151 100755 --- a/atlas/check_voms_admin +++ b/atlas/check_voms_admin @@ -80,6 +80,7 @@ def get_userid2certs_mapping(scim_url, access_token): d = response.json() if d['itemsPerPage'] == 0: break for user in d['Resources']: + if not user['active']: continue for certificate in user.get('urn:indigo-dc:scim:schemas:IndigoUser', {}).get('certificates', []): #ret.setdefault(user['id'], []).append((certificate['subjectDn'], certificate['issuerDn'], user['emails'][0]['value'])) ret.setdefault(user['id'], []).append(certificate['subjectDn']) @@ -177,7 +178,7 @@ if __name__ == '__main__': account2attrs[account['account']] = {} if account['type'] != 'USER': continue for identity in client.list_identities(account['account']): - if identity['type'] != 'X509': continue + if identity['type'] not in ['X509', 'OIDC']: continue dn2useraccount[identity['identity']] = account['account'] access_token = get_access_token(TOKEN_URL, CLIENT_ID, CLIENT_SECRET) @@ -186,6 +187,8 @@ if __name__ == '__main__': userid2accounts = {} for userid, certs in userid2certs.items(): accounts = set() + if userid in dn2useraccount: + accounts.add(dn2useraccount[userid]) for cert in certs: if cert in dn2useraccount: accounts.add(dn2useraccount[cert]) @@ -230,7 +233,7 @@ if __name__ == '__main__': account2attributes.setdefault(account, {}).update(attributes) # cloud admins from ldap egroups - for cloud in ['ca', 'de', 'es', 'fr', 'it', 'ng', 'nl', 'ru', 't0', 'tw', 'uk', 'us']: + for cloud in ['ca', 'de', 'es', 'fr', 'it', 'ng', 'nl', 'ru', 't0', 'uk', 'us']: egroup = "atlas-support-cloud-{}".format(cloud) if cloud == 'ru': egroup = 'atlas-adc-cloud-ru' From 6ceca9289dae6e10d89a624fc658647fe1340957 Mon Sep 17 00:00:00 2001 From: Petr Vokac Date: Sat, 4 Apr 2026 18:12:08 +0200 Subject: [PATCH 2/3] Skip IAM groups without corresponding Rucio account --- atlas/check_map_voms_roles | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/atlas/check_map_voms_roles b/atlas/check_map_voms_roles index 0a58938..c96686f 100755 --- a/atlas/check_map_voms_roles +++ b/atlas/check_map_voms_roles @@ -127,7 +127,7 @@ def get_account_identities(account): if identity['type'] not in ['X509', 'OIDC']: continue ret.append(identity['identity']) - except rucio.common.exception.AccountNotFound: + except AccountNotFound: return None return ret @@ -177,6 +177,7 @@ if __name__ == '__main__': account_identities = get_account_identities(ACCOUNT_MAP[account]) if account_identities == None: print("missing rucio account for {}".format(ACCOUNT_MAP[account])) + continue access_token = get_access_token(TOKEN_URL, CLIENT_ID, CLIENT_SECRET) for userid in get_groupid_members(SCIM_URL, groupid, access_token): if userid not in userid2email: @@ -253,6 +254,7 @@ if __name__ == '__main__': account_identities = get_account_identities(account) if account_identities == None: print("missing rucio account for {}".format(account)) + continue access_token = get_access_token(TOKEN_URL, CLIENT_ID, CLIENT_SECRET) for userid in get_groupid_members(SCIM_URL, groupid, access_token): if userid not in userid2email: From e78c21831a8e7f334a42d280aff668dba1fab391 Mon Sep 17 00:00:00 2001 From: Petr Vokac Date: Mon, 6 Apr 2026 10:05:51 +0200 Subject: [PATCH 3/3] Fix OIDC issuer identity structure --- atlas/check_map_voms_roles | 17 ++++++++++------- atlas/check_voms | 17 ++++++++++------- atlas/check_voms_admin | 10 ++++++---- 3 files changed, 26 insertions(+), 18 deletions(-) diff --git a/atlas/check_map_voms_roles b/atlas/check_map_voms_roles index c96686f..6b58de1 100755 --- a/atlas/check_map_voms_roles +++ b/atlas/check_map_voms_roles @@ -148,8 +148,9 @@ if __name__ == '__main__': except Exception as error: print("Failed to get also client certificate and key from rucio.cfg") sys.exit(CRITICAL) - TOKEN_URL = 'https://atlas-auth.cern.ch/token' - SCIM_URL = 'https://atlas-auth.cern.ch/scim' + ISSUER = 'https://atlas-auth.cern.ch/' + TOKEN_URL = '{}/token'.format(ISSUER) + SCIM_URL = '{}/scim'.format(ISSUER) try: CLIENT_ID = os.environ['CLIENT_ID'] CLIENT_SECRET = os.environ['CLIENT_SECRET'] @@ -187,10 +188,11 @@ if __name__ == '__main__': if userid not in account_identities: NBUSERS += 1 try: + oidc_identity = "SUB={}, ISS={}".format(userid, ISSUER) if not TEST: - CLIENT.add_identity(account=ACCOUNT_MAP[account], identity=userid, authtype='OIDC', email=userid2email[userid], default=True) + CLIENT.add_identity(account=ACCOUNT_MAP[account], identity=oidc_identity, authtype='OIDC', email=userid2email[userid], default=True) else: - print("CMD: rucio-admin identity add --account {0} --type OIDC --id '{1}' --email '{2}'".format(ACCOUNT_MAP[account], userid, userid2email[userid])) + print("CMD: rucio-admin identity add --account {0} --type OIDC --id '{1}' --email '{2}'".format(ACCOUNT_MAP[account], oidc_identity, userid2email[userid])) print('Identity {0} added to {1}'.format(userid, ACCOUNT_MAP[account])) except Duplicate: pass @@ -264,10 +266,11 @@ if __name__ == '__main__': if userid not in account_identities: NBUSERS += 1 try: + oidc_identity = "SUB={}, ISS={}".format(userid, ISSUER) if not TEST: - CLIENT.add_identity(account=account, identity=userid, authtype='OIDC', email=userid2email[userid], default=True) + CLIENT.add_identity(account=account, identity=oidc_identity, authtype='OIDC', email=userid2email[userid], default=True) else: - print("CMD: rucio-admin identity add --account {0} --type OIDC --id '{1}' --email '{2}'".format(account, userid, userid2email[userid])) + print("CMD: rucio-admin identity add --account {0} --type OIDC --id '{1}' --email '{2}'".format(account, oidc_identity, userid2email[userid])) print('Identity {0} added to {1}'.format(userid, account)) except Duplicate: pass @@ -297,7 +300,7 @@ if __name__ == '__main__': for dn in account_identities: if dn in dns: continue - itype = 'X509' if '=' in dn else 'OIDC' + itype = 'OIDC' if 'ISS=' in dn and 'SUB=' in dn else 'X509' NDUSERS += 1 try: if not TEST: diff --git a/atlas/check_voms b/atlas/check_voms index f28de0c..82f6823 100755 --- a/atlas/check_voms +++ b/atlas/check_voms @@ -221,8 +221,9 @@ if __name__ == '__main__': except Exception as error: print("Failed to get also client certificate and key from rucio.cfg") sys.exit(CRITICAL) - TOKEN_URL = 'https://atlas-auth.cern.ch/token' - SCIM_URL = 'https://atlas-auth.cern.ch/scim' + ISSUER = 'https://atlas-auth.cern.ch/' + TOKEN_URL = '{}/token'.format(ISSUER) + SCIM_URL = '{}/scim'.format(ISSUER) try: CLIENT_ID = os.environ['CLIENT_ID'] CLIENT_SECRET = os.environ['CLIENT_SECRET'] @@ -304,12 +305,13 @@ if __name__ == '__main__': except Duplicate: pass # c) add OIDC identity - if userid not in dns.get(atype, []): + oidc_identity = "SUB={}, ISS={}".format(userid, ISSUER) + if oidc_identity not in dns.get(atype, []): try: if not TEST: - client.add_identity(account=nickname, identity=userid, authtype='OIDC', email=email, default=True) + client.add_identity(account=nickname, identity=oidc_identity, authtype='OIDC', email=email, default=True) else: - print("CMD: rucio-admin identity add --account {0} --type OIDC --id '{1}' --email '{2}'".format(nickname, userid, email)) + print("CMD: rucio-admin identity add --account {0} --type OIDC --id '{1}' --email '{2}'".format(nickname, oidc_identity, email)) nbusers += 1 print('Identity {0} added to {2} account {1}'.format(userid, nickname, atype)) except Duplicate: @@ -336,10 +338,11 @@ if __name__ == '__main__': print("CMD: rucio-admin identity add --account {0} --type X509 --id '{1}' --email '{2}'".format(nickname, dn, email)) nbusers += 1 print('Identity {0} added to USER account {1}'.format(dn, nickname)) + oidc_identity = "SUB={}, ISS={}".format(userid, ISSUER) if not TEST: - client.add_identity(account=nickname, identity=userid, authtype='OIDC', email=email, default=True) + client.add_identity(account=nickname, identity=oidc_identity, authtype='OIDC', email=email, default=True) else: - print("CMD: rucio-admin identity add --account {0} --type OIDC --id '{1}' --email '{2}'".format(nickname, userid, email)) + print("CMD: rucio-admin identity add --account {0} --type OIDC --id '{1}' --email '{2}'".format(nickname, oidc_identity, email)) nbusers += 1 print('Identity {0} added to {2} account {1}'.format(userid, nickname, atype)) scope = 'user.' + nickname diff --git a/atlas/check_voms_admin b/atlas/check_voms_admin index d940151..cef18b6 100755 --- a/atlas/check_voms_admin +++ b/atlas/check_voms_admin @@ -154,8 +154,9 @@ if __name__ == '__main__': except Exception as error: print("Failed to get also client certificate and key from rucio.cfg") sys.exit(CRITICAL) - TOKEN_URL = 'https://atlas-auth.cern.ch/token' - SCIM_URL = 'https://atlas-auth.cern.ch/scim' + ISSUER = 'https://atlas-auth.cern.ch/' + TOKEN_URL = '{}/token'.format(ISSUER) + SCIM_URL = '{}/scim'.format(ISSUER) try: CLIENT_ID = os.environ['CLIENT_ID'] CLIENT_SECRET = os.environ['CLIENT_SECRET'] @@ -187,8 +188,9 @@ if __name__ == '__main__': userid2accounts = {} for userid, certs in userid2certs.items(): accounts = set() - if userid in dn2useraccount: - accounts.add(dn2useraccount[userid]) + oidc_identity = "SUB={}, ISS={}".format(userid, ISSUER) + if oidc_identity in dn2useraccount: + accounts.add(dn2useraccount[oidc_identity]) for cert in certs: if cert in dn2useraccount: accounts.add(dn2useraccount[cert])