Skip to content

ATLAS: OIDC identity synchronization #170

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
vokac wants to merge 3 commits into
base: main
Choose a base branch
from
Open

ATLAS: OIDC identity synchronization #170

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7cc3f7f
OIDC identity synchronization
Apr 4, 2026
6ceca92
Skip IAM groups without corresponding Rucio account
Apr 4, 2026
e78c218
Fix OIDC issuer identity structure
Apr 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 58 additions & 10 deletions atlas/check_map_voms_roles
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import requests.auth

from rucio.client import Client
from rucio.common.config import config_get
from rucio.common.exception import Duplicate, IdentityError
from rucio.common.exception import Duplicate, IdentityError, AccountNotFound

UNKNOWN = 3
CRITICAL = 2
Expand All @@ -46,6 +46,7 @@ def get_access_token(token_url, client_id, client_secret):

def get_userid2certs_mapping(scim_url, access_token):
ret = {}
userid2email = {}

with requests.Session() as session:
session.headers = {
Expand All @@ -62,11 +63,13 @@ def get_userid2certs_mapping(scim_url, access_token):
d = response.json()
if d['itemsPerPage'] == 0: break
for user in d['Resources']:
if not user['active']: continue
userid2email[user['id']] = user['emails'][0]['value']
for certificate in user.get('urn:indigo-dc:scim:schemas:IndigoUser', {}).get('certificates', []):
ret.setdefault(user['id'], []).append((certificate['subjectDn'], certificate['issuerDn'], user['emails'][0]['value']))
start += page_size

return ret
return ret, userid2email

def get_group2id_mapping(scim_url, access_token):
ret = {}
Expand Down Expand Up @@ -119,10 +122,13 @@ def get_groupid_members(scim_url, groupid, access_token):
def get_account_identities(account):
ret = []
client = Client()
for identity in client.list_identities(account):
if identity['type'] != 'X509':
continue
ret.append(identity['identity'])
try:
for identity in client.list_identities(account):
if identity['type'] not in ['X509', 'OIDC']:
continue
ret.append(identity['identity'])
except AccountNotFound:
return None
return ret


Expand Down Expand Up @@ -156,7 +162,7 @@ if __name__ == '__main__':
CLIENT = Client()

access_token = get_access_token(TOKEN_URL, CLIENT_ID, CLIENT_SECRET)
userid2certs = get_userid2certs_mapping(SCIM_URL, access_token)
userid2certs, userid2email = get_userid2certs_mapping(SCIM_URL, access_token)
access_token = get_access_token(TOKEN_URL, CLIENT_ID, CLIENT_SECRET)
group2id = get_group2id_mapping(SCIM_URL, access_token)

Expand All @@ -169,8 +175,28 @@ if __name__ == '__main__':
dns = []
groupid = group2id["atlas/{}".format(account)]
account_identities = get_account_identities(ACCOUNT_MAP[account])
if account_identities == None:
print("missing rucio account for {}".format(ACCOUNT_MAP[account]))
continue
access_token = get_access_token(TOKEN_URL, CLIENT_ID, CLIENT_SECRET)
for userid in get_groupid_members(SCIM_URL, groupid, access_token):
if userid not in userid2email:
print("missing or disabled account for userid {}".format(userid))
continue
dns.append(userid)
if userid not in account_identities:
NBUSERS += 1
try:
if not TEST:
CLIENT.add_identity(account=ACCOUNT_MAP[account], identity=userid, authtype='OIDC', email=userid2email[userid], default=True)
else:
print("CMD: rucio-admin identity add --account {0} --type OIDC --id '{1}' --email '{2}'".format(ACCOUNT_MAP[account], userid, userid2email[userid]))
print('Identity {0} added to {1}'.format(userid, ACCOUNT_MAP[account]))
except Duplicate:
pass
except Exception as error:
print('ERROR {0} not added to {1}: {2}'.format(userid, ACCOUNT_MAP[account], error))
STATUS = WARNING
if userid not in userid2certs:
print("missing user details for userid {}".format(userid))
continue
Expand All @@ -194,12 +220,13 @@ if __name__ == '__main__':
for dn in account_identities:
if dn in dns:
continue
itype = 'X509' if '=' in dn else 'OIDC'
NDUSERS += 1
try:
if not TEST:
CLIENT.del_identity(account=ACCOUNT_MAP[account], identity=dn, authtype='X509')
else:
print("CMD: rucio-admin identity delete --account {0} --type X509 --id '{1}'".format(ACCOUNT_MAP[account], dn))
print("CMD: rucio-admin identity delete --account {0} --type {1} --id '{2}'".format(ACCOUNT_MAP[account], itype, dn))
print('Identity {0} removed from {1}'.format(dn, ACCOUNT_MAP[account]))
except IdentityError:
pass
Expand All @@ -225,8 +252,28 @@ if __name__ == '__main__':
dns = []
groupid = group2id["atlas/{}".format(account)]
account_identities = get_account_identities(account)
if account_identities == None:
print("missing rucio account for {}".format(account))
continue
access_token = get_access_token(TOKEN_URL, CLIENT_ID, CLIENT_SECRET)
for userid in get_groupid_members(SCIM_URL, groupid, access_token):
if userid not in userid2email:
print("missing or disabled account for userid {}".format(userid))
continue
dns.append(userid)
if userid not in account_identities:
NBUSERS += 1
try:
if not TEST:
CLIENT.add_identity(account=account, identity=userid, authtype='OIDC', email=userid2email[userid], default=True)
else:
print("CMD: rucio-admin identity add --account {0} --type OIDC --id '{1}' --email '{2}'".format(account, userid, userid2email[userid]))
print('Identity {0} added to {1}'.format(userid, account))
except Duplicate:
pass
except Exception as error:
print('ERROR {0} not added to {1}: {2}'.format(userid, ACCOUNT_MAP[account], error))
STATUS = WARNING
if userid not in userid2certs:
print("missing user details for userid {}".format(userid))
continue
Expand All @@ -250,12 +297,13 @@ if __name__ == '__main__':
for dn in account_identities:
if dn in dns:
continue
itype = 'X509' if '=' in dn else 'OIDC'
NDUSERS += 1
try:
if not TEST:
CLIENT.del_identity(account=account, identity=dn, authtype='X509')
CLIENT.del_identity(account=account, identity=dn, authtype=itype)
else:
print("CMD: rucio-admin identity delete --account {0} --type X509 --id '{1}'".format(account, dn))
print("CMD: rucio-admin identity delete --account {0} --type {1} --id '{2}'".format(account, itype, dn))
print('Identity {0} removed from {1}'.format(dn, account))
except IdentityError:
pass
Expand Down
51 changes: 38 additions & 13 deletions atlas/check_voms
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,30 +54,36 @@ LDAP_PAGE_SIZE = 1000
def get_accounts_identities_fast():
from rucio.db.sqla.session import get_session
session = get_session()
query = '''select b.identity, a.account, a.email, a.account_type from atlas_rucio.accounts a, atlas_rucio.account_map b where a.account=b.account and identity_type='X509' '''
query = '''select b.identity, a.account, a.email, a.account_type, b.identity_type from atlas_rucio.accounts a, atlas_rucio.account_map b where a.account=b.account'''
dns = {}
account2identities = {}
try:
result = session.execute(query)
for dn, account, email, atype in result:
dns.setdefault(atype, {})[dn] = (account, email)
return dns
for identity, account, email, atype, itype in result:
account2identities.setdefault(account, []).append((itype, identity))
if itype not in ['X509', 'OIDC']:
continue
dns.setdefault(atype, {})[identity] = (account, email)
return dns, account2identities
except Exception as error:
print(error)


def get_accounts_identities_slow():
dns = {}
account2identities = {}
client = Client()
for account in client.list_accounts():
#if account['type'] != 'USER':
# continue
for identity in client.list_identities(account['account']):
if identity['type'] != 'X509':
account2identities.setdefault(account['account'], []).append((identity['type'], identity['identity']))
if identity['type'] not in ['X509', 'OIDC']:
continue
if identity['identity'] in dns:
print("Duplicate identity for users {} and {}: {}".format(dns[identity['identity']][0], account['account'], identity['identity']))
if account['type'] == 'USER' and identity['identity'] in dns.get(account['type'], {}):
print("Duplicate identity for users {} and {}: {}".format(dns[account['type']][identity['identity']][0], account['account'], identity['identity']))
dns.setdefault(account['type'], {})[identity['identity']] = (account['account'], account['email'])
return dns
return dns, account2identities


get_accounts_identities = get_accounts_identities_slow
Expand Down Expand Up @@ -189,11 +195,12 @@ def get_users(scim_url, access_token):
d = response.json()
if d['itemsPerPage'] == 0: break
for user in d['Resources']:
if not user['active']: continue
certs = []
for certificate in user.get('urn:indigo-dc:scim:schemas:IndigoUser', {}).get('certificates', []):
certs.append(certificate['subjectDn'])
groups = [x['display'] for x in user.get('groups', [])]
ret.append((user['userName'], user['active'], user['emails'][0]['value'], certs, groups))
ret.append((user['id'], user['userName'], user['active'], user['emails'][0]['value'], certs, groups))
start += page_size

return ret
Expand Down Expand Up @@ -229,13 +236,13 @@ if __name__ == '__main__':
client = Client()
accounts = {account['account']: account for account in client.list_accounts()}
scopes = [_ for _ in client.list_scopes()]
dns = get_accounts_identities()
dns, account2identities = get_accounts_identities()
ldap_accounts = get_ldap_identities()

# synchronize accounts, identities and scopes
synced_accounts = []
access_token = get_access_token(TOKEN_URL, CLIENT_ID, CLIENT_SECRET)
for nickname, active, email, certs, groups in get_users(SCIM_URL, access_token):
for userid, nickname, active, email, certs, groups in get_users(SCIM_URL, access_token):
if 'atlas' not in groups:
print("Skipping user {0} because it is not member of ATLAS groups".format(nickname))
continue
Expand Down Expand Up @@ -296,10 +303,21 @@ if __name__ == '__main__':
print('Identity {0} added to {2} account {1}'.format(dn, nickname, atype))
except Duplicate:
pass
# c) add OIDC identity
if userid not in dns.get(atype, []):
try:
if not TEST:
client.add_identity(account=nickname, identity=userid, authtype='OIDC', email=email, default=True)
Copy link
Copy Markdown
Contributor

@dchristidis dchristidis Apr 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OIDC identities in Rucio look like this:

SUB=01234567-89ab-cdef-0123-456789abcdef, ISS=https://atlas-auth.cern.ch/

else:
print("CMD: rucio-admin identity add --account {0} --type OIDC --id '{1}' --email '{2}'".format(nickname, userid, email))
nbusers += 1
print('Identity {0} added to {2} account {1}'.format(userid, nickname, atype))
except Duplicate:
pass
if atype == 'USER':
scope = 'user.' + nickname

elif active and nickname in ldap_accounts and len(certs) > 0:
elif active and nickname in ldap_accounts:
# no rucio account exists for this nickname
if email.lower() not in [mail.lower() for mail in ldap_accounts[nickname]['mails']]:
print('Account %s (%s) without matching email in LDAP does not exist. To create it : rucio-admin account add --type USER --email %s %s' % (nickname, ldap_accounts[nickname]['type'], email, nickname))
Expand All @@ -318,6 +336,12 @@ if __name__ == '__main__':
print("CMD: rucio-admin identity add --account {0} --type X509 --id '{1}' --email '{2}'".format(nickname, dn, email))
nbusers += 1
print('Identity {0} added to USER account {1}'.format(dn, nickname))
if not TEST:
client.add_identity(account=nickname, identity=userid, authtype='OIDC', email=email, default=True)
else:
print("CMD: rucio-admin identity add --account {0} --type OIDC --id '{1}' --email '{2}'".format(nickname, userid, email))
nbusers += 1
print('Identity {0} added to {2} account {1}'.format(userid, nickname, atype))
scope = 'user.' + nickname
except Exception:
print('Failed to add new account %s (%s)' % (nickname, ldap_accounts[nickname]['type']))
Expand All @@ -343,8 +367,9 @@ if __name__ == '__main__':
aname = account['account']
if aname not in synced_accounts:
delusers += 1
identities = account2identities.get(aname, [])
if TEST:
print('User account {0} is no longer member VO, details {1}'.format(aname, account))
print('User account {0} is no longer member VO, details {1}, identities({2}) {3}'.format(aname, account, len(identities), identities))

print('%i users extracted from VOMS, %i users updated, %i users not in VO' % (syncusers, nbusers, delusers))

Expand Down
7 changes: 5 additions & 2 deletions atlas/check_voms_admin
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ def get_userid2certs_mapping(scim_url, access_token):
d = response.json()
if d['itemsPerPage'] == 0: break
for user in d['Resources']:
if not user['active']: continue
for certificate in user.get('urn:indigo-dc:scim:schemas:IndigoUser', {}).get('certificates', []):
#ret.setdefault(user['id'], []).append((certificate['subjectDn'], certificate['issuerDn'], user['emails'][0]['value']))
ret.setdefault(user['id'], []).append(certificate['subjectDn'])
Expand Down Expand Up @@ -177,7 +178,7 @@ if __name__ == '__main__':
account2attrs[account['account']] = {}
if account['type'] != 'USER': continue
for identity in client.list_identities(account['account']):
if identity['type'] != 'X509': continue
if identity['type'] not in ['X509', 'OIDC']: continue
dn2useraccount[identity['identity']] = account['account']

access_token = get_access_token(TOKEN_URL, CLIENT_ID, CLIENT_SECRET)
Expand All @@ -186,6 +187,8 @@ if __name__ == '__main__':
userid2accounts = {}
for userid, certs in userid2certs.items():
accounts = set()
if userid in dn2useraccount:
accounts.add(dn2useraccount[userid])
for cert in certs:
if cert in dn2useraccount:
accounts.add(dn2useraccount[cert])
Expand Down Expand Up @@ -230,7 +233,7 @@ if __name__ == '__main__':
account2attributes.setdefault(account, {}).update(attributes)

# cloud admins from ldap egroups
for cloud in ['ca', 'de', 'es', 'fr', 'it', 'ng', 'nl', 'ru', 't0', 'tw', 'uk', 'us']:
for cloud in ['ca', 'de', 'es', 'fr', 'it', 'ng', 'nl', 'ru', 't0', 'uk', 'us']:
egroup = "atlas-support-cloud-{}".format(cloud)
if cloud == 'ru':
egroup = 'atlas-adc-cloud-ru'
Expand Down