Skip to content

Support multiple reviewer/area chair roles per submission #3009

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
melisabok wants to merge 18 commits into
base: master
Choose a base branch
from
Open

Support multiple reviewer/area chair roles per submission #3009

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
a413fd9
parametrize reviewers_name to create submission group invitation
melisabok Apr 9, 2026
69f0d8c
close deadline to create reviewers groups
melisabok Apr 9, 2026
e6652b1
activate submission group invitations
melisabok Apr 9, 2026
fe5bd44
set a submission committee name to deploy assignments
melisabok Apr 9, 2026
3ebe50c
test area chair roles
melisabok Apr 9, 2026
7b7cc39
create review form
melisabok Apr 9, 2026
2194e50
introduce submission_reviewer_roles to parametrize reviewer groups
melisabok Apr 10, 2026
fb1ad51
refactor
melisabok Apr 10, 2026
4a22048
do not store recruitment settings of every role in the domain
melisabok Apr 10, 2026
9a75d53
fix tests
melisabok Apr 10, 2026
ea0bbca
fix test
melisabok Apr 10, 2026
fc5c66b
more refactoring
melisabok Apr 10, 2026
2ff7d8a
add test
melisabok Apr 14, 2026
79760c3
Merge branch 'master' into fix/parametrize-reviewers-roles
melisabok Apr 23, 2026
15cf255
make sure the role is set for every review stage
melisabok Apr 23, 2026
31c7d21
remove commented code
melisabok Apr 23, 2026
92d5635
fix test
melisabok Apr 23, 2026
6636a92
remove double call
melisabok Apr 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions openreview/conference/helpers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ def set_initial_stages_v2(request_forum, venue):
venue.review_stage = openreview.stages.ReviewStage(
start_date = (submission_second_due_date if submission_second_due_date else submission_due_date) + datetime.timedelta(weeks=1),
allow_de_anonymization = (request_forum.content.get('author_and_reviewer_anonymity', {}).get('value', 'No anonymity') == 'No anonymity'),
submission_reviewer_roles = [venue.reviewers_name],
)

def get_conference(client, request_form_id, support_user='OpenReview.net/Support', setup=False):
Expand Down Expand Up @@ -128,8 +129,10 @@ def get_conference(client, request_form_id, support_user='OpenReview.net/Support
venue.senior_area_chairs_name = venue.senior_area_chair_roles[0]
venue.area_chair_roles = note.content.get('area_chair_roles', ['Area_Chairs'])
venue.area_chairs_name = venue.area_chair_roles[0]
venue.submission_area_chair_roles = [venue.area_chairs_name]
venue.reviewer_roles = note.content.get('reviewer_roles', ['Reviewers'])
venue.reviewers_name = venue.reviewer_roles[0]
venue.submission_reviewer_roles = [venue.reviewers_name]
venue.allow_gurobi_solver = venue_content.get('allow_gurobi_solver', {}).get('value', False)
venue.submission_human_verification = venue_content.get('submission_human_verification', {}).get('value')
venue.submission_license = note.content.get('submission_license', ['CC BY 4.0'])
Expand Down Expand Up @@ -176,6 +179,7 @@ def get_conference(client, request_form_id, support_user='OpenReview.net/Support

venue.submission_stage = get_submission_stage(note, venue)
venue.review_stage = get_review_stage(note)
venue.review_stage.submission_reviewer_roles = [venue.reviewers_name]
if 'bid_due_date' in note.content:
venue.bid_stages = get_bid_stages(note, reviewers_id=venue.get_reviewers_id(), area_chairs_id=venue.get_area_chairs_id(), senior_area_chairs_id=venue.get_senior_area_chairs_id())
venue.meta_review_stage = get_meta_review_stage(note)
Expand Down
32 changes: 19 additions & 13 deletions openreview/stages/venue_stages.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ class IdentityReaders(Enum):
REVIEWERS_ASSIGNED = 6

@classmethod
def get_readers(self, conference, number, identity_readers):
def get_readers(self, conference, number, identity_readers, reviewers_name=None, area_chairs_name=None):
readers = [conference.id]
if self.PROGRAM_CHAIRS in identity_readers:
readers.append(conference.get_program_chairs_id())
Expand All @@ -25,13 +25,13 @@ def get_readers(self, conference, number, identity_readers):
if self.SENIOR_AREA_CHAIRS_ASSIGNED in identity_readers:
readers.append(conference.get_senior_area_chairs_id(number))
if self.AREA_CHAIRS in identity_readers:
readers.append(conference.get_area_chairs_id())
readers.append(conference.get_area_chairs_id(name=area_chairs_name))
if self.AREA_CHAIRS_ASSIGNED in identity_readers:
readers.append(conference.get_area_chairs_id(number))
readers.append(conference.get_area_chairs_id(number, name=area_chairs_name))
if self.REVIEWERS in identity_readers:
readers.append(conference.get_reviewers_id())
readers.append(conference.get_reviewers_id(name=reviewers_name))
if self.REVIEWERS_ASSIGNED in identity_readers:
readers.append(conference.get_reviewers_id(number))
readers.append(conference.get_reviewers_id(number, name=reviewers_name))
return readers

class AuthorReorder(Enum):
Expand Down Expand Up @@ -668,6 +668,7 @@ def __init__(self,
child_invitations_name = 'Official_Review',
description = None,
submission_source=None,
submission_reviewer_roles=None,
):

self.start_date = start_date
Expand All @@ -691,18 +692,23 @@ def __init__(self,
self.preprocess_path = None
self.description = description
self.submission_source = submission_source
self.submission_reviewer_roles = submission_reviewer_roles

def _get_reviewer_roles(self, conference):
return self.submission_reviewer_roles or [conference.reviewers_name]

def _get_reviewer_readers(self, conference, number, review_signature=None):
reviewer_roles = self._get_reviewer_roles(conference)
if self.release_to_reviewers is ReviewStage.Readers.REVIEWERS:
return conference.get_reviewers_id()
return [conference.get_reviewers_id(name=name) for name in reviewer_roles]
if self.release_to_reviewers is ReviewStage.Readers.REVIEWERS_ASSIGNED:
return conference.get_reviewers_id(number = number)
return [conference.get_reviewers_id(number=number, name=name) for name in reviewer_roles]
if self.release_to_reviewers is ReviewStage.Readers.REVIEWERS_SUBMITTED:
return conference.get_reviewers_id(number = number) + '/Submitted'
return [conference.get_reviewers_id(number=number, name=name) + '/Submitted' for name in reviewer_roles]
if self.release_to_reviewers is ReviewStage.Readers.REVIEWER_SIGNATURE:
if review_signature:
return review_signature
return '{signatures}'
return [review_signature]
return ['{signatures}']
raise openreview.OpenReviewException('Unrecognized readers option')

def get_readers(self, conference, number, review_signature=None):
Expand All @@ -718,7 +724,7 @@ def get_readers(self, conference, number, review_signature=None):
if conference.use_area_chairs:
readers.append(conference.get_area_chairs_id(number = number))

readers.append(self._get_reviewer_readers(conference, number, review_signature))
readers.extend(self._get_reviewer_readers(conference, number, review_signature))

## Workaround to make the reviews visible to the author of the review when reviewers submitted is selected
if self.release_to_reviewers is ReviewStage.Readers.REVIEWERS_SUBMITTED and review_signature:
Expand Down Expand Up @@ -749,8 +755,8 @@ def get_signatures(self, conference, number):
if self.allow_de_anonymization:
return ['~.*']

return [conference.get_anon_reviewer_id(number=number, anon_id='.*')]
return [conference.get_anon_reviewer_id(number=number, anon_id='.*', name=name) for name in self._get_reviewer_roles(conference)]

def get_content(self, api_version='2', conference=None):

content = deepcopy(default_content.review_v2)
Expand Down
53 changes: 43 additions & 10 deletions openreview/venue/group.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,22 +76,22 @@ def build_groups(self, venue_id):

return groups

def get_reviewer_identity_readers(self, number):
return openreview.stages.IdentityReaders.get_readers(self.venue, number, self.venue.reviewer_identity_readers)
def get_reviewer_identity_readers(self, number, name=None):
return openreview.stages.IdentityReaders.get_readers(self.venue, number, self.venue.reviewer_identity_readers, reviewers_name=name)

def get_area_chair_identity_readers(self, number):
return openreview.stages.IdentityReaders.get_readers(self.venue, number, self.venue.area_chair_identity_readers)
def get_area_chair_identity_readers(self, number, name=None):
return openreview.stages.IdentityReaders.get_readers(self.venue, number, self.venue.area_chair_identity_readers, area_chairs_name=name)

def get_senior_area_chair_identity_readers(self, number):
return openreview.stages.IdentityReaders.get_readers(self.venue, number, self.venue.senior_area_chair_identity_readers)

def get_reviewer_paper_group_readers(self, number):
def get_reviewer_paper_group_readers(self, number, name=None):
readers=[self.venue.id]
if self.venue.use_senior_area_chairs:
readers.append(self.venue.get_senior_area_chairs_id(number))
if self.venue.use_area_chairs:
readers.append(self.venue.get_area_chairs_id(number))
readers.append(self.venue.get_reviewers_id(number))
readers.append(self.venue.get_reviewers_id(number, name=name))
return readers

def get_reviewer_paper_group_writers(self, number):
Expand All @@ -103,11 +103,11 @@ def get_reviewer_paper_group_writers(self, number):
return readers


def get_area_chair_paper_group_readers(self, number):
def get_area_chair_paper_group_readers(self, number, name=None):
readers=[self.venue.id, self.venue.get_program_chairs_id()]
if self.venue.use_senior_area_chairs:
readers.append(self.venue.get_senior_area_chairs_id(number))
readers.append(self.venue.get_area_chairs_id(number))
readers.append(self.venue.get_area_chairs_id(number, name=name))
if openreview.stages.IdentityReaders.REVIEWERS_ASSIGNED in self.venue.area_chair_identity_readers:
readers.append(self.venue.get_reviewers_id(number))
return readers
Comment on lines 109 to 113
Copy link

Copilot AI Apr 23, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In per_role layouts, self.venue.get_reviewers_id(number) can point to a per-submission group that is never created (e.g. .../Submission1/Reviewers when submissions use .../Submission1/Expert_Reviewers + .../Submission1/Technical_Reviewers). This means area-chair paper-group readers may reference a non-existent group and won't actually grant access to the assigned reviewers. Consider deriving the correct per-submission reviewer group(s) from name (e.g., map AC role -> corresponding submission_reviewer_roles entry) or include all venue.submission_reviewer_roles for that paper when REVIEWERS_ASSIGNED is enabled.

Copilot uses AI. Check for mistakes.
Expand Down Expand Up @@ -168,6 +168,7 @@ def create_venue_group(self):
'reviewers_id': { 'value': self.venue.get_reviewers_id() },
'reviewers_name': { 'value': self.venue.reviewers_name },
'reviewer_roles': { 'value': self.venue.reviewer_roles },
'submission_reviewer_roles': { 'value': self.venue.submission_reviewer_roles },
'reviewers_anon_name': { 'value': self.venue.get_anon_reviewers_name() },
'reviewers_submitted_name': { 'value': 'Submitted' },
'reviewers_custom_max_papers_id': { 'value': self.venue.get_custom_max_papers_id(self.venue.get_reviewers_id()) },
Expand Down Expand Up @@ -239,6 +240,7 @@ def create_venue_group(self):

if self.venue.use_area_chairs:
content['area_chair_roles'] = { 'value': self.venue.area_chair_roles }
content['submission_area_chair_roles'] = { 'value': self.venue.submission_area_chair_roles }
content['area_chairs_id'] = { 'value': self.venue.get_area_chairs_id() }
content['area_chairs_name'] = { 'value': self.venue.area_chairs_name }
content['area_chairs_anon_name'] = { 'value': self.venue.get_anon_area_chairs_name() }
Expand Down Expand Up @@ -511,7 +513,7 @@ def create_reviewers_group(self):
if self.venue.use_area_chairs:
area_chairs_id = self.venue.get_committee_id(self.venue.area_chair_roles[index]) if index < len(self.venue.area_chair_roles) else self.venue.get_area_chairs_id()
additional_readers.append(area_chairs_id)

self.client.post_group_edit(
invitation=f'{self.openreview_template}/-/Committee_Group',
signatures=[self.openreview_template],
Expand All @@ -521,11 +523,26 @@ def create_reviewers_group(self):
'committee_role': { 'value': 'reviewers' },
'committee_pretty_name': { 'value': pretty_name },
'committee_anon_name': { 'value': self.venue.get_anon_committee_name(role) },
'committee_submitted_name': { 'value': 'Submitted' },
'committee_submitted_name': { 'value': 'Submitted' },
'additional_readers': { 'value': additional_readers }
},
await_process=True
)

# If there are multiple reviewer roles and the category name is not
# itself one of the roles, create an umbrella group containing all
# role groups as members.
if len(self.venue.reviewer_roles) > 1 and self.venue.reviewers_name not in self.venue.reviewer_roles:
umbrella_id = self.venue.get_reviewers_id()
role_group_ids = [self.venue.get_committee_id(role) for role in self.venue.reviewer_roles]
self.post_group(Group(
id=umbrella_id,
readers=[venue_id, umbrella_id],
writers=[venue_id],
signatures=[venue_id],
signatories=[venue_id],
members=role_group_ids
))
return

for index, role in enumerate(self.venue.reviewer_roles):
Expand Down Expand Up @@ -575,6 +592,22 @@ def create_area_chairs_group(self):
},
await_process=True
)

# If there are multiple area chair roles and the category name is not
# itself one of the roles, create an umbrella group containing all
# role groups as members.
if len(self.venue.area_chair_roles) > 1 and self.venue.area_chairs_name not in self.venue.area_chair_roles:
venue_id = self.venue.id
umbrella_id = self.venue.get_area_chairs_id()
role_group_ids = [self.venue.get_committee_id(role) for role in self.venue.area_chair_roles]
self.post_group(Group(
id=umbrella_id,
readers=[venue_id, umbrella_id],
writers=[venue_id],
signatures=[venue_id],
signatories=[venue_id],
members=role_group_ids
))
return

venue_id = self.venue.id
Expand Down
2 changes: 2 additions & 0 deletions openreview/venue/helpers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ def get_venue(client, venue_id, support_user='OpenReview.net/Support'):
venue.area_chairs_name = domain.content.get('area_chairs_name', {}).get('value', venue.area_chair_roles[0])
venue.reviewer_roles = domain.content.get('reviewer_roles', {}).get('value', ['Reviewers'])
venue.reviewers_name = domain.content.get('reviewers_name', {}).get('value', venue.reviewer_roles[0])
venue.submission_reviewer_roles = domain.content.get('submission_reviewer_roles', {}).get('value', [venue.reviewers_name])
venue.submission_area_chair_roles = domain.content.get('submission_area_chair_roles', {}).get('value', [venue.area_chairs_name])
venue.allow_gurobi_solver = domain.content.get('allow_gurobi_solver', {}).get('value', False)
venue.submission_human_verification = domain.content.get('submission_human_verification', {}).get('value')

Expand Down
Loading