diff --git a/openreview/conference/helpers.py b/openreview/conference/helpers.py index 0c7c3c6913..23c691e16f 100644 --- a/openreview/conference/helpers.py +++ b/openreview/conference/helpers.py @@ -95,6 +95,7 @@ def set_initial_stages_v2(request_forum, venue): venue.review_stage = openreview.stages.ReviewStage( start_date = (submission_second_due_date if submission_second_due_date else submission_due_date) + datetime.timedelta(weeks=1), allow_de_anonymization = (request_forum.content.get('author_and_reviewer_anonymity', {}).get('value', 'No anonymity') == 'No anonymity'), + submission_reviewer_roles = [venue.reviewers_name], ) def get_conference(client, request_form_id, support_user='OpenReview.net/Support', setup=False): @@ -128,8 +129,10 @@ def get_conference(client, request_form_id, support_user='OpenReview.net/Support venue.senior_area_chairs_name = venue.senior_area_chair_roles[0] venue.area_chair_roles = note.content.get('area_chair_roles', ['Area_Chairs']) venue.area_chairs_name = venue.area_chair_roles[0] + venue.submission_area_chair_roles = [venue.area_chairs_name] venue.reviewer_roles = note.content.get('reviewer_roles', ['Reviewers']) venue.reviewers_name = venue.reviewer_roles[0] + venue.submission_reviewer_roles = [venue.reviewers_name] venue.allow_gurobi_solver = venue_content.get('allow_gurobi_solver', {}).get('value', False) venue.submission_human_verification = venue_content.get('submission_human_verification', {}).get('value') venue.submission_license = note.content.get('submission_license', ['CC BY 4.0']) @@ -176,6 +179,7 @@ def get_conference(client, request_form_id, support_user='OpenReview.net/Support venue.submission_stage = get_submission_stage(note, venue) venue.review_stage = get_review_stage(note) + venue.review_stage.submission_reviewer_roles = [venue.reviewers_name] if 'bid_due_date' in note.content: venue.bid_stages = get_bid_stages(note, reviewers_id=venue.get_reviewers_id(), area_chairs_id=venue.get_area_chairs_id(), senior_area_chairs_id=venue.get_senior_area_chairs_id()) venue.meta_review_stage = get_meta_review_stage(note) diff --git a/openreview/stages/venue_stages.py b/openreview/stages/venue_stages.py index fbae9b0878..69e7bff617 100644 --- a/openreview/stages/venue_stages.py +++ b/openreview/stages/venue_stages.py @@ -16,7 +16,7 @@ class IdentityReaders(Enum): REVIEWERS_ASSIGNED = 6 @classmethod - def get_readers(self, conference, number, identity_readers): + def get_readers(self, conference, number, identity_readers, reviewers_name=None, area_chairs_name=None): readers = [conference.id] if self.PROGRAM_CHAIRS in identity_readers: readers.append(conference.get_program_chairs_id()) @@ -25,13 +25,13 @@ def get_readers(self, conference, number, identity_readers): if self.SENIOR_AREA_CHAIRS_ASSIGNED in identity_readers: readers.append(conference.get_senior_area_chairs_id(number)) if self.AREA_CHAIRS in identity_readers: - readers.append(conference.get_area_chairs_id()) + readers.append(conference.get_area_chairs_id(name=area_chairs_name)) if self.AREA_CHAIRS_ASSIGNED in identity_readers: - readers.append(conference.get_area_chairs_id(number)) + readers.append(conference.get_area_chairs_id(number, name=area_chairs_name)) if self.REVIEWERS in identity_readers: - readers.append(conference.get_reviewers_id()) + readers.append(conference.get_reviewers_id(name=reviewers_name)) if self.REVIEWERS_ASSIGNED in identity_readers: - readers.append(conference.get_reviewers_id(number)) + readers.append(conference.get_reviewers_id(number, name=reviewers_name)) return readers class AuthorReorder(Enum): @@ -668,6 +668,7 @@ def __init__(self, child_invitations_name = 'Official_Review', description = None, submission_source=None, + submission_reviewer_roles=None, ): self.start_date = start_date @@ -691,18 +692,23 @@ def __init__(self, self.preprocess_path = None self.description = description self.submission_source = submission_source + self.submission_reviewer_roles = submission_reviewer_roles + + def _get_reviewer_roles(self, conference): + return self.submission_reviewer_roles or [conference.reviewers_name] def _get_reviewer_readers(self, conference, number, review_signature=None): + reviewer_roles = self._get_reviewer_roles(conference) if self.release_to_reviewers is ReviewStage.Readers.REVIEWERS: - return conference.get_reviewers_id() + return [conference.get_reviewers_id(name=name) for name in reviewer_roles] if self.release_to_reviewers is ReviewStage.Readers.REVIEWERS_ASSIGNED: - return conference.get_reviewers_id(number = number) + return [conference.get_reviewers_id(number=number, name=name) for name in reviewer_roles] if self.release_to_reviewers is ReviewStage.Readers.REVIEWERS_SUBMITTED: - return conference.get_reviewers_id(number = number) + '/Submitted' + return [conference.get_reviewers_id(number=number, name=name) + '/Submitted' for name in reviewer_roles] if self.release_to_reviewers is ReviewStage.Readers.REVIEWER_SIGNATURE: if review_signature: - return review_signature - return '{signatures}' + return [review_signature] + return ['{signatures}'] raise openreview.OpenReviewException('Unrecognized readers option') def get_readers(self, conference, number, review_signature=None): @@ -718,7 +724,7 @@ def get_readers(self, conference, number, review_signature=None): if conference.use_area_chairs: readers.append(conference.get_area_chairs_id(number = number)) - readers.append(self._get_reviewer_readers(conference, number, review_signature)) + readers.extend(self._get_reviewer_readers(conference, number, review_signature)) ## Workaround to make the reviews visible to the author of the review when reviewers submitted is selected if self.release_to_reviewers is ReviewStage.Readers.REVIEWERS_SUBMITTED and review_signature: @@ -749,8 +755,8 @@ def get_signatures(self, conference, number): if self.allow_de_anonymization: return ['~.*'] - return [conference.get_anon_reviewer_id(number=number, anon_id='.*')] - + return [conference.get_anon_reviewer_id(number=number, anon_id='.*', name=name) for name in self._get_reviewer_roles(conference)] + def get_content(self, api_version='2', conference=None): content = deepcopy(default_content.review_v2) diff --git a/openreview/venue/group.py b/openreview/venue/group.py index aa5058f629..d8de94c6b6 100644 --- a/openreview/venue/group.py +++ b/openreview/venue/group.py @@ -76,22 +76,22 @@ def build_groups(self, venue_id): return groups - def get_reviewer_identity_readers(self, number): - return openreview.stages.IdentityReaders.get_readers(self.venue, number, self.venue.reviewer_identity_readers) + def get_reviewer_identity_readers(self, number, name=None): + return openreview.stages.IdentityReaders.get_readers(self.venue, number, self.venue.reviewer_identity_readers, reviewers_name=name) - def get_area_chair_identity_readers(self, number): - return openreview.stages.IdentityReaders.get_readers(self.venue, number, self.venue.area_chair_identity_readers) + def get_area_chair_identity_readers(self, number, name=None): + return openreview.stages.IdentityReaders.get_readers(self.venue, number, self.venue.area_chair_identity_readers, area_chairs_name=name) def get_senior_area_chair_identity_readers(self, number): return openreview.stages.IdentityReaders.get_readers(self.venue, number, self.venue.senior_area_chair_identity_readers) - def get_reviewer_paper_group_readers(self, number): + def get_reviewer_paper_group_readers(self, number, name=None): readers=[self.venue.id] if self.venue.use_senior_area_chairs: readers.append(self.venue.get_senior_area_chairs_id(number)) if self.venue.use_area_chairs: readers.append(self.venue.get_area_chairs_id(number)) - readers.append(self.venue.get_reviewers_id(number)) + readers.append(self.venue.get_reviewers_id(number, name=name)) return readers def get_reviewer_paper_group_writers(self, number): @@ -103,11 +103,11 @@ def get_reviewer_paper_group_writers(self, number): return readers - def get_area_chair_paper_group_readers(self, number): + def get_area_chair_paper_group_readers(self, number, name=None): readers=[self.venue.id, self.venue.get_program_chairs_id()] if self.venue.use_senior_area_chairs: readers.append(self.venue.get_senior_area_chairs_id(number)) - readers.append(self.venue.get_area_chairs_id(number)) + readers.append(self.venue.get_area_chairs_id(number, name=name)) if openreview.stages.IdentityReaders.REVIEWERS_ASSIGNED in self.venue.area_chair_identity_readers: readers.append(self.venue.get_reviewers_id(number)) return readers @@ -168,6 +168,7 @@ def create_venue_group(self): 'reviewers_id': { 'value': self.venue.get_reviewers_id() }, 'reviewers_name': { 'value': self.venue.reviewers_name }, 'reviewer_roles': { 'value': self.venue.reviewer_roles }, + 'submission_reviewer_roles': { 'value': self.venue.submission_reviewer_roles }, 'reviewers_anon_name': { 'value': self.venue.get_anon_reviewers_name() }, 'reviewers_submitted_name': { 'value': 'Submitted' }, 'reviewers_custom_max_papers_id': { 'value': self.venue.get_custom_max_papers_id(self.venue.get_reviewers_id()) }, @@ -239,6 +240,7 @@ def create_venue_group(self): if self.venue.use_area_chairs: content['area_chair_roles'] = { 'value': self.venue.area_chair_roles } + content['submission_area_chair_roles'] = { 'value': self.venue.submission_area_chair_roles } content['area_chairs_id'] = { 'value': self.venue.get_area_chairs_id() } content['area_chairs_name'] = { 'value': self.venue.area_chairs_name } content['area_chairs_anon_name'] = { 'value': self.venue.get_anon_area_chairs_name() } @@ -271,6 +273,7 @@ def create_venue_group(self): if self.venue.review_stage: content['review_name'] = { 'value': self.venue.review_stage.name } + content['review_names'] = { 'value': [self.venue.review_stage.name] + [f'{role}_Review' for role in self.venue.submission_reviewer_roles[1:]] } content['review_rating'] = { 'value': self.venue.review_stage.rating_field_name } content['review_confidence'] = { 'value': self.venue.review_stage.confidence_field_name } content['review_email_pcs'] = { 'value': self.venue.review_stage.email_pcs } @@ -511,7 +514,7 @@ def create_reviewers_group(self): if self.venue.use_area_chairs: area_chairs_id = self.venue.get_committee_id(self.venue.area_chair_roles[index]) if index < len(self.venue.area_chair_roles) else self.venue.get_area_chairs_id() additional_readers.append(area_chairs_id) - + self.client.post_group_edit( invitation=f'{self.openreview_template}/-/Committee_Group', signatures=[self.openreview_template], @@ -521,11 +524,26 @@ def create_reviewers_group(self): 'committee_role': { 'value': 'reviewers' }, 'committee_pretty_name': { 'value': pretty_name }, 'committee_anon_name': { 'value': self.venue.get_anon_committee_name(role) }, - 'committee_submitted_name': { 'value': 'Submitted' }, + 'committee_submitted_name': { 'value': 'Submitted' }, 'additional_readers': { 'value': additional_readers } }, await_process=True ) + + # If there are multiple reviewer roles and the category name is not + # itself one of the roles, create an umbrella group containing all + # role groups as members. + if len(self.venue.reviewer_roles) > 1 and self.venue.reviewers_name not in self.venue.reviewer_roles: + umbrella_id = self.venue.get_reviewers_id() + role_group_ids = [self.venue.get_committee_id(role) for role in self.venue.reviewer_roles] + self.post_group(Group( + id=umbrella_id, + readers=[venue_id, umbrella_id], + writers=[venue_id], + signatures=[venue_id], + signatories=[venue_id], + members=role_group_ids + )) return for index, role in enumerate(self.venue.reviewer_roles): @@ -575,6 +593,22 @@ def create_area_chairs_group(self): }, await_process=True ) + + # If there are multiple area chair roles and the category name is not + # itself one of the roles, create an umbrella group containing all + # role groups as members. + if len(self.venue.area_chair_roles) > 1 and self.venue.area_chairs_name not in self.venue.area_chair_roles: + venue_id = self.venue.id + umbrella_id = self.venue.get_area_chairs_id() + role_group_ids = [self.venue.get_committee_id(role) for role in self.venue.area_chair_roles] + self.post_group(Group( + id=umbrella_id, + readers=[venue_id, umbrella_id], + writers=[venue_id], + signatures=[venue_id], + signatories=[venue_id], + members=role_group_ids + )) return venue_id = self.venue.id diff --git a/openreview/venue/helpers.py b/openreview/venue/helpers.py index 02d9e0d80c..04e2073141 100644 --- a/openreview/venue/helpers.py +++ b/openreview/venue/helpers.py @@ -38,6 +38,8 @@ def get_venue(client, venue_id, support_user='OpenReview.net/Support'): venue.area_chairs_name = domain.content.get('area_chairs_name', {}).get('value', venue.area_chair_roles[0]) venue.reviewer_roles = domain.content.get('reviewer_roles', {}).get('value', ['Reviewers']) venue.reviewers_name = domain.content.get('reviewers_name', {}).get('value', venue.reviewer_roles[0]) + venue.submission_reviewer_roles = domain.content.get('submission_reviewer_roles', {}).get('value', [venue.reviewers_name]) + venue.submission_area_chair_roles = domain.content.get('submission_area_chair_roles', {}).get('value', [venue.area_chairs_name]) venue.allow_gurobi_solver = domain.content.get('allow_gurobi_solver', {}).get('value', False) venue.submission_human_verification = domain.content.get('submission_human_verification', {}).get('value') diff --git a/openreview/venue/invitation.py b/openreview/venue/invitation.py index d2c3fdcf58..482a69f496 100644 --- a/openreview/venue/invitation.py +++ b/openreview/venue/invitation.py @@ -550,6 +550,11 @@ def set_review_invitation(self): venue_id = self.venue_id review_stage = self.venue.review_stage + # The set of reviewer roles invited to this review invitation. Configured on the + # ReviewStage to match how assignments were deployed (one Official_Review for + # multiple roles, or one per role). Falls back to [venue.reviewers_name] for + # backward compatibility with older callers that don't set this explicitly. + reviewer_roles = review_stage.submission_reviewer_roles or [self.venue.reviewers_name] review_invitation_id = self.venue.get_invitation_id(review_stage.name) review_cdate = tools.datetime_millis(review_stage.start_date if review_stage.start_date else datetime.datetime.now()) review_duedate = tools.datetime_millis(review_stage.due_date) if review_stage.due_date else None @@ -579,6 +584,9 @@ def set_review_invitation(self): }, 'confidence_field_name': { 'value': 'confidence' + }, + 'reviewer_roles': { + 'value': reviewer_roles } }, edit={ @@ -605,9 +613,9 @@ def set_review_invitation(self): 'invitation': { 'id': self.venue.get_invitation_id(review_stage.child_invitations_name, '${2/content/noteNumber/value}'), 'signatures': [ venue_id ], - 'readers': [venue_id, self.venue.get_reviewers_id(number='${3/content/noteNumber/value}')], + 'readers': [venue_id] + [self.venue.get_reviewers_id(number='${3/content/noteNumber/value}', name=name) for name in reviewer_roles], 'writers': [venue_id], - 'invitees': [venue_id, self.venue.get_reviewers_id(number='${3/content/noteNumber/value}')], + 'invitees': [venue_id] + [self.venue.get_reviewers_id(number='${3/content/noteNumber/value}', name=name) for name in reviewer_roles], 'maxReplies': 1, 'cdate': review_cdate, 'edit': { @@ -3293,13 +3301,22 @@ def set_assignment_invitation(self, committee_id, submission_content=None, cdate area_chairs_id = self.venue.get_area_chairs_id() senior_area_chairs_id = self.venue.get_senior_area_chairs_id() + default_submission_name = committee_name if is_reviewer: - area_chairs_id = committee_id.replace(self.venue.reviewers_name, self.venue.area_chairs_name) - senior_area_chairs_id = committee_id.replace(self.venue.reviewers_name, self.venue.senior_area_chairs_name) + area_chairs_id = committee_id.replace(committee_name, self.venue.area_chairs_name) + senior_area_chairs_id = committee_id.replace(committee_name, self.venue.senior_area_chairs_name) + if len(venue.reviewer_roles) == len(venue.submission_reviewer_roles): + default_submission_name = venue.submission_reviewer_roles[venue.reviewer_roles.index(committee_name)] + else: + default_submission_name = venue.submission_reviewer_roles[0] if is_area_chair: area_chairs_id = committee_id - senior_area_chairs_id = committee_id.replace(self.venue.area_chairs_name, self.venue.senior_area_chairs_name) + senior_area_chairs_id = committee_id.replace(committee_name, self.venue.senior_area_chairs_name) + if len(venue.area_chair_roles) == len(venue.submission_area_chair_roles): + default_submission_name = venue.submission_area_chair_roles[venue.area_chair_roles.index(committee_name)] + else: + default_submission_name = venue.submission_area_chair_roles[0] content = { 'review_name': { @@ -3309,13 +3326,16 @@ def set_assignment_invitation(self, committee_id, submission_content=None, cdate 'value': committee_id }, 'reviewers_name': { - 'value': venue.reviewers_name if is_reviewer else venue.area_chairs_name + 'value': default_submission_name }, 'reviewers_anon_name': { - 'value': venue.get_anon_reviewers_name() if is_reviewer else venue.get_anon_area_chairs_name() + 'value': venue.get_anon_committee_name(default_submission_name) }, - 'committee_role': { + 'committee_role': { 'value': venue.get_standard_committee_role(committee_id=venue.get_reviewers_id()) + }, + 'submission_committee_name': { + 'value': default_submission_name } } if committee_name == venue.area_chairs_name and venue.use_senior_area_chairs and not venue.sac_paper_assignments: @@ -3774,10 +3794,10 @@ def set_paper_recruitment_invitation(self, invitation_id, committee_id, invited_ ) self.save_invitation(paper_recruitment_invitation, replacement=True) - def set_submission_reviewer_group_invitation(self): + def set_submission_reviewer_group_invitation(self, reviewers_name=None): venue_id = self.venue_id - invitation_id = self.venue.get_invitation_id(f'{self.venue.submission_stage.name}_Group', prefix=self.venue.get_reviewers_id()) + invitation_id = self.venue.get_invitation_id(f'{self.venue.submission_stage.name}_Group', prefix=self.venue.get_reviewers_id(name=reviewers_name)) cdate=tools.datetime_millis(self.venue.submission_stage.second_due_date_exp_date if self.venue.submission_stage.second_due_date_exp_date else self.venue.submission_stage.exp_date) invitation = Invitation(id=invitation_id, @@ -3811,10 +3831,10 @@ def set_submission_reviewer_group_invitation(self): } }, 'group': { - 'id': self.venue.get_reviewers_id(number='${2/content/noteNumber/value}'), - 'readers': self.venue.group_builder.get_reviewer_paper_group_readers('${3/content/noteNumber/value}'), + 'id': self.venue.get_reviewers_id(number='${2/content/noteNumber/value}', name=reviewers_name), + 'readers': self.venue.group_builder.get_reviewer_paper_group_readers('${3/content/noteNumber/value}', name=reviewers_name), 'nonreaders': [self.venue.get_authors_id('${3/content/noteNumber/value}')], - 'deanonymizers': self.venue.group_builder.get_reviewer_identity_readers('${3/content/noteNumber/value}'), + 'deanonymizers': self.venue.group_builder.get_reviewer_identity_readers('${3/content/noteNumber/value}', name=reviewers_name), 'writers': self.venue.group_builder.get_reviewer_paper_group_writers('${3/content/noteNumber/value}'), 'signatures': [self.venue.id], 'signatories': [self.venue.id], @@ -3836,10 +3856,10 @@ def set_submission_reviewer_group_invitation(self): return invitation - def set_submission_area_chair_group_invitation(self): + def set_submission_area_chair_group_invitation(self, area_chairs_name=None): venue_id = self.venue_id - invitation_id = self.venue.get_invitation_id(f'{self.venue.submission_stage.name}_Group', prefix=self.venue.get_area_chairs_id()) + invitation_id = self.venue.get_invitation_id(f'{self.venue.submission_stage.name}_Group', prefix=self.venue.get_area_chairs_id(name=area_chairs_name)) cdate=tools.datetime_millis(self.venue.submission_stage.second_due_date_exp_date if self.venue.submission_stage.second_due_date_exp_date else self.venue.submission_stage.exp_date) @@ -3874,10 +3894,10 @@ def set_submission_area_chair_group_invitation(self): } }, 'group': { - 'id': self.venue.get_area_chairs_id(number='${2/content/noteNumber/value}'), - 'readers': self.venue.group_builder.get_area_chair_paper_group_readers('${3/content/noteNumber/value}'), + 'id': self.venue.get_area_chairs_id(number='${2/content/noteNumber/value}', name=area_chairs_name), + 'readers': self.venue.group_builder.get_area_chair_paper_group_readers('${3/content/noteNumber/value}', name=area_chairs_name), 'nonreaders': [self.venue.get_authors_id('${3/content/noteNumber/value}')], - 'deanonymizers': self.venue.group_builder.get_area_chair_identity_readers('${3/content/noteNumber/value}'), + 'deanonymizers': self.venue.group_builder.get_area_chair_identity_readers('${3/content/noteNumber/value}', name=area_chairs_name), 'writers': [self.venue.id], 'signatures': [self.venue.id], 'signatories': [self.venue.id], @@ -5281,8 +5301,11 @@ def set_submission_change_invitation(self, name, activation_date): if venue.use_senior_area_chairs: readers.append(venue.get_senior_area_chairs_id(number)) if venue.use_area_chairs: - readers.append(venue.get_area_chairs_id(number)) - readers.extend([venue.get_reviewers_id(number), venue.get_authors_id('${{2/id}/number}')]) + for ac_name in venue.submission_area_chair_roles: + readers.append(venue.get_area_chairs_id(number, name=ac_name)) + for reviewers_name in venue.submission_reviewer_roles: + readers.append(venue.get_reviewers_id(number, name=reviewers_name)) + readers.append(venue.get_authors_id('${{2/id}/number}')) invitation = Invitation( id = f'{venue_id}/-/{name}', diff --git a/openreview/venue/matching.py b/openreview/venue/matching.py index 8570954c48..f98f1d7512 100644 --- a/openreview/venue/matching.py +++ b/openreview/venue/matching.py @@ -1198,16 +1198,20 @@ def deploy_assignments(self, assignment_title, overwrite): venue = self.venue client = self.client - committee_id=self.match_group.id - role_name = committee_id.split('/')[-1] review_name = venue.review_stage.child_invitations_name if venue.review_stage else 'Official_Review' - reviewer_name = venue.reviewers_name - if role_name in venue.area_chair_roles: - reviewer_name = venue.area_chairs_name + if self.is_area_chair: review_name = venue.meta_review_stage.child_invitations_name if venue.meta_review_stage else 'Meta_Review' - elif self.is_senior_area_chair: - reviewer_name = venue.senior_area_chairs_name - + + deployed_invitation = client.get_invitation(venue.get_assignment_id(self.match_group.id, deployed=True)) + paper_committee_name = deployed_invitation.content.get('submission_committee_name', {}).get('value') + if not paper_committee_name: + if self.is_area_chair: + paper_committee_name = venue.submission_area_chair_roles[0] + elif self.is_senior_area_chair: + paper_committee_name = venue.senior_area_chairs_name + else: + paper_committee_name = venue.submission_reviewer_roles[0] + papers = self._get_submissions(details='directReplies') sac_assignment_edges = { g['id']['head']: g['values'] for g in client.get_grouped_edges(invitation=venue.get_assignment_id(self.senior_area_chairs_id, deployed=True), domain=venue.id, groupby='head', select=None)} if not venue.sac_paper_assignments else {} @@ -1216,7 +1220,7 @@ def deploy_assignments(self, assignment_title, overwrite): label=assignment_title, groupby='head', select=None)} assignment_invitation_id = venue.get_assignment_id(self.match_group.id, deployed=True) submission_group_invitation_id = venue.get_invitation_id(f'{venue.submission_stage.name}_Group', prefix=self.match_group.id) - existing_paper_committee_ids = { g.id for g in client.get_all_groups(prefix=venue.get_paper_group_prefix(), domain=venue.id) if g.id.endswith(f'/{reviewer_name}') } + existing_paper_committee_ids = { g.id for g in client.get_all_groups(prefix=venue.get_paper_group_prefix(), domain=venue.id) if g.id.endswith(f'/{paper_committee_name}') } current_assignment_edges = { g['id']['head']: g['values'] for g in client.get_grouped_edges(invitation=assignment_invitation_id, groupby='head', select=None, domain=venue.id)} print('Check if there are reviews posted') @@ -1231,7 +1235,7 @@ def deploy_assignments(self, assignment_title, overwrite): ## Remove the members from the groups based on the current assignments for paper in tqdm(papers, total=len(papers)): if paper.id in current_assignment_edges: - paper_committee_id = venue.get_committee_id(name=reviewer_name, number=paper.number) + paper_committee_id = venue.get_committee_id(name=paper_committee_name, number=paper.number) current_edges=current_assignment_edges[paper.id] for current_edge in current_edges: client.remove_members_from_group(paper_committee_id, current_edge['tail']) @@ -1243,7 +1247,7 @@ def deploy_assignments(self, assignment_title, overwrite): def process_paper_assignments(paper): paper_assignment_edges = [] if paper.id in proposed_assignment_edges: - paper_committee_id = venue.get_committee_id(name=reviewer_name, number=paper.number) + paper_committee_id = venue.get_committee_id(name=paper_committee_name, number=paper.number) proposed_edges=proposed_assignment_edges[paper.id] assigned_users = [] for proposed_edge in proposed_edges: @@ -1359,16 +1363,20 @@ def undeploy_assignments(self, assignment_title): venue = self.venue client = self.client - committee_id=self.match_group.id - role_name = committee_id.split('/')[-1] review_name = venue.review_stage.child_invitations_name if venue.review_stage else 'Official_Review' - reviewer_name = venue.reviewers_name - if role_name in venue.area_chair_roles: - reviewer_name = venue.area_chairs_name + if self.is_area_chair: review_name = venue.meta_review_stage.child_invitations_name if venue.meta_review_stage else 'Meta_Review' - elif self.is_senior_area_chair: - reviewer_name = venue.senior_area_chairs_name - + + deployed_invitation = client.get_invitation(venue.get_assignment_id(self.match_group.id, deployed=True)) + paper_committee_name = deployed_invitation.content.get('submission_committee_name', {}).get('value') + if not paper_committee_name: + if self.is_area_chair: + paper_committee_name = venue.submission_area_chair_roles[0] + elif self.is_senior_area_chair: + paper_committee_name = venue.senior_area_chairs_name + else: + paper_committee_name = venue.submission_reviewer_roles[0] + papers = self._get_submissions(details='directReplies') sac_assignment_edges = { g['id']['head']: g['values'] for g in client.get_grouped_edges(invitation=venue.get_assignment_id(self.senior_area_chairs_id, deployed=True), groupby='head', select=None, domain=venue.id)} if not venue.sac_paper_assignments else {} @@ -1387,7 +1395,7 @@ def undeploy_assignments(self, assignment_title): def process_paper_assignments(paper): if paper.id in proposed_assignment_edges: - paper_committee_id = venue.get_committee_id(name=reviewer_name, number=paper.number) + paper_committee_id = venue.get_committee_id(name=paper_committee_name, number=paper.number) proposed_edges=proposed_assignment_edges[paper.id] assigned_users = [] for proposed_edge in proposed_edges: @@ -1414,7 +1422,6 @@ def deploy(self, assignment_title, overwrite=False, enable_reviewer_reassignment recruitment_invitation_id=self.venue.get_invitation_id('Proposed_Assignment_Recruitment', prefix=self.match_group.id) self.venue.invitation_builder.expire_invitation(recruitment_invitation_id) self.venue.invitation_builder.expire_invitation(self.venue.get_assignment_id(self.match_group.id)) - ## Deploy assignments creating groups and assignment edges if self.is_senior_area_chair and not self.venue.sac_paper_assignments: @@ -1452,7 +1459,7 @@ def undeploy(self, assignment_title): if self.is_senior_area_chair and not self.venue.sac_paper_assignments: self.undeploy_sac_assignments(assignment_title) else: - self.undeploy_assignments(assignment_title) + self.undeploy_assignments(assignment_title) self.venue.invitation_builder.expire_invitation(self.venue.get_assignment_id(self.match_group.id, deployed=True)) self.venue.invitation_builder.unexpire_invitation(self.venue.get_assignment_id(self.match_group.id)) self.venue.invitation_builder.unexpire_invitation(self.venue.get_invitation_id('Proposed_Assignment_Recruitment', prefix=self.match_group.id)) diff --git a/openreview/venue/process/review_process.py b/openreview/venue/process/review_process.py index 0b845053d0..e4c6f5f451 100644 --- a/openreview/venue/process/review_process.py +++ b/openreview/venue/process/review_process.py @@ -22,10 +22,14 @@ def process(client, edit, invitation): email_reviewers = parent_invitation.get_content_value('email_reviewers') or 'submission_reviewers' in users_to_notify email_authors = parent_invitation.get_content_value('email_authors') or 'submission_authors' in users_to_notify + # The roles this review invitation is wired to (set in set_review_invitation). + # Falls back to the umbrella reviewers_name for back-compat with older invitations. + reviewer_roles = parent_invitation.get_content_value('reviewer_roles', [reviewers_name]) + submission = client.get_note(edit.note.forum) paper_group_id=f'{venue_id}/{submission_name}{submission.number}' - paper_reviewers_id = f'{paper_group_id}/{reviewers_name}' - paper_reviewers_submitted_id = f'{paper_reviewers_id}/{reviewers_submitted_name}' + paper_reviewers_ids = [f'{paper_group_id}/{role}' for role in reviewer_roles] + paper_reviewers_submitted_ids = [f'{paper_reviewers_id}/{reviewers_submitted_name}' for paper_reviewers_id in paper_reviewers_ids] paper_area_chairs_id = f'{paper_group_id}/{area_chairs_name}' paper_senior_area_chairs_id = f'{paper_group_id}/{senior_area_chairs_name}' @@ -60,7 +64,13 @@ def create_group(group_id, members=[]): ) ) - create_group(paper_reviewers_submitted_id, [review.signatures[0]]) + # Match the anon signature to its role by prefix: e.g. + # 'venue/Submission1/Expert_Reviewer_XYCV' belongs to role 'Expert_Reviewers'. + for role in reviewer_roles: + anon_prefix = f'{paper_group_id}/{role[:-1] if role.endswith("s") else role}_' + if review.signatures[0].startswith(anon_prefix): + create_group(f'{paper_group_id}/{role}/{reviewers_submitted_name}', [review.signatures[0]]) + break capital_review_name = review_name.replace('_', ' ') review_name = capital_review_name.lower() @@ -121,12 +131,15 @@ def create_group(group_id, members=[]): ) if email_reviewers: - if 'everyone' in review.readers or paper_reviewers_id in review.readers: + role_recipients = [role_id for role_id in paper_reviewers_ids if role_id in review.readers] + submitted_recipients = [submitted_id for submitted_id in paper_reviewers_submitted_ids if submitted_id in review.readers] + if 'everyone' in review.readers or role_recipients: + recipients = role_recipients if role_recipients else paper_reviewers_ids client.post_message( invitation=meta_invitation_id, signature=venue_id, sender=sender, - recipients=[paper_reviewers_id], + recipients=recipients, ignoreRecipients=ignore_groups, replyTo=contact, subject=f'''[{short_name}] {capital_review_name} posted to your assigned Paper number: {submission.number}, Paper title: "{submission.content['title']['value']}"''', @@ -139,13 +152,13 @@ def create_group(group_id, members=[]): {content} ''' ) - elif paper_reviewers_submitted_id in review.readers: + elif submitted_recipients: print('emailing reviewers who have submitted') client.post_message( invitation=meta_invitation_id, signature=venue_id, sender=sender, - recipients=[paper_reviewers_submitted_id], + recipients=submitted_recipients, ignoreRecipients=ignore_groups, replyTo=contact, subject=f'''[{short_name}] {capital_review_name} posted to your assigned Paper number: {submission.number}, Paper title: "{submission.content['title']['value']}"''', diff --git a/openreview/venue/venue.py b/openreview/venue/venue.py index fde0280249..4a1f9a8907 100644 --- a/openreview/venue/venue.py +++ b/openreview/venue/venue.py @@ -41,8 +41,10 @@ def __init__(self, client, venue_id, support_user): self.publication_chairs_name = 'Publication_Chairs' self.reviewer_roles = ['Reviewers'] self.reviewers_name = self.reviewer_roles[0] + self.submission_reviewer_roles = [self.reviewers_name] self.area_chair_roles = ['Area_Chairs'] self.area_chairs_name = self.area_chair_roles[0] + self.submission_area_chair_roles = [self.area_chairs_name] self.senior_area_chair_roles = ['Senior_Area_Chairs'] self.senior_area_chairs_name = self.senior_area_chair_roles[0] self.secondary_area_chairs_name = 'Secondary_Area_Chairs' @@ -140,8 +142,13 @@ def set_main_settings(self, request_note): elif 'reviewers_name' in request_note.content: self.reviewers_name = request_note.content['reviewers_name']['value'] self.reviewer_roles = [self.reviewers_name] + reviewer_group_layout = request_note.content.get('reviewer_group_layout', {}).get('value', 'shared') + if reviewer_group_layout == 'per_role': + self.submission_reviewer_roles = list(self.reviewer_roles) + else: + self.submission_reviewer_roles = [self.reviewers_name] preferred_email_groups = [self.get_reviewers_id(), self.get_authors_id()] - + if request_note.content.get('area_chairs_support',{}).get('value'): if 'area_chair_groups_names' in request_note.content: self.area_chair_roles = request_note.content['area_chair_groups_names']['value'] @@ -149,13 +156,18 @@ def set_main_settings(self, request_note): elif 'area_chairs_name' in request_note.content: self.area_chairs_name = request_note.content['area_chairs_name']['value'] self.area_chair_roles = [self.area_chairs_name] + area_chair_group_layout = request_note.content.get('area_chair_group_layout', {}).get('value', 'shared') + if area_chair_group_layout == 'per_role': + self.submission_area_chair_roles = list(self.area_chair_roles) + else: + self.submission_area_chair_roles = [self.area_chairs_name] self.use_area_chairs = True preferred_email_groups.append(self.get_area_chairs_id()) if 'senior_area_chairs_name' in request_note.content: ## change this once we add support for SACs self.senior_area_chairs_name = request_note.content['senior_area_chairs_name']['value'] self.use_senior_area_chairs = True - self.senior_area_chair_roles = request_note.content.get('senior_area_chair_roles', [self.senior_area_chairs_name]) + self.senior_area_chair_roles = request_note.content.get('senior_area_chair_roles', {}).get('value', [self.senior_area_chairs_name]) preferred_email_groups.append(self.get_senior_area_chairs_id()) self.preferred_emails_groups = preferred_email_groups @@ -366,7 +378,7 @@ def get_committee_id_declined(self, committee_name): def get_anon_reviewer_id(self, number, anon_id, name=None): if name == self.ethics_reviewers_name: return self.get_ethics_reviewers_id(number, True) - return self.get_reviewers_id(number, True) + return self.get_reviewers_id(number, True, name=name) def get_reviewers_name(self, pretty=True): if pretty: @@ -396,9 +408,10 @@ def get_area_chairs_name(self, pretty=True): def get_anon_area_chairs_name(self, pretty=True): return self.get_anon_committee_name(self.area_chairs_name) - def get_reviewers_id(self, number = None, anon=False, submitted=False): - rev_name = self.get_anon_reviewers_name() - reviewers_id = self.get_committee_id(f'{rev_name}.*' if anon else self.reviewers_name, number) + def get_reviewers_id(self, number = None, anon=False, submitted=False, name=None): + reviewers_name = name if name else self.reviewers_name + rev_name = self.get_anon_committee_name(reviewers_name) + reviewers_id = self.get_committee_id(f'{rev_name}.*' if anon else reviewers_name, number) if submitted: return reviewers_id + '/Submitted' return reviewers_id @@ -412,9 +425,10 @@ def get_authors_accepted_id(self, number = None): def get_program_chairs_id(self): return self.get_committee_id(self.program_chairs_name) - def get_area_chairs_id(self, number = None, anon=False): - ac_name = self.get_anon_area_chairs_name() - return self.get_committee_id(f'{ac_name}.*' if anon else self.area_chairs_name, number) + def get_area_chairs_id(self, number = None, anon=False, name=None): + area_chairs_name = name if name else self.area_chairs_name + ac_name = self.get_anon_committee_name(area_chairs_name) + return self.get_committee_id(f'{ac_name}.*' if anon else area_chairs_name, number) def get_secondary_area_chairs_id(self, number = None, anon=False): ac_name = self.get_anon_committee_name(self.secondary_area_chairs_name) @@ -699,10 +713,12 @@ def create_submission_stage(self): self.invitation_builder.set_desk_rejection_invitation() self.invitation_builder.set_post_submission_invitation() self.invitation_builder.set_pc_submission_revision_invitation() - self.invitation_builder.set_submission_reviewer_group_invitation() + for reviewers_name in self.submission_reviewer_roles: + self.invitation_builder.set_submission_reviewer_group_invitation(reviewers_name=reviewers_name) self.invitation_builder.set_submission_message_invitation() if self.use_area_chairs: - self.invitation_builder.set_submission_area_chair_group_invitation() + for area_chairs_name in self.submission_area_chair_roles: + self.invitation_builder.set_submission_area_chair_group_invitation(area_chairs_name=area_chairs_name) if self.use_senior_area_chairs: self.invitation_builder.set_submission_senior_area_chair_group_invitation() if self.expertise_selection_stage: @@ -1245,31 +1261,33 @@ def set_assignment_invitations(self, submission_deadline): invitation_prefix = self.support_user.replace('Support', 'Template') if self.use_area_chairs: - self.invitation_builder.set_assignment_invitation(committee_id=self.get_area_chairs_id(), cdate=submission_deadline + (60*60*1000*24*7*2)) + for ac_role in self.area_chair_roles: + self.invitation_builder.set_assignment_invitation(committee_id=self.get_area_chairs_id(name=ac_role), cdate=submission_deadline + (60*60*1000*24*7*2)) + + self.client.post_invitation_edit( + invitations=f'{invitation_prefix}/-/Reviewer_Assignment_Deployment', + signatures=[invitation_prefix], + content={ + 'venue_id': { 'value': self.venue_id }, + 'name': { 'value': f'{ac_role}_Assignment_Deployment' }, + 'activation_date': { 'value': submission_deadline + (60*60*1000*24*7*2.1) }, + 'committee_name': { 'value': ac_role }, + 'committee_pretty_name': { 'value': self.get_committee_name(ac_role, pretty=True) } + }, + await_process=True + ) + for reviewer_role in self.reviewer_roles: + self.invitation_builder.set_assignment_invitation(committee_id=self.get_reviewers_id(name=reviewer_role), cdate=submission_deadline + (60*60*1000*24*7*2.2)) self.client.post_invitation_edit( invitations=f'{invitation_prefix}/-/Reviewer_Assignment_Deployment', signatures=[invitation_prefix], content={ 'venue_id': { 'value': self.venue_id }, - 'name': { 'value': f'{self.area_chairs_name}_Assignment_Deployment' }, - 'activation_date': { 'value': submission_deadline + (60*60*1000*24*7*2.1) }, - 'committee_name': { 'value': self.area_chairs_name }, - 'committee_pretty_name': { 'value': self.get_area_chairs_name(pretty=True) } - }, - await_process=True - ) - - self.invitation_builder.set_assignment_invitation(committee_id=self.get_reviewers_id(), cdate=submission_deadline + (60*60*1000*24*7*2.2)) - self.client.post_invitation_edit( - invitations=f'{invitation_prefix}/-/Reviewer_Assignment_Deployment', - signatures=[invitation_prefix], - content={ - 'venue_id': { 'value': self.venue_id }, - 'name': { 'value': f'{self.reviewers_name}_Assignment_Deployment' }, + 'name': { 'value': f'{reviewer_role}_Assignment_Deployment' }, 'activation_date': { 'value': submission_deadline + (60*60*1000*24*7*2.3) }, - 'committee_name': { 'value': self.reviewers_name }, - 'committee_pretty_name': { 'value': self.get_reviewers_name(pretty=True) } + 'committee_name': { 'value': reviewer_role }, + 'committee_pretty_name': { 'value': self.get_committee_name(reviewer_role, pretty=True) } }, await_process=True ) @@ -1282,11 +1300,13 @@ def setup_matching_invitations(self): :meth:`setup_committee_matching` to also compute scores and conflicts. """ if self.use_area_chairs: - venue_matching = matching.Matching(self, self.client.get_group(self.get_area_chairs_id())) - venue_matching.setup_matching_invitations() + for ac_role in self.area_chair_roles: + venue_matching = matching.Matching(self, self.client.get_group(self.get_area_chairs_id(name=ac_role))) + venue_matching.setup_matching_invitations() - venue_matching = matching.Matching(self, self.client.get_group(self.get_reviewers_id())) - venue_matching.setup_matching_invitations() + for reviewer_role in self.reviewer_roles: + venue_matching = matching.Matching(self, self.client.get_group(self.get_reviewers_id(name=reviewer_role))) + venue_matching.setup_matching_invitations() def setup_all_committees_matching(self): """Run full matching setup (invitations, affinity scores, conflicts) for all committees. @@ -1295,11 +1315,13 @@ def setup_all_committees_matching(self): computing affinity scores and conflicts with default settings. """ if self.use_area_chairs: - venue_matching = matching.Matching(self, self.client.get_group(self.get_area_chairs_id())) - venue_matching.setup() + for ac_role in self.area_chair_roles: + venue_matching = matching.Matching(self, self.client.get_group(self.get_area_chairs_id(name=ac_role))) + venue_matching.setup() - venue_matching = matching.Matching(self, self.client.get_group(self.get_reviewers_id())) - venue_matching.setup() + for reviewer_role in self.reviewer_roles: + venue_matching = matching.Matching(self, self.client.get_group(self.get_reviewers_id(name=reviewer_role))) + venue_matching.setup() def setup_committee_matching(self, committee_id=None, compute_affinity_scores=False, compute_conflicts=False, compute_conflicts_n_years=None, alternate_matching_group=None, submission_track=None): """Set up paper matching for a specific committee, optionally computing affinity scores and conflicts. @@ -1338,8 +1360,10 @@ def set_assignments(self, assignment_title, committee_id, enable_reviewer_reassi Copies edges from the proposed assignment configuration (identified by ``assignment_title``) to the deployed assignment invitation, creating - per-paper committee member groups. Optionally enables reviewer - reassignment for area chairs after deployment. + per-paper committee member groups. The target per-submission group name + is read from the ``submission_committee_name`` content field on the + Assignment invitation. Optionally enables reviewer reassignment for + area chairs after deployment. :param assignment_title: Label of the proposed assignment configuration to deploy. :type assignment_title: str @@ -1356,13 +1380,14 @@ def set_assignments(self, assignment_title, committee_id, enable_reviewer_reassi assignment_invitation = self.client.get_invitation(self.get_assignment_id(match_group.id)) conference_matching = matching.Matching(self, match_group, submission_content=assignment_invitation.edit.get('head', {}).get('param', {}).get('withContent')) return conference_matching.deploy(assignment_title, overwrite, enable_reviewer_reassignment) - + def unset_assignments(self, assignment_title, committee_id): """Revert deployed assignments back to proposed state for a committee. Removes the deployed assignment edges and per-paper committee member groups, restoring the assignment configuration to its pre-deployment - state. + state. The target per-submission group name is read from the + ``submission_committee_name`` content field on the Assignment invitation. :param assignment_title: Label of the assignment configuration to undeploy. :type assignment_title: str @@ -1373,7 +1398,7 @@ def unset_assignments(self, assignment_title, committee_id): """ match_group = self.client.get_group(committee_id) conference_matching = matching.Matching(self, match_group) - return conference_matching.undeploy(assignment_title) + return conference_matching.undeploy(assignment_title) def setup_assignment_recruitment(self, committee_id, hash_seed, due_date, assignment_title=None, invitation_labels={}, email_template=None): """Set up invite-based assignment recruitment for external or emergency reviewers. diff --git a/openreview/venue/webfield/programChairsWebfield.js b/openreview/venue/webfield/programChairsWebfield.js index 3bf5bf96cc..1f8111d503 100644 --- a/openreview/venue/webfield/programChairsWebfield.js +++ b/openreview/venue/webfield/programChairsWebfield.js @@ -54,6 +54,7 @@ return { withdrawnVenueId: domain.content.withdrawn_venue_id?.value, deskRejectedVenueId: domain.content.desk_rejected_venue_id?.value, officialReviewName: domain.content.review_name?.value || "Official_Review", + officialReviewNames: domain.content.review_names?.value || [domain.content.review_name?.value || "Official_Review"], commentName: domain.content.comment_name?.value || "Official_Comment", officialMetaReviewName: domain.content.meta_review_name?.value || "Meta_Review", diff --git a/openreview/workflows/process/committee_recruitment_request_edit_reminder_process.py b/openreview/workflows/process/committee_recruitment_request_edit_reminder_process.py index 77aff78c7d..e571aa7a10 100644 --- a/openreview/workflows/process/committee_recruitment_request_edit_reminder_process.py +++ b/openreview/workflows/process/committee_recruitment_request_edit_reminder_process.py @@ -2,13 +2,10 @@ def process(client, edit, invitation): domain = client.get_group(invitation.domain) committee_id = invitation.content['committee_id']['value'] - committee_group = client.get_group(committee_id) - committee_role = committee_group.content['committee_role']['value'] - committee_invited_id= domain.content[f'{committee_role}_invited_id']['value'] - committee_id = domain.content[f'{committee_role}_id']['value'] - committee_declined_id = domain.content[f'{committee_role}_declined_id']['value'] - committee_invited_response_id = domain.content[f'{committee_role}_recruitment_id']['value'] - committee_invited_message_id = domain.content[f'{committee_role}_invited_message_id']['value'] + committee_invited_id = f'{committee_id}/Invited' + committee_declined_id = f'{committee_id}/Declined' + committee_invited_response_id = f'{committee_id}/-/Recruitment_Response' + committee_invited_message_id = f'{committee_id}/Invited/-/Message' committee_invited_response_invitation = client.get_invitation(committee_invited_response_id) contact_email = domain.get_content_value('contact') diff --git a/openreview/workflows/process/committee_recruitment_request_process.py b/openreview/workflows/process/committee_recruitment_request_process.py index 14fa4a1d12..d934be5f13 100644 --- a/openreview/workflows/process/committee_recruitment_request_process.py +++ b/openreview/workflows/process/committee_recruitment_request_process.py @@ -5,12 +5,11 @@ def process(client, edit, invitation): meta_invitation_id = domain.content['meta_invitation_id']['value'] committee_id = invitation.content['committee_id']['value'] committee_group = client.get_group(committee_id) - committee_role = committee_group.content['committee_role']['value'] committee_name = committee_group.content['committee_name']['value'] - invited_group = client.get_group(domain.content[f'{committee_role}_invited_id']['value']) - group_id = domain.content[f'{committee_role}_id']['value'] - committee_invited_response_id = domain.content[f'{committee_role}_recruitment_id']['value'] - committee_invited_message_id = domain.content[f'{committee_role}_invited_message_id']['value'] + group_id = committee_id + invited_group = client.get_group(f'{committee_id}/Invited') + committee_invited_response_id = f'{committee_id}/-/Recruitment_Response' + committee_invited_message_id = f'{committee_id}/Invited/-/Message' committee_invited_response_invitation = client.get_invitation(committee_invited_response_id) contact_email = domain.get_content_value('contact') diff --git a/openreview/workflows/process/committee_recruitment_request_reminder_process.py b/openreview/workflows/process/committee_recruitment_request_reminder_process.py index fc580d42cb..30145100ee 100644 --- a/openreview/workflows/process/committee_recruitment_request_reminder_process.py +++ b/openreview/workflows/process/committee_recruitment_request_reminder_process.py @@ -2,13 +2,10 @@ def process(client, edit, invitation): domain = client.get_group(invitation.domain) committee_id = invitation.content['committee_id']['value'] - committee_group = client.get_group(committee_id) - committee_role = committee_group.content['committee_role']['value'] - committee_invited_id= domain.content[f'{committee_role}_invited_id']['value'] - committee_id = domain.content[f'{committee_role}_id']['value'] - committee_declined_id = domain.content[f'{committee_role}_declined_id']['value'] - committee_invited_response_id = domain.content[f'{committee_role}_recruitment_id']['value'] - committee_invited_message_id = domain.content[f'{committee_role}_invited_message_id']['value'] + committee_invited_id = f'{committee_id}/Invited' + committee_declined_id = f'{committee_id}/Declined' + committee_invited_response_id = f'{committee_id}/-/Recruitment_Response' + committee_invited_message_id = f'{committee_id}/Invited/-/Message' committee_invited_response_invitation = client.get_invitation(committee_invited_response_id) contact_email = domain.get_content_value('contact') diff --git a/openreview/workflows/process/committee_recruitment_response_process.py b/openreview/workflows/process/committee_recruitment_response_process.py index 7824e466cd..93f64d7bd5 100644 --- a/openreview/workflows/process/committee_recruitment_response_process.py +++ b/openreview/workflows/process/committee_recruitment_response_process.py @@ -1,12 +1,10 @@ def process(client, edit, invitation): domain = client.get_group(invitation.domain) - - committee_group = client.get_group(invitation.content['committee_id']['value']) - committee_role = committee_group.content['committee_role']['value'] - committee_id = domain.content[f'{committee_role}_id']['value'] - committee_declined_id = domain.content[f'{committee_role}_declined_id']['value'] - committee_invited_message_id = domain.content[f'{committee_role}_invited_message_id']['value'] + + committee_id = invitation.content['committee_id']['value'] + committee_declined_id = f'{committee_id}/Declined' + committee_invited_message_id = f'{committee_id}/Invited/-/Message' note = edit.note user=note.content.get('user', {}).get('value', note.signatures[0]) diff --git a/openreview/workflows/workflow_process/committee_group_template_process.py b/openreview/workflows/workflow_process/committee_group_template_process.py index d644ea6988..692c5e1ee6 100644 --- a/openreview/workflows/workflow_process/committee_group_template_process.py +++ b/openreview/workflows/workflow_process/committee_group_template_process.py @@ -5,21 +5,7 @@ def process(client, edit, invitation): domain = client.get_group(venue_id) committee_role = edit.content['committee_role']['value'] - committee_name = edit.content['committee_name']['value'] committee_pretty_name = edit.content['committee_pretty_name']['value'] - committee_anon_name = edit.content.get('committee_anon_name', {}).get('value', False) - committee_submitted_name = edit.content.get('committee_submitted_name', {}).get('value', False) - - content = { - f'{committee_role}_id': { 'value': edit.group.id }, - f'{committee_role}_name': { 'value': committee_name }, - } - - if committee_anon_name: - content[f'{committee_role}_anon_name'] = { 'value': committee_anon_name } - - if committee_submitted_name: - content[f'{committee_role}_submitted_name'] = { 'value': committee_submitted_name } client.post_group_edit( invitation=domain.content['meta_invitation_id']['value'], @@ -76,9 +62,6 @@ def process(client, edit, invitation): await_process=True ) - content[f'{committee_role}_declined_id'] = { 'value': declined_group_edit['group']['id'] } - content[f'{committee_role}_invited_id'] = { 'value': invited_group_id['group']['id'] } - invitation_edit = client.post_invitation_edit( invitations=f'{invitation.domain}/-/Committee_Recruitment_Request', signatures=[invitation.domain], @@ -125,15 +108,4 @@ def process(client, edit, invitation): ) edit_invitations_builder.set_edit_dates_one_level_invitation(invitation_edit['invitation']['id'], include_due_date=True, include_exp_date=True) - edit_invitations_builder.set_edit_committee_recruitment_invitation(invitation_edit['invitation']['id']) - - content[f'{committee_role}_recruitment_id'] = { 'value': invitation_edit['invitation']['id'] } - - client.post_group_edit( - invitation=domain.content['meta_invitation_id']['value'], - signatures=[invitation.domain], - group=openreview.api.Group( - id=venue_id, - content=content - ) - ) \ No newline at end of file + edit_invitations_builder.set_edit_committee_recruitment_invitation(invitation_edit['invitation']['id']) \ No newline at end of file diff --git a/openreview/workflows/workflow_process/committee_invited_group_template_process.py b/openreview/workflows/workflow_process/committee_invited_group_template_process.py index b241a38e0f..4991828c1f 100644 --- a/openreview/workflows/workflow_process/committee_invited_group_template_process.py +++ b/openreview/workflows/workflow_process/committee_invited_group_template_process.py @@ -21,14 +21,6 @@ def process(client, edit, invitation): await_process=True ) - content = {} - content[f'{committee_role}_invited_message_id'] = { 'value': invitation_edit['invitation']['id'] } - - client.post_group_edit( - invitation=domain.content['meta_invitation_id']['value'], - signatures=[invitation.domain], - group=openreview.api.Group( - id=venue_id, - content=content - ) - ) \ No newline at end of file + # The invited message invitation id ({committee_id}/Invited/-/Message) is now + # derived from committee_id directly in the recruitment process functions, + # so we no longer need to store it on the domain. \ No newline at end of file diff --git a/openreview/workflows/workflow_process/conference_review_workflow_deployment.py b/openreview/workflows/workflow_process/conference_review_workflow_deployment.py index 8b71a69caa..8800002111 100644 --- a/openreview/workflows/workflow_process/conference_review_workflow_deployment.py +++ b/openreview/workflows/workflow_process/conference_review_workflow_deployment.py @@ -49,7 +49,8 @@ def process(client, edit, invitation): venue.review_stage = openreview.stages.ReviewStage( start_date=submission_deadline_datetime + datetime.timedelta(weeks=3.5), - due_date=submission_deadline_datetime + datetime.timedelta(weeks=5) + due_date=submission_deadline_datetime + datetime.timedelta(weeks=5), + submission_reviewer_roles=[venue.submission_reviewer_roles[0]] ) venue.comment_stage = openreview.stages.CommentStage( @@ -119,6 +120,19 @@ def process(client, edit, invitation): venue.create_review_stage() venue.create_comment_stage() + # When reviewer groups are configured per-role, create one additional review + # invitation per secondary reviewer role so each role gets its own form. + for additional_role in venue.submission_reviewer_roles[1:]: + review_name = f'{additional_role}_Review' + venue.review_stage = openreview.stages.ReviewStage( + name=review_name, + child_invitations_name=review_name, + start_date=submission_deadline_datetime + datetime.timedelta(weeks=3.5), + due_date=submission_deadline_datetime + datetime.timedelta(weeks=5), + submission_reviewer_roles=[additional_role] + ) + venue.create_review_stage() + client.post_invitation_edit( invitations=f'{invitation_prefix}/-/Note_Release', signatures=[invitation_prefix], diff --git a/openreview/workflows/workflows.py b/openreview/workflows/workflows.py index c47dda9ad4..fac388218d 100644 --- a/openreview/workflows/workflows.py +++ b/openreview/workflows/workflows.py @@ -240,8 +240,21 @@ def set_conference_review_request(self): } } }, - 'area_chairs_support': { + 'reviewer_group_layout': { 'order': 13, + 'description': 'How are reviewers grouped per submission? "shared" means all reviewer roles are assigned into a single per-submission group (using the primary reviewer name). "per_role" means each reviewer role gets its own per-submission group.', + 'value': { + 'param': { + 'type': 'string', + 'enum': ['shared', 'per_role'], + 'default': 'shared', + 'input': 'radio', + 'optional': True + } + } + }, + 'area_chairs_support': { + 'order': 14, 'description': "Does your venue have area chairs? Leave unchecked if your venue does not have area chairs.", 'value': { 'param': { @@ -254,7 +267,7 @@ def set_conference_review_request(self): } }, 'area_chair_groups_names': { - 'order': 14, + 'order': 15, 'description': 'Please provide the designated name to be used for area chairs. Use underscores for spaces and capitalize as needed. Default is "Area_Chairs". Ignore if your venue does not have area chairs.', 'value': { 'param': { @@ -264,8 +277,21 @@ def set_conference_review_request(self): } } }, + 'area_chair_group_layout': { + 'order': 16, + 'description': 'How are area chairs grouped per submission? "shared" means all area chair roles are assigned into a single per-submission group (using the primary area chair name). "per_role" means each area chair role gets its own per-submission group.', + 'value': { + 'param': { + 'type': 'string', + 'enum': ['shared', 'per_role'], + 'default': 'shared', + 'input': 'radio', + 'optional': True + } + } + }, 'colocated': { - 'order': 15, + 'order': 17, 'description': 'Please provide the name of the conference, organization, or academic institution with which your event is colocated. If your event is independent of a conference or organization, you can leave this blank or write "independent"', 'value': { 'param': { @@ -277,7 +303,7 @@ def set_conference_review_request(self): } }, 'previous_venue': { - 'order': 16, + 'order': 18, 'description': 'If possible, please provide a link to the previous iteration of this venue on OpenReview.', 'value': { 'param': { @@ -289,7 +315,7 @@ def set_conference_review_request(self): } }, 'expected_submissions': { - 'order': 17, + 'order': 19, 'description': 'How many submissions do you expect to receive for this venue? Please provide a number. This will help us plan for the expected load on our servers.', 'value': { 'param': { @@ -299,7 +325,7 @@ def set_conference_review_request(self): } }, 'how_did_you_hear_about_us': { - 'order': 18, + 'order': 20, 'description': 'How did you hear about OpenReview?', 'value': { 'param': { @@ -312,7 +338,7 @@ def set_conference_review_request(self): } }, 'other_important_information': { - 'order': 19, + 'order': 21, 'description': 'Please provide any other important information about your venue that you would like to share with OpenReview. Please use this space to clarify any questions for which you could not use any of the provided options, and to clarify any other information that you think we may need.', 'value': { 'param': { @@ -325,7 +351,7 @@ def set_conference_review_request(self): } }, 'venue_organizer_agreement': { - 'order': 20, + 'order': 22, 'description': 'In order to use OpenReview, venue chairs must agree to the following:', 'value': { 'param': { diff --git a/tests/test_acs_and_reviewers.py b/tests/test_acs_and_reviewers.py index be294138cb..7ae21962fd 100644 --- a/tests/test_acs_and_reviewers.py +++ b/tests/test_acs_and_reviewers.py @@ -185,23 +185,17 @@ def test_setup(self, openreview_client, helpers): assert openreview.tools.get_invitation(openreview_client, 'EFGH.cc/2025/Conference/Reviewers/-/Custom_User_Demands') domain_content = openreview.tools.get_group(openreview_client, 'EFGH.cc/2025/Conference').content - assert domain_content['reviewers_invited_id']['value'] == 'EFGH.cc/2025/Conference/Reviewers/Invited' - assert domain_content['reviewers_declined_id']['value'] == 'EFGH.cc/2025/Conference/Reviewers/Declined' assert domain_content['reviewers_id']['value'] == 'EFGH.cc/2025/Conference/Reviewers' assert domain_content['reviewers_name']['value'] == 'Reviewers' assert domain_content['reviewers_anon_name']['value'] == 'Reviewer_' assert domain_content['reviewers_submitted_name']['value'] == 'Submitted' assert domain_content['reviewers_recruitment_id']['value'] == 'EFGH.cc/2025/Conference/Reviewers/-/Recruitment_Response' - assert domain_content['reviewers_invited_message_id']['value'] == 'EFGH.cc/2025/Conference/Reviewers/Invited/-/Message' - assert domain_content['area_chairs_invited_id']['value'] == 'EFGH.cc/2025/Conference/Action_Editors/Invited' - assert domain_content['area_chairs_declined_id']['value'] == 'EFGH.cc/2025/Conference/Action_Editors/Declined' assert domain_content['area_chairs_id']['value'] == 'EFGH.cc/2025/Conference/Action_Editors' assert domain_content['area_chairs_name']['value'] == 'Action_Editors' assert domain_content['area_chairs_anon_name']['value'] == 'Action_Editor_' assert domain_content.get('area_chairs_submitted_name') is None assert domain_content['area_chairs_recruitment_id']['value'] == 'EFGH.cc/2025/Conference/Action_Editors/-/Recruitment_Response' - assert domain_content['area_chairs_invited_message_id']['value'] == 'EFGH.cc/2025/Conference/Action_Editors/Invited/-/Message' assert openreview.tools.get_invitation(openreview_client, 'EFGH.cc/2025/Conference/-/Submission') invitation = openreview.tools.get_invitation(openreview_client, 'EFGH.cc/2025/Conference/-/Submission_Change_Before_Bidding') @@ -234,23 +228,17 @@ def test_setup(self, openreview_client, helpers): # check domain object domain_content = openreview_client.get_group('EFGH.cc/2025/Conference').content - assert domain_content['reviewers_invited_id']['value'] == 'EFGH.cc/2025/Conference/Reviewers/Invited' - assert domain_content['reviewers_declined_id']['value'] == 'EFGH.cc/2025/Conference/Reviewers/Declined' assert domain_content['reviewers_id']['value'] == 'EFGH.cc/2025/Conference/Reviewers' assert domain_content['reviewers_name']['value'] == 'Reviewers' assert domain_content['reviewers_anon_name']['value'] == 'Reviewer_' assert domain_content['reviewers_submitted_name']['value'] == 'Submitted' assert domain_content['reviewers_recruitment_id']['value'] == 'EFGH.cc/2025/Conference/Reviewers/-/Recruitment_Response' - assert domain_content['reviewers_invited_message_id']['value'] == 'EFGH.cc/2025/Conference/Reviewers/Invited/-/Message' - assert domain_content['area_chairs_invited_id']['value'] == 'EFGH.cc/2025/Conference/Action_Editors/Invited' - assert domain_content['area_chairs_declined_id']['value'] == 'EFGH.cc/2025/Conference/Action_Editors/Declined' assert domain_content['area_chairs_id']['value'] == 'EFGH.cc/2025/Conference/Action_Editors' assert domain_content['area_chairs_name']['value'] == 'Action_Editors' assert domain_content['area_chairs_anon_name']['value'] == 'Action_Editor_' assert 'area_chairs_submitted_name' not in domain_content assert domain_content['area_chairs_recruitment_id']['value'] == 'EFGH.cc/2025/Conference/Action_Editors/-/Recruitment_Response' - assert domain_content['area_chairs_invited_message_id']['value'] == 'EFGH.cc/2025/Conference/Action_Editors/Invited/-/Message' def test_recruit_area_chairs(self, openreview_client, selenium, request_page, helpers): diff --git a/tests/test_change_venue_email.py b/tests/test_change_venue_email.py index 9c617662c6..31fe53a57d 100644 --- a/tests/test_change_venue_email.py +++ b/tests/test_change_venue_email.py @@ -98,7 +98,8 @@ def test_setup(self, openreview_client, helpers): venue_group = openreview.tools.get_group(openreview_client, 'VenueEmail.cc/2025/Conference') assert venue_group and venue_group.content['reviewers_recruitment_id']['value'] == 'VenueEmail.cc/2025/Conference/Reviewers/-/Recruitment_Response' - assert all(key in venue_group.content for key in ['reviewers_declined_id', 'reviewers_invited_id', 'reviewers_invited_message_id']) + assert openreview_client.get_group('VenueEmail.cc/2025/Conference/Reviewers/Invited') + assert openreview_client.get_group('VenueEmail.cc/2025/Conference/Reviewers/Declined') request_note = openreview_client.get_note(request.id) assert request_note.domain == 'openreview.net/Support' diff --git a/tests/test_icml_conference_with_templates.py b/tests/test_icml_conference_with_templates.py index 71e8c6b257..16350fa8ab 100644 --- a/tests/test_icml_conference_with_templates.py +++ b/tests/test_icml_conference_with_templates.py @@ -70,6 +70,7 @@ def test_create_conference(self, openreview_client, helpers): venue.review_stage = openreview.stages.ReviewStage( start_date = due_date + datetime.timedelta(weeks=1), allow_de_anonymization = False, + submission_reviewer_roles = [venue.reviewers_name], ) venue.meta_review_stage = openreview.stages.MetaReviewStage( diff --git a/tests/test_matching_v2.py b/tests/test_matching_v2.py index 229cc28f0d..5396a8a5f3 100644 --- a/tests/test_matching_v2.py +++ b/tests/test_matching_v2.py @@ -36,8 +36,10 @@ def venue(self, openreview_client, helpers): venue.use_area_chairs = True venue.area_chairs_name = 'Senior_Program_Committee' venue.area_chair_roles = ['Senior_Program_Committee'] + venue.submission_area_chair_roles = ['Senior_Program_Committee'] venue.reviewers_name = 'Program_Committee' venue.reviewer_roles = ['Program_Committee'] + venue.submission_reviewer_roles = ['Program_Committee'] now = datetime.datetime.now() venue.submission_stage = openreview.stages.SubmissionStage( due_date = now + datetime.timedelta(minutes = 40), diff --git a/tests/test_merged_committee_roles.py b/tests/test_merged_committee_roles.py new file mode 100644 index 0000000000..48bba50453 --- /dev/null +++ b/tests/test_merged_committee_roles.py @@ -0,0 +1,266 @@ +import datetime +import openreview +import openreview.venue + + +class TestMergedCommitteeRoles(): + + def test_setup_venue(self, openreview_client, helpers): + """Setup a venue with two reviewer roles and two area chair roles + configured in the 'shared' (merged) layout. With this layout, all + roles share a single per-submission group named after the primary + role.""" + + support_group_id = 'openreview.net/Support' + + helpers.create_user('programchair@mrg.cc', 'ProgramChair', 'MRG') + helpers.create_user('expert_one@mrg.cc', 'ExpertOne', 'MRG') + helpers.create_user('expert_two@mrg.cc', 'ExpertTwo', 'MRG') + helpers.create_user('technical_one@mrg.cc', 'TechnicalOne', 'MRG') + helpers.create_user('technical_two@mrg.cc', 'TechnicalTwo', 'MRG') + helpers.create_user('expert_ac@mrg.cc', 'ExpertAC', 'MRG') + helpers.create_user('technical_ac@mrg.cc', 'TechnicalAC', 'MRG') + + pc_client = openreview.api.OpenReviewClient(username='programchair@mrg.cc', password=helpers.strong_password) + + now = datetime.datetime.now() + due_date = now + datetime.timedelta(days=2) + + request = pc_client.post_note_edit( + invitation='openreview.net/Support/Venue_Request/-/Conference_Review_Workflow', + signatures=['~ProgramChair_MRG1'], + note=openreview.api.Note( + content={ + 'official_venue_name': { 'value': 'The MRG Conference' }, + 'abbreviated_venue_name': { 'value': 'MRG 2025' }, + 'venue_website_url': { 'value': 'https://mrg.cc/Conferences/2025' }, + 'location': { 'value': 'Amherst, Massachusetts' }, + 'venue_start_date': { 'value': openreview.tools.datetime_millis(now + datetime.timedelta(weeks=52)) }, + 'program_chair_emails': { 'value': ['programchair@mrg.cc'] }, + 'contact_email': { 'value': 'mrg2025.programchairs@gmail.com' }, + 'submission_start_date': { 'value': openreview.tools.datetime_millis(now) }, + 'submission_deadline': { 'value': openreview.tools.datetime_millis(due_date) }, + 'reviewer_groups_names': { 'value': ['Expert_Reviewers', 'Technical_Reviewers'] }, + 'reviewer_group_layout': { 'value': 'shared' }, + 'area_chairs_support': { 'value': True }, + 'area_chair_groups_names': { 'value': ['Area_Chairs', 'Technical_Area_Chairs'] }, + 'area_chair_group_layout': { 'value': 'shared' }, + 'expected_submissions': { 'value': 100 }, + 'venue_organizer_agreement': { + 'value': [ + 'OpenReview natively supports a wide variety of reviewing workflow configurations. However, if we want significant reviewing process customizations or experiments, we will detail these requests to the OpenReview staff at least three months in advance.', + 'We will ask authors and reviewers to create an OpenReview Profile at least two weeks in advance of the paper submission deadlines.', + 'When assembling our group of reviewers, we will only include email addresses or OpenReview Profile IDs of people we know to have authored publications relevant to our venue. (We will not solicit new reviewers using an open web form, because unfortunately some malicious actors sometimes try to create "fake ids" aiming to be assigned to review their own paper submissions.)', + 'We acknowledge that, if our venue\'s reviewing workflow is non-standard, or if our venue is expecting more than a few hundred submissions for any one deadline, we should designate our own Workflow Chair, who will read the OpenReview documentation and manage our workflow configurations throughout the reviewing process.', + 'We acknowledge that OpenReview staff work Monday-Friday during standard business hours US Eastern time, and we cannot expect support responses outside those times. For this reason, we recommend setting submission and reviewing deadlines Monday through Thursday.', + 'We will treat the OpenReview staff with kindness and consideration.', + 'We acknowledge that authors and reviewers will be required to share their preferred email.', + 'We acknowledge that review counts will be collected for all the reviewers and publicly available in OpenReview.', + 'We acknowledge that metadata for accepted papers will be publicly released in OpenReview.' + ] + } + } + ) + ) + helpers.await_queue_edit(openreview_client, edit_id=request['id']) + + request = openreview_client.get_note(request['note']['id']) + + # deploy the venue + edit = openreview_client.post_note_edit( + invitation='openreview.net/Support/Venue_Request/Conference_Review_Workflow/-/Deployment', + signatures=[support_group_id], + note=openreview.api.Note( + id=request.id, + content={ 'venue_id': { 'value': 'MRG.cc/2025/Conference' } } + ) + ) + helpers.await_queue_edit(openreview_client, edit_id=edit['id']) + + venue_group = openreview_client.get_group('MRG.cc/2025/Conference') + assert venue_group + assert venue_group.content['reviewers_name']['value'] == 'Expert_Reviewers' + + # Both reviewer role top-level groups exist so each role can have its + # own matching, but the shared layout means both assignment invitations + # point to the same per-submission group name (the primary). + assert openreview_client.get_group('MRG.cc/2025/Conference/Expert_Reviewers') + assert openreview_client.get_group('MRG.cc/2025/Conference/Technical_Reviewers') + + expert_assignment = openreview_client.get_invitation('MRG.cc/2025/Conference/Expert_Reviewers/-/Assignment') + assert expert_assignment.content['reviewers_name']['value'] == 'Expert_Reviewers' + assert expert_assignment.content['submission_committee_name']['value'] == 'Expert_Reviewers' + + technical_assignment = openreview_client.get_invitation('MRG.cc/2025/Conference/Technical_Reviewers/-/Assignment') + # Shared layout: technical role reuses the primary submission group name + assert technical_assignment.content['reviewers_name']['value'] == 'Expert_Reviewers' + assert technical_assignment.content['submission_committee_name']['value'] == 'Expert_Reviewers' + + # AC Assignment invitations - also share the primary AC submission name + ac_assignment = openreview_client.get_invitation('MRG.cc/2025/Conference/Area_Chairs/-/Assignment') + assert ac_assignment.content['submission_committee_name']['value'] == 'Area_Chairs' + technical_ac_assignment = openreview_client.get_invitation('MRG.cc/2025/Conference/Technical_Area_Chairs/-/Assignment') + assert technical_ac_assignment.content['submission_committee_name']['value'] == 'Area_Chairs' + + # Domain has both reviewer roles but a single shared submission role + assert venue_group.content['reviewer_roles']['value'] == ['Expert_Reviewers', 'Technical_Reviewers'] + assert venue_group.content['submission_reviewer_roles']['value'] == ['Expert_Reviewers'] + assert venue_group.content['area_chair_roles']['value'] == ['Area_Chairs', 'Technical_Area_Chairs'] + assert venue_group.content['submission_area_chair_roles']['value'] == ['Area_Chairs'] + + # Only the primary Official_Review invitation is auto-created for shared layout + assert venue_group.content['review_names']['value'] == ['Official_Review'] + assert openreview_client.get_invitation('MRG.cc/2025/Conference/-/Official_Review') + + # Populate committee groups + openreview_client.post_group_edit( + invitation='MRG.cc/2025/Conference/Expert_Reviewers/-/Members', + signatures=['MRG.cc/2025/Conference'], + group=openreview.api.Group(members={ 'append': ['~ExpertOne_MRG1', '~ExpertTwo_MRG1'] }) + ) + openreview_client.post_group_edit( + invitation='MRG.cc/2025/Conference/Technical_Reviewers/-/Members', + signatures=['MRG.cc/2025/Conference'], + group=openreview.api.Group(members={ 'append': ['~TechnicalOne_MRG1', '~TechnicalTwo_MRG1'] }) + ) + openreview_client.post_group_edit( + invitation='MRG.cc/2025/Conference/Area_Chairs/-/Members', + signatures=['MRG.cc/2025/Conference'], + group=openreview.api.Group(members={ 'append': ['~ExpertAC_MRG1'] }) + ) + openreview_client.post_group_edit( + invitation='MRG.cc/2025/Conference/Technical_Area_Chairs/-/Members', + signatures=['MRG.cc/2025/Conference'], + group=openreview.api.Group(members={ 'append': ['~TechnicalAC_MRG1'] }) + ) + + def test_post_submissions(self, openreview_client, test_client, helpers): + + test_client = openreview.api.OpenReviewClient(token=test_client.token) + + for i in range(1, 4): + note = openreview.api.Note( + license='CC BY 4.0', + content={ + 'title': { 'value': f'MRG Paper title {i}' }, + 'abstract': { 'value': f'MRG abstract {i}' }, + 'authorids': { 'value': ['~SomeFirstName_User1'] }, + 'authors': { 'value': ['SomeFirstName User'] }, + 'keywords': { 'value': ['key'] }, + 'pdf': { 'value': '/pdf/' + 'p' * 40 + '.pdf' }, + 'email_sharing': { 'value': 'We authorize the sharing of all author emails with Program Chairs.' }, + 'data_release': { 'value': 'We authorize the release of our submission and author names to the public in the event of acceptance.' }, + } + ) + test_client.post_note_edit( + invitation='MRG.cc/2025/Conference/-/Submission', + signatures=['~SomeFirstName_User1'], + note=note + ) + + helpers.await_queue_edit(openreview_client, invitation='MRG.cc/2025/Conference/-/Submission', count=3) + + submissions = openreview_client.get_notes(invitation='MRG.cc/2025/Conference/-/Submission', sort='number:asc') + assert len(submissions) == 3 + + def test_close_submissions_and_create_groups(self, openreview_client, helpers): + """Close the submission deadline and verify that only the primary + reviewer role's Submission_Group invitation creates per-submission + groups (the secondary role shares the primary's group in this layout).""" + + pc_client = openreview.api.OpenReviewClient(username='programchair@mrg.cc', password=helpers.strong_password) + now = datetime.datetime.now() + + pc_client.post_invitation_edit( + invitations='MRG.cc/2025/Conference/Expert_Reviewers/-/Submission_Group/Dates', + content={ + 'activation_date': { 'value': openreview.tools.datetime_millis(now - datetime.timedelta(minutes=30)) } + } + ) + helpers.await_queue_edit(openreview_client, edit_id='MRG.cc/2025/Conference/Expert_Reviewers/-/Submission_Group-0-1', count=2) + + submissions = openreview_client.get_notes(invitation='MRG.cc/2025/Conference/-/Submission', sort='number:asc') + assert len(submissions) == 3 + + for submission in submissions: + # Only the primary per-submission reviewers group exists + assert openreview_client.get_group(f'MRG.cc/2025/Conference/Submission{submission.number}/Expert_Reviewers') + technical_group = openreview.tools.get_group(openreview_client, f'MRG.cc/2025/Conference/Submission{submission.number}/Technical_Reviewers') + assert technical_group is None + + def test_setup_matching_for_both_roles(self, openreview_client, helpers): + """Each reviewer role runs its own matching (distinct match_group) but + both assignment invitations write into the shared per-submission group.""" + + submissions = openreview_client.get_notes(invitation='MRG.cc/2025/Conference/-/Submission', sort='number:asc') + + for role, reviewers in [ + ('Expert_Reviewers', ['~ExpertOne_MRG1', '~ExpertTwo_MRG1']), + ('Technical_Reviewers', ['~TechnicalOne_MRG1', '~TechnicalTwo_MRG1']) + ]: + label = f'{role.lower()}-matching-1' + openreview_client.post_note_edit( + invitation=f'MRG.cc/2025/Conference/{role}/-/Assignment_Configuration', + signatures=['MRG.cc/2025/Conference'], + note=openreview.api.Note( + content={ + 'title': { 'value': label }, + 'user_demand': { 'value': '1' }, + 'max_papers': { 'value': '5' }, + 'min_papers': { 'value': '0' }, + 'alternates': { 'value': '0' }, + 'paper_invitation': { 'value': 'MRG.cc/2025/Conference/-/Submission&content.venueid=MRG.cc/2025/Conference/Submission' }, + 'match_group': { 'value': f'MRG.cc/2025/Conference/{role}' }, + 'aggregate_score_invitation': { 'value': f'MRG.cc/2025/Conference/{role}/-/Aggregate_Score' }, + 'conflicts_invitation': { 'value': f'MRG.cc/2025/Conference/{role}/-/Conflict' }, + 'solver': { 'value': 'FairFlow' }, + 'status': { 'value': 'Complete' } + } + ) + ) + + for index, submission in enumerate(submissions): + openreview_client.post_edge(openreview.api.Edge( + invitation=f'MRG.cc/2025/Conference/{role}/-/Proposed_Assignment', + head=submission.id, + tail=reviewers[index % len(reviewers)], + signatures=['MRG.cc/2025/Conference/Program_Chairs'], + weight=1, + label=label + )) + + def test_deploy_assignments_share_one_group(self, openreview_client, helpers): + """Deploy assignments for both reviewer roles and verify that reviewers + from both roles end up in the same per-submission reviewers group.""" + + venue = openreview.venue.helpers.get_venue(openreview_client, 'MRG.cc/2025/Conference', support_user='openreview.net/Support') + + venue.set_assignments(assignment_title='expert_reviewers-matching-1', committee_id='MRG.cc/2025/Conference/Expert_Reviewers') + venue.set_assignments(assignment_title='technical_reviewers-matching-1', committee_id='MRG.cc/2025/Conference/Technical_Reviewers') + + submissions = openreview_client.get_notes(invitation='MRG.cc/2025/Conference/-/Submission', sort='number:asc') + + all_expert_members = {'~ExpertOne_MRG1', '~ExpertTwo_MRG1'} + all_technical_members = {'~TechnicalOne_MRG1', '~TechnicalTwo_MRG1'} + + merged_members = set() + for submission in submissions: + reviewers_group = openreview_client.get_group(f'MRG.cc/2025/Conference/Submission{submission.number}/Expert_Reviewers') + + # The shared per-submission group holds reviewers from BOTH roles + expert_from_group = set(reviewers_group.members) & all_expert_members + technical_from_group = set(reviewers_group.members) & all_technical_members + assert expert_from_group, f'Submission {submission.number} missing an Expert reviewer' + assert technical_from_group, f'Submission {submission.number} missing a Technical reviewer' + + # Only reviewers from the two configured role groups should be present + for member in reviewers_group.members: + assert member in all_expert_members | all_technical_members + merged_members |= set(reviewers_group.members) + + # No separate Technical_Reviewers per-submission group is created + assert openreview.tools.get_group(openreview_client, f'MRG.cc/2025/Conference/Submission{submission.number}/Technical_Reviewers') is None + + # Sanity: at least one member from each role was assigned somewhere + assert merged_members & all_expert_members + assert merged_members & all_technical_members diff --git a/tests/test_reviewers_only.py b/tests/test_reviewers_only.py index 465df3ff4d..2b0755b3c8 100644 --- a/tests/test_reviewers_only.py +++ b/tests/test_reviewers_only.py @@ -228,7 +228,6 @@ def test_setup(self, openreview_client, helpers): venue_group = openreview.tools.get_group(openreview_client, 'ABCD.cc/2025/Conference') assert venue_group and venue_group.content['reviewers_recruitment_id']['value'] == 'ABCD.cc/2025/Conference/Program_Committee/-/Recruitment_Response' - assert all(key in venue_group.content for key in ['reviewers_declined_id', 'reviewers_invited_id', 'reviewers_invited_message_id']) assert venue_group.content['status_invitation_id']['value'] == f'openreview.net/Support/Venue_Request/Conference_Review_Workflow/-/Status' @@ -248,7 +247,6 @@ def test_setup(self, openreview_client, helpers): venue_group = openreview.tools.get_group(openreview_client, 'ABCD.cc/2025/Conference') assert venue_group and venue_group.content['reviewers_recruitment_id']['value'] == 'ABCD.cc/2025/Conference/Program_Committee/-/Recruitment_Response' - assert all(key in venue_group.content for key in ['reviewers_declined_id', 'reviewers_invited_id', 'reviewers_invited_message_id']) #after deployment, check domain hasn't changed request_note = openreview_client.get_note(request.id) @@ -376,14 +374,11 @@ def test_setup(self, openreview_client, helpers): # check domain object domain_content = openreview_client.get_group('ABCD.cc/2025/Conference').content - assert domain_content['reviewers_invited_id']['value'] == 'ABCD.cc/2025/Conference/Program_Committee/Invited' - assert domain_content['reviewers_declined_id']['value'] == 'ABCD.cc/2025/Conference/Program_Committee/Declined' assert domain_content['reviewers_id']['value'] == 'ABCD.cc/2025/Conference/Program_Committee' assert domain_content['reviewers_name']['value'] == 'Program_Committee' assert domain_content['reviewers_anon_name']['value'] == 'Program_Committee_' assert domain_content['reviewers_submitted_name']['value'] == 'Submitted' assert domain_content['reviewers_recruitment_id']['value'] == 'ABCD.cc/2025/Conference/Program_Committee/-/Recruitment_Response' - assert domain_content['reviewers_invited_message_id']['value'] == 'ABCD.cc/2025/Conference/Program_Committee/Invited/-/Message' request_form = pc_client.get_note(request.id) assert request_form diff --git a/tests/test_tasks.py b/tests/test_tasks.py index 784736fded..7d1402b6f1 100644 --- a/tests/test_tasks.py +++ b/tests/test_tasks.py @@ -101,7 +101,8 @@ def test_setup(self, openreview_client, helpers): venue_group = openreview.tools.get_group(openreview_client, 'Tasks.cc/2025/Conference') assert venue_group and venue_group.content['reviewers_recruitment_id']['value'] == 'Tasks.cc/2025/Conference/Program_Committee/-/Recruitment_Response' - assert all(key in venue_group.content for key in ['reviewers_declined_id', 'reviewers_invited_id', 'reviewers_invited_message_id']) + assert openreview_client.get_group('Tasks.cc/2025/Conference/Program_Committee/Invited') + assert openreview_client.get_group('Tasks.cc/2025/Conference/Program_Committee/Declined') request_note = openreview_client.get_note(request.id) assert request_note.domain == 'openreview.net/Support' diff --git a/tests/test_two_submission_committee_roles.py b/tests/test_two_submission_committee_roles.py new file mode 100644 index 0000000000..4241537502 --- /dev/null +++ b/tests/test_two_submission_committee_roles.py @@ -0,0 +1,548 @@ +import datetime +import openreview +import openreview.venue + + +class TestTwoSubmissionCommitteeRoles(): + + def test_setup_venue(self, openreview_client, helpers): + """Setup a venue using the new workflow configuration with the primary + reviewer role being 'Expert_Reviewers'.""" + + support_group_id = 'openreview.net/Support' + + helpers.create_user('programchair@xyzw.cc', 'ProgramChair', 'XYZW') + helpers.create_user('expert_one@xyzw.cc', 'ExpertOne', 'XYZW') + helpers.create_user('expert_two@xyzw.cc', 'ExpertTwo', 'XYZW') + helpers.create_user('technical_one@xyzw.cc', 'TechnicalOne', 'XYZW') + helpers.create_user('technical_two@xyzw.cc', 'TechnicalTwo', 'XYZW') + helpers.create_user('expert_ac@xyzw.cc', 'ExpertAC', 'XYZW') + helpers.create_user('technical_ac@xyzw.cc', 'TechnicalAC', 'XYZW') + + pc_client = openreview.api.OpenReviewClient(username='programchair@xyzw.cc', password=helpers.strong_password) + + now = datetime.datetime.now() + due_date = now + datetime.timedelta(days=2) + + request = pc_client.post_note_edit( + invitation='openreview.net/Support/Venue_Request/-/Conference_Review_Workflow', + signatures=['~ProgramChair_XYZW1'], + note=openreview.api.Note( + content={ + 'official_venue_name': { 'value': 'The XYZW Conference' }, + 'abbreviated_venue_name': { 'value': 'XYZW 2025' }, + 'venue_website_url': { 'value': 'https://xyzw.cc/Conferences/2025' }, + 'location': { 'value': 'Amherst, Massachusetts' }, + 'venue_start_date': { 'value': openreview.tools.datetime_millis(now + datetime.timedelta(weeks=52)) }, + 'program_chair_emails': { 'value': ['programchair@xyzw.cc'] }, + 'contact_email': { 'value': 'xyzw2025.programchairs@gmail.com' }, + 'submission_start_date': { 'value': openreview.tools.datetime_millis(now) }, + 'submission_deadline': { 'value': openreview.tools.datetime_millis(due_date) }, + 'reviewer_groups_names': { 'value': ['Expert_Reviewers', 'Technical_Reviewers'] }, + 'reviewer_group_layout': { 'value': 'per_role' }, + 'area_chairs_support': { 'value': True }, + 'area_chair_groups_names': { 'value': ['Area_Chairs', 'Technical_Area_Chairs'] }, + 'area_chair_group_layout': { 'value': 'per_role' }, + 'expected_submissions': { 'value': 100 }, + 'venue_organizer_agreement': { + 'value': [ + 'OpenReview natively supports a wide variety of reviewing workflow configurations. However, if we want significant reviewing process customizations or experiments, we will detail these requests to the OpenReview staff at least three months in advance.', + 'We will ask authors and reviewers to create an OpenReview Profile at least two weeks in advance of the paper submission deadlines.', + 'When assembling our group of reviewers, we will only include email addresses or OpenReview Profile IDs of people we know to have authored publications relevant to our venue. (We will not solicit new reviewers using an open web form, because unfortunately some malicious actors sometimes try to create "fake ids" aiming to be assigned to review their own paper submissions.)', + 'We acknowledge that, if our venue\'s reviewing workflow is non-standard, or if our venue is expecting more than a few hundred submissions for any one deadline, we should designate our own Workflow Chair, who will read the OpenReview documentation and manage our workflow configurations throughout the reviewing process.', + 'We acknowledge that OpenReview staff work Monday-Friday during standard business hours US Eastern time, and we cannot expect support responses outside those times. For this reason, we recommend setting submission and reviewing deadlines Monday through Thursday.', + 'We will treat the OpenReview staff with kindness and consideration.', + 'We acknowledge that authors and reviewers will be required to share their preferred email.', + 'We acknowledge that review counts will be collected for all the reviewers and publicly available in OpenReview.', + 'We acknowledge that metadata for accepted papers will be publicly released in OpenReview.' + ] + } + } + ) + ) + helpers.await_queue_edit(openreview_client, edit_id=request['id']) + + request = openreview_client.get_note(request['note']['id']) + + # deploy the venue + edit = openreview_client.post_note_edit( + invitation='openreview.net/Support/Venue_Request/Conference_Review_Workflow/-/Deployment', + signatures=[support_group_id], + note=openreview.api.Note( + id=request.id, + content={ 'venue_id': { 'value': 'XYZW.cc/2025/Conference' } } + ) + ) + helpers.await_queue_edit(openreview_client, edit_id=edit['id']) + + venue_group = openreview_client.get_group('XYZW.cc/2025/Conference') + assert venue_group + assert venue_group.content['reviewers_name']['value'] == 'Expert_Reviewers' + assert openreview_client.get_group('XYZW.cc/2025/Conference/Expert_Reviewers') + assert openreview_client.get_invitation('XYZW.cc/2025/Conference/Expert_Reviewers/-/Submission_Group') + + expert_assignment = openreview_client.get_invitation('XYZW.cc/2025/Conference/Expert_Reviewers/-/Assignment') + assert expert_assignment.content['review_name']['value'] == 'Official_Review' + assert expert_assignment.content['reviewers_id']['value'] == 'XYZW.cc/2025/Conference/Expert_Reviewers' + assert expert_assignment.content['reviewers_name']['value'] == 'Expert_Reviewers' + assert expert_assignment.content['reviewers_anon_name']['value'] == 'Expert_Reviewer_' + assert expert_assignment.content['committee_role']['value'] == 'reviewers' + assert expert_assignment.content['submission_committee_name']['value'] == 'Expert_Reviewers' + assert openreview_client.get_invitation('XYZW.cc/2025/Conference/Expert_Reviewers/-/Proposed_Assignment') + assert openreview_client.get_invitation('XYZW.cc/2025/Conference/Expert_Reviewers/-/Assignment_Configuration') + + # Second reviewer role auto-created by deployment via reviewer_groups_names + assert openreview_client.get_group('XYZW.cc/2025/Conference/Technical_Reviewers') + assert openreview_client.get_invitation('XYZW.cc/2025/Conference/Technical_Reviewers/-/Submission_Group') + technical_assignment = openreview_client.get_invitation('XYZW.cc/2025/Conference/Technical_Reviewers/-/Assignment') + assert technical_assignment.content['review_name']['value'] == 'Official_Review' + assert technical_assignment.content['reviewers_id']['value'] == 'XYZW.cc/2025/Conference/Technical_Reviewers' + assert technical_assignment.content['reviewers_name']['value'] == 'Technical_Reviewers' + assert technical_assignment.content['reviewers_anon_name']['value'] == 'Technical_Reviewer_' + assert technical_assignment.content['committee_role']['value'] == 'reviewers' + assert technical_assignment.content['submission_committee_name']['value'] == 'Technical_Reviewers' + assert openreview_client.get_invitation('XYZW.cc/2025/Conference/Technical_Reviewers/-/Proposed_Assignment') + assert openreview_client.get_invitation('XYZW.cc/2025/Conference/Technical_Reviewers/-/Assignment_Configuration') + + # AC Assignment invitations + ac_assignment = openreview_client.get_invitation('XYZW.cc/2025/Conference/Area_Chairs/-/Assignment') + assert ac_assignment.content['submission_committee_name']['value'] == 'Area_Chairs' + technical_ac_assignment = openreview_client.get_invitation('XYZW.cc/2025/Conference/Technical_Area_Chairs/-/Assignment') + assert technical_ac_assignment.content['submission_committee_name']['value'] == 'Technical_Area_Chairs' + + # Domain has both reviewer roles + submission_reviewer_roles configured per_role + assert venue_group.content['reviewer_roles']['value'] == ['Expert_Reviewers', 'Technical_Reviewers'] + assert venue_group.content['submission_reviewer_roles']['value'] == ['Expert_Reviewers', 'Technical_Reviewers'] + + # Second AC role auto-created as well + assert openreview_client.get_group('XYZW.cc/2025/Conference/Technical_Area_Chairs') + assert openreview_client.get_invitation('XYZW.cc/2025/Conference/Technical_Area_Chairs/-/Submission_Group') + assert venue_group.content['area_chair_roles']['value'] == ['Area_Chairs', 'Technical_Area_Chairs'] + assert venue_group.content['submission_area_chair_roles']['value'] == ['Area_Chairs', 'Technical_Area_Chairs'] + + # Populate committee groups + openreview_client.post_group_edit( + invitation='XYZW.cc/2025/Conference/Expert_Reviewers/-/Members', + signatures=['XYZW.cc/2025/Conference'], + group=openreview.api.Group(members={ 'append': ['~ExpertOne_XYZW1', '~ExpertTwo_XYZW1'] }) + ) + openreview_client.post_group_edit( + invitation='XYZW.cc/2025/Conference/Technical_Reviewers/-/Members', + signatures=['XYZW.cc/2025/Conference'], + group=openreview.api.Group(members={ 'append': ['~TechnicalOne_XYZW1', '~TechnicalTwo_XYZW1'] }) + ) + openreview_client.post_group_edit( + invitation='XYZW.cc/2025/Conference/Area_Chairs/-/Members', + signatures=['XYZW.cc/2025/Conference'], + group=openreview.api.Group(members={ 'append': ['~ExpertAC_XYZW1'] }) + ) + openreview_client.post_group_edit( + invitation='XYZW.cc/2025/Conference/Technical_Area_Chairs/-/Members', + signatures=['XYZW.cc/2025/Conference'], + group=openreview.api.Group(members={ 'append': ['~TechnicalAC_XYZW1'] }) + ) + + def test_post_submissions(self, openreview_client, test_client, helpers): + + test_client = openreview.api.OpenReviewClient(token=test_client.token) + + for i in range(1, 4): + note = openreview.api.Note( + license='CC BY 4.0', + content={ + 'title': { 'value': f'XYZW Paper title {i}' }, + 'abstract': { 'value': f'XYZW abstract {i}' }, + 'authorids': { 'value': ['~SomeFirstName_User1'] }, + 'authors': { 'value': ['SomeFirstName User'] }, + 'keywords': { 'value': ['key'] }, + 'pdf': { 'value': '/pdf/' + 'p' * 40 + '.pdf' }, + 'email_sharing': { 'value': 'We authorize the sharing of all author emails with Program Chairs.' }, + 'data_release': { 'value': 'We authorize the release of our submission and author names to the public in the event of acceptance.' }, + } + ) + test_client.post_note_edit( + invitation='XYZW.cc/2025/Conference/-/Submission', + signatures=['~SomeFirstName_User1'], + note=note + ) + + helpers.await_queue_edit(openreview_client, invitation='XYZW.cc/2025/Conference/-/Submission', count=3) + + submissions = openreview_client.get_notes(invitation='XYZW.cc/2025/Conference/-/Submission', sort='number:asc') + assert len(submissions) == 3 + + def test_close_submissions_and_create_groups(self, openreview_client, helpers): + """Close the submission deadline and verify both reviewer roles' + Submission_Group invitations run and create per-submission groups.""" + + pc_client = openreview.api.OpenReviewClient(username='programchair@xyzw.cc', password=helpers.strong_password) + now = datetime.datetime.now() + + pc_client.post_invitation_edit( + invitations='XYZW.cc/2025/Conference/Expert_Reviewers/-/Submission_Group/Dates', + content={ + 'activation_date': { 'value': openreview.tools.datetime_millis(now - datetime.timedelta(minutes=30)) } + } + ) + helpers.await_queue_edit(openreview_client, edit_id='XYZW.cc/2025/Conference/Expert_Reviewers/-/Submission_Group-0-1', count=2) + + pc_client.post_invitation_edit( + invitations='XYZW.cc/2025/Conference/Technical_Reviewers/-/Submission_Group/Dates', + content={ + 'activation_date': { 'value': openreview.tools.datetime_millis(now - datetime.timedelta(minutes=30)) } + } + ) + helpers.await_queue_edit(openreview_client, edit_id='XYZW.cc/2025/Conference/Technical_Reviewers/-/Submission_Group-0-1', count=2) + + submissions = openreview_client.get_notes(invitation='XYZW.cc/2025/Conference/-/Submission', sort='number:asc') + assert len(submissions) == 3 + + for submission in submissions: + assert openreview_client.get_group(f'XYZW.cc/2025/Conference/Submission{submission.number}/Expert_Reviewers') + assert openreview_client.get_group(f'XYZW.cc/2025/Conference/Submission{submission.number}/Technical_Reviewers') + + def test_setup_matching_for_both_roles(self, openreview_client, helpers): + """Post Assignment_Configuration notes and Proposed_Assignment edges for + both reviewer roles.""" + + submissions = openreview_client.get_notes(invitation='XYZW.cc/2025/Conference/-/Submission', sort='number:asc') + + for role, reviewers in [ + ('Expert_Reviewers', ['~ExpertOne_XYZW1', '~ExpertTwo_XYZW1']), + ('Technical_Reviewers', ['~TechnicalOne_XYZW1', '~TechnicalTwo_XYZW1']) + ]: + label = f'{role.lower()}-matching-1' + openreview_client.post_note_edit( + invitation=f'XYZW.cc/2025/Conference/{role}/-/Assignment_Configuration', + signatures=['XYZW.cc/2025/Conference'], + note=openreview.api.Note( + content={ + 'title': { 'value': label }, + 'user_demand': { 'value': '1' }, + 'max_papers': { 'value': '5' }, + 'min_papers': { 'value': '0' }, + 'alternates': { 'value': '0' }, + 'paper_invitation': { 'value': 'XYZW.cc/2025/Conference/-/Submission&content.venueid=XYZW.cc/2025/Conference/Submission' }, + 'match_group': { 'value': f'XYZW.cc/2025/Conference/{role}' }, + 'aggregate_score_invitation': { 'value': f'XYZW.cc/2025/Conference/{role}/-/Aggregate_Score' }, + 'conflicts_invitation': { 'value': f'XYZW.cc/2025/Conference/{role}/-/Conflict' }, + 'solver': { 'value': 'FairFlow' }, + 'status': { 'value': 'Complete' } + } + ) + ) + + for index, submission in enumerate(submissions): + openreview_client.post_edge(openreview.api.Edge( + invitation=f'XYZW.cc/2025/Conference/{role}/-/Proposed_Assignment', + head=submission.id, + tail=reviewers[index % len(reviewers)], + signatures=['XYZW.cc/2025/Conference/Program_Chairs'], + weight=1, + label=label + )) + + def test_deploy_assignments_for_both_roles(self, openreview_client, helpers): + """Deploy assignments for both reviewer roles and verify each role's + per-submission group has only the reviewers from its own matching.""" + + venue = openreview.venue.helpers.get_venue(openreview_client, 'XYZW.cc/2025/Conference', support_user='openreview.net/Support') + + venue.set_assignments(assignment_title='expert_reviewers-matching-1', committee_id='XYZW.cc/2025/Conference/Expert_Reviewers') + venue.set_assignments(assignment_title='technical_reviewers-matching-1', committee_id='XYZW.cc/2025/Conference/Technical_Reviewers') + + submissions = openreview_client.get_notes(invitation='XYZW.cc/2025/Conference/-/Submission', sort='number:asc') + + expert_members = set() + technical_members = set() + for submission in submissions: + expert_group = openreview_client.get_group(f'XYZW.cc/2025/Conference/Submission{submission.number}/Expert_Reviewers') + technical_group = openreview_client.get_group(f'XYZW.cc/2025/Conference/Submission{submission.number}/Technical_Reviewers') + + # The Expert_Reviewers per-submission group must only contain expert reviewers + for member in expert_group.members: + assert member in ['~ExpertOne_XYZW1', '~ExpertTwo_XYZW1'] + expert_members.add(member) + + # The Technical_Reviewers per-submission group must only contain technical reviewers + for member in technical_group.members: + assert member in ['~TechnicalOne_XYZW1', '~TechnicalTwo_XYZW1'] + technical_members.add(member) + + # Sanity: at least one of each role's reviewers was assigned somewhere + assert expert_members + assert technical_members + + def test_undeploy_assignments_for_both_roles(self, openreview_client, helpers): + """Undeploy assignments for both reviewer roles and verify each role's + per-submission group is emptied of its assigned members.""" + + venue = openreview.venue.helpers.get_venue(openreview_client, 'XYZW.cc/2025/Conference', support_user='openreview.net/Support') + + venue.unset_assignments(assignment_title='expert_reviewers-matching-1', committee_id='XYZW.cc/2025/Conference/Expert_Reviewers') + venue.unset_assignments(assignment_title='technical_reviewers-matching-1', committee_id='XYZW.cc/2025/Conference/Technical_Reviewers') + + submissions = openreview_client.get_notes(invitation='XYZW.cc/2025/Conference/-/Submission', sort='number:asc') + + for submission in submissions: + expert_group = openreview_client.get_group(f'XYZW.cc/2025/Conference/Submission{submission.number}/Expert_Reviewers') + technical_group = openreview_client.get_group(f'XYZW.cc/2025/Conference/Submission{submission.number}/Technical_Reviewers') + assert expert_group.members == [] + assert technical_group.members == [] + + def test_trigger_area_chair_submission_groups(self, openreview_client, helpers): + """Trigger both Area_Chairs Submission_Group invitations to create + per-paper AC groups. Matching + Assignment invitations for both AC + roles were auto-created at deployment.""" + + pc_client = openreview.api.OpenReviewClient(username='programchair@xyzw.cc', password=helpers.strong_password) + now = datetime.datetime.now() + for role in ['Area_Chairs', 'Technical_Area_Chairs']: + pc_client.post_invitation_edit( + invitations=f'XYZW.cc/2025/Conference/{role}/-/Submission_Group/Dates', + content={ + 'activation_date': { 'value': openreview.tools.datetime_millis(now - datetime.timedelta(minutes=30)) } + } + ) + helpers.await_queue_edit(openreview_client, edit_id=f'XYZW.cc/2025/Conference/{role}/-/Submission_Group-0-1', count=2) + + submissions = openreview_client.get_notes(invitation='XYZW.cc/2025/Conference/-/Submission', sort='number:asc') + for submission in submissions: + assert openreview_client.get_group(f'XYZW.cc/2025/Conference/Submission{submission.number}/Area_Chairs') + assert openreview_client.get_group(f'XYZW.cc/2025/Conference/Submission{submission.number}/Technical_Area_Chairs') + + assert openreview_client.get_invitation('XYZW.cc/2025/Conference/Area_Chairs/-/Assignment') + assert openreview_client.get_invitation('XYZW.cc/2025/Conference/Technical_Area_Chairs/-/Assignment') + assert openreview_client.get_invitation('XYZW.cc/2025/Conference/Area_Chairs/-/Proposed_Assignment') + assert openreview_client.get_invitation('XYZW.cc/2025/Conference/Technical_Area_Chairs/-/Proposed_Assignment') + + def test_setup_ac_matching_for_both_roles(self, openreview_client, helpers): + """Post Assignment_Configuration notes and Proposed_Assignment edges for + both area chair roles.""" + + submissions = openreview_client.get_notes(invitation='XYZW.cc/2025/Conference/-/Submission', sort='number:asc') + + for role, acs in [ + ('Area_Chairs', ['~ExpertAC_XYZW1']), + ('Technical_Area_Chairs', ['~TechnicalAC_XYZW1']) + ]: + label = f'{role.lower()}-matching-1' + openreview_client.post_note_edit( + invitation=f'XYZW.cc/2025/Conference/{role}/-/Assignment_Configuration', + signatures=['XYZW.cc/2025/Conference'], + note=openreview.api.Note( + content={ + 'title': { 'value': label }, + 'user_demand': { 'value': '1' }, + 'max_papers': { 'value': '5' }, + 'min_papers': { 'value': '0' }, + 'alternates': { 'value': '0' }, + 'paper_invitation': { 'value': 'XYZW.cc/2025/Conference/-/Submission&content.venueid=XYZW.cc/2025/Conference/Submission' }, + 'match_group': { 'value': f'XYZW.cc/2025/Conference/{role}' }, + 'aggregate_score_invitation': { 'value': f'XYZW.cc/2025/Conference/{role}/-/Aggregate_Score' }, + 'conflicts_invitation': { 'value': f'XYZW.cc/2025/Conference/{role}/-/Conflict' }, + 'solver': { 'value': 'FairFlow' }, + 'status': { 'value': 'Complete' } + } + ) + ) + + for index, submission in enumerate(submissions): + openreview_client.post_edge(openreview.api.Edge( + invitation=f'XYZW.cc/2025/Conference/{role}/-/Proposed_Assignment', + head=submission.id, + tail=acs[index % len(acs)], + signatures=['XYZW.cc/2025/Conference/Program_Chairs'], + weight=1, + label=label + )) + + def test_deploy_ac_assignments_for_both_roles(self, openreview_client, helpers): + """Deploy AC assignments for both area chair roles and verify each role's + per-submission group has only its own AC.""" + + venue = openreview.venue.helpers.get_venue(openreview_client, 'XYZW.cc/2025/Conference', support_user='openreview.net/Support') + + venue.set_assignments(assignment_title='area_chairs-matching-1', committee_id='XYZW.cc/2025/Conference/Area_Chairs') + venue.set_assignments(assignment_title='technical_area_chairs-matching-1', committee_id='XYZW.cc/2025/Conference/Technical_Area_Chairs') + + submissions = openreview_client.get_notes(invitation='XYZW.cc/2025/Conference/-/Submission', sort='number:asc') + + for submission in submissions: + ac_group = openreview_client.get_group(f'XYZW.cc/2025/Conference/Submission{submission.number}/Area_Chairs') + technical_ac_group = openreview_client.get_group(f'XYZW.cc/2025/Conference/Submission{submission.number}/Technical_Area_Chairs') + for member in ac_group.members: + assert member == '~ExpertAC_XYZW1' + for member in technical_ac_group.members: + assert member == '~TechnicalAC_XYZW1' + + def test_setup_review_stage_for_both_roles(self, openreview_client, helpers): + """Verify the default Official_Review only targets the primary reviewer + role, then add a second review form for the second reviewer role and + verify the per-paper child invitations for each form invite the right + reviewer group.""" + + pc_client = openreview.api.OpenReviewClient(username='programchair@xyzw.cc', password=helpers.strong_password) + now = datetime.datetime.now() + new_cdate = openreview.tools.datetime_millis(now) + new_duedate = openreview.tools.datetime_millis(now + datetime.timedelta(days=3)) + + # Trigger the default Official_Review invitation (wired to Expert_Reviewers) + # to create per-paper child invitations. + pc_client.post_invitation_edit( + invitations='XYZW.cc/2025/Conference/-/Official_Review/Dates', + content={ + 'activation_date': { 'value': new_cdate }, + 'due_date': { 'value': new_duedate }, + 'expiration_date': { 'value': new_duedate } + } + ) + helpers.await_queue_edit(openreview_client, edit_id='XYZW.cc/2025/Conference/-/Official_Review-0-1', count=2) + + submissions = openreview_client.get_notes(invitation='XYZW.cc/2025/Conference/-/Submission', sort='number:asc') + for submission in submissions: + child = openreview_client.get_invitation(f'XYZW.cc/2025/Conference/Submission{submission.number}/-/Official_Review') + assert f'XYZW.cc/2025/Conference/Submission{submission.number}/Expert_Reviewers' in child.invitees + assert f'XYZW.cc/2025/Conference/Submission{submission.number}/Technical_Reviewers' not in child.invitees + # Only Expert_Reviewer anon signatures are allowed + signatures_items = child.edit['signatures']['param']['items'] + assert any('Expert_Reviewer_' in item.get('prefix', '') for item in signatures_items) + assert not any('Technical_Reviewer_' in item.get('prefix', '') for item in signatures_items) + + # The second review form for Technical_Reviewers is auto-created by the + # venue deployment when the reviewer group layout is per-role. Trigger + # its /Dates to activate it now and verify the per-paper children. + pc_client.post_invitation_edit( + invitations='XYZW.cc/2025/Conference/-/Technical_Reviewers_Review/Dates', + content={ + 'activation_date': { 'value': new_cdate }, + 'due_date': { 'value': new_duedate }, + 'expiration_date': { 'value': new_duedate } + } + ) + helpers.await_queue_edit(openreview_client, edit_id='XYZW.cc/2025/Conference/-/Technical_Reviewers_Review-0-1', count=2) + + for submission in submissions: + child = openreview_client.get_invitation(f'XYZW.cc/2025/Conference/Submission{submission.number}/-/Technical_Reviewers_Review') + assert f'XYZW.cc/2025/Conference/Submission{submission.number}/Technical_Reviewers' in child.invitees + assert f'XYZW.cc/2025/Conference/Submission{submission.number}/Expert_Reviewers' not in child.invitees + signatures_items = child.edit['signatures']['param']['items'] + assert any('Technical_Reviewer_' in item.get('prefix', '') for item in signatures_items) + assert not any('Expert_Reviewer_' in item.get('prefix', '') for item in signatures_items) + + # Reviewer assignments were undeployed earlier; redeploy them so reviewers + # can actually post reviews for submission 1. + venue = openreview.venue.helpers.get_venue(openreview_client, 'XYZW.cc/2025/Conference', support_user='openreview.net/Support') + venue.set_assignments(assignment_title='expert_reviewers-matching-1', committee_id='XYZW.cc/2025/Conference/Expert_Reviewers') + venue.set_assignments(assignment_title='technical_reviewers-matching-1', committee_id='XYZW.cc/2025/Conference/Technical_Reviewers') + + # Run Submission_Change_Before_Reviewing so that submissions become + # readable by both per-paper reviewer groups. + pc_client.post_invitation_edit( + invitations='XYZW.cc/2025/Conference/-/Submission_Change_Before_Reviewing/Dates', + content={ + 'activation_date': { 'value': openreview.tools.datetime_millis(now - datetime.timedelta(minutes=30)) } + } + ) + helpers.await_queue_edit(openreview_client, edit_id='XYZW.cc/2025/Conference/-/Submission_Change_Before_Reviewing-0-1', count=2) + + submission1 = openreview_client.get_note(submissions[0].id) + assert f'XYZW.cc/2025/Conference/Submission{submission1.number}/Expert_Reviewers' in submission1.readers + assert f'XYZW.cc/2025/Conference/Submission{submission1.number}/Technical_Reviewers' in submission1.readers + + # Post one Official_Review as an assigned Expert reviewer for submission 1 + expert_client = openreview.api.OpenReviewClient(username='expert_one@xyzw.cc', password=helpers.strong_password) + expert_anon_groups = expert_client.get_groups(prefix=f'XYZW.cc/2025/Conference/Submission{submission1.number}/Expert_Reviewer_.*', signatory='~ExpertOne_XYZW1') + if not expert_anon_groups: + expert_client = openreview.api.OpenReviewClient(username='expert_two@xyzw.cc', password=helpers.strong_password) + expert_anon_groups = expert_client.get_groups(prefix=f'XYZW.cc/2025/Conference/Submission{submission1.number}/Expert_Reviewer_.*', signatory='~ExpertTwo_XYZW1') + assert len(expert_anon_groups) == 1 + + expert_review = expert_client.post_note_edit( + invitation=f'XYZW.cc/2025/Conference/Submission{submission1.number}/-/Official_Review', + signatures=[expert_anon_groups[0].id], + note=openreview.api.Note( + content={ + 'title': { 'value': 'Expert review' }, + 'review': { 'value': 'Solid contribution from an expert perspective.' }, + 'rating': { 'value': 8 }, + 'confidence': { 'value': 4 } + } + ) + ) + helpers.await_queue_edit(openreview_client, edit_id=expert_review['id']) + + # Post one Technical_Reviewers_Review as an assigned Technical reviewer for submission 1 + technical_client = openreview.api.OpenReviewClient(username='technical_one@xyzw.cc', password=helpers.strong_password) + technical_anon_groups = technical_client.get_groups(prefix=f'XYZW.cc/2025/Conference/Submission{submission1.number}/Technical_Reviewer_.*', signatory='~TechnicalOne_XYZW1') + if not technical_anon_groups: + technical_client = openreview.api.OpenReviewClient(username='technical_two@xyzw.cc', password=helpers.strong_password) + technical_anon_groups = technical_client.get_groups(prefix=f'XYZW.cc/2025/Conference/Submission{submission1.number}/Technical_Reviewer_.*', signatory='~TechnicalTwo_XYZW1') + assert len(technical_anon_groups) == 1 + + technical_review_note = technical_client.post_note_edit( + invitation=f'XYZW.cc/2025/Conference/Submission{submission1.number}/-/Technical_Reviewers_Review', + signatures=[technical_anon_groups[0].id], + note=openreview.api.Note( + content={ + 'title': { 'value': 'Technical review' }, + 'review': { 'value': 'Technically sound implementation.' }, + 'rating': { 'value': 7 }, + 'confidence': { 'value': 4 } + } + ) + ) + helpers.await_queue_edit(openreview_client, edit_id=technical_review_note['id']) + + official_reviews = openreview_client.get_notes(invitation=f'XYZW.cc/2025/Conference/Submission{submission1.number}/-/Official_Review') + assert len(official_reviews) == 1 + technical_reviews = openreview_client.get_notes(invitation=f'XYZW.cc/2025/Conference/Submission{submission1.number}/-/Technical_Reviewers_Review') + assert len(technical_reviews) == 1 + + # Each reviewer's anon signature should land only in their own role's Submitted group. + expert_submitted_group = openreview_client.get_group(f'XYZW.cc/2025/Conference/Submission{submission1.number}/Expert_Reviewers/Submitted') + assert expert_anon_groups[0].id in expert_submitted_group.members + assert technical_anon_groups[0].id not in expert_submitted_group.members + + technical_submitted_group = openreview_client.get_group(f'XYZW.cc/2025/Conference/Submission{submission1.number}/Technical_Reviewers/Submitted') + assert technical_anon_groups[0].id in technical_submitted_group.members + assert expert_anon_groups[0].id not in technical_submitted_group.members + + # Program chairs are notified for each posted review. The subject uses + # the domain-level review_name ('Official_Review'), so both posts share the subject. + pc_messages = openreview_client.get_messages( + to='programchair@xyzw.cc', + subject=f'[XYZW 2025] A official review has been received on Paper number: {submission1.number}, Paper title: "XYZW Paper title 1"' + ) + assert len(pc_messages) == 2 + + # The reviewer who posted each review gets the "Your review has been received" confirmation. + expert_reviewer_id = openreview_client.get_group(expert_anon_groups[0].id).members[0] + technical_reviewer_id = openreview_client.get_group(technical_anon_groups[0].id).members[0] + expert_tauthor_email = openreview_client.get_messages( + to=openreview_client.get_profile(expert_reviewer_id).content['preferredEmail'], + subject=f'[XYZW 2025] Your official review has been received on your assigned Paper number: {submission1.number}, Paper title: "XYZW Paper title 1"' + ) + assert len(expert_tauthor_email) == 1 + technical_tauthor_email = openreview_client.get_messages( + to=openreview_client.get_profile(technical_reviewer_id).content['preferredEmail'], + subject=f'[XYZW 2025] Your official review has been received on your assigned Paper number: {submission1.number}, Paper title: "XYZW Paper title 1"' + ) + assert len(technical_tauthor_email) == 1 + + def test_undeploy_ac_assignments_for_both_roles(self, openreview_client, helpers): + """Undeploy AC assignments for both area chair roles and verify each + role's per-submission group is emptied.""" + + venue = openreview.venue.helpers.get_venue(openreview_client, 'XYZW.cc/2025/Conference', support_user='openreview.net/Support') + + venue.unset_assignments(assignment_title='area_chairs-matching-1', committee_id='XYZW.cc/2025/Conference/Area_Chairs') + venue.unset_assignments(assignment_title='technical_area_chairs-matching-1', committee_id='XYZW.cc/2025/Conference/Technical_Area_Chairs') + + submissions = openreview_client.get_notes(invitation='XYZW.cc/2025/Conference/-/Submission', sort='number:asc') + + for submission in submissions: + ac_group = openreview_client.get_group(f'XYZW.cc/2025/Conference/Submission{submission.number}/Area_Chairs') + technical_ac_group = openreview_client.get_group(f'XYZW.cc/2025/Conference/Submission{submission.number}/Technical_Area_Chairs') + assert ac_group.members == [] + assert technical_ac_group.members == []