ARR: Add way for ACs to evaluate Review Issue Reports

openreview/arr/helpers.py

@@ -473,6 +473,129 @@ def _build_preprint_release_edit(client, venue, builder, request_form):
             edit['note']['content'] = note_content
 
         return {'edit': edit}
+
+    @staticmethod
+    def _build_report_evaluation_edit(client, venue, builder, request_form):
+        venue_id = venue.id
+        invitation_name = builder.REPORT_EVALUATION_NAME
+
+        invitees = [venue.get_area_chairs_id(number='${3/content/noteNumber/value}')]
+        readers = [
+            venue_id,
+            venue.get_program_chairs_id(),
+            venue.get_area_chairs_id(number='${3/content/noteNumber/value}')
+        ]
+        note_readers = [
+            venue_id,
+            venue.get_program_chairs_id(),
+            venue.get_area_chairs_id(number='${5/content/noteNumber/value}')
+        ]
+        signature_items = [
+            {
+                'prefix': venue.get_area_chairs_id(number='${7/content/noteNumber/value}', anon=True),
+                'optional': True
+            },
+            {
+                'value': venue.get_program_chairs_id(),
+                'optional': True
+            },
+            {
+                'value': venue_id,
+                'optional': True
+            }
+        ]
+
+        if venue.use_senior_area_chairs:
+            readers.append(venue.get_senior_area_chairs_id(number='${3/content/noteNumber/value}'))
+            note_readers.append(venue.get_senior_area_chairs_id(number='${5/content/noteNumber/value}'))
+
+        child_invitation_id = venue.get_invitation_id(
+            invitation_name,
+            prefix=(
+                f"{venue.get_paper_group_prefix('${2/content/noteNumber/value}')}"
+                f"/Review_Issue_Report${{{{2/content/replyId/value}}/number}}"
+            )
+        )
+        with_invitation = venue.get_invitation_id(
+            invitation_name,
+            prefix=(
+                f"{venue.get_paper_group_prefix('${6/content/noteNumber/value}')}"
+                f"/Review_Issue_Report${{{{6/content/replyId/value}}/number}}"
+            )
+        )
+
+        return {
+            'description': 'Create a per-report Area Chair evaluation invitation for a submitted review issue report.',
+            'edit': {
+                'signatures': [venue_id],
+                'readers': [venue_id],
+                'writers': [venue_id],
+                'content': {
+                    'noteNumber': {
+                        'value': {
+                            'param': {
+                                'type': 'integer'
+                            }
+                        }
+                    },
+                    'replyId': {
+                        'value': {
+                            'param': {
+                                'type': 'string'
+                            }
+                        }
+                    },
+                    'expdate': {
+                        'value': {
+                            'param': {
+                                'type': 'integer',
+                                'range': [0, 9999999999999]
+                            }
+                        }
+                    },
+                    'content': {
+                        'value': {
+                            'param': {
+                                'type': 'content'
+                            }
+                        }
+                    }
+                },
+                'replacement': True,
+                'invitation': {
+                    'id': child_invitation_id,
+                    'signatures': [venue_id],
+                    'readers': readers,
+                    'writers': [venue_id],
+                    'invitees': invitees,
+                    'maxReplies': 1,
+                    'expdate': '${2/content/expdate/value}',
+                    'edit': {
+                        'signatures': {
+                            'param': {
+                                'items': signature_items
+                            }
+                        },
+                        'readers': ['${2/note/readers}'],
+                        'writers': [venue_id, '${2/signatures}'],
+                        'note': {
+                            'id': {
+                                'param': {
+                                    'withInvitation': with_invitation,
+                                    'optional': True
+                                }
+                            },
+                            'forum': '${{4/content/replyId/value}/forum}',
+                            'replyto': '${4/content/replyId/value}',
+                            'signatures': ['${3/signatures}'],
+                            'readers': note_readers,
+                            'writers': [venue_id, '${3/signatures}'],
+                            'content': '${4/content/content/value}'
+                        }
+                    }
+                }
+            }
+        }
 
     @staticmethod
     def _extend_desk_reject_verification(client, venue, builder, request_form):
@@ -1055,6 +1178,15 @@ def __init__(self, client_v2, venue, configuration_note, request_form_id, suppor
                 process='../arr/process/verification_process.py',
                 extend=ARRWorkflow._extend_desk_reject_verification
             ),
+            ARRStage(
+                type=ARRStage.Type.PROCESS_INVITATION,
+                required_fields=['review_issue_start_date', 'review_issue_exp_date'],
+                super_invitation_id=f"{self.venue_id}/-/{self.invitation_builder.REPORT_EVALUATION_NAME}",
+                stage_arguments={},
+                start_date=self.configuration_note.content.get('review_issue_start_date'),
+                exp_date=self.configuration_note.content.get('review_issue_exp_date'),
+                build_edit=ARRWorkflow._build_report_evaluation_edit
+            ),
             ARRStage(
                 type=ARRStage.Type.CUSTOM_STAGE,
                 required_fields=['review_issue_start_date', 'review_issue_exp_date'],
@@ -1074,7 +1206,8 @@ def __init__(self, client_v2, venue, configuration_note, request_form_id, suppor
                     'email_sacs': False
                 },
                 start_date=self.configuration_note.content.get('review_issue_start_date'),
-                exp_date=self.configuration_note.content.get('review_issue_exp_date')
+                exp_date=self.configuration_note.content.get('review_issue_exp_date'),
+                process='../arr/process/review_issue_report_process.py'
             ),
             ARRStage(
                 type=ARRStage.Type.CUSTOM_STAGE,

openreview/arr/invitation.py

@@ -22,6 +22,7 @@ class InvitationBuilder(object):
     REVIEWER_LICENSE_NAME = 'License_Agreement'
     METAREVIEWER_LICENSE_NAME = 'Metareview_License_Agreement'
     RECOGNITION_NAME = 'Recognition_Request'
+    REPORT_EVALUATION_NAME = 'Report_Evaluation'
     SUBMITTED_AUTHORS_NAME = 'Submitted_Author_Form'
 
     def __init__(self, venue, update_wait_time=5000):
@@ -163,6 +164,27 @@ def set_process_invitation(self, arr_stage):
         venue_id = self.venue_id
         process_invitation_id = arr_stage.super_invitation_id
 
+        process_invitation_arguments = {
+            'id': process_invitation_id,
+            'invitees': [venue_id],
+            'readers': [venue_id],
+            'signatures': ['~Super_User1'] if arr_stage.process else [venue_id],
+            'writers': ['~Super_User1'] if arr_stage.process else [venue_id]
+        }
+
+        if arr_stage.start_date:
+            process_invitation_arguments['cdate'] = openreview.tools.datetime_millis(arr_stage.start_date)
+        if arr_stage.due_date:
+            process_invitation_arguments['duedate'] = openreview.tools.datetime_millis(arr_stage.due_date)
+        if arr_stage.exp_date:
+            process_invitation_arguments['expdate'] = openreview.tools.datetime_millis(arr_stage.exp_date)
+
+        if arr_stage.process:
+            process_invitation_arguments['date_processes'] = [{
+                'dates': ["#{4/cdate}", self.update_date_string],
+                'script': self.get_process_content(arr_stage.process)
+            }]
+
         # Build base date_processes with initial trigger
         date_processes = [{
             'dates': ["#{4/cdate}", self.update_date_string],
@@ -177,14 +199,7 @@ def set_process_invitation(self, arr_stage):
             })
 
         process_invitation = Invitation(
-            id=process_invitation_id,
-            invitees = [venue_id],
-            signatures = ['~Super_User1'],
-            readers = [venue_id],
-            writers = ['~Super_User1'],
-            cdate = openreview.tools.datetime_millis(arr_stage.start_date),
-            expdate = openreview.tools.datetime_millis(arr_stage.exp_date) if arr_stage.exp_date else None,
-            date_processes=date_processes,
+            **process_invitation_arguments,
             **arr_stage.stage_arguments
         )
 

openreview/arr/process/review_issue_report_process.py

@@ -0,0 +1,93 @@
+from openreview.stages.arr_content import arr_review_rating_content
+
+REPORT_EVALUATION_NAME = 'Report_Evaluation'
+REPORT_EVALUATION_WINDOW_DAYS = 30
+REPORT_EVALUATION_WINDOW_MILLIS = REPORT_EVALUATION_WINDOW_DAYS * 24 * 60 * 60 * 1000
+
+
+def _get_issue_options(report):
+    issue_options = []
+    for field_name in arr_review_rating_content:
+        if field_name == 'justification':
+            continue
+
+        field_value = report.content.get(field_name, {}).get('value')
+        if isinstance(field_value, list):
+            field_value = field_value[0] if field_value else None
+
+        if field_value:
+            issue_code = field_name.split('_')[0]
+            issue_options.append(f'{issue_code}. {field_value}')
+
+    return issue_options
+
+
+def _build_report_evaluation_content(report):
+    issue_options = _get_issue_options(report)
+    if not issue_options:
+        return None
+
+    return {
+        'justified_issues': {
+            'value': {
+                'param': {
+                    'type': 'string[]',
+                    'input': 'checkbox',
+                    'enum': issue_options,
+                    'optional': True,
+                    'deletable': True
+                }
+            },
+            'description': 'Select the reported review issues that you found justified.',
+            'order': 1
+        },
+        'justification': {
+            'value': {
+                'param': {
+                    'type': 'string',
+                    'input': 'textarea',
+                    'maxLength': 5000,
+                    'markdown': True,
+                    'optional': True
+                }
+            },
+            'description': 'Optional comments for Program Chairs and Senior Area Chairs about this evaluation.',
+            'order': 2
+        }
+    }
+
+
+def process(client, edit, invitation):
+    domain = client.get_group(edit.domain)
+    venue_id = domain.id
+    report = client.get_note(edit.note.id)
+
+    if report.ddate or report.tcdate != report.tmdate:
+        return
+
+    submission = client.get_note(report.forum)
+    evaluation_content = _build_report_evaluation_content(report)
+    if not evaluation_content:
+        return
+
+    client.post_invitation_edit(
+        invitations=f'{venue_id}/-/{REPORT_EVALUATION_NAME}',
+        readers=[venue_id],
+        writers=[venue_id],
+        signatures=[venue_id],
+        content={
+            'noteNumber': {
+                'value': submission.number
+            },
+            'replyId': {
+                'value': report.id
+            },
+            'expdate': {
+                'value': report.cdate + REPORT_EVALUATION_WINDOW_MILLIS
+            },
+            'content': {
+                'value': evaluation_content
+            }
+        },
+        invitation=openreview.api.Invitation()
+    )

tests/test_arr_venue_v2.py

@@ -6328,6 +6328,7 @@ def test_review_issue_forms(self, client, openreview_client, helpers, test_clien
         helpers.await_queue()
 
         assert openreview_client.get_invitation('aclweb.org/ACL/ARR/2023/August/-/Review_Issue_Report')
+        assert openreview_client.get_invitation(f'aclweb.org/ACL/ARR/2023/August/-/{invitation_builder.REPORT_EVALUATION_NAME}')
 
         helpers.await_queue_edit(openreview_client, 'aclweb.org/ACL/ARR/2023/August/-/Review_Issue_Report-0-1')
 
@@ -6344,25 +6345,52 @@ def test_review_issue_forms(self, client, openreview_client, helpers, test_clien
             signatures=['aclweb.org/ACL/ARR/2023/August/Submission3/Authors'],
             note=openreview.api.Note(
                 content = {
-                    "I1_not_specific": {"value": 'The review is not specific enough.'},
                     "I2_reviewer_heuristics": {"value": 'The review exhibits one or more of the reviewer heuristics discussed in the ARR reviewer guidelines: https://aclrollingreview.org/reviewertutorial'},
-                    "I3_score_mismatch": {"value": 'The review score(s) do not match the text of the review.'},
-                    "I4_unprofessional_tone": {"value": 'The tone of the review does not conform to professional conduct standards.'},
                     "I5_expertise": {"value": 'The review does not evince expertise.'},
-                    "I6_type_mismatch": {"value": "The review does not match the type of paper."},
-                    "I7_contribution_mismatch": {"value": "The review does not match the type of contribution."},
-                    "I8_missing_review": {"value": "The review is missing or is uninformative."},
-                    "I9_late_review": {"value": "The review was late."},
-                    "I10_unreasonable_requests": {"value": "The reviewer requests experiments that are not needed to demonstrate the stated claim."},
-                    "I11_non_response": {"value": "The review does not acknowledge critical evidence in the author response."},
-                    "I12_revisions_unacknowledged": {"value": "The review does not acknowledge the revisions"},
-                    "I13_other": {"value": "Some other technical violation of the peer review process."},
                     "justification": {"value": "required justification"},
                 }
             )
         )
 
+        helpers.await_queue_edit(openreview_client, edit_id=rating_edit['id'])
+
         assert test_client.get_note(rating_edit['note']['id'])
+        review_issue_note = test_client.get_note(rating_edit['note']['id'])
+
+        expected_issue_options = [
+            'I2. The review exhibits one or more of the reviewer heuristics discussed in the ARR reviewer guidelines: https://aclrollingreview.org/reviewertutorial',
+            'I5. The review does not evince expertise.'
+        ]
+        report_evaluation_invitation_id = f'aclweb.org/ACL/ARR/2023/August/Submission3/Review_Issue_Report{review_issue_note.number}/-/{invitation_builder.REPORT_EVALUATION_NAME}'
+        report_evaluation_invitation = openreview_client.get_invitation(report_evaluation_invitation_id)
+        assert report_evaluation_invitation
+        assert report_evaluation_invitation.expdate == review_issue_note.cdate + (30 * 24 * 60 * 60 * 1000)
+        assert report_evaluation_invitation.edit['note']['replyto'] == review_issue_note.id
+        assert report_evaluation_invitation.edit['note']['content']['justified_issues']['value']['param']['enum'] == expected_issue_options
+
+        ac_client = openreview.api.OpenReviewClient(username='ac2@aclrollingreview.com', password=helpers.strong_password)
+        anon_groups = ac_client.get_groups(
+            prefix='aclweb.org/ACL/ARR/2023/August/Submission3/Area_Chair_',
+            signatory='~AC_ARRTwo1'
+        )
+        ac_signature = anon_groups[0].id
+
+        report_evaluation_edit = ac_client.post_note_edit(
+            invitation=report_evaluation_invitation_id,
+            signatures=[ac_signature],
+            note=openreview.api.Note(
+                content={
+                    'justified_issues': {'value': [expected_issue_options[0]]},
+                    'justification': {'value': 'The reviewer heuristic complaint is justified.'}
+                }
+            )
+        )
+
+        report_evaluation_note = openreview_client.get_note(report_evaluation_edit['note']['id'])
+        assert report_evaluation_note.replyto == review_issue_note.id
+        assert report_evaluation_note.signatures == [ac_signature]
+        assert report_evaluation_note.content['justified_issues']['value'] == [expected_issue_options[0]]
+        assert report_evaluation_note.content['justification']['value'] == 'The reviewer heuristic complaint is justified.'
 
         meta_review_rating_edit = test_client.post_note_edit(
             invitation='aclweb.org/ACL/ARR/2023/August/Submission4/Meta_Review4/-/Meta-Review_Issue_Report',