diff --git a/openreview/arr/helpers.py b/openreview/arr/helpers.py index db7d0f6bc..ca641948a 100644 --- a/openreview/arr/helpers.py +++ b/openreview/arr/helpers.py @@ -473,6 +473,129 @@ def _build_preprint_release_edit(client, venue, builder, request_form): edit['note']['content'] = note_content return {'edit': edit} + + @staticmethod + def _build_report_evaluation_edit(client, venue, builder, request_form): + venue_id = venue.id + invitation_name = builder.REPORT_EVALUATION_NAME + + invitees = [venue.get_area_chairs_id(number='${3/content/noteNumber/value}')] + readers = [ + venue_id, + venue.get_program_chairs_id(), + venue.get_area_chairs_id(number='${3/content/noteNumber/value}') + ] + note_readers = [ + venue_id, + venue.get_program_chairs_id(), + venue.get_area_chairs_id(number='${5/content/noteNumber/value}') + ] + signature_items = [ + { + 'prefix': venue.get_area_chairs_id(number='${7/content/noteNumber/value}', anon=True), + 'optional': True + }, + { + 'value': venue.get_program_chairs_id(), + 'optional': True + }, + { + 'value': venue_id, + 'optional': True + } + ] + + if venue.use_senior_area_chairs: + readers.append(venue.get_senior_area_chairs_id(number='${3/content/noteNumber/value}')) + note_readers.append(venue.get_senior_area_chairs_id(number='${5/content/noteNumber/value}')) + + child_invitation_id = venue.get_invitation_id( + invitation_name, + prefix=( + f"{venue.get_paper_group_prefix('${2/content/noteNumber/value}')}" + f"/Review_Issue_Report${{{{2/content/replyId/value}}/number}}" + ) + ) + with_invitation = venue.get_invitation_id( + invitation_name, + prefix=( + f"{venue.get_paper_group_prefix('${6/content/noteNumber/value}')}" + f"/Review_Issue_Report${{{{6/content/replyId/value}}/number}}" + ) + ) + + return { + 'description': 'Create a per-report Area Chair evaluation invitation for a submitted review issue report.', + 'edit': { + 'signatures': [venue_id], + 'readers': [venue_id], + 'writers': [venue_id], + 'content': { + 'noteNumber': { + 'value': { + 'param': { + 'type': 'integer' + } + } + }, + 'replyId': { + 'value': { + 'param': { + 'type': 'string' + } + } + }, + 'expdate': { + 'value': { + 'param': { + 'type': 'integer', + 'range': [0, 9999999999999] + } + } + }, + 'content': { + 'value': { + 'param': { + 'type': 'content' + } + } + } + }, + 'replacement': True, + 'invitation': { + 'id': child_invitation_id, + 'signatures': [venue_id], + 'readers': readers, + 'writers': [venue_id], + 'invitees': invitees, + 'maxReplies': 1, + 'expdate': '${2/content/expdate/value}', + 'edit': { + 'signatures': { + 'param': { + 'items': signature_items + } + }, + 'readers': ['${2/note/readers}'], + 'writers': [venue_id, '${2/signatures}'], + 'note': { + 'id': { + 'param': { + 'withInvitation': with_invitation, + 'optional': True + } + }, + 'forum': '${{4/content/replyId/value}/forum}', + 'replyto': '${4/content/replyId/value}', + 'signatures': ['${3/signatures}'], + 'readers': note_readers, + 'writers': [venue_id, '${3/signatures}'], + 'content': '${4/content/content/value}' + } + } + } + } + } @staticmethod def _extend_desk_reject_verification(client, venue, builder, request_form): @@ -1055,6 +1178,15 @@ def __init__(self, client_v2, venue, configuration_note, request_form_id, suppor process='../arr/process/verification_process.py', extend=ARRWorkflow._extend_desk_reject_verification ), + ARRStage( + type=ARRStage.Type.PROCESS_INVITATION, + required_fields=['review_issue_start_date', 'review_issue_exp_date'], + super_invitation_id=f"{self.venue_id}/-/{self.invitation_builder.REPORT_EVALUATION_NAME}", + stage_arguments={}, + start_date=self.configuration_note.content.get('review_issue_start_date'), + exp_date=self.configuration_note.content.get('review_issue_exp_date'), + build_edit=ARRWorkflow._build_report_evaluation_edit + ), ARRStage( type=ARRStage.Type.CUSTOM_STAGE, required_fields=['review_issue_start_date', 'review_issue_exp_date'], @@ -1074,7 +1206,8 @@ def __init__(self, client_v2, venue, configuration_note, request_form_id, suppor 'email_sacs': False }, start_date=self.configuration_note.content.get('review_issue_start_date'), - exp_date=self.configuration_note.content.get('review_issue_exp_date') + exp_date=self.configuration_note.content.get('review_issue_exp_date'), + process='../arr/process/review_issue_report_process.py' ), ARRStage( type=ARRStage.Type.CUSTOM_STAGE, diff --git a/openreview/arr/invitation.py b/openreview/arr/invitation.py index e533e32d3..35c35eb13 100644 --- a/openreview/arr/invitation.py +++ b/openreview/arr/invitation.py @@ -22,6 +22,7 @@ class InvitationBuilder(object): REVIEWER_LICENSE_NAME = 'License_Agreement' METAREVIEWER_LICENSE_NAME = 'Metareview_License_Agreement' RECOGNITION_NAME = 'Recognition_Request' + REPORT_EVALUATION_NAME = 'Report_Evaluation' SUBMITTED_AUTHORS_NAME = 'Submitted_Author_Form' def __init__(self, venue, update_wait_time=5000): @@ -163,6 +164,27 @@ def set_process_invitation(self, arr_stage): venue_id = self.venue_id process_invitation_id = arr_stage.super_invitation_id + process_invitation_arguments = { + 'id': process_invitation_id, + 'invitees': [venue_id], + 'readers': [venue_id], + 'signatures': ['~Super_User1'] if arr_stage.process else [venue_id], + 'writers': ['~Super_User1'] if arr_stage.process else [venue_id] + } + + if arr_stage.start_date: + process_invitation_arguments['cdate'] = openreview.tools.datetime_millis(arr_stage.start_date) + if arr_stage.due_date: + process_invitation_arguments['duedate'] = openreview.tools.datetime_millis(arr_stage.due_date) + if arr_stage.exp_date: + process_invitation_arguments['expdate'] = openreview.tools.datetime_millis(arr_stage.exp_date) + + if arr_stage.process: + process_invitation_arguments['date_processes'] = [{ + 'dates': ["#{4/cdate}", self.update_date_string], + 'script': self.get_process_content(arr_stage.process) + }] + # Build base date_processes with initial trigger date_processes = [{ 'dates': ["#{4/cdate}", self.update_date_string], @@ -177,14 +199,7 @@ def set_process_invitation(self, arr_stage): }) process_invitation = Invitation( - id=process_invitation_id, - invitees = [venue_id], - signatures = ['~Super_User1'], - readers = [venue_id], - writers = ['~Super_User1'], - cdate = openreview.tools.datetime_millis(arr_stage.start_date), - expdate = openreview.tools.datetime_millis(arr_stage.exp_date) if arr_stage.exp_date else None, - date_processes=date_processes, + **process_invitation_arguments, **arr_stage.stage_arguments ) diff --git a/openreview/arr/process/review_issue_report_process.py b/openreview/arr/process/review_issue_report_process.py new file mode 100644 index 000000000..c80757876 --- /dev/null +++ b/openreview/arr/process/review_issue_report_process.py @@ -0,0 +1,93 @@ +from openreview.stages.arr_content import arr_review_rating_content + +REPORT_EVALUATION_NAME = 'Report_Evaluation' +REPORT_EVALUATION_WINDOW_DAYS = 30 +REPORT_EVALUATION_WINDOW_MILLIS = REPORT_EVALUATION_WINDOW_DAYS * 24 * 60 * 60 * 1000 + + +def _get_issue_options(report): + issue_options = [] + for field_name in arr_review_rating_content: + if field_name == 'justification': + continue + + field_value = report.content.get(field_name, {}).get('value') + if isinstance(field_value, list): + field_value = field_value[0] if field_value else None + + if field_value: + issue_code = field_name.split('_')[0] + issue_options.append(f'{issue_code}. {field_value}') + + return issue_options + + +def _build_report_evaluation_content(report): + issue_options = _get_issue_options(report) + if not issue_options: + return None + + return { + 'justified_issues': { + 'value': { + 'param': { + 'type': 'string[]', + 'input': 'checkbox', + 'enum': issue_options, + 'optional': True, + 'deletable': True + } + }, + 'description': 'Select the reported review issues that you found justified.', + 'order': 1 + }, + 'justification': { + 'value': { + 'param': { + 'type': 'string', + 'input': 'textarea', + 'maxLength': 5000, + 'markdown': True, + 'optional': True + } + }, + 'description': 'Optional comments for Program Chairs and Senior Area Chairs about this evaluation.', + 'order': 2 + } + } + + +def process(client, edit, invitation): + domain = client.get_group(edit.domain) + venue_id = domain.id + report = client.get_note(edit.note.id) + + if report.ddate or report.tcdate != report.tmdate: + return + + submission = client.get_note(report.forum) + evaluation_content = _build_report_evaluation_content(report) + if not evaluation_content: + return + + client.post_invitation_edit( + invitations=f'{venue_id}/-/{REPORT_EVALUATION_NAME}', + readers=[venue_id], + writers=[venue_id], + signatures=[venue_id], + content={ + 'noteNumber': { + 'value': submission.number + }, + 'replyId': { + 'value': report.id + }, + 'expdate': { + 'value': report.cdate + REPORT_EVALUATION_WINDOW_MILLIS + }, + 'content': { + 'value': evaluation_content + } + }, + invitation=openreview.api.Invitation() + ) diff --git a/tests/test_arr_venue_v2.py b/tests/test_arr_venue_v2.py index 8b1e28784..a8120f66a 100644 --- a/tests/test_arr_venue_v2.py +++ b/tests/test_arr_venue_v2.py @@ -6328,6 +6328,7 @@ def test_review_issue_forms(self, client, openreview_client, helpers, test_clien helpers.await_queue() assert openreview_client.get_invitation('aclweb.org/ACL/ARR/2023/August/-/Review_Issue_Report') + assert openreview_client.get_invitation(f'aclweb.org/ACL/ARR/2023/August/-/{invitation_builder.REPORT_EVALUATION_NAME}') helpers.await_queue_edit(openreview_client, 'aclweb.org/ACL/ARR/2023/August/-/Review_Issue_Report-0-1') @@ -6344,25 +6345,52 @@ def test_review_issue_forms(self, client, openreview_client, helpers, test_clien signatures=['aclweb.org/ACL/ARR/2023/August/Submission3/Authors'], note=openreview.api.Note( content = { - "I1_not_specific": {"value": 'The review is not specific enough.'}, "I2_reviewer_heuristics": {"value": 'The review exhibits one or more of the reviewer heuristics discussed in the ARR reviewer guidelines: https://aclrollingreview.org/reviewertutorial'}, - "I3_score_mismatch": {"value": 'The review score(s) do not match the text of the review.'}, - "I4_unprofessional_tone": {"value": 'The tone of the review does not conform to professional conduct standards.'}, "I5_expertise": {"value": 'The review does not evince expertise.'}, - "I6_type_mismatch": {"value": "The review does not match the type of paper."}, - "I7_contribution_mismatch": {"value": "The review does not match the type of contribution."}, - "I8_missing_review": {"value": "The review is missing or is uninformative."}, - "I9_late_review": {"value": "The review was late."}, - "I10_unreasonable_requests": {"value": "The reviewer requests experiments that are not needed to demonstrate the stated claim."}, - "I11_non_response": {"value": "The review does not acknowledge critical evidence in the author response."}, - "I12_revisions_unacknowledged": {"value": "The review does not acknowledge the revisions"}, - "I13_other": {"value": "Some other technical violation of the peer review process."}, "justification": {"value": "required justification"}, } ) ) + helpers.await_queue_edit(openreview_client, edit_id=rating_edit['id']) + assert test_client.get_note(rating_edit['note']['id']) + review_issue_note = test_client.get_note(rating_edit['note']['id']) + + expected_issue_options = [ + 'I2. The review exhibits one or more of the reviewer heuristics discussed in the ARR reviewer guidelines: https://aclrollingreview.org/reviewertutorial', + 'I5. The review does not evince expertise.' + ] + report_evaluation_invitation_id = f'aclweb.org/ACL/ARR/2023/August/Submission3/Review_Issue_Report{review_issue_note.number}/-/{invitation_builder.REPORT_EVALUATION_NAME}' + report_evaluation_invitation = openreview_client.get_invitation(report_evaluation_invitation_id) + assert report_evaluation_invitation + assert report_evaluation_invitation.expdate == review_issue_note.cdate + (30 * 24 * 60 * 60 * 1000) + assert report_evaluation_invitation.edit['note']['replyto'] == review_issue_note.id + assert report_evaluation_invitation.edit['note']['content']['justified_issues']['value']['param']['enum'] == expected_issue_options + + ac_client = openreview.api.OpenReviewClient(username='ac2@aclrollingreview.com', password=helpers.strong_password) + anon_groups = ac_client.get_groups( + prefix='aclweb.org/ACL/ARR/2023/August/Submission3/Area_Chair_', + signatory='~AC_ARRTwo1' + ) + ac_signature = anon_groups[0].id + + report_evaluation_edit = ac_client.post_note_edit( + invitation=report_evaluation_invitation_id, + signatures=[ac_signature], + note=openreview.api.Note( + content={ + 'justified_issues': {'value': [expected_issue_options[0]]}, + 'justification': {'value': 'The reviewer heuristic complaint is justified.'} + } + ) + ) + + report_evaluation_note = openreview_client.get_note(report_evaluation_edit['note']['id']) + assert report_evaluation_note.replyto == review_issue_note.id + assert report_evaluation_note.signatures == [ac_signature] + assert report_evaluation_note.content['justified_issues']['value'] == [expected_issue_options[0]] + assert report_evaluation_note.content['justification']['value'] == 'The reviewer heuristic complaint is justified.' meta_review_rating_edit = test_client.post_note_edit( invitation='aclweb.org/ACL/ARR/2023/August/Submission4/Meta_Review4/-/Meta-Review_Issue_Report',