From b9fdee6ecd8100e8235f021ed9017778e61410d7 Mon Sep 17 00:00:00 2001 From: Shayan Khan Date: Fri, 2 Feb 2024 17:19:38 -0600 Subject: [PATCH 001/328] Testing dynamic forms for digital rocks --- .gitignore | 1 + .../DataFilesModals/DataFilesFormModal.jsx | 67 +++++++++++++++++++ .../DataFilesFormModal.module.scss | 0 .../DataFilesModals/DataFilesModals.jsx | 2 + .../DataFilesProjectFileListing.jsx | 14 ++++ .../_common/Form/DynamicForm/DynamicForm.jsx | 35 ++++++++++ .../_common/Form/DynamicForm/DynamicForm.scss | 0 .../_common/Form/DynamicForm/index.js | 1 + .../src/redux/reducers/datafiles.reducers.js | 2 + server/portal/apps/forms/__init__.py | 0 server/portal/apps/forms/urls.py | 11 +++ server/portal/apps/forms/views.py | 12 ++++ server/portal/settings/settings.py | 6 ++ server/portal/urls.py | 1 + 14 files changed, 152 insertions(+) create mode 100644 client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx create mode 100644 client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.module.scss create mode 100644 client/src/components/_common/Form/DynamicForm/DynamicForm.jsx create mode 100644 client/src/components/_common/Form/DynamicForm/DynamicForm.scss create mode 100644 client/src/components/_common/Form/DynamicForm/index.js create mode 100644 server/portal/apps/forms/__init__.py create mode 100644 server/portal/apps/forms/urls.py create mode 100644 server/portal/apps/forms/views.py diff --git a/.gitignore b/.gitignore index ad43acffd9..297742961c 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ server/docs *settings_secret.py *settings_custom.py *settings_local.py +*settings_forms.py docker_repo.var server/src diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx new file mode 100644 index 0000000000..350cc4eeea --- /dev/null +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx @@ -0,0 +1,67 @@ +import React, { useCallback } from 'react'; +import { useDispatch, useSelector, shallowEqual } from 'react-redux'; +import { Modal, ModalHeader, ModalBody } from 'reactstrap'; +import { DynamicForm } from '_common/Form/DynamicForm'; +import { useQuery } from 'react-query'; +import { fetchUtil } from 'utils/fetchUtil'; +import { Formik, Form } from 'formik'; +import * as Yup from 'yup'; + +const DataFilesFormModal = () => { + const dispatch = useDispatch(); + + const { formName } = useSelector( + (state) => state.files.modalProps.dynamicform + ); + const isOpen = useSelector((state) => state.files.modals.dynamicform); + + const getFormFields = async (formName) => { + const response = await fetchUtil({ + url: 'api/forms', + params: { + form_name: formName, + }, + }); + return response; + }; + + const useFormFields = (formName) => { + const query = useQuery(['form', formName], () => getFormFields(formName), { + enabled: !!formName, + }); + return query; + }; + + const { data, isLoading } = useFormFields(formName); + + const toggle = useCallback(() => { + dispatch({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { operation: 'dynamicform', props: {} }, + }); + }, []); + + return ( +
+ + +
+ + {formName} + + + + +
+ + +
+ ); +}; + +export default DataFilesFormModal; diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.module.scss b/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.module.scss new file mode 100644 index 0000000000..e69de29bb2 diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesModals.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesModals.jsx index 247c67e851..8d2083c18b 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesModals.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesModals.jsx @@ -17,6 +17,7 @@ import DataFilesShowPathModal from './DataFilesShowPathModal'; import DataFilesMakePublicModal from './DataFilesMakePublicModal'; import DataFilesDownloadMessageModal from './DataFilesDownloadMessageModal'; import './DataFilesModals.scss'; +import DataFilesFormModal from './DataFilesFormModal'; export default function DataFilesModals() { return ( @@ -38,6 +39,7 @@ export default function DataFilesModals() { + ); } diff --git a/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx b/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx index cf11554f8a..3c45e28cef 100644 --- a/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx +++ b/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx @@ -64,6 +64,16 @@ const DataFilesProjectFileListing = ({ system, path }) => { }); }; + const onAddSampleData = () => { + dispatch({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { + operation: 'dynamicform', + props: { formName: 'ADD_SAMPLE_DATA' }, + }, + }); + }; + if (metadata.loading) { return (
@@ -99,6 +109,10 @@ const DataFilesProjectFileListing = ({ system, path }) => { + | +
} manualContent diff --git a/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx b/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx new file mode 100644 index 0000000000..49feb6b6de --- /dev/null +++ b/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx @@ -0,0 +1,35 @@ +import React from 'react'; +import FormField from '../FormField'; + +const DynamicForm = ({ formFields }) => { + const renderFormField = (field) => { + switch (field.type) { + case 'text': + return ; + case 'textarea': + return + case 'select': + return ( + + {field.options.map((option) => { + return ( + + ); + })} + + ); + } + }; + + return ( + <> + {formFields.map((field) => ( +
{renderFormField(field)}
+ ))} + + ); +}; + +export default DynamicForm; diff --git a/client/src/components/_common/Form/DynamicForm/DynamicForm.scss b/client/src/components/_common/Form/DynamicForm/DynamicForm.scss new file mode 100644 index 0000000000..e69de29bb2 diff --git a/client/src/components/_common/Form/DynamicForm/index.js b/client/src/components/_common/Form/DynamicForm/index.js new file mode 100644 index 0000000000..f3a00fc9da --- /dev/null +++ b/client/src/components/_common/Form/DynamicForm/index.js @@ -0,0 +1 @@ +export { default as DynamicForm } from './DynamicForm'; diff --git a/client/src/redux/reducers/datafiles.reducers.js b/client/src/redux/reducers/datafiles.reducers.js index b8e28b4cfa..752e5d313c 100644 --- a/client/src/redux/reducers/datafiles.reducers.js +++ b/client/src/redux/reducers/datafiles.reducers.js @@ -161,6 +161,7 @@ export const initialFilesState = { editproject: false, makePublic: false, downloadMessage: false, + dynamicform: false, }, modalProps: { preview: {}, @@ -174,6 +175,7 @@ export const initialFilesState = { showpath: {}, makePublic: {}, downloadMessage: {}, + dynamicform: {}, }, refs: {}, preview: { diff --git a/server/portal/apps/forms/__init__.py b/server/portal/apps/forms/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/server/portal/apps/forms/urls.py b/server/portal/apps/forms/urls.py new file mode 100644 index 0000000000..2c0b16cfa1 --- /dev/null +++ b/server/portal/apps/forms/urls.py @@ -0,0 +1,11 @@ +""" +.. module:: portal.apps.forms.urls + :synopsis: Forms URLs +""" +from django.urls import re_path +from portal.apps.forms.views import FormsView + +app_name = 'workbench' +urlpatterns = [ + re_path('', FormsView.as_view(), name='form'), +] diff --git a/server/portal/apps/forms/views.py b/server/portal/apps/forms/views.py new file mode 100644 index 0000000000..36f0173e1d --- /dev/null +++ b/server/portal/apps/forms/views.py @@ -0,0 +1,12 @@ +from portal.views.base import BaseApiView +from django.conf import settings +from django.http import JsonResponse, HttpResponseForbidden + + +class FormsView(BaseApiView): + + def get(self, request): + form_name = request.GET.get('form_name') + form = settings.FORMS.get(form_name) + + return JsonResponse(form, safe=False) diff --git a/server/portal/settings/settings.py b/server/portal/settings/settings.py index 6c4fe98368..13021262b0 100644 --- a/server/portal/settings/settings.py +++ b/server/portal/settings/settings.py @@ -26,6 +26,10 @@ else: from portal.settings import settings_default as settings_custom +if os.path.isfile(os.path.join(BASE_DIR, 'settings', 'settings_forms.py')): + from portal.settings import settings_forms + + FIXTURE_DIRS = [ os.path.join(BASE_DIR, 'fixtures'), ] @@ -528,6 +532,8 @@ PORTAL_DATA_DEPOT_PAGE_SIZE = 100 +FORMS = getattr(settings_forms, '_FORMS', {}) + """ SETTINGS: EXTERNAL DATA RESOURCES """ diff --git a/server/portal/urls.py b/server/portal/urls.py index f19e911660..26eeac78fb 100644 --- a/server/portal/urls.py +++ b/server/portal/urls.py @@ -89,6 +89,7 @@ path('api/jupyter_mounts/', include('portal.apps.jupyter_mounts.api.urls', namespace='jupyter_mounts_api')), path('api/projects/', include('portal.apps.projects.urls', namespace='projects')), path('api/site-search/', include('portal.apps.site_search.api.urls', namespace='site_search_api')), + path('api/forms/', include('portal.apps.forms.urls', namespace='forms')), # webhooks path('webhooks/', include('portal.apps.webhooks.urls', namespace='webhooks')), From 050f7fb62c29776cf79141d68f9e134f582c7902 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Tue, 6 Feb 2024 09:59:58 -0600 Subject: [PATCH 002/328] Added dataset forms, improved form formatting --- .../DataFilesModals/DataFilesFormModal.jsx | 34 ++++++++++++++----- .../DataFilesFormModal.module.scss | 6 ++++ .../DataFilesProjectFileListing.jsx | 16 ++++++--- .../_common/Form/DynamicForm/DynamicForm.jsx | 26 +++++++++++--- .../_common/Form/DynamicForm/DynamicForm.scss | 8 +++++ server/portal/apps/forms/views.py | 2 +- 6 files changed, 75 insertions(+), 17 deletions(-) diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx index 350cc4eeea..5147e05471 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx @@ -1,11 +1,12 @@ import React, { useCallback } from 'react'; import { useDispatch, useSelector, shallowEqual } from 'react-redux'; -import { Modal, ModalHeader, ModalBody } from 'reactstrap'; +import { Modal, ModalHeader, ModalBody, ModalFooter } from 'reactstrap'; import { DynamicForm } from '_common/Form/DynamicForm'; import { useQuery } from 'react-query'; import { fetchUtil } from 'utils/fetchUtil'; import { Formik, Form } from 'formik'; import * as Yup from 'yup'; +import styles from './DataFilesFormModal.module.scss'; const DataFilesFormModal = () => { const dispatch = useDispatch(); @@ -32,7 +33,12 @@ const DataFilesFormModal = () => { return query; }; - const { data, isLoading } = useFormFields(formName); + const { data: form, isLoading } = useFormFields(formName); + + const initialValues = form?.form_fields.reduce((acc, field) => { + acc[field.name] = field.value || ''; + return acc; + }, {}) const toggle = useCallback(() => { dispatch({ @@ -41,22 +47,34 @@ const DataFilesFormModal = () => { }); }, []); + const handleSubmit = (values) => { + console.log("FORM SUBMIT", values) + } + return (
- +
- {formName} + {form?.heading} - - + + + {form?.footer && ( + + + + )}
diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.module.scss b/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.module.scss index e69de29bb2..597b315157 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.module.scss +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.module.scss @@ -0,0 +1,6 @@ + +.modal-body-container { + width: 100%; + max-height: 70vh; + overflow: auto; + } \ No newline at end of file diff --git a/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx b/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx index 3c45e28cef..9ba37cf7af 100644 --- a/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx +++ b/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx @@ -64,15 +64,15 @@ const DataFilesProjectFileListing = ({ system, path }) => { }); }; - const onAddSampleData = () => { + const onOpenFormModal = (form_name) => { dispatch({ type: 'DATA_FILES_TOGGLE_MODAL', payload: { operation: 'dynamicform', - props: { formName: 'ADD_SAMPLE_DATA' }, + props: { formName: form_name }, }, }); - }; + } if (metadata.loading) { return ( @@ -110,9 +110,17 @@ const DataFilesProjectFileListing = ({ system, path }) => { {readOnlyTeam ? 'View' : 'Manage'} Team | - + | + + | +
} manualContent diff --git a/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx b/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx index 49feb6b6de..656c0c6dd3 100644 --- a/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx +++ b/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx @@ -1,25 +1,43 @@ import React from 'react'; import FormField from '../FormField'; +import Button from '_common/Button'; const DynamicForm = ({ formFields }) => { const renderFormField = (field) => { switch (field.type) { case 'text': - return ; + case 'number': + return ; case 'textarea': - return + return case 'select': return ( - + {field.options.map((option) => { return ( ); })} ); + case 'radio': + return ( +
+ + {field.options.map((option) => ( +
+ + +
+ ))} +
+ ); + case 'submit': + return ( + + ); } }; diff --git a/client/src/components/_common/Form/DynamicForm/DynamicForm.scss b/client/src/components/_common/Form/DynamicForm/DynamicForm.scss index e69de29bb2..7860e4260f 100644 --- a/client/src/components/_common/Form/DynamicForm/DynamicForm.scss +++ b/client/src/components/_common/Form/DynamicForm/DynamicForm.scss @@ -0,0 +1,8 @@ + +.bold-label { + font-weight: bold; +} + +.radio-input { + margin-right: 10px; +} \ No newline at end of file diff --git a/server/portal/apps/forms/views.py b/server/portal/apps/forms/views.py index 36f0173e1d..e8a313f369 100644 --- a/server/portal/apps/forms/views.py +++ b/server/portal/apps/forms/views.py @@ -9,4 +9,4 @@ def get(self, request): form_name = request.GET.get('form_name') form = settings.FORMS.get(form_name) - return JsonResponse(form, safe=False) + return JsonResponse(form) From 0401ff51d2655a089e586885e79db0f646d407e1 Mon Sep 17 00:00:00 2001 From: Shayan Khan Date: Wed, 14 Feb 2024 10:21:31 -0600 Subject: [PATCH 003/328] Added custom components --- .../DataFilesAddProjectModal.jsx | 4 +- .../DataFilesProjectFileListing.jsx | 38 ++++++++----------- .../DataFilesAddProjectFormAddon.jsx | 14 +++++++ .../DataFilesProjectListingAddon.jsx | 38 +++++++++++++++++++ .../DataFilesProjectListingAddon.module.scss | 5 +++ server/portal/settings/settings_default.py | 1 + 6 files changed, 77 insertions(+), 23 deletions(-) create mode 100644 client/src/components/_custom/drp/DataFilesAddProjectFormAddon/DataFilesAddProjectFormAddon.jsx create mode 100644 client/src/components/_custom/drp/DataFilesProjectListingAddon/DataFilesProjectListingAddon.jsx create mode 100644 client/src/components/_custom/drp/DataFilesProjectListingAddon/DataFilesProjectListingAddon.module.scss diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesAddProjectModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesAddProjectModal.jsx index 0ed9aaa37d..8bbdd617c9 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesAddProjectModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesAddProjectModal.jsx @@ -1,12 +1,13 @@ import React, { useState, useEffect } from 'react'; import { useDispatch, useSelector } from 'react-redux'; import * as Yup from 'yup'; -import { Formik, Form } from 'formik'; +import { Formik, Form, FieldArray } from 'formik'; import FormField from '_common/Form/FormField'; import { Button, InlineMessage } from '_common'; import { Modal, ModalHeader, ModalBody, ModalFooter } from 'reactstrap'; import { useHistory, useRouteMatch } from 'react-router-dom'; import DataFilesProjectMembers from '../DataFilesProjectMembers/DataFilesProjectMembers'; +import DataFilesProjectFormAddon from '../../_custom/drp/DataFilesAddProjectFormAddon/DataFilesAddProjectFormAddon'; const DataFilesAddProjectModal = () => { const history = useHistory(); @@ -114,6 +115,7 @@ const DataFilesAddProjectModal = () => { } /> + { const dispatch = useDispatch(); + const [addon, setAddon] = useState(null); const { fetchListing } = useFileListing('FilesListing'); useEffect(() => { dispatch({ @@ -39,6 +40,19 @@ const DataFilesProjectFileListing = ({ system, path }) => { .map((currentUser) => currentUser.access === 'owner')[0] ); + const { hasProjectFileListingToolbarAddon, portalName } = useSelector((state) => ({ + hasProjectFileListingToolbarAddon: state.workbench.config.hasProjectFileListingAddon, + portalName: state.workbench.portalName + }) + ) + + useEffect(async () => { + if (hasProjectFileListingToolbarAddon) { + const module = await import(`../../_custom/${portalName}/DataFilesProjectListingAddon/DataFilesProjectListingAddon`); + setAddon(module.default) + } + }) + const readOnlyTeam = useSelector((state) => { const projectSystem = state.systems.storage.configuration.find( (s) => s.scheme === 'projects' @@ -64,15 +78,6 @@ const DataFilesProjectFileListing = ({ system, path }) => { }); }; - const onOpenFormModal = (form_name) => { - dispatch({ - type: 'DATA_FILES_TOGGLE_MODAL', - payload: { - operation: 'dynamicform', - props: { formName: form_name }, - }, - }); - } if (metadata.loading) { return ( @@ -109,18 +114,7 @@ const DataFilesProjectFileListing = ({ system, path }) => { - | - - | - - | - + {hasProjectFileListingToolbarAddon ? addon : null} } manualContent diff --git a/client/src/components/_custom/drp/DataFilesAddProjectFormAddon/DataFilesAddProjectFormAddon.jsx b/client/src/components/_custom/drp/DataFilesAddProjectFormAddon/DataFilesAddProjectFormAddon.jsx new file mode 100644 index 0000000000..417e8c2375 --- /dev/null +++ b/client/src/components/_custom/drp/DataFilesAddProjectFormAddon/DataFilesAddProjectFormAddon.jsx @@ -0,0 +1,14 @@ +import { FormField } from '_common/Form'; +import React, { useEffect } from 'react'; + +const DataFilesProjectFormAddon = () => { + + return ( +
+ +
+ ) + +} + +export default DataFilesProjectFormAddon \ No newline at end of file diff --git a/client/src/components/_custom/drp/DataFilesProjectListingAddon/DataFilesProjectListingAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectListingAddon/DataFilesProjectListingAddon.jsx new file mode 100644 index 0000000000..5fd9694b0c --- /dev/null +++ b/client/src/components/_custom/drp/DataFilesProjectListingAddon/DataFilesProjectListingAddon.jsx @@ -0,0 +1,38 @@ +import React, { useEffect } from 'react'; +import { Button } from '_common'; +import { useDispatch, useSelector } from 'react-redux'; +import styles from './DataFilesProjectListingAddon.module.scss'; + +const DataFilesProjectFileListingAddon = () => { + + const dispatch = useDispatch(); + + const onOpenFormModal = (form_name) => { + dispatch({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { + operation: 'dynamicform', + props: { formName: form_name }, + }, + }); + } + + return ( + <> + | + + | + + | + + + ) +} + +export default DataFilesProjectFileListingAddon; \ No newline at end of file diff --git a/client/src/components/_custom/drp/DataFilesProjectListingAddon/DataFilesProjectListingAddon.module.scss b/client/src/components/_custom/drp/DataFilesProjectListingAddon/DataFilesProjectListingAddon.module.scss new file mode 100644 index 0000000000..6ed1445e4e --- /dev/null +++ b/client/src/components/_custom/drp/DataFilesProjectListingAddon/DataFilesProjectListingAddon.module.scss @@ -0,0 +1,5 @@ +.separator { + font-weight: bold; + padding-left: 0.25em; + padding-right: 0.25em; + } \ No newline at end of file diff --git a/server/portal/settings/settings_default.py b/server/portal/settings/settings_default.py index 9ce91759ab..4a008c3342 100644 --- a/server/portal/settings/settings_default.py +++ b/server/portal/settings/settings_default.py @@ -239,6 +239,7 @@ "hideManageAccount": False, "hideSystemStatus": False, "hasUserGuide": True, + "hasProjectFileListingAddon": True, "onboardingCompleteRedirect": '/workbench/', "noPHISystem": "", "customDashboardSection": None, From 61183ada2ad18c64fce9e92cd99221529b90bc34 Mon Sep 17 00:00:00 2001 From: Shayan Khan Date: Thu, 15 Feb 2024 12:35:16 -0600 Subject: [PATCH 004/328] fix warning and errors, ran linting --- .../DataFilesAddProjectModal.jsx | 2 +- .../DataFilesModals/DataFilesFormModal.jsx | 65 ++++++++++--------- .../DataFilesFormModal.module.scss | 9 ++- .../DataFilesProjectFileListing.jsx | 31 +++++---- .../_common/Form/DynamicForm/DynamicForm.jsx | 43 ++++++++++-- .../_common/Form/DynamicForm/DynamicForm.scss | 7 +- .../DataFilesAddProjectFormAddon.jsx | 20 +++--- .../DataFilesProjectListingAddon.jsx | 60 ++++++++--------- .../DataFilesProjectListingAddon.module.scss | 8 +-- 9 files changed, 146 insertions(+), 99 deletions(-) diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesAddProjectModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesAddProjectModal.jsx index 8bbdd617c9..0e2ef6dc1a 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesAddProjectModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesAddProjectModal.jsx @@ -115,7 +115,7 @@ const DataFilesAddProjectModal = () => { } /> - + { const initialValues = form?.form_fields.reduce((acc, field) => { acc[field.name] = field.value || ''; return acc; - }, {}) + }, {}); const toggle = useCallback(() => { dispatch({ @@ -48,37 +48,42 @@ const DataFilesFormModal = () => { }, []); const handleSubmit = (values) => { - console.log("FORM SUBMIT", values) - } + console.log('FORM SUBMIT', values); + }; return ( -
- - -
- - {form?.heading} - - - - - {form?.footer && ( - - - - )} -
- - -
+ <> + {form && ( +
+ + +
+ + {form.heading} + + + + + {form?.footer && ( + + + + )} +
+ + +
+ )} + ); }; diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.module.scss b/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.module.scss index 597b315157..044246cfb1 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.module.scss +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.module.scss @@ -1,6 +1,5 @@ - .modal-body-container { - width: 100%; - max-height: 70vh; - overflow: auto; - } \ No newline at end of file + width: 100%; + max-height: 70vh; + overflow: auto; +} diff --git a/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx b/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx index 3616b0e954..28ac36c947 100644 --- a/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx +++ b/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx @@ -14,7 +14,7 @@ import styles from './DataFilesProjectFileListing.module.scss'; const DataFilesProjectFileListing = ({ system, path }) => { const dispatch = useDispatch(); - const [addon, setAddon] = useState(null); + const [AddonComponent, setAddonComponent] = useState(null); const { fetchListing } = useFileListing('FilesListing'); useEffect(() => { dispatch({ @@ -40,18 +40,26 @@ const DataFilesProjectFileListing = ({ system, path }) => { .map((currentUser) => currentUser.access === 'owner')[0] ); - const { hasProjectFileListingToolbarAddon, portalName } = useSelector((state) => ({ - hasProjectFileListingToolbarAddon: state.workbench.config.hasProjectFileListingAddon, - portalName: state.workbench.portalName + const { hasProjectFileListingToolbarAddon, portalName } = useSelector( + (state) => ({ + hasProjectFileListingToolbarAddon: + state.workbench.config.hasProjectFileListingAddon, + portalName: state.workbench.portalName, }) - ) + ); + + useEffect(() => { + const loadAddonComponent = async () => { + const module = await import( + `../../_custom/${portalName}/DataFilesProjectListingAddon/DataFilesProjectListingAddon` + ); + setAddonComponent(() => module.default); + }; - useEffect(async () => { if (hasProjectFileListingToolbarAddon) { - const module = await import(`../../_custom/${portalName}/DataFilesProjectListingAddon/DataFilesProjectListingAddon`); - setAddon(module.default) + loadAddonComponent(); } - }) + }, [hasProjectFileListingToolbarAddon, portalName]); const readOnlyTeam = useSelector((state) => { const projectSystem = state.systems.storage.configuration.find( @@ -78,7 +86,6 @@ const DataFilesProjectFileListing = ({ system, path }) => { }); }; - if (metadata.loading) { return (
@@ -114,7 +121,9 @@ const DataFilesProjectFileListing = ({ system, path }) => { - {hasProjectFileListingToolbarAddon ? addon : null} + {hasProjectFileListingToolbarAddon && AddonComponent && ( + + )}
} manualContent diff --git a/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx b/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx index 656c0c6dd3..ce4f7aaa77 100644 --- a/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx +++ b/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx @@ -7,12 +7,35 @@ const DynamicForm = ({ formFields }) => { switch (field.type) { case 'text': case 'number': - return ; + return ( + + ); case 'textarea': - return + return ( + + ); case 'select': return ( - + {field.options.map((option) => { return ( - ); - case 'radio': + ); + case 'radio': return (
{field.options.map((option) => (
{
))}
+ + ); + case 'file': // Adding support for file type + return ( +
+ +
); case 'submit': return ( diff --git a/client/src/components/_custom/drp/DataFilesAddProjectFormAddon/DataFilesAddProjectFormAddon.jsx b/client/src/components/_custom/drp/DataFilesAddProjectFormAddon/DataFilesAddProjectFormAddon.jsx index 78f7b5cbaf..745a5a2c26 100644 --- a/client/src/components/_custom/drp/DataFilesAddProjectFormAddon/DataFilesAddProjectFormAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesAddProjectFormAddon/DataFilesAddProjectFormAddon.jsx @@ -1,16 +1,25 @@ -import { FormField } from '_common/Form'; -import React, { useEffect } from 'react'; - +import React from 'react'; +import { useQuery } from 'react-query'; +import { fetchUtil } from 'utils/fetchUtil'; +import { DynamicForm } from '_common/Form/DynamicForm'; const DataFilesProjectFormAddon = () => { + const { data: form, isLoading } = useQuery('form_ADD_PROJECT', () => + fetchUtil({ + url: 'api/forms', + params: { + form_name: 'ADD_PROJECT', + }, + }) + ); return (
- + {isLoading ? ( +

Loading form...

+ ) : ( + + )}
- ); + ); }; export default DataFilesProjectFormAddon; diff --git a/client/src/components/_custom/drp/DataFilesProjectListingAddon/DataFilesProjectListingAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectListingAddon/DataFilesProjectListingAddon.jsx index a6c9c83ca8..1f58ab216f 100644 --- a/client/src/components/_custom/drp/DataFilesProjectListingAddon/DataFilesProjectListingAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectListingAddon/DataFilesProjectListingAddon.jsx @@ -18,6 +18,9 @@ const DataFilesProjectFileListingAddon = () => { return ( <> + | + | ); }; diff --git a/client/src/hooks/datafiles/index.js b/client/src/hooks/datafiles/index.js index d238f754bf..9b20825c94 100644 --- a/client/src/hooks/datafiles/index.js +++ b/client/src/hooks/datafiles/index.js @@ -1,5 +1,6 @@ export { default as useSystemDisplayName } from './useSystemDisplayName'; export { default as useSelectedFiles } from './useSelectedFiles'; export { default as useFileListing } from './useFileListing'; +export { default as useAddonComponents } from './useAddonComponents'; export { default as useSystems } from './useSystems'; export { default as useModal } from './useModal'; diff --git a/client/src/hooks/datafiles/useAddonComponents.js b/client/src/hooks/datafiles/useAddonComponents.js new file mode 100644 index 0000000000..e396d6bf2f --- /dev/null +++ b/client/src/hooks/datafiles/useAddonComponents.js @@ -0,0 +1,29 @@ +import { useState, useEffect } from 'react'; +import { useSelector } from 'react-redux'; + +function useAddonComponents({ portalName }) { + const [addonComponents, setAddonComponents] = useState({}); + const hasAddons = useSelector(state => state.workbench.config.hasAddons); + useEffect(() => { + const loadAddonComponents = async () => { + try { + const addOns = {}; + for (const addOnName of hasAddons) { + const module = await import(`../../components/_custom/${portalName}/${addOnName}/${addOnName}.jsx`); + addOns[addOnName] = module.default; + } + setAddonComponents(addOns); + } catch (error) { + console.error('Error loading addon components:', error); + } + }; + + if (hasAddons) { + loadAddonComponents(); + } + }, []); + console.log("truuu", addonComponents) + return addonComponents; +} + +export default useAddonComponents; \ No newline at end of file diff --git a/server/portal/settings/settings_default.py b/server/portal/settings/settings_default.py index 4a008c3342..ce4f7c4cef 100644 --- a/server/portal/settings/settings_default.py +++ b/server/portal/settings/settings_default.py @@ -9,7 +9,7 @@ _DEBUG = True # Namespace for portal -_PORTAL_NAMESPACE = 'CEP' +_PORTAL_NAMESPACE = 'DRP' # NOTE: set _WH_BASE_URL to ngrok redirect for local dev testing (i.e. _WH_BASE_URL = 'https://12345.ngrock.io', see https://ngrok.com/) _WH_BASE_URL = '' @@ -240,6 +240,7 @@ "hideSystemStatus": False, "hasUserGuide": True, "hasProjectFileListingAddon": True, + "hasAddons": ['DataFilesProjectListingAddon', 'DataFilesAddProjectFormAddon'], "onboardingCompleteRedirect": '/workbench/', "noPHISystem": "", "customDashboardSection": None, From 2399b8b9d76eea6efa4762452f6c6d018a3ad38f Mon Sep 17 00:00:00 2001 From: Asim Regmi Date: Tue, 27 Feb 2024 17:08:43 -0600 Subject: [PATCH 006/328] linting fixed --- .../DataFilesAddProjectModal.jsx | 12 +++-- .../DataFilesProjectFileListing.jsx | 12 ++--- .../_common/Form/DynamicForm/DynamicForm.jsx | 17 ++++--- .../DataFilesAddProjectFormAddon.jsx | 2 +- .../src/hooks/datafiles/useAddonComponents.js | 46 ++++++++++--------- 5 files changed, 46 insertions(+), 43 deletions(-) diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesAddProjectModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesAddProjectModal.jsx index 88f5d4acd0..c8bc4080e6 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesAddProjectModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesAddProjectModal.jsx @@ -9,7 +9,6 @@ import { useHistory, useRouteMatch } from 'react-router-dom'; import DataFilesProjectMembers from '../DataFilesProjectMembers/DataFilesProjectMembers'; import { useAddonComponents, useFileListing } from 'hooks/datafiles'; - const DataFilesAddProjectModal = () => { const history = useHistory(); const match = useRouteMatch(); @@ -20,8 +19,9 @@ const DataFilesAddProjectModal = () => { ); // logic to render addonComponents for DRP - const portalName = useSelector(state => state.workbench.portalName); - const addonComponents = portalName === 'DRP' && useAddonComponents({ portalName: 'DRP' }); + const portalName = useSelector((state) => state.workbench.portalName); + const addonComponents = + portalName === 'DRP' && useAddonComponents({ portalName: 'DRP' }); useEffect(() => { setMembers([ @@ -90,7 +90,6 @@ const DataFilesAddProjectModal = () => { .required('Please enter a title.'), }); - return ( <> { } /> - {addonComponents && addonComponents.DataFilesAddProjectFormAddon && } + {addonComponents && + addonComponents.DataFilesAddProjectFormAddon && ( + + )} { const dispatch = useDispatch(); const { fetchListing } = useFileListing('FilesListing'); - + // logic to render addonComponents for DRP - const portalName = useSelector(state => state.workbench.portalName); - const addonComponents = portalName === 'DRP' && useAddonComponents({ portalName: 'DRP' }); + const portalName = useSelector((state) => state.workbench.portalName); + const addonComponents = + portalName === 'DRP' && useAddonComponents({ portalName: 'DRP' }); useEffect(() => { dispatch({ @@ -44,8 +45,6 @@ const DataFilesProjectFileListing = ({ system, path }) => { .map((currentUser) => currentUser.access === 'owner')[0] ); - - // const { hasProjectFileListingToolbarAddon, portalName } = useSelector( // (state) => ({ // hasProjectFileListingToolbarAddon: @@ -119,7 +118,8 @@ const DataFilesProjectFileListing = ({ system, path }) => { {canEditSystem ? ( <> {/* AddonComponent for DRP portal */} - {addonComponents && addonComponents.DataFilesProjectListingAddon ? ( + {addonComponents && + addonComponents.DataFilesProjectListingAddon ? ( ) : ( <> diff --git a/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx b/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx index 66ac0f6ced..3feb32326d 100644 --- a/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx +++ b/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx @@ -44,8 +44,8 @@ const DynamicForm = ({ formFields }) => { ); })} - ); - case 'radio': + ); + case 'radio': return (
@@ -61,18 +61,17 @@ const DynamicForm = ({ formFields }) => {
))} - ); case 'file': // Adding support for file type return (
+ name={field.name} + label={field.label} + type="file" + description={field?.description} + required={field?.validation?.required} + />
); case 'submit': diff --git a/client/src/components/_custom/drp/DataFilesAddProjectFormAddon/DataFilesAddProjectFormAddon.jsx b/client/src/components/_custom/drp/DataFilesAddProjectFormAddon/DataFilesAddProjectFormAddon.jsx index 745a5a2c26..fc1abf6cd5 100644 --- a/client/src/components/_custom/drp/DataFilesAddProjectFormAddon/DataFilesAddProjectFormAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesAddProjectFormAddon/DataFilesAddProjectFormAddon.jsx @@ -19,7 +19,7 @@ const DataFilesProjectFormAddon = () => { )} - ); + ); }; export default DataFilesProjectFormAddon; diff --git a/client/src/hooks/datafiles/useAddonComponents.js b/client/src/hooks/datafiles/useAddonComponents.js index e396d6bf2f..3b72f625f4 100644 --- a/client/src/hooks/datafiles/useAddonComponents.js +++ b/client/src/hooks/datafiles/useAddonComponents.js @@ -2,28 +2,30 @@ import { useState, useEffect } from 'react'; import { useSelector } from 'react-redux'; function useAddonComponents({ portalName }) { - const [addonComponents, setAddonComponents] = useState({}); - const hasAddons = useSelector(state => state.workbench.config.hasAddons); - useEffect(() => { - const loadAddonComponents = async () => { - try { - const addOns = {}; - for (const addOnName of hasAddons) { - const module = await import(`../../components/_custom/${portalName}/${addOnName}/${addOnName}.jsx`); - addOns[addOnName] = module.default; - } - setAddonComponents(addOns); - } catch (error) { - console.error('Error loading addon components:', error); - } - }; - - if (hasAddons) { - loadAddonComponents(); + const [addonComponents, setAddonComponents] = useState({}); + const hasAddons = useSelector((state) => state.workbench.config.hasAddons); + useEffect(() => { + const loadAddonComponents = async () => { + try { + const addOns = {}; + for (const addOnName of hasAddons) { + const module = await import( + `../../components/_custom/${portalName}/${addOnName}/${addOnName}.jsx` + ); + addOns[addOnName] = module.default; } - }, []); - console.log("truuu", addonComponents) - return addonComponents; + setAddonComponents(addOns); + } catch (error) { + console.error('Error loading addon components:', error); + } + }; + + if (hasAddons) { + loadAddonComponents(); + } + }, []); + console.log('truuu', addonComponents); + return addonComponents; } -export default useAddonComponents; \ No newline at end of file +export default useAddonComponents; From 5d5b7b7a3f3bd1c58348b8b4b6a04e968397a220 Mon Sep 17 00:00:00 2001 From: Shayan Khan Date: Wed, 28 Feb 2024 14:00:45 -0600 Subject: [PATCH 007/328] Added custom saga, added initial metadata support in the backend --- .../DataFilesAddProjectModal.jsx | 2 - .../DataFilesModals/DataFilesFormModal.jsx | 15 +++- client/src/components/Workbench/AppRouter.jsx | 8 +++ .../DataFilesAddProjectFormAddon.jsx | 16 ----- client/src/redux/sagas/_custom/drp.sagas.js | 68 +++++++++++++++++++ client/src/redux/sagas/datafiles.sagas.js | 4 +- client/src/redux/sagas/index.js | 21 +++++- .../migrations/0003_datafilesmetadata.py | 20 ++++++ server/portal/apps/datafiles/models.py | 8 +++ server/portal/libs/agave/operations.py | 29 +++++++- server/portal/settings/settings_default.py | 3 +- 11 files changed, 168 insertions(+), 26 deletions(-) delete mode 100644 client/src/components/_custom/drp/DataFilesAddProjectFormAddon/DataFilesAddProjectFormAddon.jsx create mode 100644 client/src/redux/sagas/_custom/drp.sagas.js create mode 100644 server/portal/apps/datafiles/migrations/0003_datafilesmetadata.py diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesAddProjectModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesAddProjectModal.jsx index 0e2ef6dc1a..b9d7dd96ae 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesAddProjectModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesAddProjectModal.jsx @@ -7,7 +7,6 @@ import { Button, InlineMessage } from '_common'; import { Modal, ModalHeader, ModalBody, ModalFooter } from 'reactstrap'; import { useHistory, useRouteMatch } from 'react-router-dom'; import DataFilesProjectMembers from '../DataFilesProjectMembers/DataFilesProjectMembers'; -import DataFilesProjectFormAddon from '../../_custom/drp/DataFilesAddProjectFormAddon/DataFilesAddProjectFormAddon'; const DataFilesAddProjectModal = () => { const history = useHistory(); @@ -115,7 +114,6 @@ const DataFilesAddProjectModal = () => { } /> - { const dispatch = useDispatch(); + const history = useHistory(); + const location = useLocation(); + + const reloadPage = () => { + history.push(location.pathname); + }; const { formName } = useSelector( (state) => state.files.modalProps.dynamicform ); const isOpen = useSelector((state) => state.files.modals.dynamicform); + const { params } = useFileListing('FilesListing'); const getFormFields = async (formName) => { const response = await fetchUtil({ @@ -48,7 +58,10 @@ const DataFilesFormModal = () => { }, []); const handleSubmit = (values) => { - console.log('FORM SUBMIT', values); + dispatch({ + type: formName, + payload: { params, values, reloadPage }, + }); }; return ( diff --git a/client/src/components/Workbench/AppRouter.jsx b/client/src/components/Workbench/AppRouter.jsx index 1a1b6fab71..3ced0725f1 100644 --- a/client/src/components/Workbench/AppRouter.jsx +++ b/client/src/components/Workbench/AppRouter.jsx @@ -16,6 +16,7 @@ function AppRouter() { const authenticatedUser = useSelector( (state) => state.authenticatedUser.user ); + const hasCustomSagas = useSelector((state) => state.workbench.config.hasCustomSagas); useEffect(() => { dispatch({ type: 'FETCH_AUTHENTICATED_USER' }); @@ -29,6 +30,13 @@ function AppRouter() { dispatch({ type: 'FETCH_CUSTOM_MESSAGES' }); } }, [authenticatedUser]); + + useEffect(() => { + if (hasCustomSagas) { + dispatch({ type: 'START_CUSTOM_SAGA' }); + } + }, [hasCustomSagas]); + return ( diff --git a/client/src/components/_custom/drp/DataFilesAddProjectFormAddon/DataFilesAddProjectFormAddon.jsx b/client/src/components/_custom/drp/DataFilesAddProjectFormAddon/DataFilesAddProjectFormAddon.jsx deleted file mode 100644 index 78f7b5cbaf..0000000000 --- a/client/src/components/_custom/drp/DataFilesAddProjectFormAddon/DataFilesAddProjectFormAddon.jsx +++ /dev/null @@ -1,16 +0,0 @@ -import { FormField } from '_common/Form'; -import React, { useEffect } from 'react'; - -const DataFilesProjectFormAddon = () => { - return ( -
- -
- ); -}; - -export default DataFilesProjectFormAddon; diff --git a/client/src/redux/sagas/_custom/drp.sagas.js b/client/src/redux/sagas/_custom/drp.sagas.js new file mode 100644 index 0000000000..8257ead763 --- /dev/null +++ b/client/src/redux/sagas/_custom/drp.sagas.js @@ -0,0 +1,68 @@ +import { + takeLatest, + takeLeading, + put, + call, + all, + race, + take, + select, + } from 'redux-saga/effects'; +import { mkdirUtil } from '../datafiles.sagas'; + + +export function* addSampleData(action) { + console.log('ADD SAMPLE DATA', action); + + const { params, values, reloadPage: reloadCallback } = action.payload + yield call(mkdir, params, { ...values, data_type: 'sample' }, reloadCallback); +} + +// goes to rename +export function* editSampleData(action) { + console.log('EDIT SAMPLE DATA', action); +} + +export function* addAnalysisDataset(action) { + console.log('ADD ANALYSIS DATA', action); + + const { params, values, reloadPage: reloadCallback } = action.payload + yield call(mkdir, params, { ...values, data_type: 'analysis_data' }, reloadCallback); +} + +export function* addOriginDataset(action) { + console.log('ADD ORIGIN DATA', action); + + const { params, values, reloadPage: reloadCallback } = action.payload + yield call(mkdir, params, { ...values, data_type: 'origin_data' }, reloadCallback); +} + + +function* mkdir(params, values, reloadCallback) { + yield call( + mkdirUtil, + params.api, + params.scheme, + params.system, + params.path, + values.name, + values + ) + + yield call(reloadCallback); + + yield put({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { + operation: 'dynamicform', + props: {}, + }, + }); +} + +export default function* watchDRP() { + yield takeLatest('ADD_SAMPLE_DATA', addSampleData); + yield takeLatest('EDIT_SAMPLE_DATA', editSampleData); + yield takeLatest('ADD_ANALYSIS_DATASET', addAnalysisDataset); + yield takeLatest('ADD_ORIGIN_DATASET', addOriginDataset); +} diff --git a/client/src/redux/sagas/datafiles.sagas.js b/client/src/redux/sagas/datafiles.sagas.js index 1ccb630af6..d6b9b4992f 100644 --- a/client/src/redux/sagas/datafiles.sagas.js +++ b/client/src/redux/sagas/datafiles.sagas.js @@ -596,7 +596,7 @@ export async function previewUtil(api, scheme, system, path) { return requestJson.data; } -export async function mkdirUtil(api, scheme, system, path, dirname) { +export async function mkdirUtil(api, scheme, system, path, dirname, metadata) { let apiPath = !path || path[0] === '/' ? path : `/${path}`; if (apiPath === '/') { apiPath = ''; @@ -610,7 +610,7 @@ export async function mkdirUtil(api, scheme, system, path, dirname) { method: 'PUT', headers: { 'X-CSRFToken': Cookies.get('csrftoken') }, credentials: 'same-origin', - body: JSON.stringify({ dir_name: dirname }), + body: JSON.stringify({ dir_name: dirname, metadata: metadata ?? null }), }); return request; diff --git a/client/src/redux/sagas/index.js b/client/src/redux/sagas/index.js index 6a879a9051..9631694ead 100644 --- a/client/src/redux/sagas/index.js +++ b/client/src/redux/sagas/index.js @@ -1,4 +1,4 @@ -import { all } from 'redux-saga/effects'; +import { all, select, takeEvery, fork } from 'redux-saga/effects'; import { watchJobs, watchJobDetails } from './jobs.sagas'; import watchApps from './apps.sagas'; import watchSystems from './systems.sagas'; @@ -53,6 +53,24 @@ import { watchProjects } from './projects.sagas'; import { watchUsers } from './users.sagas'; import { watchSiteSearch } from './siteSearch.sagas'; +function* watchStartCustomSaga() { + yield takeEvery('START_CUSTOM_SAGA', startCustomSaga); +} + +function* startCustomSaga(action) { + const portalName = yield select((state) => { + return state.workbench.portalName; + }); + + const { default: customSaga } = yield import( + `./_custom/${portalName.toLowerCase()}.sagas` + ); + + if (customSaga) { + yield fork(customSaga); + } +} + export default function* rootSaga() { yield all([ watchJobs(), @@ -102,5 +120,6 @@ export default function* rootSaga() { watchProjects(), watchUsers(), watchSiteSearch(), + watchStartCustomSaga(), ]); } diff --git a/server/portal/apps/datafiles/migrations/0003_datafilesmetadata.py b/server/portal/apps/datafiles/migrations/0003_datafilesmetadata.py new file mode 100644 index 0000000000..db042217f7 --- /dev/null +++ b/server/portal/apps/datafiles/migrations/0003_datafilesmetadata.py @@ -0,0 +1,20 @@ +# Generated by Django 4.2.7 on 2024-02-20 18:55 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('datafiles', '0002_auto_20230317_2209'), + ] + + operations = [ + migrations.CreateModel( + name='DataFilesMetadata', + fields=[ + ('path', models.CharField(max_length=255, primary_key=True, serialize=False)), + ('metadata', models.JSONField()), + ], + ), + ] diff --git a/server/portal/apps/datafiles/models.py b/server/portal/apps/datafiles/models.py index d35f2ec64f..0d02dc1472 100644 --- a/server/portal/apps/datafiles/models.py +++ b/server/portal/apps/datafiles/models.py @@ -21,3 +21,11 @@ def to_dict(self): 'updated': str(self.updated), 'expiration': str(self.expiration) } + +class DataFilesMetadata(models.Model): + + path = models.CharField(max_length=255, primary_key=True) + metadata = models.JSONField() + + def __str__(self): + return self.path \ No newline at end of file diff --git a/server/portal/libs/agave/operations.py b/server/portal/libs/agave/operations.py index 83264a1a80..7c9d7e5dd1 100644 --- a/server/portal/libs/agave/operations.py +++ b/server/portal/libs/agave/operations.py @@ -10,10 +10,19 @@ from portal.libs.agave.filter_mapping import filter_mapping from pathlib import Path from tapipy.errors import BaseTapyException +from portal.apps.datafiles.models import DataFilesMetadata logger = logging.getLogger(__name__) +def get_datafile_metadata(system, path): + try: + metadata_record = DataFilesMetadata.objects.get(path=f'{system}/{path.strip("/")}') + return metadata_record.metadata + except: + return None + + def listing(client, system, path, offset=0, limit=100, *args, **kwargs): """ Perform a Tapis file listing @@ -41,6 +50,9 @@ def listing(client, system, path, offset=0, limit=100, *args, **kwargs): path=path, offset=int(offset), limit=int(limit)) + + + folder_metadata = get_datafile_metadata(system=system, path=path) try: # Convert file objects to dicts for serialization. @@ -55,14 +67,16 @@ def listing(client, system, path, offset=0, limit=100, *args, **kwargs): 'lastModified': f.lastModified, '_links': { 'self': {'href': f.url} - }}, raw_listing)) + }, + 'metadata': get_datafile_metadata(system=system, path=f.path) + }, raw_listing)) except IndexError: # Return [] if the listing is empty. listing = [] # Update Elasticsearch after each listing. tapis_listing_indexer.delay(listing) - return {'listing': listing, 'reachedEnd': len(listing) < int(limit)} + return {'listing': listing, 'reachedEnd': len(listing) < int(limit), 'folder_metadata': folder_metadata} def iterate_listing(client, system, path, limit=100): @@ -176,7 +190,7 @@ def download(client, system, path, max_uses=3, lifetime=600, **kwargs): return redeemUrl -def mkdir(client, system, path, dir_name): +def mkdir(client, system, path, dir_name, metadata): """Create a new directory. Params @@ -198,6 +212,15 @@ def mkdir(client, system, path, dir_name): path_input = str(Path(path) / Path(dir_name)) client.files.mkdir(systemId=system, path=path_input) + if metadata is not None: + files_metadata = DataFilesMetadata( + path = f'{system}/{path_input.strip("/")}', + metadata = metadata + ) + + files_metadata.save() + print(f'File Metadata for path {path_input} saved successfully') + tapis_indexer.apply_async(kwargs={'access_token': client.access_token.access_token, 'systemId': system, 'filePath': path, diff --git a/server/portal/settings/settings_default.py b/server/portal/settings/settings_default.py index 4a008c3342..db8914ae45 100644 --- a/server/portal/settings/settings_default.py +++ b/server/portal/settings/settings_default.py @@ -9,7 +9,7 @@ _DEBUG = True # Namespace for portal -_PORTAL_NAMESPACE = 'CEP' +_PORTAL_NAMESPACE = 'DRP' # NOTE: set _WH_BASE_URL to ngrok redirect for local dev testing (i.e. _WH_BASE_URL = 'https://12345.ngrock.io', see https://ngrok.com/) _WH_BASE_URL = '' @@ -240,6 +240,7 @@ "hideSystemStatus": False, "hasUserGuide": True, "hasProjectFileListingAddon": True, + "hasCustomSagas": True, "onboardingCompleteRedirect": '/workbench/', "noPHISystem": "", "customDashboardSection": None, From 4b776b0c8b01a0874cf5f7bc4f7b8125e2e140f2 Mon Sep 17 00:00:00 2001 From: Shayan Khan Date: Fri, 1 Mar 2024 12:01:59 -0600 Subject: [PATCH 008/328] Updated and simplified components --- .../DataFilesAddProjectModal.jsx | 8 ++--- .../DataFilesProjectFileListing.jsx | 31 +++---------------- .../_common/Form/DynamicForm/DynamicForm.jsx | 6 ++++ .../DataFilesAddProjectModalAddon.jsx} | 8 +++-- .../DataFilesProjectFileListingAddon.jsx} | 2 +- ...aFilesProjectFileListingAddon.module.scss} | 0 .../src/hooks/datafiles/useAddonComponents.js | 17 +++++----- server/portal/settings/settings_default.py | 2 +- 8 files changed, 28 insertions(+), 46 deletions(-) rename client/src/components/_custom/drp/{DataFilesAddProjectFormAddon/DataFilesAddProjectFormAddon.jsx => DataFilesAddProjectModalAddon/DataFilesAddProjectModalAddon.jsx} (78%) rename client/src/components/_custom/drp/{DataFilesProjectListingAddon/DataFilesProjectListingAddon.jsx => DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx} (94%) rename client/src/components/_custom/drp/{DataFilesProjectListingAddon/DataFilesProjectListingAddon.module.scss => DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.module.scss} (100%) diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesAddProjectModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesAddProjectModal.jsx index c8bc4080e6..e2c8e48a46 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesAddProjectModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesAddProjectModal.jsx @@ -20,8 +20,7 @@ const DataFilesAddProjectModal = () => { // logic to render addonComponents for DRP const portalName = useSelector((state) => state.workbench.portalName); - const addonComponents = - portalName === 'DRP' && useAddonComponents({ portalName: 'DRP' }); + const { DataFilesAddProjectModalAddon } = useAddonComponents({portalName}) useEffect(() => { setMembers([ @@ -120,10 +119,7 @@ const DataFilesAddProjectModal = () => { } /> - {addonComponents && - addonComponents.DataFilesAddProjectFormAddon && ( - - )} + {DataFilesAddProjectModalAddon && } { // logic to render addonComponents for DRP const portalName = useSelector((state) => state.workbench.portalName); - const addonComponents = - portalName === 'DRP' && useAddonComponents({ portalName: 'DRP' }); + + const { DataFilesProjectFileListingAddon } = useAddonComponents({portalName}) useEffect(() => { dispatch({ @@ -45,27 +45,6 @@ const DataFilesProjectFileListing = ({ system, path }) => { .map((currentUser) => currentUser.access === 'owner')[0] ); - // const { hasProjectFileListingToolbarAddon, portalName } = useSelector( - // (state) => ({ - // hasProjectFileListingToolbarAddon: - // state.workbench.config.hasProjectFileListingAddon, - // portalName: state.workbench.portalName, - // }) - // ); - - // useEffect(() => { - // const loadAddonComponent = async () => { - // const module = await import( - // `../../_custom/${portalName}/DataFilesProjectListingAddon/DataFilesProjectListingAddon.jsx` - // ); - // setAddonComponent(() => module.default); - // }; - - // if (hasProjectFileListingToolbarAddon) { - // loadAddonComponent(); - // } - // }, [hasProjectFileListingToolbarAddon, portalName]); - const readOnlyTeam = useSelector((state) => { const projectSystem = state.systems.storage.configuration.find( (s) => s.scheme === 'projects' @@ -118,10 +97,8 @@ const DataFilesProjectFileListing = ({ system, path }) => { {canEditSystem ? ( <> {/* AddonComponent for DRP portal */} - {addonComponents && - addonComponents.DataFilesProjectListingAddon ? ( - - ) : ( + {DataFilesProjectFileListingAddon ? + : ( <> - | - - )} + + | ) : null} + { DataFilesProjectFileListingAddon && } } manualContent diff --git a/client/src/components/_custom/drp/DataFilesProjectEditDescriptionModalAddon/DataFilesProjectEditDescriptionModalAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectEditDescriptionModalAddon/DataFilesProjectEditDescriptionModalAddon.jsx new file mode 100644 index 0000000000..629c20ad6d --- /dev/null +++ b/client/src/components/_custom/drp/DataFilesProjectEditDescriptionModalAddon/DataFilesProjectEditDescriptionModalAddon.jsx @@ -0,0 +1,23 @@ +import React from 'react'; +import { useQuery } from 'react-query'; +import { fetchUtil } from 'utils/fetchUtil'; +import { DynamicForm } from '_common/Form/DynamicForm'; + +const DataFilesProjectEditDescriptionModalAddon = () => { + + const { data: form, isLoading } = useQuery('form_EDIT_PROJECT', () => + fetchUtil({ + url: 'api/forms', + params: { + form_name: 'EDIT_PROJECT_ADDON', + }, + }) + ); + return ( +
+ +
+ ); +}; + +export default DataFilesProjectEditDescriptionModalAddon; \ No newline at end of file diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx index d5c9f91b61..d3654cc865 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx @@ -18,9 +18,6 @@ const DataFilesProjectFileListingAddon = () => { return ( <> - | ); diff --git a/client/src/components/_common/Form/DynamicForm/DynamicForm.scss b/client/src/components/_common/Form/DynamicForm/DynamicForm.scss index 9b537668de..394eedd9fb 100644 --- a/client/src/components/_common/Form/DynamicForm/DynamicForm.scss +++ b/client/src/components/_common/Form/DynamicForm/DynamicForm.scss @@ -3,5 +3,6 @@ } .radio-input { - margin-right: 10px; + margin-left: 1rem; + margin-bottom: 0rem; } diff --git a/client/src/redux/sagas/datafiles.sagas.js b/client/src/redux/sagas/datafiles.sagas.js index 2ab1722fcf..3009dc5d30 100644 --- a/client/src/redux/sagas/datafiles.sagas.js +++ b/client/src/redux/sagas/datafiles.sagas.js @@ -1171,3 +1171,19 @@ export function* doMakePublic(action) { export function* watchMakePublic() { yield takeLeading('DATA_FILES_MAKE_PUBLIC', doMakePublic); } + +export async function updateMetadataUtil(api, scheme, system, path, oldName, newName, metadata) { + const url = `/api/datafiles/${api}/update_metadata/${scheme}/${system}/${path}/`; + const response = await fetch(url, { + method: 'PUT', + headers: { 'X-CSRFToken': Cookies.get('csrftoken') }, + credentials: 'same-origin', + body: JSON.stringify({ old_name: oldName, new_name: newName, metadata: metadata ?? null }), + }); + if (!response.ok) { + throw new Error(response.status); + } + + const responseJson = await response.json(); + return responseJson.data; +} \ No newline at end of file diff --git a/server/portal/apps/datafiles/handlers/tapis_handlers.py b/server/portal/apps/datafiles/handlers/tapis_handlers.py index 379cd5e9be..f429abd528 100644 --- a/server/portal/apps/datafiles/handlers/tapis_handlers.py +++ b/server/portal/apps/datafiles/handlers/tapis_handlers.py @@ -11,7 +11,7 @@ 'public': ['listing', 'search', 'copy', 'download', 'preview'], 'community': ['listing', 'search', 'copy', 'download', 'preview'], 'projects': ['listing', 'search', 'copy', 'download', 'mkdir', - 'move', 'rename', 'trash', 'preview', 'upload', 'makepublic'] + 'move', 'rename', 'trash', 'preview', 'upload', 'makepublic', 'update_metadata'] } diff --git a/server/portal/libs/agave/operations.py b/server/portal/libs/agave/operations.py index aa603dd31c..3434858afd 100644 --- a/server/portal/libs/agave/operations.py +++ b/server/portal/libs/agave/operations.py @@ -28,8 +28,9 @@ def get_datafile_metadata(system, path): def update_datafile_metadata(system, name, old_path, new_path, metadata): files_metadata = DataFilesMetadata.objects.get(path=f'{system}/{old_path.strip("/")}') files_metadata.name = name - files_metadata.path = f'{system}/{new_path}' + files_metadata.path = f"{system}/{new_path.strip('/')}" files_metadata.metadata = metadata + files_metadata.metadata['name'] = name files_metadata.save() for child in DataFilesMetadata.objects.filter(parent=files_metadata.id): @@ -578,8 +579,20 @@ def download_bytes(client, system, path): io.BytesIO BytesIO object representing the downloaded file. """ - file_name = os.path.basename(path) + file_name = os.path.basename(path) resp = client.files.getContents(systemId=system, path=path) result = io.BytesIO(resp) result.name = file_name return result + +@transaction.atomic +def update_metadata(client, system, path, old_name, new_name, metadata): + new_path = os.path.dirname(path) + + if old_name != new_name: + move_result = move(client, src_system=system, src_path=path, + dest_system=system, dest_path=new_path, file_name=new_name) + move_message = move_result['message'].split('DestinationPath: ', 1)[1] + new_name = ('/' + move_message).rsplit('/', 1)[1] + + update_datafile_metadata(system=system, name=new_name, old_path=path, new_path=f'{new_path.strip("/")}/{new_name}', metadata=metadata) From ac42a97a48a2fb85c6f7838bbc000a5683032974 Mon Sep 17 00:00:00 2001 From: Shayan Khan Date: Mon, 18 Mar 2024 12:03:46 -0500 Subject: [PATCH 015/328] Added edit forms and functionality --- .../DataFilesModals/DataFilesFormModal.jsx | 9 ++-- .../DataFilesProjectFileListingAddon.jsx | 52 ++++++++++++++----- client/src/redux/sagas/_custom/drp.sagas.js | 43 +++++++++++++-- 3 files changed, 81 insertions(+), 23 deletions(-) diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx index b76561b1ad..d52dc97a96 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx @@ -7,7 +7,7 @@ import { fetchUtil } from 'utils/fetchUtil'; import { Formik, Form } from 'formik'; import * as Yup from 'yup'; import styles from './DataFilesFormModal.module.scss'; -import { useFileListing } from 'hooks/datafiles'; +import { useFileListing, useSelectedFiles } from 'hooks/datafiles'; import { useHistory, useLocation } from 'react-router-dom'; @@ -20,7 +20,7 @@ const DataFilesFormModal = () => { history.push(location.pathname); }; - const { formName } = useSelector( + const { formName, selectedFile } = useSelector( (state) => state.files.modalProps.dynamicform ); const isOpen = useSelector((state) => state.files.modals.dynamicform); @@ -44,9 +44,10 @@ const DataFilesFormModal = () => { }; const { data: form, isLoading } = useFormFields(formName); + const { selectedFiles } = useSelectedFiles(); const initialValues = form?.form_fields.reduce((acc, field) => { - acc[field.name] = field.value || ''; + acc[field.name] = selectedFile ? selectedFile.metadata[field.name] : '' return acc; }, {}); @@ -60,7 +61,7 @@ const DataFilesFormModal = () => { const handleSubmit = (values) => { dispatch({ type: formName, - payload: { params, values, reloadPage }, + payload: { params, values, reloadPage, selectedFile: selectedFiles[0] }, }); }; diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx index d3654cc865..c1cfe4f7f4 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx @@ -2,37 +2,61 @@ import React, { useEffect } from 'react'; import { Button } from '_common'; import { useDispatch, useSelector } from 'react-redux'; import styles from './DataFilesProjectFileListingAddon.module.scss'; +import { useSelectedFiles } from 'hooks/datafiles'; const DataFilesProjectFileListingAddon = () => { const dispatch = useDispatch(); - const onOpenFormModal = (form_name) => { + const onOpenFormModal = (formName, selectedFile) => { dispatch({ type: 'DATA_FILES_TOGGLE_MODAL', payload: { operation: 'dynamicform', - props: { formName: form_name }, + props: { formName, selectedFile }, }, }); }; + const { selectedFiles } = useSelectedFiles(); + return ( <> | - + {selectedFiles.length == 1 && selectedFiles[0]?.metadata['data_type'] === 'sample' ? + + : + + } | - + {selectedFiles.length == 1 && selectedFiles[0]?.metadata['data_type'] === 'origin_data' ? + + : + + } | - + {selectedFiles.length == 1 && selectedFiles[0]?.metadata['data_type'] === 'analysis_data' ? + + : + + } | ); diff --git a/client/src/redux/sagas/_custom/drp.sagas.js b/client/src/redux/sagas/_custom/drp.sagas.js index 8257ead763..05d9334738 100644 --- a/client/src/redux/sagas/_custom/drp.sagas.js +++ b/client/src/redux/sagas/_custom/drp.sagas.js @@ -8,7 +8,7 @@ import { take, select, } from 'redux-saga/effects'; -import { mkdirUtil } from '../datafiles.sagas'; +import { mkdirUtil, renameFileUtil, updateMetadataUtil } from '../datafiles.sagas'; export function* addSampleData(action) { @@ -18,9 +18,9 @@ export function* addSampleData(action) { yield call(mkdir, params, { ...values, data_type: 'sample' }, reloadCallback); } -// goes to rename export function* editSampleData(action) { - console.log('EDIT SAMPLE DATA', action); + const { params, values, reloadPage: reloadCallback, selectedFile } = action.payload + yield call(update, params, { ...values, data_type: 'sample' }, reloadCallback, selectedFile); } export function* addAnalysisDataset(action) { @@ -30,13 +30,21 @@ export function* addAnalysisDataset(action) { yield call(mkdir, params, { ...values, data_type: 'analysis_data' }, reloadCallback); } -export function* addOriginDataset(action) { - console.log('ADD ORIGIN DATA', action); +export function* editAnalysisDataset(action) { + const { params, values, reloadPage: reloadCallback, selectedFile } = action.payload + yield call(update, params, { ...values, data_type: 'analysis_data' }, reloadCallback, selectedFile); +} +export function* addOriginDataset(action) { const { params, values, reloadPage: reloadCallback } = action.payload yield call(mkdir, params, { ...values, data_type: 'origin_data' }, reloadCallback); } +export function* editOriginDataset(action) { + const { params, values, reloadPage: reloadCallback, selectedFile } = action.payload + yield call(update, params, { ...values, data_type: 'origin_data' }, reloadCallback, selectedFile); +} + function* mkdir(params, values, reloadCallback) { yield call( @@ -60,9 +68,34 @@ function* mkdir(params, values, reloadCallback) { }); } +function* update(params, values, reloadCallback, file) { + yield call( + updateMetadataUtil, + params.api, + params.scheme, + params.system, + '/' + file.path, + file.name, + values.name, + values + ) + + yield call(reloadCallback); + + yield put({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { + operation: 'dynamicform', + props: {}, + }, + }); +} + export default function* watchDRP() { yield takeLatest('ADD_SAMPLE_DATA', addSampleData); yield takeLatest('EDIT_SAMPLE_DATA', editSampleData); yield takeLatest('ADD_ANALYSIS_DATASET', addAnalysisDataset); + yield takeLatest('EDIT_ANALYSIS_DATASET', editAnalysisDataset); yield takeLatest('ADD_ORIGIN_DATASET', addOriginDataset); + yield takeLatest('EDIT_ORIGIN_DATASET', editOriginDataset); } From 656eef6781f4126ca6aeb2789d2edef50c19e13b Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Tue, 19 Mar 2024 15:44:35 -0500 Subject: [PATCH 016/328] Added project edit functionality --- ...aFilesProjectEditDescriptionModalAddon.jsx | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/client/src/components/_custom/drp/DataFilesProjectEditDescriptionModalAddon/DataFilesProjectEditDescriptionModalAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectEditDescriptionModalAddon/DataFilesProjectEditDescriptionModalAddon.jsx index 629c20ad6d..6436e738e7 100644 --- a/client/src/components/_custom/drp/DataFilesProjectEditDescriptionModalAddon/DataFilesProjectEditDescriptionModalAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectEditDescriptionModalAddon/DataFilesProjectEditDescriptionModalAddon.jsx @@ -1,9 +1,14 @@ -import React from 'react'; +import React, { useEffect } from 'react'; import { useQuery } from 'react-query'; import { fetchUtil } from 'utils/fetchUtil'; import { DynamicForm } from '_common/Form/DynamicForm'; +import { useSelector } from 'react-redux'; +import { useFormikContext } from 'formik' + const DataFilesProjectEditDescriptionModalAddon = () => { + + const { setFieldValue } = useFormikContext(); const { data: form, isLoading } = useQuery('form_EDIT_PROJECT', () => fetchUtil({ @@ -13,9 +18,24 @@ const DataFilesProjectEditDescriptionModalAddon = () => { }, }) ); + + const { metadata } = useSelector(state => state.projects) + + useEffect(() => { + if (!isLoading && form && metadata) { + form.form_fields.forEach(field => { + if (metadata.hasOwnProperty(field.name)) { + setFieldValue(field.name, metadata[field.name]) + } + }); + } + }, [form]) + return (
+ {!isLoading && form && + }
); }; From 56cef6e6dfe3603b785a95522180163d5800c9b7 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Wed, 20 Mar 2024 12:15:59 -0500 Subject: [PATCH 017/328] Added a comment --- server/portal/libs/agave/operations.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/portal/libs/agave/operations.py b/server/portal/libs/agave/operations.py index 3434858afd..b5768b66c8 100644 --- a/server/portal/libs/agave/operations.py +++ b/server/portal/libs/agave/operations.py @@ -592,6 +592,7 @@ def update_metadata(client, system, path, old_name, new_name, metadata): if old_name != new_name: move_result = move(client, src_system=system, src_path=path, dest_system=system, dest_path=new_path, file_name=new_name) + # update the name in the case it was changed during the move operation move_message = move_result['message'].split('DestinationPath: ', 1)[1] new_name = ('/' + move_message).rsplit('/', 1)[1] From 6dd485f68207137d81f4251d1f58d8149cbee04a Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Wed, 27 Mar 2024 12:14:13 -0500 Subject: [PATCH 018/328] update migrations --- .../portal/apps/datafiles/migrations/0003_datafilesmetadata.py | 2 +- .../{0003_projectsmetadata.py => 0004_projectsmetadata.py} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename server/portal/apps/projects/migrations/{0003_projectsmetadata.py => 0004_projectsmetadata.py} (86%) diff --git a/server/portal/apps/datafiles/migrations/0003_datafilesmetadata.py b/server/portal/apps/datafiles/migrations/0003_datafilesmetadata.py index c88aa23967..1aa49f9559 100644 --- a/server/portal/apps/datafiles/migrations/0003_datafilesmetadata.py +++ b/server/portal/apps/datafiles/migrations/0003_datafilesmetadata.py @@ -7,7 +7,7 @@ class Migration(migrations.Migration): dependencies = [ - ('projects', '0003_projectsmetadata'), + ('projects', '0004_projectsmetadata'), ('datafiles', '0002_auto_20230317_2209'), ] diff --git a/server/portal/apps/projects/migrations/0003_projectsmetadata.py b/server/portal/apps/projects/migrations/0004_projectsmetadata.py similarity index 86% rename from server/portal/apps/projects/migrations/0003_projectsmetadata.py rename to server/portal/apps/projects/migrations/0004_projectsmetadata.py index 5049a92109..dbfdc202b5 100644 --- a/server/portal/apps/projects/migrations/0003_projectsmetadata.py +++ b/server/portal/apps/projects/migrations/0004_projectsmetadata.py @@ -6,7 +6,7 @@ class Migration(migrations.Migration): dependencies = [ - ('projects', '0002_auto_20210312_1743'), + ('projects', '0003_alter_abstractprojectmetadata_co_pis_and_more'), ] operations = [ From e137731cd9c3990f31e293f90696afc4ecf74577 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Wed, 27 Mar 2024 13:38:18 -0500 Subject: [PATCH 019/328] Added custom drp endpoint, modified forms to populate retrieved values --- .../DataFilesModals/DataFilesFormModal.jsx | 52 +++--- .../_common/Form/DynamicForm/DynamicForm.jsx | 53 +++--- .../DataFilesProjectFileListingAddon.jsx | 134 ++++++++++++--- client/src/redux/sagas/_custom/drp.sagas.js | 160 +++++++++--------- client/src/redux/sagas/datafiles.sagas.js | 4 +- server/portal/apps/_custom/drp/__init__.py | 0 server/portal/apps/_custom/drp/urls.py | 11 ++ server/portal/apps/_custom/drp/views.py | 20 +++ server/portal/libs/agave/operations.py | 6 +- server/portal/settings/settings_default.py | 1 + server/portal/urls.py | 9 + 11 files changed, 299 insertions(+), 151 deletions(-) create mode 100644 server/portal/apps/_custom/drp/__init__.py create mode 100644 server/portal/apps/_custom/drp/urls.py create mode 100644 server/portal/apps/_custom/drp/views.py diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx index d241e265fb..79137f6149 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx @@ -10,7 +10,6 @@ import styles from './DataFilesFormModal.module.scss'; import { useFileListing, useSelectedFiles } from 'hooks/datafiles'; import { useHistory, useLocation } from 'react-router-dom'; - const DataFilesFormModal = () => { const dispatch = useDispatch(); const history = useHistory(); @@ -20,35 +19,24 @@ const DataFilesFormModal = () => { history.push(location.pathname); }; - const { formName, selectedFile } = useSelector( + const { form, selectedFile, formName, additionalData } = useSelector( (state) => state.files.modalProps.dynamicform ); const isOpen = useSelector((state) => state.files.modals.dynamicform); const { params } = useFileListing('FilesListing'); - const getFormFields = async (formName) => { - const response = await fetchUtil({ - url: 'api/forms', - params: { - form_name: formName, - }, - }); - return response; - }; - - const useFormFields = (formName) => { - const query = useQuery(['form', formName], () => getFormFields(formName), { - enabled: !!formName, - }); - return query; - }; - - const { data: form, isLoading } = useFormFields(formName); const { selectedFiles } = useSelectedFiles(); const initialValues = form?.form_fields.reduce((acc, field) => { - const value = field.options && field.options.length > 0 ? field.options[0].value : ''; - acc[field.name] = selectedFile ? selectedFile.metadata[field.name] : value + let value = ''; + if (field.optgroups) { + value = field.optgroups[0].options[0]?.value; + } else { + value = + field.options && field.options.length > 0 ? field.options[0].value : ''; + } + + acc[field.name] = selectedFile ? selectedFile.metadata[field.name] : value; return acc; }, {}); @@ -62,18 +50,26 @@ const DataFilesFormModal = () => { const handleSubmit = (values) => { dispatch({ type: formName, - payload: { params, values, reloadPage, selectedFile: selectedFiles[0] }, + payload: { + params, + values, + reloadPage, + selectedFile: selectedFiles[0], + additionalData, + }, }); }; const validationSchema = Yup.object().shape({ ...(form?.form_fields ?? []).reduce((schema, field) => { if (field.validation?.required) { - schema[field.name] = Yup.string().required(`${field.label} is required`); + schema[field.name] = Yup.string().required( + `${field.label} is required` + ); } return schema; }, {}), - }); + }); return ( <> @@ -85,7 +81,11 @@ const DataFilesFormModal = () => { toggle={toggle} className={styles['modal-dialog']} > - + {form.heading} diff --git a/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx b/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx index 507a531f94..1f4f2dbdeb 100644 --- a/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx +++ b/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx @@ -1,13 +1,14 @@ import React, { useState } from 'react'; import FormField from '../FormField'; import { Button } from '_common'; -import { useFormikContext } from 'formik' +import { useFormikContext } from 'formik'; import { FormGroup, Input } from 'reactstrap'; import './DynamicForm.scss'; const DynamicForm = ({ formFields }) => { // For file processing - const { setFieldValue, values, handleChange, handleBlur } = useFormikContext(); + const { setFieldValue, values, handleChange, handleBlur } = + useFormikContext(); const renderFormField = (field) => { switch (field.type) { @@ -42,31 +43,43 @@ const DynamicForm = ({ formFields }) => { description={field?.description} required={field?.validation?.required} > - {field.options.map((option) => ( - - ))} + {field.optgroups + ? field.optgroups.map((optgroup) => { + return ( + + {optgroup.options.map((option) => ( + + ))} + + ); + }) + : field.options.map((option) => ( + + ))} ); case 'radio': return ( - + {field.options.map((option) => ( - - - - + + + + ))} ); diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx index c1cfe4f7f4..84bb8d885f 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx @@ -1,62 +1,158 @@ -import React, { useEffect } from 'react'; +import React, { useEffect, useState } from 'react'; import { Button } from '_common'; import { useDispatch, useSelector } from 'react-redux'; import styles from './DataFilesProjectFileListingAddon.module.scss'; import { useSelectedFiles } from 'hooks/datafiles'; +import { useQuery } from 'react-query'; +import { fetchUtil } from 'utils/fetchUtil'; const DataFilesProjectFileListingAddon = () => { const dispatch = useDispatch(); + const portalName = useSelector((state) => state.workbench.portalName); + const { projectId } = useSelector((state) => state.projects.metadata); + const { selectedFiles } = useSelectedFiles(); + + const getFormFields = async (formName) => { + const response = await fetchUtil({ + url: 'api/forms', + params: { + form_name: formName, + }, + }); + return response; + }; + + const getSamples = async (projectId, getOriginData = false) => { + const response = await fetchUtil({ + url: `api/${portalName.toLowerCase()}`, + params: { + project_id: projectId, + get_origin_data: getOriginData, + }, + }); + return response; + }; + + const onOpenSampleModal = async (formName, selectedFile) => { + + const form = await getFormFields(formName); - const onOpenFormModal = (formName, selectedFile) => { dispatch({ type: 'DATA_FILES_TOGGLE_MODAL', payload: { operation: 'dynamicform', - props: { formName, selectedFile }, + props: { form, selectedFile, formName }, }, }); }; - const { selectedFiles } = useSelectedFiles(); + const onOpenOriginDataModal = async (formName, selectedFile) => { + const form = await getFormFields(formName); + const samples = await getSamples(projectId); + + form.form_fields.map((field) => { + if (field.name === 'sample') { + field.options = samples.map((sample) => { + return { + value: parseInt(sample.id), + label: sample.name, + }; + }); + } + }); + + dispatch({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { + operation: 'dynamicform', + props: { selectedFile, form, formName, additionalData: samples }, + }, + }); + }; + + const onOpenAnalysisDataModal = async (formName, selectedFile) => { + const form = await getFormFields(formName); + const samples = await getSamples(projectId, true); + + form.form_fields.map((field) => { + if (field.name === 'base_origin_data') { + field.optgroups = samples.map((sample) => { + return { + label: sample.name, + options: sample.origin_data.map((originData) => { + return { + value: parseInt(originData.id), + label: originData.name, + }; + }), + }; + }); + } + }); + + dispatch({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { + operation: 'dynamicform', + props: { selectedFile, form, formName, additionalData: samples }, + }, + }); + }; return ( <> | - {selectedFiles.length == 1 && selectedFiles[0]?.metadata['data_type'] === 'sample' ? - - : - - } + )} | - {selectedFiles.length == 1 && selectedFiles[0]?.metadata['data_type'] === 'origin_data' ? - - : - - } + )} | - {selectedFiles.length == 1 && selectedFiles[0]?.metadata['data_type'] === 'analysis_data' ? + {selectedFiles.length == 1 && + selectedFiles[0]?.metadata['data_type'] === 'analysis_data' ? ( - : + ) : ( - } + )} | ); diff --git a/client/src/redux/sagas/_custom/drp.sagas.js b/client/src/redux/sagas/_custom/drp.sagas.js index 05d9334738..fb28ee12da 100644 --- a/client/src/redux/sagas/_custom/drp.sagas.js +++ b/client/src/redux/sagas/_custom/drp.sagas.js @@ -1,61 +1,40 @@ import { - takeLatest, - takeLeading, - put, - call, - all, - race, - take, - select, - } from 'redux-saga/effects'; -import { mkdirUtil, renameFileUtil, updateMetadataUtil } from '../datafiles.sagas'; - - -export function* addSampleData(action) { - console.log('ADD SAMPLE DATA', action); - - const { params, values, reloadPage: reloadCallback } = action.payload - yield call(mkdir, params, { ...values, data_type: 'sample' }, reloadCallback); -} - -export function* editSampleData(action) { - const { params, values, reloadPage: reloadCallback, selectedFile } = action.payload - yield call(update, params, { ...values, data_type: 'sample' }, reloadCallback, selectedFile); -} - -export function* addAnalysisDataset(action) { - console.log('ADD ANALYSIS DATA', action); - - const { params, values, reloadPage: reloadCallback } = action.payload - yield call(mkdir, params, { ...values, data_type: 'analysis_data' }, reloadCallback); -} - -export function* editAnalysisDataset(action) { - const { params, values, reloadPage: reloadCallback, selectedFile } = action.payload - yield call(update, params, { ...values, data_type: 'analysis_data' }, reloadCallback, selectedFile); -} - -export function* addOriginDataset(action) { - const { params, values, reloadPage: reloadCallback } = action.payload - yield call(mkdir, params, { ...values, data_type: 'origin_data' }, reloadCallback); -} - -export function* editOriginDataset(action) { - const { params, values, reloadPage: reloadCallback, selectedFile } = action.payload - yield call(update, params, { ...values, data_type: 'origin_data' }, reloadCallback, selectedFile); -} - - -function* mkdir(params, values, reloadCallback) { - yield call( - mkdirUtil, - params.api, - params.scheme, - params.system, - params.path, - values.name, - values - ) + takeLatest, + put, + call, +} from 'redux-saga/effects'; +import { + mkdirUtil,updateMetadataUtil, +} from '../datafiles.sagas'; +import { useHistory, useLocation } from 'react-router-dom'; + + +function* executeOperation(isEdit, params, values, reloadCallback, file = null, path = '') { + + const location = useLocation(); + if (file && isEdit) { + yield call( + updateMetadataUtil, + params.api, + params.scheme, + params.system, + '/' + file.path, + '/' + path, + file.name, + values.name, + values + ); + } else { + yield call( + mkdirUtil, + params.api, + params.scheme, + params.system, + path, + values.name, + values + ); + } yield call(reloadCallback); @@ -66,36 +45,57 @@ function* mkdir(params, values, reloadCallback) { props: {}, }, }); + } -function* update(params, values, reloadCallback, file) { +function* handleSampleData(action, isEdit) { + const { params, values, reloadPage: reloadCallback, selectedFile } = action.payload; yield call( - updateMetadataUtil, - params.api, - params.scheme, - params.system, - '/' + file.path, - file.name, - values.name, - values + executeOperation, isEdit, params, {...values, data_type: 'sample'}, reloadCallback, selectedFile ) +} - yield call(reloadCallback); +function* handleOriginData(action, isEdit) { + const { params, values, reloadPage: reloadCallback, selectedFile, additionalData: samples } = action.payload; - yield put({ - type: 'DATA_FILES_TOGGLE_MODAL', - payload: { - operation: 'dynamicform', - props: {}, - }, - }); + const sample = samples.find( + (sample) => sample.id === parseInt(values.sample) + ); + + // get the path without system name + const path = sample.path.split('/').slice(1).join('/'); + yield call( + executeOperation, isEdit, params, { ...values, data_type: 'origin_data' }, reloadCallback, selectedFile, path + ) +} + +function* handleAnalysisData(action, isEdit) { + const { params, values, reloadPage: reloadCallback, selectedFile, additionalData: samples } = action.payload; + + let { originDataId, path, sampleId } = samples.reduce((acc, sample) => { + const originData = sample.origin_data.find( + (originData) => originData.id === parseInt(values.base_origin_data) + ); + if (originData) { + acc.originDataId = originData.id; + acc.path = originData.path; + acc.sampleId = sample.id; + } + return acc; + }, {}); + + // get the path without system name + path = path.split('/').slice(1).join('/'); + yield call( + executeOperation, isEdit, params, {...values, data_type: 'analysis_data', sample: sampleId, base_origin_data: originDataId}, reloadCallback, selectedFile, path + ) } export default function* watchDRP() { - yield takeLatest('ADD_SAMPLE_DATA', addSampleData); - yield takeLatest('EDIT_SAMPLE_DATA', editSampleData); - yield takeLatest('ADD_ANALYSIS_DATASET', addAnalysisDataset); - yield takeLatest('EDIT_ANALYSIS_DATASET', editAnalysisDataset); - yield takeLatest('ADD_ORIGIN_DATASET', addOriginDataset); - yield takeLatest('EDIT_ORIGIN_DATASET', editOriginDataset); + yield takeLatest('ADD_SAMPLE_DATA', action => handleSampleData(action, false)); + yield takeLatest('EDIT_SAMPLE_DATA', action => handleSampleData(action, true)); + yield takeLatest('ADD_ANALYSIS_DATASET', action => handleAnalysisData(action, false)); + yield takeLatest('EDIT_ANALYSIS_DATASET', action => handleAnalysisData(action, true)); + yield takeLatest('ADD_ORIGIN_DATASET', action => handleOriginData(action, false)); + yield takeLatest('EDIT_ORIGIN_DATASET', action => handleOriginData(action, true)); } diff --git a/client/src/redux/sagas/datafiles.sagas.js b/client/src/redux/sagas/datafiles.sagas.js index a3fec2926f..e2d0a6618a 100644 --- a/client/src/redux/sagas/datafiles.sagas.js +++ b/client/src/redux/sagas/datafiles.sagas.js @@ -1172,13 +1172,13 @@ export function* watchMakePublic() { yield takeLeading('DATA_FILES_MAKE_PUBLIC', doMakePublic); } -export async function updateMetadataUtil(api, scheme, system, path, oldName, newName, metadata) { +export async function updateMetadataUtil(api, scheme, system, path, newPath, oldName, newName, metadata) { const url = `/api/datafiles/${api}/update_metadata/${scheme}/${system}/${path}/`; const response = await fetch(url, { method: 'PUT', headers: { 'X-CSRFToken': Cookies.get('csrftoken') }, credentials: 'same-origin', - body: JSON.stringify({ old_name: oldName, new_name: newName, metadata: metadata ?? null }), + body: JSON.stringify({ new_path: newPath, old_name: oldName, new_name: newName, metadata: metadata ?? null }), }); if (!response.ok) { throw new Error(response.status); diff --git a/server/portal/apps/_custom/drp/__init__.py b/server/portal/apps/_custom/drp/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/server/portal/apps/_custom/drp/urls.py b/server/portal/apps/_custom/drp/urls.py new file mode 100644 index 0000000000..a409819c5a --- /dev/null +++ b/server/portal/apps/_custom/drp/urls.py @@ -0,0 +1,11 @@ +""" +.. module:: portal.apps.forms.urls + :synopsis: Forms URLs +""" +from django.urls import re_path +from portal.apps._custom.drp.views import DigitalRocksSampleView + +app_name = 'custom' +urlpatterns = [ + re_path('', DigitalRocksSampleView.as_view(), name='drp'), +] diff --git a/server/portal/apps/_custom/drp/views.py b/server/portal/apps/_custom/drp/views.py new file mode 100644 index 0000000000..b0fc6f8de2 --- /dev/null +++ b/server/portal/apps/_custom/drp/views.py @@ -0,0 +1,20 @@ +from portal.views.base import BaseApiView +from django.conf import settings +from django.http import JsonResponse, HttpResponseForbidden +from portal.apps.datafiles.models import DataFilesMetadata + + +class DigitalRocksSampleView(BaseApiView): + + def get(self, request): + project_id = request.GET.get('project_id') + get_origin_data = request.GET.get('get_origin_data') + + samples = DataFilesMetadata.objects.filter(project_id=f'{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}', metadata__data_type='sample').values('id', 'name', 'path') + + if get_origin_data: + for sample in samples: + sample['origin_data'] = list(DataFilesMetadata.objects.filter(parent_id=sample['id'], metadata__data_type='origin_data').values('id', 'name', 'path')) + + print(f'SAMPLES RETRIEVED: {samples}') + return JsonResponse(list(samples), safe=False) \ No newline at end of file diff --git a/server/portal/libs/agave/operations.py b/server/portal/libs/agave/operations.py index b5768b66c8..68cf468d9a 100644 --- a/server/portal/libs/agave/operations.py +++ b/server/portal/libs/agave/operations.py @@ -586,10 +586,8 @@ def download_bytes(client, system, path): return result @transaction.atomic -def update_metadata(client, system, path, old_name, new_name, metadata): - new_path = os.path.dirname(path) - - if old_name != new_name: +def update_metadata(client, system, path, new_path, old_name, new_name, metadata): + if (old_name != new_name) or (os.path.dirname(path) != new_path): move_result = move(client, src_system=system, src_path=path, dest_system=system, dest_path=new_path, file_name=new_name) # update the name in the case it was changed during the move operation diff --git a/server/portal/settings/settings_default.py b/server/portal/settings/settings_default.py index b1aba94384..f05c1f9a60 100644 --- a/server/portal/settings/settings_default.py +++ b/server/portal/settings/settings_default.py @@ -241,6 +241,7 @@ "hasUserGuide": True, "hasProjectFileListingAddon": True, "hasCustomSagas": True, + "hasCustomEndpoints": True, "addons": ['DataFilesProjectFileListingAddon', 'DataFilesAddProjectModalAddon', 'DataFilesProjectEditDescriptionModalAddon'], "onboardingCompleteRedirect": '/workbench/', "noPHISystem": "", diff --git a/server/portal/urls.py b/server/portal/urls.py index 26eeac78fb..5dbe668176 100644 --- a/server/portal/urls.py +++ b/server/portal/urls.py @@ -119,3 +119,12 @@ ] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) + +# custom endpoint +if settings.WORKBENCH_SETTINGS.get('hasCustomEndpoints'): + urlpatterns.append( + path( + f'api/{settings.PORTAL_NAMESPACE.lower()}/', + include(f'portal.apps._custom.{settings.PORTAL_NAMESPACE.lower()}.urls', namespace='custom') + ) + ) \ No newline at end of file From b5798d5e5b7a000a16dd506adf711b55bd53f4d1 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Wed, 27 Mar 2024 13:39:31 -0500 Subject: [PATCH 020/328] small fix --- client/src/redux/sagas/_custom/drp.sagas.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/client/src/redux/sagas/_custom/drp.sagas.js b/client/src/redux/sagas/_custom/drp.sagas.js index fb28ee12da..e54044a5bb 100644 --- a/client/src/redux/sagas/_custom/drp.sagas.js +++ b/client/src/redux/sagas/_custom/drp.sagas.js @@ -10,8 +10,6 @@ import { useHistory, useLocation } from 'react-router-dom'; function* executeOperation(isEdit, params, values, reloadCallback, file = null, path = '') { - - const location = useLocation(); if (file && isEdit) { yield call( updateMetadataUtil, From 9d1a72c41b8d71e34b7f7f37443b1c6da7d33fa3 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Thu, 28 Mar 2024 16:14:32 -0500 Subject: [PATCH 021/328] Added trash and delete functionality for projects --- .../DataFilesToolbar/DataFilesToolbar.jsx | 29 ++++++++++++++----- .../DataFilesProjectFileListingAddon.jsx | 6 ++-- client/src/redux/sagas/datafiles.sagas.js | 9 +++--- server/portal/libs/agave/operations.py | 14 ++++++--- 4 files changed, 39 insertions(+), 19 deletions(-) diff --git a/client/src/components/DataFiles/DataFilesToolbar/DataFilesToolbar.jsx b/client/src/components/DataFiles/DataFilesToolbar/DataFilesToolbar.jsx index fee8929a85..dbb9b848a9 100644 --- a/client/src/components/DataFiles/DataFilesToolbar/DataFilesToolbar.jsx +++ b/client/src/components/DataFiles/DataFilesToolbar/DataFilesToolbar.jsx @@ -52,8 +52,15 @@ const DataFilesToolbar = ({ scheme, api }) => { shallowEqual ); + // A project system has different fields than a regular system const selectedSystem = systemList.find( - (sys) => sys.system === params.system && sys.scheme === params.scheme + (sys) => { + if (params.scheme === 'projects') { + return params.api === sys.api && sys.scheme === params.scheme; + } else { + return sys.system === params.system && sys.scheme === params.scheme + } + } ); const { projectId } = useSelector((state) => state.projects.metadata); @@ -70,13 +77,19 @@ const DataFilesToolbar = ({ scheme, api }) => { const isGuest = authenticatedUserQuery?.data?.role === 'GUEST'; const inTrash = useSelector((state) => { - // remove leading slash from homeDir value - const homeDir = selectedSystem?.homeDir?.slice(1); - if (!homeDir) return false; - - return state.files.params.FilesListing.path.startsWith( - `${homeDir}/${state.workbench.config.trashPath}` - ); + if (selectedSystem?.scheme === 'projects') { + return state.files.params.FilesListing.path.startsWith( + `${state.workbench.config.trashPath}` + ); + } else { + // remove leading slash from homeDir value + const homeDir = selectedSystem?.homeDir?.slice(1); + if (!homeDir) return false; + + return state.files.params.FilesListing.path.startsWith( + `${homeDir}/${state.workbench.config.trashPath}` + ); + } }); const trashedFiles = useSelector((state) => diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx index 84bb8d885f..87f3b7a5eb 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx @@ -103,7 +103,7 @@ const DataFilesProjectFileListingAddon = () => { <> | {selectedFiles.length == 1 && - selectedFiles[0]?.metadata['data_type'] === 'sample' ? ( + selectedFiles[0]?.metadata?.['data_type'] === 'sample' ? ( - { DataFilesProjectFileListingAddon && } + {DataFilesProjectFileListingAddon && ( + + )} } manualContent @@ -126,8 +132,17 @@ const DataFilesProjectFileListing = ({ system, path }) => { - (D) __both__ (A) or (B) __and__ (C) */}
- { folderMetadata?.description || metadata.description && {folderMetadata?.description || metadata.description} } - { folderMetadata && } + {folderMetadata?.description || + (metadata.description && ( + + {folderMetadata?.description || metadata.description} + + ))} + {folderMetadata && ( + + )}
Date: Tue, 2 Apr 2024 14:53:57 -0500 Subject: [PATCH 025/328] fixed DataFilesProjectFileListingMetadataAddon bug --- .../DataFilesProjectFileListing/DataFilesProjectFileListing.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx b/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx index 625d94b821..730afe7987 100644 --- a/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx +++ b/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx @@ -138,7 +138,7 @@ const DataFilesProjectFileListing = ({ system, path }) => { {folderMetadata?.description || metadata.description} ))} - {folderMetadata && ( + {folderMetadata && DataFilesProjectFileListingMetadataAddon && ( From 8a14acfe0be604ce83c3e79fc73fd51c976b5bd9 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Tue, 2 Apr 2024 17:29:07 -0500 Subject: [PATCH 026/328] Cleanup datafiles area, support different names for Shared Workspaces --- .../DataFilesDropdown/DataFilesDropdown.jsx | 11 +++- .../DataFilesAddProjectModal.jsx | 17 ++++-- .../DataFilesProjectsList.jsx | 19 ++++--- .../DataFilesSidebar/DataFilesSidebar.jsx | 2 +- client/src/utils/systems.js | 8 ++- server/portal/apps/datafiles/views.py | 2 +- server/portal/settings/settings_default.py | 53 +------------------ 7 files changed, 44 insertions(+), 68 deletions(-) diff --git a/client/src/components/DataFiles/DataFilesDropdown/DataFilesDropdown.jsx b/client/src/components/DataFiles/DataFilesDropdown/DataFilesDropdown.jsx index 246f72788c..7f6f45ce3d 100644 --- a/client/src/components/DataFiles/DataFilesDropdown/DataFilesDropdown.jsx +++ b/client/src/components/DataFiles/DataFilesDropdown/DataFilesDropdown.jsx @@ -1,4 +1,5 @@ import React, { useState } from 'react'; +import { useSelector, shallowEqual } from 'react-redux'; import PropTypes from 'prop-types'; import { Link, useLocation } from 'react-router-dom'; import { Button } from '_common'; @@ -62,6 +63,12 @@ const BreadcrumbsDropdown = ({ const overlapIndex = pathComponents.findIndex( (component, index) => startingPathComponents[index] !== component ); + const systems = useSelector( + (state) => state.systems.storage.configuration.filter((s) => !s.hidden), + shallowEqual + ); + + const sharedWorkspacesDisplayName = systems.find((e) => e.scheme === 'projects')?.name; let currentPath = startingPath; pathComponents.slice(overlapIndex).forEach((component) => { @@ -125,8 +132,8 @@ const BreadcrumbsDropdown = ({ {scheme === 'projects' - ? 'Shared Workspaces' - : systemName || 'Shared Workspaces'} + ? sharedWorkspacesDisplayName + : systemName || sharedWorkspacesDisplayName} {homeDir ? Root : null} diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesAddProjectModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesAddProjectModal.jsx index ddf94ba641..63696f93ab 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesAddProjectModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesAddProjectModal.jsx @@ -1,5 +1,5 @@ import React, { useState, useEffect } from 'react'; -import { useDispatch, useSelector } from 'react-redux'; +import { useDispatch, useSelector, shallowEqual } from 'react-redux'; import * as Yup from 'yup'; import { Formik, Form, FieldArray } from 'formik'; import FormField from '_common/Form/FormField'; @@ -45,6 +45,13 @@ const DataFilesAddProjectModal = () => { state.projects.operation.error ); }); + + const systems = useSelector( + (state) => state.systems.storage.configuration.filter((s) => !s.hidden), + shallowEqual + ); + + const sharedWorkspacesDisplayName = systems.find((e) => e.scheme === 'projects')?.name; const toggle = () => { dispatch({ @@ -107,14 +114,14 @@ const DataFilesAddProjectModal = () => { > - Add Shared Workspace + Add {sharedWorkspacesDisplayName} - Workspace Title{' '} + {sharedWorkspacesDisplayName} Title{' '} (Maximum 150 characters) @@ -131,7 +138,7 @@ const DataFilesAddProjectModal = () => { {error ? ( - Your shared workspace could not be created + Your {sharedWorkspacesDisplayName} could not be created ) : null} diff --git a/client/src/components/DataFiles/DataFilesProjectsList/DataFilesProjectsList.jsx b/client/src/components/DataFiles/DataFilesProjectsList/DataFilesProjectsList.jsx index f90f807917..02569e1417 100644 --- a/client/src/components/DataFiles/DataFilesProjectsList/DataFilesProjectsList.jsx +++ b/client/src/components/DataFiles/DataFilesProjectsList/DataFilesProjectsList.jsx @@ -1,7 +1,7 @@ import React, { useEffect, useCallback } from 'react'; import PropTypes from 'prop-types'; import { Link, useLocation } from 'react-router-dom'; -import { useSelector, useDispatch } from 'react-redux'; +import { useSelector, useDispatch, shallowEqual } from 'react-redux'; import queryStringParser from 'query-string'; import { InfiniteScrollTable, @@ -19,6 +19,13 @@ const DataFilesProjectsList = ({ modal }) => { const modalProps = useSelector((state) => state.files.modalProps[modal]); const query = queryStringParser.parse(useLocation().search); + const systems = useSelector( + (state) => state.systems.storage.configuration.filter((s) => !s.hidden), + shallowEqual + ); + + const sharedWorkspacesDisplayName = systems.find((e) => e.scheme === 'projects')?.name; + const infiniteScrollCallback = useCallback(() => {}); const dispatch = useDispatch(); @@ -59,7 +66,7 @@ const DataFilesProjectsList = ({ modal }) => { const columns = [ { - Header: 'Workspace Title', + Header: `${sharedWorkspacesDisplayName} Title`, headerStyle: { textAlign: 'left' }, accessor: 'title', Cell: (el) => ( @@ -92,14 +99,14 @@ const DataFilesProjectsList = ({ modal }) => { ]; const noDataText = query.query_string - ? 'No Shared Workspaces match your search term.' - : "You don't have any Shared Workspaces."; + ? `No ${sharedWorkspacesDisplayName} match your search term.` + : `You don't have any ${sharedWorkspacesDisplayName}`; if (error) { return (
- There was a problem retrieving your Shared Workspaces. + There was a problem retrieving your {sharedWorkspacesDisplayName}.
); @@ -123,7 +130,7 @@ const DataFilesProjectsList = ({ modal }) => { diff --git a/client/src/components/DataFiles/DataFilesSidebar/DataFilesSidebar.jsx b/client/src/components/DataFiles/DataFilesSidebar/DataFilesSidebar.jsx index 6ac81f652d..c177d11a82 100644 --- a/client/src/components/DataFiles/DataFilesSidebar/DataFilesSidebar.jsx +++ b/client/src/components/DataFiles/DataFilesSidebar/DataFilesSidebar.jsx @@ -86,7 +86,7 @@ const DataFilesAddButton = ({ readOnly }) => { {sharedWorkspaces && !sharedWorkspaces.readOnly && ( - Shared Workspace + {sharedWorkspaces.name} )} system.scheme === 'projects') + return projectSystem ? projectSystem.name : ''; + } else { + return project; + } default: return findSystemDisplayName(systemList, system, isRoot, scheme, path); } diff --git a/server/portal/apps/datafiles/views.py b/server/portal/apps/datafiles/views.py index 69fa02fdb1..b66e63e15b 100644 --- a/server/portal/apps/datafiles/views.py +++ b/server/portal/apps/datafiles/views.py @@ -49,7 +49,7 @@ def get(self, request): ] default_system = settings.PORTAL_DATAFILES_DEFAULT_STORAGE_SYSTEM or settings.PORTAL_DATAFILES_STORAGE_SYSTEMS[0] - if default_system: + if default_system and default_system.get('scheme') != 'projects': system_id = default_system.get('system') system_def = request.user.tapis_oauth.client.systems.getSystem(systemId=system_id, select='host') response['default_host'] = system_def.host diff --git a/server/portal/settings/settings_default.py b/server/portal/settings/settings_default.py index f05c1f9a60..de6d663e62 100644 --- a/server/portal/settings/settings_default.py +++ b/server/portal/settings/settings_default.py @@ -60,64 +60,13 @@ _PORTAL_DATAFILES_STORAGE_SYSTEMS = [ { - 'name': 'My Data (Work)', - 'system': 'frontera', - 'scheme': 'private', - 'api': 'tapis', - 'homeDir': '/work/{tasdir}', - 'icon': None, - 'default': True - }, - { - 'name': 'My Data (Scratch)', - 'system': 'frontera', - 'scheme': 'private', - 'api': 'tapis', - 'homeDir': '/scratch1/{tasdir}', - 'icon': None - }, - { - 'name': 'My Data (Frontera)', - 'system': 'frontera', - 'scheme': 'private', - 'api': 'tapis', - 'homeDir': '/home1/{tasdir}', - 'icon': None, - }, - { - 'name': 'Community Data', - 'system': 'cloud.data', - 'scheme': 'community', - 'api': 'tapis', - 'homeDir': '/corral/tacc/aci/CEP/community', - 'icon': None, - 'siteSearchPriority': 1 - }, - { - 'name': 'Public Data', - 'system': 'cloud.data', - 'scheme': 'public', - 'api': 'tapis', - 'homeDir': '/corral/tacc/aci/CEP/public', - 'icon': 'publications', - 'siteSearchPriority': 0 - }, - { - 'name': 'Shared Workspaces', + 'name': 'Project', 'scheme': 'projects', 'api': 'tapis', 'icon': 'publications', 'readOnly': False, 'hideSearchBar': False }, - { - 'name': 'Google Drive', - 'system': 'googledrive', - 'scheme': 'private', - 'api': 'googledrive', - 'icon': None, - 'integration': 'portal.apps.googledrive_integration' - } ] ######################## From a6c8a831fea9e570b387fd0451dfdd1407aa1ee5 Mon Sep 17 00:00:00 2001 From: Asim Regmi Date: Mon, 8 Apr 2024 11:23:24 -0500 Subject: [PATCH 027/328] Add Edit Data button --- ...esProjectFileListingMetadataTitleAddon.jsx | 156 ++++++++++++++++++ ...tFileListingMetadataTitleAddon.module.scss | 9 + 2 files changed, 165 insertions(+) create mode 100644 client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx create mode 100644 client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.module.scss diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx new file mode 100644 index 0000000000..deee97c657 --- /dev/null +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx @@ -0,0 +1,156 @@ +import React, { useEffect, useState } from 'react'; +import { useDispatch, useSelector } from 'react-redux'; +import PropTypes from 'prop-types'; +import styles from './DataFilesProjectFileListingMetadataTitleAddon.module.scss'; +import { Button } from '_common'; +import { useSelectedFiles } from 'hooks/datafiles'; +import { fetchUtil } from 'utils/fetchUtil'; + +const DataFilesProjectFileListingMetadataTitleAddon = ({ folderMetadata, system, path }) => { + const dispatch = useDispatch(); + const portalName = useSelector((state) => state.workbench.portalName); + const { projectId } = useSelector((state) => state.projects.metadata); + + // reconstruct editFile to mimic SelectedFile object + const editFile = { + "format": "folder", + "id" : system + "/" + path, + "metadata": folderMetadata, + "name": path, + "system": system, + "path": path, + "type": "dir", + "_links": { + "self": { + "href": "tapis://" + system + "/" + path, + }, + } + } + + const getFormFields = async (formName) => { + const response = await fetchUtil({ + url: 'api/forms', + params: { + form_name: formName, + }, + }); + return response; + }; + + const getSamples = async (projectId, getOriginData = false) => { + const response = await fetchUtil({ + url: `api/${portalName.toLowerCase()}`, + params: { + project_id: projectId, + get_origin_data: getOriginData, + }, + }); + return response; + }; + + const onOpenSampleModal = async (formName, selectedFile) => { + console.log("Selected file in MetadataTitleAddon", selectedFile) + const form = await getFormFields(formName); + + dispatch({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { + operation: 'dynamicform', + props: { form, selectedFile: editFile, formName }, + }, + }); + }; + + const onOpenOriginDataModal = async (formName, selectedFile) => { + const form = await getFormFields(formName); + const samples = await getSamples(projectId); + + form.form_fields.map((field) => { + if (field.name === 'sample') { + field.options = samples.map((sample) => { + return { + value: parseInt(sample.id), + label: sample.name, + }; + }); + } + }); + + dispatch({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { + operation: 'dynamicform', + props: { selectedFile, form, formName, additionalData: samples }, + }, + }); + }; + + const onOpenAnalysisDataModal = async (formName, selectedFile) => { + const form = await getFormFields(formName); + const samples = await getSamples(projectId, true); + + form.form_fields.map((field) => { + if (field.name === 'base_origin_data') { + field.optgroups = samples.map((sample) => { + return { + label: sample.name, + options: sample.origin_data.map((originData) => { + return { + value: parseInt(originData.id), + label: originData.name, + }; + }), + }; + }); + } + }); + + dispatch({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { + operation: 'dynamicform', + props: { selectedFile, form, formName, additionalData: samples }, + }, + }); + }; + + const onEditData = (dataType) => { + switch (dataType) { + case 'sample': + onOpenSampleModal('EDIT_SAMPLE_DATA', editFile); + break; + case 'origin_data': + onOpenOriginDataModal('EDIT_ORIGIN_DATASET', null); + break; + case 'analysis_data': + onOpenAnalysisDataModal('EDIT_ANALYSIS_DATASET', null); + break; + default: + break; + } + } + + return ( + <> + {folderMetadata.name} + {folderMetadata?.data_type && ( + + {folderMetadata.data_type} + + )} + + + ); +}; + +DataFilesProjectFileListingMetadataTitleAddon.propTypes = { + folderMetadata: PropTypes.object.isRequired, +}; + +export default DataFilesProjectFileListingMetadataTitleAddon; \ No newline at end of file diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.module.scss b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.module.scss new file mode 100644 index 0000000000..5de8857669 --- /dev/null +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.module.scss @@ -0,0 +1,9 @@ +.dataTypeBox { + display: inline-block; + background-color: #f0f0f0; /* Light grey background */ + color: #333; /* Dark text color */ + padding: 2px 8px; /* Small top/bottom padding, larger left/right padding */ + border-radius: 10px; /* Rounded corners */ + font-size: 0.875rem; /* Smaller font size */ + margin-left: 8px; /* Space from the title */ + } \ No newline at end of file From 9b546fd77fe13e33092b788fae184cf9b395ff91 Mon Sep 17 00:00:00 2001 From: Asim Regmi Date: Mon, 8 Apr 2024 11:30:37 -0500 Subject: [PATCH 028/328] render DataFilesProjectFileListingMetadataTitleAddon within ProjectFileListing --- .../DataFilesProjectFileListing.jsx | 44 +++++++++++-------- .../DataFilesProjectFileListing.module.scss | 10 ----- .../DataFilesProjectListing.scss | 4 -- ...esProjectFileListingMetadataTitleAddon.jsx | 6 +-- 4 files changed, 28 insertions(+), 36 deletions(-) diff --git a/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx b/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx index 730afe7987..d5f0882183 100644 --- a/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx +++ b/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx @@ -8,7 +8,7 @@ import { SectionMessage, SectionTableWrapper, } from '_common'; -import { useAddonComponents, useFileListing } from 'hooks/datafiles'; +import { useAddonComponents, useFileListing, useSelectedFiles } from 'hooks/datafiles'; import DataFilesListing from '../DataFilesListing/DataFilesListing'; import styles from './DataFilesProjectFileListing.module.scss'; @@ -21,6 +21,7 @@ const DataFilesProjectFileListing = ({ system, path }) => { const { DataFilesProjectFileListingAddon, DataFilesProjectFileListingMetadataAddon, + DataFilesProjectFileListingMetadataTitleAddon, } = useAddonComponents({ portalName }); useEffect(() => { dispatch({ @@ -73,8 +74,8 @@ const DataFilesProjectFileListing = ({ system, path }) => { payload: { operation: 'manageproject', props: {} }, }); }; - const isLoading = metadata.loading || folderMetadata?.loading; - if (isLoading) { + + if (metadata.loading) { return (
@@ -97,14 +98,18 @@ const DataFilesProjectFileListing = ({ system, path }) => { className={styles.root} header={
- {folderMetadata?.name || metadata.title} - {folderMetadata?.data_type && ( - - {folderMetadata.data_type} - + {path !== '/' ? ( + folderMetadata && DataFilesProjectFileListingMetadataTitleAddon && + + ) : ( + metadata.title )}
} + headerActions={
{canEditSystem ? ( @@ -132,17 +137,18 @@ const DataFilesProjectFileListing = ({ system, path }) => { - (D) __both__ (A) or (B) __and__ (C) */}
- {folderMetadata?.description || - (metadata.description && ( - - {folderMetadata?.description || metadata.description} - - ))} - {folderMetadata && DataFilesProjectFileListingMetadataAddon && ( - - )} + <> + {path !== '/' + ? + {folderMetadata && DataFilesProjectFileListingMetadataAddon && ( + + )} + + : {metadata.description} + } +
onOpenSampleModal('EDIT_SAMPLE_DATA', editFile)} + onClick={() => onEditData(folderMetadata.data_type)} > Edit Data From 761944452c99a7b58d314c58f10d4ced34fe9c1d Mon Sep 17 00:00:00 2001 From: Asim Regmi Date: Mon, 8 Apr 2024 14:04:49 -0500 Subject: [PATCH 029/328] add DataFilesProjectFileListingMetadataTitleAddon in settings_default --- server/portal/settings/settings_default.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/portal/settings/settings_default.py b/server/portal/settings/settings_default.py index 34beffd82a..74555a4106 100644 --- a/server/portal/settings/settings_default.py +++ b/server/portal/settings/settings_default.py @@ -242,7 +242,7 @@ "hasProjectFileListingAddon": True, "hasCustomSagas": True, "hasCustomEndpoints": True, - "addons": ['DataFilesProjectFileListingAddon', 'DataFilesAddProjectModalAddon', 'DataFilesProjectEditDescriptionModalAddon', 'DataFilesProjectFileListingMetadataAddon'], + "addons": ['DataFilesProjectFileListingAddon', 'DataFilesAddProjectModalAddon', 'DataFilesProjectEditDescriptionModalAddon', 'DataFilesProjectFileListingMetadataAddon', 'DataFilesProjectFileListingMetadataTitleAddon'], "onboardingCompleteRedirect": '/workbench/', "noPHISystem": "", "customDashboardSection": None, From e4d9b38f926a6bc7bd1bd1ebcc7e3c1c33d59eb0 Mon Sep 17 00:00:00 2001 From: Asim Regmi Date: Mon, 8 Apr 2024 15:30:58 -0500 Subject: [PATCH 030/328] fix description not loading issue --- .../DataFilesProjectFileListingMetadataAddon.jsx | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx index 73fbebfa81..33417dfedb 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx @@ -1,6 +1,6 @@ import React from 'react'; import PropTypes from 'prop-types'; -import { Section, SectionContent } from '_common'; +import { Section, SectionContent, } from '_common'; import styles from './DataFilesProjectFileListingMetadataAddon.module.scss'; const DataDisplay = ({ data }) => { @@ -38,7 +38,10 @@ const DataDisplay = ({ data }) => { const DataFilesProjectFileListingMetadataAddon = ({ folderMetadata }) => { return ( - + <> + {folderMetadata.description} + + ); }; From 480b8a3e8e6246bf828d2b3e26c426a1d693b5fa Mon Sep 17 00:00:00 2001 From: Asim Regmi Date: Mon, 8 Apr 2024 15:32:09 -0500 Subject: [PATCH 031/328] fix Showmore component --- client/src/components/_common/ShowMore/ShowMore.jsx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/client/src/components/_common/ShowMore/ShowMore.jsx b/client/src/components/_common/ShowMore/ShowMore.jsx index f553349de3..fa29ced360 100644 --- a/client/src/components/_common/ShowMore/ShowMore.jsx +++ b/client/src/components/_common/ShowMore/ShowMore.jsx @@ -1,4 +1,4 @@ -import React, { useState, useCallback } from 'react'; +import React, { useState, useEffect, useCallback } from 'react'; import { useResizeDetector } from 'react-resize-detector'; import PropTypes from 'prop-types'; import { Button } from '_common'; @@ -13,8 +13,10 @@ const ShowMore = ({ className, children }) => { const { height, ref } = useResizeDetector(); + // overflowThreshold to account for minor differences in height for example 84.5 and 85 + const overflowThreshold = 1; const hasOverflow = - ref && ref.current ? ref.current.scrollHeight > height : false; + ref && ref.current ? (ref.current.scrollHeight - height) > overflowThreshold : false; return ( <> From 025acc588a69de68e616e40fc6d40489c25cbd9e Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Tue, 9 Apr 2024 15:12:02 -0500 Subject: [PATCH 032/328] convert portalName to lower case --- client/src/hooks/datafiles/useAddonComponents.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/hooks/datafiles/useAddonComponents.js b/client/src/hooks/datafiles/useAddonComponents.js index 8d88e35b46..55c0db2178 100644 --- a/client/src/hooks/datafiles/useAddonComponents.js +++ b/client/src/hooks/datafiles/useAddonComponents.js @@ -9,7 +9,7 @@ const useAddonComponents = ({ portalName }) => { try { for (const addonName of addons) { const module = await import( - `../../components/_custom/${portalName}/${addonName}/${addonName}.jsx` + `../../components/_custom/${portalName.toLowerCase()}/${addonName}/${addonName}.jsx` ); setAddonComponents(prevComponents => ({ ...prevComponents, From 88f0420df55da8db68661cb7236aaaf23b405068 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Tue, 9 Apr 2024 15:23:46 -0500 Subject: [PATCH 033/328] added extension to saga dynamic import --- client/src/redux/sagas/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/redux/sagas/index.js b/client/src/redux/sagas/index.js index 9631694ead..928da69c95 100644 --- a/client/src/redux/sagas/index.js +++ b/client/src/redux/sagas/index.js @@ -63,7 +63,7 @@ function* startCustomSaga(action) { }); const { default: customSaga } = yield import( - `./_custom/${portalName.toLowerCase()}.sagas` + `./_custom/${portalName.toLowerCase()}.sagas.js` ); if (customSaga) { From a7d00058dd6f4676a2c98f0b509f1876d503e7d3 Mon Sep 17 00:00:00 2001 From: Asim Regmi Date: Tue, 9 Apr 2024 16:54:43 -0500 Subject: [PATCH 034/328] fix Edit Data button --- .../DataFilesModals/DataFilesFormModal.jsx | 9 +++-- ...esProjectFileListingMetadataTitleAddon.jsx | 36 +++++++++---------- client/src/redux/sagas/_custom/drp.sagas.js | 9 ++++- 3 files changed, 29 insertions(+), 25 deletions(-) diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx index 79137f6149..763f12e3d7 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx @@ -15,8 +15,9 @@ const DataFilesFormModal = () => { const history = useHistory(); const location = useLocation(); - const reloadPage = () => { - history.push(location.pathname); + const reloadPage = (updatedPath = "") => { + const path = updatedPath ? location.pathname.replace(/\/[^\/]+\/?$/, '/' + updatedPath) : location.pathname; + history.push(path); }; const { form, selectedFile, formName, additionalData } = useSelector( @@ -25,8 +26,6 @@ const DataFilesFormModal = () => { const isOpen = useSelector((state) => state.files.modals.dynamicform); const { params } = useFileListing('FilesListing'); - const { selectedFiles } = useSelectedFiles(); - const initialValues = form?.form_fields.reduce((acc, field) => { let value = ''; if (field.optgroups) { @@ -54,7 +53,7 @@ const DataFilesFormModal = () => { params, values, reloadPage, - selectedFile: selectedFiles[0], + selectedFile, additionalData, }, }); diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx index 4968286cd0..fdafe0a51c 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx @@ -11,22 +11,6 @@ const DataFilesProjectFileListingMetadataTitleAddon = ({ folderMetadata, system, const portalName = useSelector((state) => state.workbench.portalName); const { projectId } = useSelector((state) => state.projects.metadata); - // reconstruct editFile to mimic SelectedFile object - const editFile = { - "format": "folder", - "id" : system + "/" + path, - "metadata": folderMetadata, - "name": path, - "system": system, - "path": path, - "type": "dir", - "_links": { - "self": { - "href": "tapis://" + system + "/" + path, - }, - } - } - const getFormFields = async (formName) => { const response = await fetchUtil({ url: 'api/forms', @@ -49,14 +33,12 @@ const DataFilesProjectFileListingMetadataTitleAddon = ({ folderMetadata, system, }; const onOpenSampleModal = async (formName, selectedFile) => { - console.log("Selected file in MetadataTitleAddon", selectedFile) const form = await getFormFields(formName); - dispatch({ type: 'DATA_FILES_TOGGLE_MODAL', payload: { operation: 'dynamicform', - props: { form, selectedFile: editFile, formName }, + props: { form, selectedFile, formName }, }, }); }; @@ -115,6 +97,22 @@ const DataFilesProjectFileListingMetadataTitleAddon = ({ folderMetadata, system, }; const onEditData = (dataType) => { + const name = path.split('/').pop(); + // reconstruct editFile to mimic SelectedFile object + const editFile = { + "format": "folder", + "id" : system + "/" + path, + "metadata": folderMetadata, + "name": name, + "system": system, + "path": path, + "type": "dir", + "_links": { + "self": { + "href": "tapis://" + system + "/" + path, + }, + } + }; switch (dataType) { case 'sample': onOpenSampleModal('EDIT_SAMPLE_DATA', editFile); diff --git a/client/src/redux/sagas/_custom/drp.sagas.js b/client/src/redux/sagas/_custom/drp.sagas.js index e54044a5bb..481ec1aba3 100644 --- a/client/src/redux/sagas/_custom/drp.sagas.js +++ b/client/src/redux/sagas/_custom/drp.sagas.js @@ -34,7 +34,14 @@ function* executeOperation(isEdit, params, values, reloadCallback, file = null, ); } - yield call(reloadCallback); + if (isEdit && file.path === params.path) { + // reload to new URL if name/path is changed + console.log("Reloading with new path") + yield call (reloadCallback, values.name) + } + else { + yield call(reloadCallback); + } yield put({ type: 'DATA_FILES_TOGGLE_MODAL', From b3abad219e264e1c8c6add19c748d1d9e8ff65c9 Mon Sep 17 00:00:00 2001 From: Asim Regmi Date: Tue, 9 Apr 2024 17:05:53 -0500 Subject: [PATCH 035/328] fix css --- .../DataFilesProjectFileListingMetadataTitleAddon.module.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.module.scss b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.module.scss index 5de8857669..49443e216a 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.module.scss +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.module.scss @@ -5,5 +5,5 @@ padding: 2px 8px; /* Small top/bottom padding, larger left/right padding */ border-radius: 10px; /* Rounded corners */ font-size: 0.875rem; /* Smaller font size */ - margin-left: 8px; /* Space from the title */ + margin: 8px 8px; /* Space from the title */ } \ No newline at end of file From 78237989c8184808d6fa9299bffae2dda2740bab Mon Sep 17 00:00:00 2001 From: Asim Regmi Date: Tue, 9 Apr 2024 17:08:36 -0500 Subject: [PATCH 036/328] remove unused import --- .../DataFilesProjectFileListing/DataFilesProjectFileListing.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx b/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx index d5f0882183..e9c2166c0c 100644 --- a/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx +++ b/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx @@ -8,7 +8,7 @@ import { SectionMessage, SectionTableWrapper, } from '_common'; -import { useAddonComponents, useFileListing, useSelectedFiles } from 'hooks/datafiles'; +import { useAddonComponents, useFileListing } from 'hooks/datafiles'; import DataFilesListing from '../DataFilesListing/DataFilesListing'; import styles from './DataFilesProjectFileListing.module.scss'; From 2f50199cc342ebdf1e87a6e611d1d5798598c2a1 Mon Sep 17 00:00:00 2001 From: Asim Regmi Date: Wed, 10 Apr 2024 11:38:24 -0500 Subject: [PATCH 037/328] remove unused imports and logging --- client/src/components/_common/ShowMore/ShowMore.jsx | 2 +- .../DataFilesProjectFileListingMetadataTitleAddon.jsx | 1 - client/src/redux/sagas/_custom/drp.sagas.js | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/client/src/components/_common/ShowMore/ShowMore.jsx b/client/src/components/_common/ShowMore/ShowMore.jsx index fa29ced360..59abefecf6 100644 --- a/client/src/components/_common/ShowMore/ShowMore.jsx +++ b/client/src/components/_common/ShowMore/ShowMore.jsx @@ -1,4 +1,4 @@ -import React, { useState, useEffect, useCallback } from 'react'; +import React, { useState, useCallback } from 'react'; import { useResizeDetector } from 'react-resize-detector'; import PropTypes from 'prop-types'; import { Button } from '_common'; diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx index fdafe0a51c..5b63b08e2f 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx @@ -140,7 +140,6 @@ const DataFilesProjectFileListingMetadataTitleAddon = ({ folderMetadata, system, type="link" onClick={() => onEditData(folderMetadata.data_type)} > - Edit Data diff --git a/client/src/redux/sagas/_custom/drp.sagas.js b/client/src/redux/sagas/_custom/drp.sagas.js index 481ec1aba3..a590bfef56 100644 --- a/client/src/redux/sagas/_custom/drp.sagas.js +++ b/client/src/redux/sagas/_custom/drp.sagas.js @@ -36,7 +36,6 @@ function* executeOperation(isEdit, params, values, reloadCallback, file = null, if (isEdit && file.path === params.path) { // reload to new URL if name/path is changed - console.log("Reloading with new path") yield call (reloadCallback, values.name) } else { From 38fb4865e2fb067735e3fac3e95566bffe66c1ad Mon Sep 17 00:00:00 2001 From: Asim Regmi Date: Wed, 10 Apr 2024 15:44:46 -0500 Subject: [PATCH 038/328] add comments and minor improvements --- .../DataFilesModals/DataFilesFormModal.jsx | 1 + .../DataFilesProjectFileListingMetadataAddon.jsx | 14 ++++++++++++-- ...taFilesProjectFileListingMetadataTitleAddon.jsx | 9 +++++++-- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx index 763f12e3d7..ac6a7546bf 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx @@ -16,6 +16,7 @@ const DataFilesFormModal = () => { const location = useLocation(); const reloadPage = (updatedPath = "") => { + // using regex to replace last segment of pathname with optional parameter 'updatedPath' to navigate to a new path const path = updatedPath ? location.pathname.replace(/\/[^\/]+\/?$/, '/' + updatedPath) : location.pathname; history.push(path); }; diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx index 33417dfedb..195840d4fd 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx @@ -4,7 +4,14 @@ import { Section, SectionContent, } from '_common'; import styles from './DataFilesProjectFileListingMetadataAddon.module.scss'; const DataDisplay = ({ data }) => { - const formatLabel = (key) => key.split('_').map(word => word.charAt(0).toUpperCase() + word.slice(1)).join(' '); + + // Function to format the dict key from snake_case to Label Case i.e. data_type -> Data Type + const formatLabel = (key) => + key.split('_') + .map(word => word.charAt(0).toUpperCase() + word.slice(1)) + .join(' '); + + //filter out empty values and unwanted keys const processedData = Object.entries(data) .filter(([key, value]) => value !== "" && key !== "name" && key !== "description") .map(([key, value]) => ({ @@ -12,11 +19,13 @@ const DataDisplay = ({ data }) => { value: value, })); + // Divide processed data into chunks for two-column layout display const chunkSize = Math.ceil(processedData.length / 2); const chunks = []; for (let i = 0; i < processedData.length; i += chunkSize) { chunks.push(processedData.slice(i, i + chunkSize)); } + const renderDataEntries = (entries) => ( entries.map(({ label, value }, index) => (
@@ -24,7 +33,8 @@ const DataDisplay = ({ data }) => {
)) ); - + + // Render each data entry within its chunk for two-column layout return (
{chunks.map((chunk, index) => ( diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx index 5b63b08e2f..92a87a5ed4 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx @@ -3,7 +3,6 @@ import { useDispatch, useSelector } from 'react-redux'; import PropTypes from 'prop-types'; import styles from './DataFilesProjectFileListingMetadataTitleAddon.module.scss'; import { Button } from '_common'; -import { useSelectedFiles } from 'hooks/datafiles'; import { fetchUtil } from 'utils/fetchUtil'; const DataFilesProjectFileListingMetadataTitleAddon = ({ folderMetadata, system, path }) => { @@ -128,12 +127,18 @@ const DataFilesProjectFileListingMetadataTitleAddon = ({ folderMetadata, system, } } + // Function to format the data_type value from snake_case to Label Case i.e. origin_data -> Origin Data + const formatDatatype = (data_type) => + data_type.split('_') + .map(word => word.charAt(0).toUpperCase() + word.slice(1)) + .join(' '); + return ( <> {folderMetadata.name} {folderMetadata?.data_type && ( - {folderMetadata.data_type} + {formatDatatype(folderMetadata.data_type)} )} )} - | ); }; diff --git a/client/src/redux/sagas/_custom/drp.sagas.js b/client/src/redux/sagas/_custom/drp.sagas.js index e54044a5bb..521d9e149e 100644 --- a/client/src/redux/sagas/_custom/drp.sagas.js +++ b/client/src/redux/sagas/_custom/drp.sagas.js @@ -48,8 +48,14 @@ function* executeOperation(isEdit, params, values, reloadCallback, file = null, function* handleSampleData(action, isEdit) { const { params, values, reloadPage: reloadCallback, selectedFile } = action.payload; + + const metadata = { + ...values, + data_type: 'sample', + } + yield call( - executeOperation, isEdit, params, {...values, data_type: 'sample'}, reloadCallback, selectedFile + executeOperation, isEdit, params, metadata, reloadCallback, selectedFile ) } @@ -60,32 +66,42 @@ function* handleOriginData(action, isEdit) { (sample) => sample.id === parseInt(values.sample) ); + const metadata = { + ...values, + data_type: 'origin_data', + sample: sample.id, + } + // get the path without system name const path = sample.path.split('/').slice(1).join('/'); yield call( - executeOperation, isEdit, params, { ...values, data_type: 'origin_data' }, reloadCallback, selectedFile, path + executeOperation, isEdit, params, metadata, reloadCallback, selectedFile, path ) } function* handleAnalysisData(action, isEdit) { - const { params, values, reloadPage: reloadCallback, selectedFile, additionalData: samples } = action.payload; + const { params, values, reloadPage: reloadCallback, selectedFile, additionalData: { samples, originDatasets } } = action.payload; - let { originDataId, path, sampleId } = samples.reduce((acc, sample) => { - const originData = sample.origin_data.find( - (originData) => originData.id === parseInt(values.base_origin_data) - ); - if (originData) { - acc.originDataId = originData.id; - acc.path = originData.path; - acc.sampleId = sample.id; - } - return acc; - }, {}); + const sample = samples.find( + (sample) => sample.id === parseInt(values.sample) + ); + + const originData = originDatasets.find( + (originData) => originData.id === parseInt(values.base_origin_data) + ); + + let path = originData ? originData.path : sample.path; + path = path.split('/').slice(1).join('/') + + const metadata = { + ...values, + data_type: 'analysis_data', + sample: sample.id, + base_origin_data: originData ? originData.id : '' + } - // get the path without system name - path = path.split('/').slice(1).join('/'); yield call( - executeOperation, isEdit, params, {...values, data_type: 'analysis_data', sample: sampleId, base_origin_data: originDataId}, reloadCallback, selectedFile, path + executeOperation, isEdit, params, metadata, reloadCallback, selectedFile, path ) } diff --git a/server/portal/apps/_custom/drp/views.py b/server/portal/apps/_custom/drp/views.py index b0fc6f8de2..565857a8d4 100644 --- a/server/portal/apps/_custom/drp/views.py +++ b/server/portal/apps/_custom/drp/views.py @@ -3,18 +3,23 @@ from django.http import JsonResponse, HttpResponseForbidden from portal.apps.datafiles.models import DataFilesMetadata - class DigitalRocksSampleView(BaseApiView): def get(self, request): project_id = request.GET.get('project_id') get_origin_data = request.GET.get('get_origin_data') - samples = DataFilesMetadata.objects.filter(project_id=f'{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}', metadata__data_type='sample').values('id', 'name', 'path') + full_project_id = f'{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}' + + samples = DataFilesMetadata.objects.filter(project_id=full_project_id, metadata__data_type='sample').values('id', 'name', 'path') + origin_data = [] + + if get_origin_data == 'true': + origin_data = DataFilesMetadata.objects.filter(project_id=full_project_id, metadata__data_type='origin_data').values('id', 'name', 'path', 'metadata') - if get_origin_data: - for sample in samples: - sample['origin_data'] = list(DataFilesMetadata.objects.filter(parent_id=sample['id'], metadata__data_type='origin_data').values('id', 'name', 'path')) + response_data = { + 'samples': list(samples), + 'origin_data': list(origin_data) + } - print(f'SAMPLES RETRIEVED: {samples}') - return JsonResponse(list(samples), safe=False) \ No newline at end of file + return JsonResponse(response_data) \ No newline at end of file From 3976ee3418185189ad1b5c8328acb97e62bc5c1f Mon Sep 17 00:00:00 2001 From: Asim Regmi Date: Tue, 16 Apr 2024 13:59:17 -0500 Subject: [PATCH 040/328] fix metadata addons --- .../DataFilesProjectFileListing.jsx | 40 ++++---- .../DataFilesProjectFileListingAddon.jsx | 98 +++++++++---------- ...taFilesProjectFileListingMetadataAddon.jsx | 18 +++- ...esProjectFileListingMetadataTitleAddon.jsx | 42 ++++---- 4 files changed, 110 insertions(+), 88 deletions(-) diff --git a/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx b/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx index e9c2166c0c..d062d437c7 100644 --- a/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx +++ b/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx @@ -92,21 +92,23 @@ const DataFilesProjectFileListing = ({ system, path }) => {
); } - + return ( - {path !== '/' ? ( - folderMetadata && DataFilesProjectFileListingMetadataTitleAddon && + { + DataFilesProjectFileListingMetadataTitleAddon ? - ) : ( - metadata.title - )} + folderMetadata={folderMetadata} + metadata={metadata} + system={system} + path={path}/> + + : metadata.title + } +
} @@ -138,15 +140,17 @@ const DataFilesProjectFileListing = ({ system, path }) => { */}
<> - {path !== '/' - ? - {folderMetadata && DataFilesProjectFileListingMetadataAddon && ( - - )} - - : {metadata.description} + + {DataFilesProjectFileListingMetadataTitleAddon ? + + + + + : {metadata.description} }
diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx index 84bb8d885f..2d433e0afc 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx @@ -102,57 +102,57 @@ const DataFilesProjectFileListingAddon = () => { return ( <> | - {selectedFiles.length == 1 && - selectedFiles[0]?.metadata['data_type'] === 'sample' ? ( - - ) : ( - - )} + {selectedFiles.length == 1 && selectedFiles[0]?.metadata && + selectedFiles[0].metadata['data_type'] === 'sample' ? ( + + ) : ( + + )} | - {selectedFiles.length == 1 && - selectedFiles[0]?.metadata['data_type'] === 'origin_data' ? ( - - ) : ( - - )} + {selectedFiles.length == 1 && selectedFiles[0]?.metadata && + selectedFiles[0].metadata['data_type'] === 'origin_data' ? ( + + ) : ( + + )} | - {selectedFiles.length == 1 && - selectedFiles[0]?.metadata['data_type'] === 'analysis_data' ? ( - - ) : ( - - )} + {selectedFiles.length == 1 && selectedFiles[0]?.metadata && + selectedFiles[0].metadata['data_type'] === 'analysis_data' ? ( + + ) : ( + + )} | ); diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx index 195840d4fd..646e82bc16 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx @@ -46,14 +46,24 @@ const DataDisplay = ({ data }) => { ); } -const DataFilesProjectFileListingMetadataAddon = ({ folderMetadata }) => { +const DataFilesProjectFileListingMetadataAddon = ({ folderMetadata, metadata, path }) => { return ( <> - {folderMetadata.description} - + {path !== '/' ? ( + <> + {folderMetadata && ( + <> + {folderMetadata.description} + + + )} + + ) : ( + metadata.description + )} ); -}; +} DataFilesProjectFileListingMetadataAddon.propTypes = { folderMetadata: PropTypes.object.isRequired, diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx index 92a87a5ed4..5eb6cf5f0d 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx @@ -5,7 +5,7 @@ import styles from './DataFilesProjectFileListingMetadataTitleAddon.module.scss' import { Button } from '_common'; import { fetchUtil } from 'utils/fetchUtil'; -const DataFilesProjectFileListingMetadataTitleAddon = ({ folderMetadata, system, path }) => { +const DataFilesProjectFileListingMetadataTitleAddon = ({ folderMetadata, metadata, system, path }) => { const dispatch = useDispatch(); const portalName = useSelector((state) => state.workbench.portalName); const { projectId } = useSelector((state) => state.projects.metadata); @@ -133,22 +133,30 @@ const DataFilesProjectFileListingMetadataTitleAddon = ({ folderMetadata, system, .map(word => word.charAt(0).toUpperCase() + word.slice(1)) .join(' '); - return ( - <> - {folderMetadata.name} - {folderMetadata?.data_type && ( - - {formatDatatype(folderMetadata.data_type)} - - )} - - - ); + return ( + <> + {path !== '/' ? ( + <> + {folderMetadata && ( + <> + {folderMetadata.name} + + {formatDatatype(folderMetadata.data_type)} + + + + )} + + ) : ( + metadata.title + )} + + ); }; DataFilesProjectFileListingMetadataTitleAddon.propTypes = { From 7994b3485d3fa96ef461dd6423480cfc1be5850e Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Wed, 17 Apr 2024 15:05:49 -0500 Subject: [PATCH 041/328] updates and improved loading --- .../DataFilesProjectFileListing.jsx | 4 +- ...taFilesProjectFileListingMetadataAddon.jsx | 20 +++++---- ...esProjectFileListingMetadataTitleAddon.jsx | 41 ++++++++++--------- 3 files changed, 34 insertions(+), 31 deletions(-) diff --git a/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx b/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx index d062d437c7..d3a0a4ec33 100644 --- a/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx +++ b/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx @@ -108,7 +108,6 @@ const DataFilesProjectFileListing = ({ system, path }) => { : metadata.title } - } @@ -140,8 +139,7 @@ const DataFilesProjectFileListing = ({ system, path }) => { */}
<> - - {DataFilesProjectFileListingMetadataTitleAddon ? + {DataFilesProjectFileListingMetadataAddon ? { @@ -47,20 +48,21 @@ const DataDisplay = ({ data }) => { } const DataFilesProjectFileListingMetadataAddon = ({ folderMetadata, metadata, path }) => { + + const { loading } = useFileListing('FilesListing'); + return ( <> - {path !== '/' ? ( - <> - {folderMetadata && ( + {!loading && ( + folderMetadata ? ( <> {folderMetadata.description} - )} - - ) : ( - metadata.description - )} + ) : ( + metadata.description + ) + )} ); } diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx index 5eb6cf5f0d..dca11fac5c 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx @@ -2,14 +2,17 @@ import React, { useEffect, useState } from 'react'; import { useDispatch, useSelector } from 'react-redux'; import PropTypes from 'prop-types'; import styles from './DataFilesProjectFileListingMetadataTitleAddon.module.scss'; -import { Button } from '_common'; +import { Button, LoadingSpinner } from '_common'; import { fetchUtil } from 'utils/fetchUtil'; +import { useFileListing } from 'hooks/datafiles'; const DataFilesProjectFileListingMetadataTitleAddon = ({ folderMetadata, metadata, system, path }) => { const dispatch = useDispatch(); const portalName = useSelector((state) => state.workbench.portalName); const { projectId } = useSelector((state) => state.projects.metadata); + const { loading } = useFileListing('FilesListing'); + const getFormFields = async (formName) => { const response = await fetchUtil({ url: 'api/forms', @@ -135,25 +138,25 @@ const DataFilesProjectFileListingMetadataTitleAddon = ({ folderMetadata, metadat return ( <> - {path !== '/' ? ( - <> - {folderMetadata && ( - <> - {folderMetadata.name} - - {formatDatatype(folderMetadata.data_type)} - - - - )} - + {loading ? ( + ) : ( - metadata.title + folderMetadata ? ( + <> + {folderMetadata.name} + + {formatDatatype(folderMetadata.data_type)} + + + + ) : ( + metadata.title + ) )} ); From af9e464c5955e227ab849191dfb485afe8aeef08 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Thu, 18 Apr 2024 14:15:47 -0500 Subject: [PATCH 042/328] update DynamicForm dependent field logic --- .../_common/Form/DynamicForm/DynamicForm.jsx | 51 ++++++++++--------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx b/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx index 6613f719f4..8d4beea8b5 100644 --- a/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx +++ b/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx @@ -5,40 +5,43 @@ import { useFormikContext } from 'formik'; import { FormGroup, Input } from 'reactstrap'; import './DynamicForm.scss'; -const DynamicForm = ({ initialFormFields }) => { +const updateFormFieldsBasedOnDependency = (formFields, values) => { + return formFields.map((field) => { - const [formFields, setFormFields] = useState(initialFormFields); - // For file processing - const { setFieldValue, values, handleChange, handleBlur } = useFormikContext(); + const { dependency } = field; - // This function updates and filters any dependant fields. Field dependency is described in the form config file - const handleDependentFieldUpdate = (value) => { - const updatedFormFields = formFields.map((field) => { - - if (field.dependency?.type === 'filter') { - const filteredOptions = field.options.filter(option => option.dependentId == value); + if (dependency) { + if (dependency.type === 'filter') { + const filteredOptions = field.options.filter(option => option.dependentId == values[dependency.name]); const updatedOptions = [{ value: '', label: '' }, ...filteredOptions]; - return { ...field, hidden: false, filteredOptions: updatedOptions}; - } else if (field.dependency?.type === 'visibility') { - if (field.dependency.value) { - return { ...field, hidden: field.dependency.value !== value}; + return { ...field, hidden: false, filteredOptions: updatedOptions }; + } else if (dependency.type === 'visibility') { + if (dependency.value) { + return { ...field, hidden: field.dependency.value !== values[field.dependency.name] }; } else { - return { ...field, hidden: !field.hidden}; + return { ...field, hidden: !field.hidden }; } } + } + return field; + }); +}; - return field; - }); +const DynamicForm = ({ initialFormFields }) => { + const [formFields, setFormFields] = useState(initialFormFields); + // For file processing + const { setFieldValue, values, handleChange, handleBlur } = useFormikContext(); + + // This function updates and filters any dependant fields. Field dependency is described in the form config file + const handleDependentFieldUpdate = (value, modifiedField) => { + const updatedFormFields = updateFormFieldsBasedOnDependency(formFields, { ...values, [modifiedField.name]: value }); setFormFields(updatedFormFields); - } + }; useEffect(() => { - formFields.forEach((field) => { - if (values[field.name]) { - handleDependentFieldUpdate(values[field.name]); - } - }); + const updatedFormFields = updateFormFieldsBasedOnDependency(formFields, values); + setFormFields(updatedFormFields); }, []); const renderFormField = (field) => { @@ -80,7 +83,7 @@ const DynamicForm = ({ initialFormFields }) => { required={field?.validation?.required} onChange={(event) => { setFieldValue(field.name, event.target.value); - handleDependentFieldUpdate(event.target.value); + handleDependentFieldUpdate(event.target.value, field); }} > {/* If we have a select with optgroup */} From c288aa84ca801d6ddbee616c18e6c9308c736068 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Thu, 18 Apr 2024 16:03:08 -0500 Subject: [PATCH 043/328] added input sanitization + bug fix --- .../DataFiles/DataFilesModals/DataFilesFormModal.jsx | 5 +++++ .../DataFilesAddProjectModalAddon.jsx | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx index 0e638c809a..384aa3a7bb 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx @@ -48,6 +48,11 @@ const DataFilesFormModal = () => { }, []); const handleSubmit = (values) => { + + Object.keys(values).forEach(key => { + values[key] = typeof(values[key]) === 'string' ? values[key].trim() : values[key]; + }); + dispatch({ type: formName, payload: { diff --git a/client/src/components/_custom/drp/DataFilesAddProjectModalAddon/DataFilesAddProjectModalAddon.jsx b/client/src/components/_custom/drp/DataFilesAddProjectModalAddon/DataFilesAddProjectModalAddon.jsx index 6f7463360f..e4fc9e3b31 100644 --- a/client/src/components/_custom/drp/DataFilesAddProjectModalAddon/DataFilesAddProjectModalAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesAddProjectModalAddon/DataFilesAddProjectModalAddon.jsx @@ -18,7 +18,7 @@ const DataFilesAddProjectModalAddon = () => { {isLoading ? (

Loading form...

) : ( - + )}
); From b40eb1c9b4a40188be35fa301dfaf1a08832fb33 Mon Sep 17 00:00:00 2001 From: Shayan Khan Date: Thu, 18 Apr 2024 16:43:01 -0500 Subject: [PATCH 044/328] small update --- .../DataFilesProjectFileListingAddon.jsx | 1 - 1 file changed, 1 deletion(-) diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx index c09911e87a..6ee6b0fa09 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx @@ -158,7 +158,6 @@ const DataFilesProjectFileListingAddon = () => { Add Analysis Dataset )} - | ); }; From 997912515b45c6d45329c30742bba47b03b31a14 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Tue, 23 Apr 2024 15:17:34 -0500 Subject: [PATCH 045/328] small css update --- .../DataFilesProjectFileListingMetadataAddon.jsx | 2 +- .../DataFilesProjectFileListingMetadataAddon.module.scss | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx index 0d31456232..630e41288f 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx @@ -37,7 +37,7 @@ const DataDisplay = ({ data }) => { // Render each data entry within its chunk for two-column layout return ( -
+
{chunks.map((chunk, index) => ( {renderDataEntries(chunk)} diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.module.scss b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.module.scss index e69de29bb2..74b98a2efc 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.module.scss +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.module.scss @@ -0,0 +1,4 @@ + +.metadata-section main { + padding-left: 0; +} \ No newline at end of file From 29fd74ea1c61b4cdbfd93e3ee938b348563005b5 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Tue, 23 Apr 2024 16:49:15 -0500 Subject: [PATCH 046/328] Added links to sample and origin data inside metadata display --- ...taFilesProjectFileListingMetadataAddon.jsx | 86 ++++++++++++------- ...rojectFileListingMetadataAddon.module.scss | 12 ++- 2 files changed, 68 insertions(+), 30 deletions(-) diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx index 630e41288f..80bef885c8 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx @@ -1,10 +1,15 @@ import React from 'react'; import PropTypes from 'prop-types'; import { Section, SectionContent, LoadingSpinner } from '_common'; +import { useLocation, Link } from 'react-router-dom'; import styles from './DataFilesProjectFileListingMetadataAddon.module.scss'; import { useFileListing } from 'hooks/datafiles'; -const DataDisplay = ({ data }) => { +const excludeKeys = ['name', 'description', 'data_type', 'sample', 'origin_data']; + +const DataDisplay = ({ data, path }) => { + + const location = useLocation(); // Function to format the dict key from snake_case to Label Case i.e. data_type -> Data Type const formatLabel = (key) => @@ -14,37 +19,60 @@ const DataDisplay = ({ data }) => { //filter out empty values and unwanted keys const processedData = Object.entries(data) - .filter(([key, value]) => value !== "" && key !== "name" && key !== "description") + .filter(([key, value]) => value !== "" && !excludeKeys.includes(key)) .map(([key, value]) => ({ label: formatLabel(key), - value: value, + value: typeof(value) === 'string' ? formatLabel(value) : value })); - // Divide processed data into chunks for two-column layout display - const chunkSize = Math.ceil(processedData.length / 2); - const chunks = []; - for (let i = 0; i < processedData.length; i += chunkSize) { - chunks.push(processedData.slice(i, i + chunkSize)); - } - - const renderDataEntries = (entries) => ( - entries.map(({ label, value }, index) => ( -
- {label}: {value} -
- )) - ); - - // Render each data entry within its chunk for two-column layout - return ( -
- {chunks.map((chunk, index) => ( - - {renderDataEntries(chunk)} - - ))} -
- ); + // use the path to get sample and origin data names + const sample = data.sample ? path.split('/')[0] : null; + const origin_data = data.base_origin_data ? path.split('/')[1] : null; + + // remove trailing / from pathname + const locationPathname = location.pathname.endsWith('/') ? location.pathname.slice(0, -1) : location.pathname; + + const locationPathnameParts = locationPathname.split('/'); + + // construct urls for sample and origin data and add to processed data + + if (origin_data) { + const originDataUrl = locationPathnameParts.slice(0, -1).join('/'); + processedData.unshift({ label: 'Origin Data', value: {origin_data} }); + + } + + if (sample) { + const sampleUrl = locationPathnameParts.slice(0, data.base_origin_data ? -2 : -1).join('/') + processedData.unshift({ label: 'Sample', value: {sample} }); + } + + + // Divide processed data into chunks for two-column layout display + const chunkSize = Math.ceil(processedData.length / 2); + const chunks = []; + for (let i = 0; i < processedData.length; i += chunkSize) { + chunks.push(processedData.slice(i, i + chunkSize)); + } + + const renderDataEntries = (entries) => ( + entries.map(({ label, value }, index) => ( +
+ {label}: {value} +
+ )) + ); + + // Render each data entry within its chunk for two-column layout + return ( +
+ {chunks.map((chunk, index) => ( + + {renderDataEntries(chunk)} + + ))} +
+ ); } const DataFilesProjectFileListingMetadataAddon = ({ folderMetadata, metadata, path }) => { @@ -57,7 +85,7 @@ const DataFilesProjectFileListingMetadataAddon = ({ folderMetadata, metadata, pa folderMetadata ? ( <> {folderMetadata.description} - + ) : ( metadata.description diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.module.scss b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.module.scss index 74b98a2efc..06a134e2cc 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.module.scss +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.module.scss @@ -1,4 +1,14 @@ .metadata-section main { padding-left: 0; -} \ No newline at end of file +} + +.dataset-link { + composes: c-button from '../../../../styles/components/c-button--new.css'; + composes: c-button--size-small from '../../../../styles/components/c-button--new.css'; + composes: c-button__text from '../../../../styles/components/c-button--new.css'; + composes: c-button--as-link from '../../../../styles/components/c-button--new.css'; + font-size: 14px; + padding: 0; + margin-bottom: -3px; + } \ No newline at end of file From 972aba16ceaa44768e336ffe0e1564eb3edcac21 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Tue, 23 Apr 2024 17:15:44 -0500 Subject: [PATCH 047/328] fixed edit project bug --- .../DataFilesProjectEditDescriptionModalAddon.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/components/_custom/drp/DataFilesProjectEditDescriptionModalAddon/DataFilesProjectEditDescriptionModalAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectEditDescriptionModalAddon/DataFilesProjectEditDescriptionModalAddon.jsx index 6436e738e7..1be9a68047 100644 --- a/client/src/components/_custom/drp/DataFilesProjectEditDescriptionModalAddon/DataFilesProjectEditDescriptionModalAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectEditDescriptionModalAddon/DataFilesProjectEditDescriptionModalAddon.jsx @@ -34,7 +34,7 @@ const DataFilesProjectEditDescriptionModalAddon = () => { return (
{!isLoading && form && - + }
); From afe32e9e12730c324cae5ff498671f809cbdc749 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Wed, 24 Apr 2024 14:43:27 -0500 Subject: [PATCH 048/328] Added metadata for uploaded files --- .../DataFilesModals/DataFilesUploadModal.jsx | 13 +++- .../DataFilesUploadModal.module.scss | 5 ++ .../_common/Form/DynamicForm/DynamicForm.jsx | 6 +- .../DataFilesUploadModalAddon.jsx | 72 +++++++++++++++++++ .../DataFilesUploadModalAddon.module.scss | 9 +++ client/src/redux/sagas/datafiles.sagas.js | 10 +-- server/portal/apps/datafiles/views.py | 5 +- server/portal/libs/agave/operations.py | 17 ++++- server/portal/settings/settings_default.py | 2 +- 9 files changed, 129 insertions(+), 10 deletions(-) create mode 100644 client/src/components/_custom/drp/DataFilesUploadModalAddon/DataFilesUploadModalAddon.jsx create mode 100644 client/src/components/_custom/drp/DataFilesUploadModalAddon/DataFilesUploadModalAddon.module.scss diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesUploadModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesUploadModal.jsx index e7fb3b3ba4..bf0d3a627c 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesUploadModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesUploadModal.jsx @@ -1,6 +1,6 @@ /* FP-993: Create and use a common Uploader component */ import React, { useState } from 'react'; -import { useDispatch } from 'react-redux'; +import { useDispatch, useSelector } from 'react-redux'; import { useHistory, useLocation } from 'react-router-dom'; import PropTypes from 'prop-types'; import { v4 as uuidv4 } from 'uuid'; @@ -10,6 +10,7 @@ import { useSystemDisplayName, useFileListing, useModal, + useAddonComponents, } from 'hooks/datafiles'; import { useUpload } from 'hooks/datafiles/mutations'; import DataFilesUploadModalListingTable from './DataFilesUploadModalListing/DataFilesUploadModalListingTable'; @@ -30,6 +31,8 @@ const DataFilesUploadModal = ({ className, layout }) => { const reloadCallback = () => { history.push(location.pathname); }; + const portalName = useSelector((state) => state.workbench.portalName); + const { DataFilesUploadModalAddon } = useAddonComponents({portalName}) const { getStatus: getModalStatus, toggle } = useModal(); const isOpen = getModalStatus('upload'); @@ -118,7 +121,7 @@ const DataFilesUploadModal = ({ className, layout }) => { > Upload Files - +
{ />
)} + {DataFilesUploadModalAddon && ( + + )} )} + {DataFilesPreviewModalAddon && params.metadata && !isLoading && } ); diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesPreviewModal.module.scss b/client/src/components/DataFiles/DataFilesModals/DataFilesPreviewModal.module.scss index f38b95c41d..60ed3525f0 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesPreviewModal.module.scss +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesPreviewModal.module.scss @@ -28,6 +28,10 @@ height: 100%; } +.error-condensed { + height: 20%; +} + .button { background-color: var(--global-color-accent--normal); border-color: var(--global-color-accent--normal); @@ -43,3 +47,8 @@ font-size: 1.3em; } } + +.modal-body { + max-height: 80vh; + overflow: auto; +} diff --git a/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.jsx b/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.jsx new file mode 100644 index 0000000000..035a8bfcd4 --- /dev/null +++ b/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.jsx @@ -0,0 +1,98 @@ +import React, { useEffect, useState } from 'react'; +import { useQuery } from 'react-query'; +import { fetchUtil } from 'utils/fetchUtil'; +import { DynamicForm } from '_common/Form/DynamicForm'; +import { Form, Formik } from 'formik'; +import { LoadingSpinner, Section, SectionHeader } from '_common'; +import styles from './DataFilesPreviewModalAddon.module.scss'; +import { useFileListing } from 'hooks/datafiles'; +import { useDispatch, useSelector } from 'react-redux'; +import { useHistory, useLocation } from 'react-router-dom'; + +const DataFilesPreviewModalAddon = ({ metadata }) => { + + const dispatch = useDispatch(); + const history = useHistory(); + const location = useLocation(); + + const { params } = useFileListing('FilesListing'); + + const file = useSelector( + (state) => state.files.modalProps.preview + ); + + const { data: form, isLoading } = useQuery('form_UPLOAD_FILE', () => + fetchUtil({ + url: 'api/forms', + params: { + form_name: 'EDIT_FILE', + }, + })); + + const initialValues = form?.form_fields.reduce((acc, field) => { + let value = ''; + if (field.optgroups) { + value = field.optgroups[0].options[0]?.value; + } else { + value = + field.options && field.options.length > 0 ? field.options[0].value : ''; + } + + acc[field.name] = metadata ? metadata[field.name] : value; + return acc; + }, {}); + + const reloadPage = (updatedPath = "") => { + // using regex to replace last segment of pathname with optional parameter 'updatedPath' to navigate to a new path + const path = updatedPath ? location.pathname.replace(/\/[^\/]+\/?$/, '/' + updatedPath) : location.pathname; + history.push(path); + }; + + const handleSubmit = (values) => { + + Object.keys(values).forEach(key => { + values[key] = typeof(values[key]) === 'string' ? values[key].trim() : values[key]; + }); + + dispatch({ + type: 'EDIT_FILE', + payload: { + params, + values, + reloadPage, + selectedFile: file + }, + }); + }; + + return ( + !isLoading && form && + <> + +
+ Metadata +
+ + + } + /> + {form?.footer && ( +
+ +
+ )} + + + + + ) + +}; + +export default DataFilesPreviewModalAddon; \ No newline at end of file diff --git a/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.module.scss b/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.module.scss new file mode 100644 index 0000000000..80aaf5bbcb --- /dev/null +++ b/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.module.scss @@ -0,0 +1,14 @@ +.section { + border: 1px solid black; + padding: 20px 0px; +} + +.listing-header { + font-size: 20px; + } + +.footer { + display: flex; + justify-content: flex-end; + padding: 10px 0px; +} \ No newline at end of file diff --git a/client/src/redux/sagas/_custom/drp.sagas.js b/client/src/redux/sagas/_custom/drp.sagas.js index 35291c1f26..3815b37e58 100644 --- a/client/src/redux/sagas/_custom/drp.sagas.js +++ b/client/src/redux/sagas/_custom/drp.sagas.js @@ -111,6 +111,20 @@ function* handleAnalysisData(action, isEdit) { ) } +function* handleFile(action, isEdit) { + + const { params, values, reloadPage: reloadCallback, selectedFile } = action.payload; + + const metadata = { + ...values, + data_type: 'file', + } + + yield call( + executeOperation, isEdit, params, metadata, reloadCallback, selectedFile, params.path + ) +} + export default function* watchDRP() { yield takeLatest('ADD_SAMPLE_DATA', action => handleSampleData(action, false)); yield takeLatest('EDIT_SAMPLE_DATA', action => handleSampleData(action, true)); @@ -118,4 +132,5 @@ export default function* watchDRP() { yield takeLatest('EDIT_ANALYSIS_DATASET', action => handleAnalysisData(action, true)); yield takeLatest('ADD_ORIGIN_DATASET', action => handleOriginData(action, false)); yield takeLatest('EDIT_ORIGIN_DATASET', action => handleOriginData(action, true)); + yield takeLatest('EDIT_FILE', action => handleFile(action, true)) } diff --git a/server/portal/settings/settings_default.py b/server/portal/settings/settings_default.py index 46993f87a4..90c3d966df 100644 --- a/server/portal/settings/settings_default.py +++ b/server/portal/settings/settings_default.py @@ -191,7 +191,7 @@ "hasProjectFileListingAddon": True, "hasCustomSagas": True, "hasCustomEndpoints": True, - "addons": ['DataFilesProjectFileListingAddon', 'DataFilesAddProjectModalAddon', 'DataFilesProjectEditDescriptionModalAddon', 'DataFilesProjectFileListingMetadataAddon', 'DataFilesProjectFileListingMetadataTitleAddon', 'DataFilesUploadModalAddon'], + "addons": ['DataFilesProjectFileListingAddon', 'DataFilesAddProjectModalAddon', 'DataFilesProjectEditDescriptionModalAddon', 'DataFilesProjectFileListingMetadataAddon', 'DataFilesProjectFileListingMetadataTitleAddon', 'DataFilesUploadModalAddon', 'DataFilesPreviewModalAddon'], "onboardingCompleteRedirect": '/workbench/', "noPHISystem": "", "customDashboardSection": None, From 9826c671a4cd3450147eb2073143041c7b70e69b Mon Sep 17 00:00:00 2001 From: Asim Regmi Date: Fri, 26 Apr 2024 11:52:31 -0500 Subject: [PATCH 050/328] remove toolbar buttons and add label indicators --- .../DataFilesListing/DataFilesListing.jsx | 17 +++++++++++++++++ .../DataFilesListingCells.jsx | 19 +++++++++++++++++++ .../DataFilesListingCells.scss | 12 ++++++++++++ .../DataFilesToolbar/DataFilesToolbar.jsx | 18 ++++++++++++------ server/portal/settings/settings_default.py | 1 + 5 files changed, 61 insertions(+), 6 deletions(-) diff --git a/client/src/components/DataFiles/DataFilesListing/DataFilesListing.jsx b/client/src/components/DataFiles/DataFilesListing/DataFilesListing.jsx index 53d2e26160..20a56bcdd8 100644 --- a/client/src/components/DataFiles/DataFilesListing/DataFilesListing.jsx +++ b/client/src/components/DataFiles/DataFilesListing/DataFilesListing.jsx @@ -16,6 +16,7 @@ import { LastModifiedCell, FileIconCell, ViewPathCell, + DataTypeCell, } from './DataFilesListingCells'; import DataFilesTable from '../DataFilesTable/DataFilesTable'; import Searchbar from '_common/Searchbar'; @@ -53,6 +54,11 @@ const DataFilesListing = ({ api, scheme, system, path, isPublic }) => { api === 'tapis' && state.workbench && state.workbench.config.viewPath ); + const showDataType = useSelector( + (state) => state.workbench.portalName === 'DRP' + ); + + const { data: files, loading, @@ -62,6 +68,8 @@ const DataFilesListing = ({ api, scheme, system, path, isPublic }) => { } = useFileListing('FilesListing'); const { selectFile } = useSelectedFiles(); + console.log("data", files) + useLayoutEffect(() => { fetchListing({ api, scheme, system, path }); }, [api, scheme, system, path, location]); @@ -144,6 +152,15 @@ const DataFilesListing = ({ api, scheme, system, path, isPublic }) => { width: 0.1, Cell: (el) => , }); + } + + if (showDataType) { + cells.splice(3, 0, { // Inserting at index 3 after 'Name' + Header: 'Data Type', + accessor: 'metadata.data_type', + Cell: DataTypeCell, + width: 0.2, + }); } return cells; }, [api, showViewPath, fileNavCellCallback]); diff --git a/client/src/components/DataFiles/DataFilesListing/DataFilesListingCells.jsx b/client/src/components/DataFiles/DataFilesListing/DataFilesListingCells.jsx index 77464c6575..ea85f7cc44 100644 --- a/client/src/components/DataFiles/DataFilesListing/DataFilesListingCells.jsx +++ b/client/src/components/DataFiles/DataFilesListing/DataFilesListingCells.jsx @@ -181,3 +181,22 @@ export const ViewPathCell = ({ file }) => { ViewPathCell.propTypes = { file: PropTypes.shape({}).isRequired, }; + +export const DataTypeCell = ({ cell }) => { + const dataType = cell.value; + + const formatDatatype = (data_type) => + data_type.split('_') + .map(word => word.charAt(0).toUpperCase() + word.slice(1)) + .join(' '); + + return dataType ? ( + + {formatDatatype(dataType)} + + ) : null; +} + +DataTypeCell.propTypes = { + cell: PropTypes.shape({ value: PropTypes.string }).isRequired, +}; \ No newline at end of file diff --git a/client/src/components/DataFiles/DataFilesListing/DataFilesListingCells.scss b/client/src/components/DataFiles/DataFilesListing/DataFilesListingCells.scss index 3d92c60b43..6faa53f140 100644 --- a/client/src/components/DataFiles/DataFilesListing/DataFilesListingCells.scss +++ b/client/src/components/DataFiles/DataFilesListing/DataFilesListingCells.scss @@ -35,3 +35,15 @@ color: #707070; display: flex; } + + +.dataTypeBox { + display: inline-block; + background-color: #808080; /* Light grey background */ + color: #fff; /* Dark text color */ + padding: 2px 8px; /* Small top/bottom padding, larger left/right padding */ + border-radius: 10px; /* Rounded corners */ + font-size: 0.875rem; /* Smaller font size */ + margin: 8px 8px; + font-weight: bold; +} diff --git a/client/src/components/DataFiles/DataFilesToolbar/DataFilesToolbar.jsx b/client/src/components/DataFiles/DataFilesToolbar/DataFilesToolbar.jsx index fee8929a85..909a40e726 100644 --- a/client/src/components/DataFiles/DataFilesToolbar/DataFilesToolbar.jsx +++ b/client/src/components/DataFiles/DataFilesToolbar/DataFilesToolbar.jsx @@ -58,6 +58,8 @@ const DataFilesToolbar = ({ scheme, api }) => { const { projectId } = useSelector((state) => state.projects.metadata); + const allowedFileActions = useSelector((state) => state.workbench.config.allowedFileActions); + const authenticatedUser = useSelector( (state) => state.authenticatedUser.user.username ); @@ -88,13 +90,15 @@ const DataFilesToolbar = ({ scheme, api }) => { const modifiableUserData = api === 'tapis' && scheme !== 'public' && scheme !== 'community'; - const showMove = modifiableUserData; + const showCopy = allowedFileActions.includes('copy'); + + const showMove = modifiableUserData && allowedFileActions.includes('move'); - const showRename = modifiableUserData; + const showRename = modifiableUserData && allowedFileActions.includes('rename'); - const showTrash = modifiableUserData; + const showTrash = modifiableUserData && allowedFileActions.includes('trash'); - const showDownload = api === 'tapis'; + const showDownload = api === 'tapis' && allowedFileActions.includes('download'); const showMakeLink = useSelector( (state) => @@ -105,11 +109,11 @@ const DataFilesToolbar = ({ scheme, api }) => { ); const showCompress = !!useSelector( - (state) => state.workbench.config.extractApp && modifiableUserData + (state) => state.workbench.config.extractApp && modifiableUserData && allowedFileActions.includes('compress') ); const showExtract = !!useSelector( - (state) => state.workbench.config.compressApp && modifiableUserData + (state) => state.workbench.config.compressApp && modifiableUserData && allowedFileActions.includes('extract') ); const showMakePublic = useSelector( @@ -236,12 +240,14 @@ const DataFilesToolbar = ({ scheme, api }) => { disabled={!canMove} /> )} + {showCopy && ( + )} {showDownload && ( Date: Fri, 26 Apr 2024 11:56:55 -0500 Subject: [PATCH 051/328] remove console.log --- .../components/DataFiles/DataFilesListing/DataFilesListing.jsx | 3 --- 1 file changed, 3 deletions(-) diff --git a/client/src/components/DataFiles/DataFilesListing/DataFilesListing.jsx b/client/src/components/DataFiles/DataFilesListing/DataFilesListing.jsx index 20a56bcdd8..ba99a9551d 100644 --- a/client/src/components/DataFiles/DataFilesListing/DataFilesListing.jsx +++ b/client/src/components/DataFiles/DataFilesListing/DataFilesListing.jsx @@ -57,7 +57,6 @@ const DataFilesListing = ({ api, scheme, system, path, isPublic }) => { const showDataType = useSelector( (state) => state.workbench.portalName === 'DRP' ); - const { data: files, @@ -68,8 +67,6 @@ const DataFilesListing = ({ api, scheme, system, path, isPublic }) => { } = useFileListing('FilesListing'); const { selectFile } = useSelectedFiles(); - console.log("data", files) - useLayoutEffect(() => { fetchListing({ api, scheme, system, path }); }, [api, scheme, system, path, location]); From 1e3c16998719d1183b4c4ecc0c4e877a61c1d9ee Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Wed, 1 May 2024 13:20:08 -0500 Subject: [PATCH 052/328] removed default selection for select form fields --- .../DataFilesProjectFileListingAddon.jsx | 16 +++++++--------- ...FilesProjectFileListingMetadataTitleAddon.jsx | 14 ++++++-------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx index 6ee6b0fa09..f19c53730a 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx @@ -49,15 +49,15 @@ const DataFilesProjectFileListingAddon = () => { const onOpenOriginDataModal = async (formName, selectedFile) => { const form = await getFormFields(formName); const { samples } = await getDatasets(projectId); - + form.form_fields.map((field) => { if (field.name === 'sample') { - field.options = samples.map((sample) => { + field.options.push(...samples.map((sample) => { return { value: parseInt(sample.id), label: sample.name, }; - }); + })); } }); @@ -76,22 +76,20 @@ const DataFilesProjectFileListingAddon = () => { form.form_fields.map((field) => { if (field.name === 'sample') { - field.options = samples.map((sample) => { + field.options.push(...samples.map((sample) => { return { value: parseInt(sample.id), label: sample.name, }; - }); + })); } else if (field.name === 'base_origin_data') { - field.options = originDatasets.map((originData) => { + field.options.push(...originDatasets.map((originData) => { return { value: parseInt(originData.id), label: originData.name, dependentId: parseInt(originData.metadata.sample), }; - }) - // Add a blank option to the select - field.options.unshift({ value: '', label: '' }); + })); } }); diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx index caa7e9339d..be6a92ed58 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx @@ -51,12 +51,12 @@ const DataFilesProjectFileListingMetadataTitleAddon = ({ folderMetadata, metadat form.form_fields.map((field) => { if (field.name === 'sample') { - field.options = samples.map((sample) => { + field.options.push(...samples.map((sample) => { return { value: parseInt(sample.id), label: sample.name, }; - }); + })); } }); @@ -75,22 +75,20 @@ const DataFilesProjectFileListingMetadataTitleAddon = ({ folderMetadata, metadat form.form_fields.map((field) => { if (field.name === 'sample') { - field.options = samples.map((sample) => { + field.options.push(...samples.map((sample) => { return { value: parseInt(sample.id), label: sample.name, }; - }); + })); } else if (field.name === 'base_origin_data') { - field.options = originDatasets.map((originData) => { + field.options.push(...originDatasets.map((originData) => { return { value: parseInt(originData.id), label: originData.name, dependentId: parseInt(originData.metadata.sample), }; - }) - // Add a blank option to the select - field.options.unshift({ value: '', label: '' }); + })); } }); From d68d43cd31d6b9617f317393c20f68f865c33df6 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Wed, 1 May 2024 16:26:14 -0500 Subject: [PATCH 053/328] made css and object tweaks --- .../DataFiles/DataFilesListing/DataFilesListing.jsx | 9 +++------ .../DataFiles/DataFilesListing/DataFilesListingCells.jsx | 7 ++++--- .../DataFilesListing/DataFilesListingCells.scss | 4 ++-- .../DataFilesProjectFileListingMetadataAddon.jsx | 4 ++-- .../DataFilesProjectFileListingMetadataTitleAddon.jsx | 2 +- server/portal/settings/settings_default.py | 1 + 6 files changed, 13 insertions(+), 14 deletions(-) diff --git a/client/src/components/DataFiles/DataFilesListing/DataFilesListing.jsx b/client/src/components/DataFiles/DataFilesListing/DataFilesListing.jsx index ba99a9551d..e8c7178a46 100644 --- a/client/src/components/DataFiles/DataFilesListing/DataFilesListing.jsx +++ b/client/src/components/DataFiles/DataFilesListing/DataFilesListing.jsx @@ -54,9 +54,7 @@ const DataFilesListing = ({ api, scheme, system, path, isPublic }) => { api === 'tapis' && state.workbench && state.workbench.config.viewPath ); - const showDataType = useSelector( - (state) => state.workbench.portalName === 'DRP' - ); + const { showDataFileType } = useSelector((state) => state.workbench.config); const { data: files, @@ -151,11 +149,10 @@ const DataFilesListing = ({ api, scheme, system, path, isPublic }) => { }); } - if (showDataType) { + if (showDataFileType) { cells.splice(3, 0, { // Inserting at index 3 after 'Name' Header: 'Data Type', - accessor: 'metadata.data_type', - Cell: DataTypeCell, + Cell: (el) => , width: 0.2, }); } diff --git a/client/src/components/DataFiles/DataFilesListing/DataFilesListingCells.jsx b/client/src/components/DataFiles/DataFilesListing/DataFilesListingCells.jsx index ea85f7cc44..591b16203c 100644 --- a/client/src/components/DataFiles/DataFilesListing/DataFilesListingCells.jsx +++ b/client/src/components/DataFiles/DataFilesListing/DataFilesListingCells.jsx @@ -182,8 +182,9 @@ ViewPathCell.propTypes = { file: PropTypes.shape({}).isRequired, }; -export const DataTypeCell = ({ cell }) => { - const dataType = cell.value; +export const DataTypeCell = ({ file }) => { + + const dataType = file.metadata ? file.metadata.data_type : file.type; const formatDatatype = (data_type) => data_type.split('_') @@ -198,5 +199,5 @@ export const DataTypeCell = ({ cell }) => { } DataTypeCell.propTypes = { - cell: PropTypes.shape({ value: PropTypes.string }).isRequired, + file: PropTypes.shape({}).isRequired, }; \ No newline at end of file diff --git a/client/src/components/DataFiles/DataFilesListing/DataFilesListingCells.scss b/client/src/components/DataFiles/DataFilesListing/DataFilesListingCells.scss index 6faa53f140..6868deabc4 100644 --- a/client/src/components/DataFiles/DataFilesListing/DataFilesListingCells.scss +++ b/client/src/components/DataFiles/DataFilesListing/DataFilesListingCells.scss @@ -42,8 +42,8 @@ background-color: #808080; /* Light grey background */ color: #fff; /* Dark text color */ padding: 2px 8px; /* Small top/bottom padding, larger left/right padding */ - border-radius: 10px; /* Rounded corners */ + border-radius: 12px; /* Rounded corners */ font-size: 0.875rem; /* Smaller font size */ - margin: 8px 8px; + margin: 8px 0px; font-weight: bold; } diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx index 80bef885c8..249b451356 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx @@ -67,7 +67,7 @@ const DataDisplay = ({ data, path }) => { return (
{chunks.map((chunk, index) => ( - + {renderDataEntries(chunk)} ))} @@ -96,7 +96,7 @@ const DataFilesProjectFileListingMetadataAddon = ({ folderMetadata, metadata, pa } DataFilesProjectFileListingMetadataAddon.propTypes = { - folderMetadata: PropTypes.object.isRequired, + folderMetadata: PropTypes.shape({}).isRequired, }; export default DataFilesProjectFileListingMetadataAddon; \ No newline at end of file diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx index be6a92ed58..fa2e656cf0 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx @@ -166,7 +166,7 @@ const DataFilesProjectFileListingMetadataTitleAddon = ({ folderMetadata, metadat }; DataFilesProjectFileListingMetadataTitleAddon.propTypes = { - folderMetadata: PropTypes.object.isRequired, + folderMetadata: PropTypes.shape({}).isRequired, }; export default DataFilesProjectFileListingMetadataTitleAddon; \ No newline at end of file diff --git a/server/portal/settings/settings_default.py b/server/portal/settings/settings_default.py index 7cba2ace68..e33463664c 100644 --- a/server/portal/settings/settings_default.py +++ b/server/portal/settings/settings_default.py @@ -193,6 +193,7 @@ "hasCustomEndpoints": True, "addons": ['DataFilesProjectFileListingAddon', 'DataFilesAddProjectModalAddon', 'DataFilesProjectEditDescriptionModalAddon', 'DataFilesProjectFileListingMetadataAddon', 'DataFilesProjectFileListingMetadataTitleAddon'], "allowedFileActions": ['trash', 'download'], + "showDataFileType": True, "onboardingCompleteRedirect": '/workbench/', "noPHISystem": "", "customDashboardSection": None, From d5bca9cc9ed635fe030d58b66b23a2b62af2dc6d Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Wed, 1 May 2024 16:36:54 -0500 Subject: [PATCH 054/328] small tweaks --- .../DataFiles/DataFilesToolbar/DataFilesToolbar.jsx | 4 ++-- server/portal/settings/settings_default.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/client/src/components/DataFiles/DataFilesToolbar/DataFilesToolbar.jsx b/client/src/components/DataFiles/DataFilesToolbar/DataFilesToolbar.jsx index 909a40e726..a4ff70c37a 100644 --- a/client/src/components/DataFiles/DataFilesToolbar/DataFilesToolbar.jsx +++ b/client/src/components/DataFiles/DataFilesToolbar/DataFilesToolbar.jsx @@ -58,7 +58,7 @@ const DataFilesToolbar = ({ scheme, api }) => { const { projectId } = useSelector((state) => state.projects.metadata); - const allowedFileActions = useSelector((state) => state.workbench.config.allowedFileActions); + const { allowedFileActions } = useSelector((state) => state.workbench.config); const authenticatedUser = useSelector( (state) => state.authenticatedUser.user.username @@ -106,7 +106,7 @@ const DataFilesToolbar = ({ scheme, api }) => { state.workbench.config.makeLink && api === 'tapis' && (scheme === 'private' || scheme === 'projects') - ); + ) && allowedFileActions.includes('link'); const showCompress = !!useSelector( (state) => state.workbench.config.extractApp && modifiableUserData && allowedFileActions.includes('compress') diff --git a/server/portal/settings/settings_default.py b/server/portal/settings/settings_default.py index e33463664c..0464d72b62 100644 --- a/server/portal/settings/settings_default.py +++ b/server/portal/settings/settings_default.py @@ -188,7 +188,6 @@ "hideManageAccount": False, "hideSystemStatus": False, "hasUserGuide": True, - "hasProjectFileListingAddon": True, "hasCustomSagas": True, "hasCustomEndpoints": True, "addons": ['DataFilesProjectFileListingAddon', 'DataFilesAddProjectModalAddon', 'DataFilesProjectEditDescriptionModalAddon', 'DataFilesProjectFileListingMetadataAddon', 'DataFilesProjectFileListingMetadataTitleAddon'], From 68c4e9ca135907bfa8adc593468db34515f6f3b3 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Fri, 3 May 2024 14:29:21 -0500 Subject: [PATCH 055/328] added redirect to correct path for form modal, fixed analysis data bug --- .../DataFiles/DataFilesModals/DataFilesFormModal.jsx | 8 +++++--- .../_common/Form/DynamicForm/DynamicForm.jsx | 12 +++++++++--- .../DataFilesProjectFileListingMetadataAddon.jsx | 2 +- client/src/redux/sagas/_custom/drp.sagas.js | 8 +------- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx index 384aa3a7bb..2fe0c3d764 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx @@ -15,9 +15,11 @@ const DataFilesFormModal = () => { const history = useHistory(); const location = useLocation(); - const reloadPage = (updatedPath = "") => { - // using regex to replace last segment of pathname with optional parameter 'updatedPath' to navigate to a new path - const path = updatedPath ? location.pathname.replace(/\/[^\/]+\/?$/, '/' + updatedPath) : location.pathname; + const reloadPage = (updatedPath = '') => { + // using regex to get the url up until the project name + const projectUrl = location.pathname.replace(/(\/projects\/[^/]+\/).*/, '$1'); + + const path = updatedPath ? `${projectUrl}${updatedPath}` : `${projectUrl}`; history.push(path); }; diff --git a/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx b/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx index 8d4beea8b5..ce43fcf8c5 100644 --- a/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx +++ b/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx @@ -5,7 +5,7 @@ import { useFormikContext } from 'formik'; import { FormGroup, Input } from 'reactstrap'; import './DynamicForm.scss'; -const updateFormFieldsBasedOnDependency = (formFields, values) => { +const updateFormFieldsBasedOnDependency = (formFields, values, setFieldValue, modifiedField) => { return formFields.map((field) => { const { dependency } = field; @@ -14,7 +14,13 @@ const updateFormFieldsBasedOnDependency = (formFields, values) => { if (dependency.type === 'filter') { const filteredOptions = field.options.filter(option => option.dependentId == values[dependency.name]); const updatedOptions = [{ value: '', label: '' }, ...filteredOptions]; - return { ...field, hidden: false, filteredOptions: updatedOptions }; + + // only update the field value if the modified field is the dependency field + if (modifiedField && modifiedField.name === dependency.name) { + setFieldValue(field.name, updatedOptions[0].value); + } + + return { ...field, hidden: false, filteredOptions: updatedOptions, value: '' }; } else if (dependency.type === 'visibility') { if (dependency.value) { return { ...field, hidden: field.dependency.value !== values[field.dependency.name] }; @@ -35,7 +41,7 @@ const DynamicForm = ({ initialFormFields }) => { // This function updates and filters any dependant fields. Field dependency is described in the form config file const handleDependentFieldUpdate = (value, modifiedField) => { - const updatedFormFields = updateFormFieldsBasedOnDependency(formFields, { ...values, [modifiedField.name]: value }); + const updatedFormFields = updateFormFieldsBasedOnDependency(formFields, { ...values, [modifiedField.name]: value }, setFieldValue, modifiedField); setFormFields(updatedFormFields); }; diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx index 249b451356..c3ccda4f59 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx @@ -5,7 +5,7 @@ import { useLocation, Link } from 'react-router-dom'; import styles from './DataFilesProjectFileListingMetadataAddon.module.scss'; import { useFileListing } from 'hooks/datafiles'; -const excludeKeys = ['name', 'description', 'data_type', 'sample', 'origin_data']; +const excludeKeys = ['name', 'description', 'data_type', 'sample', 'base_origin_data']; const DataDisplay = ({ data, path }) => { diff --git a/client/src/redux/sagas/_custom/drp.sagas.js b/client/src/redux/sagas/_custom/drp.sagas.js index 35291c1f26..9868752a9c 100644 --- a/client/src/redux/sagas/_custom/drp.sagas.js +++ b/client/src/redux/sagas/_custom/drp.sagas.js @@ -34,13 +34,7 @@ function* executeOperation(isEdit, params, values, reloadCallback, file = null, ); } - if (isEdit && file.path === params.path) { - // reload to new URL if name/path is changed - yield call (reloadCallback, values.name) - } - else { - yield call(reloadCallback); - } + yield call(reloadCallback, path); yield put({ type: 'DATA_FILES_TOGGLE_MODAL', From b066831cf679c842bdbe199034a281b61ac4737f Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Fri, 3 May 2024 15:11:18 -0500 Subject: [PATCH 056/328] bug fix for modal redirect --- .../DataFiles/DataFilesModals/DataFilesFormModal.jsx | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx index 2fe0c3d764..b78fe52814 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx @@ -17,9 +17,13 @@ const DataFilesFormModal = () => { const reloadPage = (updatedPath = '') => { // using regex to get the url up until the project name - const projectUrl = location.pathname.replace(/(\/projects\/[^/]+\/).*/, '$1'); + let projectUrl = location.pathname.replace(/(\/projects\/[^/]+\/).*/, '$1'); + + if (projectUrl.endsWith('/')) { + projectUrl = projectUrl.slice(0, -1); + } - const path = updatedPath ? `${projectUrl}${updatedPath}` : `${projectUrl}`; + const path = updatedPath ? `${projectUrl}/${updatedPath}` : `${projectUrl}`; history.push(path); }; From aece33f18fcf8086dc1e7afc0a142844bd0c8761 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Mon, 6 May 2024 14:19:53 -0500 Subject: [PATCH 057/328] Added regex to check uploaded file type --- .../DataFilesUploadModalAddon.jsx | 7 +++++-- client/src/redux/sagas/datafiles.sagas.js | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/client/src/components/_custom/drp/DataFilesUploadModalAddon/DataFilesUploadModalAddon.jsx b/client/src/components/_custom/drp/DataFilesUploadModalAddon/DataFilesUploadModalAddon.jsx index cd478e3e9f..1149cfebe0 100644 --- a/client/src/components/_custom/drp/DataFilesUploadModalAddon/DataFilesUploadModalAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesUploadModalAddon/DataFilesUploadModalAddon.jsx @@ -12,8 +12,11 @@ const DataFilesUploadModalAddon = ({ uploadedFiles, setUploadedFiles }) => { const [uploadedFilesWithMetadata, setUploadedFilesWithMetadata] = useState([]); + // regex from old digitalrocks portal + const standardImageType = /(\.|\/)(gif|jpe?g|png|tiff?)$/i; + useEffect(() => { - setUploadedFilesWithMetadata(uploadedFiles.filter((file) => file.data.name.split('.').pop() === 'raw')); + setUploadedFilesWithMetadata(uploadedFiles.filter((file) => !standardImageType.test(file.data.name))); }, [uploadedFiles]) @@ -31,7 +34,7 @@ const DataFilesUploadModalAddon = ({ uploadedFiles, setUploadedFiles }) => { if (uploadedFile.data.name === file.data.name) { return { ...uploadedFile, - metadata: values + metadata: {name: uploadedFile.data.name, ...values} } } return uploadedFile; diff --git a/client/src/redux/sagas/datafiles.sagas.js b/client/src/redux/sagas/datafiles.sagas.js index 83090c8404..d1f49432f6 100644 --- a/client/src/redux/sagas/datafiles.sagas.js +++ b/client/src/redux/sagas/datafiles.sagas.js @@ -485,7 +485,7 @@ export async function uploadFileUtil(api, scheme, system, path, file, metadata) } const formData = new FormData(); formData.append('uploaded_file', file); - formData.append('metadata', metadata ? JSON.stringify(metadata) : null); + formData.append('metadata', metadata ? JSON.stringify({data_type: 'file', ...metadata}) : null); const url = removeDuplicateSlashes( `/api/datafiles/${api}/upload/${scheme}/${system}/${apiPath}/` From 35b9317573d653582be874f4c66c88b1236dea40 Mon Sep 17 00:00:00 2001 From: Shayan Khan Date: Tue, 7 May 2024 16:15:18 -0500 Subject: [PATCH 058/328] centralized dataset modal handling code, removed duplicated code --- .../DataFilesProjectFileListingAddon.jsx | 105 ++-------------- ...esProjectFileListingMetadataTitleAddon.jsx | 97 ++------------- .../_custom/drp/utils/datasetFormHandlers.js | 112 ++++++++++++++++++ 3 files changed, 129 insertions(+), 185 deletions(-) create mode 100644 client/src/components/_custom/drp/utils/datasetFormHandlers.js diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx index f19c53730a..95d71834cd 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx @@ -5,6 +5,7 @@ import styles from './DataFilesProjectFileListingAddon.module.scss'; import { useSelectedFiles } from 'hooks/datafiles'; import { useQuery } from 'react-query'; import { fetchUtil } from 'utils/fetchUtil'; +import { createSampleModalHandler, createOriginDataModalHandler, createAnalysisDataModalHandler } from '../utils/datasetFormHandlers'; const DataFilesProjectFileListingAddon = () => { const dispatch = useDispatch(); @@ -12,95 +13,9 @@ const DataFilesProjectFileListingAddon = () => { const { projectId } = useSelector((state) => state.projects.metadata); const { selectedFiles } = useSelectedFiles(); - const getFormFields = async (formName) => { - const response = await fetchUtil({ - url: 'api/forms', - params: { - form_name: formName, - }, - }); - return response; - }; - - const getDatasets = async (projectId, getOriginData = false) => { - const response = await fetchUtil({ - url: `api/${portalName.toLowerCase()}`, - params: { - project_id: projectId, - get_origin_data: getOriginData, - }, - }); - return response; - }; - - const onOpenSampleModal = async (formName, selectedFile) => { - - const form = await getFormFields(formName); - - dispatch({ - type: 'DATA_FILES_TOGGLE_MODAL', - payload: { - operation: 'dynamicform', - props: { form, selectedFile, formName }, - }, - }); - }; - - const onOpenOriginDataModal = async (formName, selectedFile) => { - const form = await getFormFields(formName); - const { samples } = await getDatasets(projectId); - - form.form_fields.map((field) => { - if (field.name === 'sample') { - field.options.push(...samples.map((sample) => { - return { - value: parseInt(sample.id), - label: sample.name, - }; - })); - } - }); - - dispatch({ - type: 'DATA_FILES_TOGGLE_MODAL', - payload: { - operation: 'dynamicform', - props: { selectedFile, form, formName, additionalData: samples }, - }, - }); - }; - - const onOpenAnalysisDataModal = async (formName, selectedFile) => { - const form = await getFormFields(formName); - const { samples, origin_data: originDatasets } = await getDatasets(projectId, true); - - form.form_fields.map((field) => { - if (field.name === 'sample') { - field.options.push(...samples.map((sample) => { - return { - value: parseInt(sample.id), - label: sample.name, - }; - })); - } else if (field.name === 'base_origin_data') { - field.options.push(...originDatasets.map((originData) => { - return { - value: parseInt(originData.id), - label: originData.name, - dependentId: parseInt(originData.metadata.sample), - }; - })); - } - }); - - dispatch({ - type: 'DATA_FILES_TOGGLE_MODAL', - payload: { - operation: 'dynamicform', - props: { selectedFile, form, formName, additionalData: { samples, originDatasets } }, - }, - }); - }; + const handleOpenSampleModal = createSampleModalHandler(dispatch); + const handleOriginDataModal = createOriginDataModalHandler(dispatch, projectId, portalName); + const handleAnalysisDataModal = createAnalysisDataModalHandler(dispatch, projectId, portalName); return ( <> @@ -109,12 +24,12 @@ const DataFilesProjectFileListingAddon = () => { selectedFiles[0].metadata['data_type'] === 'sample' ? ( ) : ( - )} @@ -124,7 +39,7 @@ const DataFilesProjectFileListingAddon = () => { @@ -143,7 +58,7 @@ const DataFilesProjectFileListingAddon = () => { diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx index fa2e656cf0..d56cc95d3e 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx @@ -5,6 +5,7 @@ import styles from './DataFilesProjectFileListingMetadataTitleAddon.module.scss' import { Button, LoadingSpinner } from '_common'; import { fetchUtil } from 'utils/fetchUtil'; import { useFileListing } from 'hooks/datafiles'; +import { createSampleModalHandler, createOriginDataModalHandler, createAnalysisDataModalHandler } from '../utils/datasetFormHandlers'; const DataFilesProjectFileListingMetadataTitleAddon = ({ folderMetadata, metadata, system, path }) => { const dispatch = useDispatch(); @@ -13,93 +14,9 @@ const DataFilesProjectFileListingMetadataTitleAddon = ({ folderMetadata, metadat const { loading } = useFileListing('FilesListing'); - const getFormFields = async (formName) => { - const response = await fetchUtil({ - url: 'api/forms', - params: { - form_name: formName, - }, - }); - return response; - }; - - const getDatasets = async (projectId, getOriginData = false) => { - const response = await fetchUtil({ - url: `api/${portalName.toLowerCase()}`, - params: { - project_id: projectId, - get_origin_data: getOriginData, - }, - }); - return response; - }; - - const onOpenSampleModal = async (formName, selectedFile) => { - const form = await getFormFields(formName); - dispatch({ - type: 'DATA_FILES_TOGGLE_MODAL', - payload: { - operation: 'dynamicform', - props: { form, selectedFile, formName }, - }, - }); - }; - - const onOpenOriginDataModal = async (formName, selectedFile) => { - const form = await getFormFields(formName); - const { samples } = await getDatasets(projectId); - - form.form_fields.map((field) => { - if (field.name === 'sample') { - field.options.push(...samples.map((sample) => { - return { - value: parseInt(sample.id), - label: sample.name, - }; - })); - } - }); - - dispatch({ - type: 'DATA_FILES_TOGGLE_MODAL', - payload: { - operation: 'dynamicform', - props: { selectedFile, form, formName, additionalData: samples }, - }, - }); - }; - - const onOpenAnalysisDataModal = async (formName, selectedFile) => { - const form = await getFormFields(formName); - const { samples, origin_data: originDatasets } = await getDatasets(projectId, true); - - form.form_fields.map((field) => { - if (field.name === 'sample') { - field.options.push(...samples.map((sample) => { - return { - value: parseInt(sample.id), - label: sample.name, - }; - })); - } else if (field.name === 'base_origin_data') { - field.options.push(...originDatasets.map((originData) => { - return { - value: parseInt(originData.id), - label: originData.name, - dependentId: parseInt(originData.metadata.sample), - }; - })); - } - }); - - dispatch({ - type: 'DATA_FILES_TOGGLE_MODAL', - payload: { - operation: 'dynamicform', - props: { selectedFile, form, formName, additionalData: { samples, originDatasets } }, - }, - }); - }; + const handleSampleModal = createSampleModalHandler(dispatch); + const handleOriginDataModal = createOriginDataModalHandler(dispatch, projectId, portalName); + const handleAnalysisDataModal = createAnalysisDataModalHandler(dispatch, projectId, portalName); const onEditData = (dataType) => { const name = path.split('/').pop(); @@ -120,13 +37,13 @@ const DataFilesProjectFileListingMetadataTitleAddon = ({ folderMetadata, metadat }; switch (dataType) { case 'sample': - onOpenSampleModal('EDIT_SAMPLE_DATA', editFile); + handleSampleModal('EDIT_SAMPLE_DATA', editFile); break; case 'origin_data': - onOpenOriginDataModal('EDIT_ORIGIN_DATASET', editFile); + handleOriginDataModal('EDIT_ORIGIN_DATASET', editFile); break; case 'analysis_data': - onOpenAnalysisDataModal('EDIT_ANALYSIS_DATASET', editFile); + handleAnalysisDataModal('EDIT_ANALYSIS_DATASET', editFile); break; default: break; diff --git a/client/src/components/_custom/drp/utils/datasetFormHandlers.js b/client/src/components/_custom/drp/utils/datasetFormHandlers.js new file mode 100644 index 0000000000..6b426fee7d --- /dev/null +++ b/client/src/components/_custom/drp/utils/datasetFormHandlers.js @@ -0,0 +1,112 @@ +import { fetchUtil } from 'utils/fetchUtil'; + +const getFormFields = async (formName) => { + const response = await fetchUtil({ + url: 'api/forms', + params: { + form_name: formName, + }, + }); + return response; +}; + +const getDatasets = async (projectId, portalName, getOriginData = false) => { + const response = await fetchUtil({ + url: `api/${portalName.toLowerCase()}`, + params: { + project_id: projectId, + get_origin_data: getOriginData, + }, + }); + + return response; +}; + +export const createSampleModalHandler = + (dispatch) => + async (formName, selectedFile = null) => { + const form = await getFormFields(formName); + + dispatch({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { + operation: 'dynamicform', + props: { form, selectedFile, formName }, + }, + }); + }; + +export const createOriginDataModalHandler = + (dispatch, projectId, portalName) => + async (formName, selectedFile = null) => { + const form = await getFormFields(formName); + const { samples } = await getDatasets(projectId, portalName); + + form.form_fields.map((field) => { + if (field.name === 'sample') { + field.options.push( + ...samples.map((sample) => { + return { + value: parseInt(sample.id), + label: sample.name, + }; + }) + ); + } + }); + + dispatch({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { + operation: 'dynamicform', + props: { selectedFile, form, formName, additionalData: samples }, + }, + }); + }; + +export const createAnalysisDataModalHandler = + (dispatch, projectId, portalName) => + async (formName, selectedFile = null) => { + const form = await getFormFields(formName); + const { samples, origin_data: originDatasets } = await getDatasets( + projectId, + portalName, + true + ); + + form.form_fields.map((field) => { + if (field.name === 'sample') { + field.options.push( + ...samples.map((sample) => { + return { + value: parseInt(sample.id), + label: sample.name, + }; + }) + ); + } else if (field.name === 'base_origin_data') { + field.options.push( + ...originDatasets.map((originData) => { + return { + value: parseInt(originData.id), + label: originData.name, + dependentId: parseInt(originData.metadata.sample), + }; + }) + ); + } + }); + + dispatch({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { + operation: 'dynamicform', + props: { + selectedFile, + form, + formName, + additionalData: { samples, originDatasets }, + }, + }, + }); + }; From bdd8587b9c373d6146b7c556d51b51fdeaa339e3 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Wed, 8 May 2024 11:24:06 -0500 Subject: [PATCH 059/328] Added project metadata under description, some refactoring --- ...taFilesProjectFileListingMetadataAddon.jsx | 88 ++++-------------- .../drp/utils/DataDisplay/DataDisplay.jsx | 89 +++++++++++++++++++ .../utils/DataDisplay/DataDisplay.module.scss | 13 +++ server/portal/settings/settings_default.py | 4 +- 4 files changed, 123 insertions(+), 71 deletions(-) create mode 100644 client/src/components/_custom/drp/utils/DataDisplay/DataDisplay.jsx create mode 100644 client/src/components/_custom/drp/utils/DataDisplay/DataDisplay.module.scss diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx index c3ccda4f59..4ae6cb7098 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx @@ -1,83 +1,28 @@ import React from 'react'; import PropTypes from 'prop-types'; -import { Section, SectionContent, LoadingSpinner } from '_common'; -import { useLocation, Link } from 'react-router-dom'; import styles from './DataFilesProjectFileListingMetadataAddon.module.scss'; import { useFileListing } from 'hooks/datafiles'; +import DataDisplay from '../utils/DataDisplay/DataDisplay'; +import { formatDate } from 'utils/timeFormat'; const excludeKeys = ['name', 'description', 'data_type', 'sample', 'base_origin_data']; -const DataDisplay = ({ data, path }) => { - - const location = useLocation(); - - // Function to format the dict key from snake_case to Label Case i.e. data_type -> Data Type - const formatLabel = (key) => - key.split('_') - .map(word => word.charAt(0).toUpperCase() + word.slice(1)) - .join(' '); - - //filter out empty values and unwanted keys - const processedData = Object.entries(data) - .filter(([key, value]) => value !== "" && !excludeKeys.includes(key)) - .map(([key, value]) => ({ - label: formatLabel(key), - value: typeof(value) === 'string' ? formatLabel(value) : value - })); - - // use the path to get sample and origin data names - const sample = data.sample ? path.split('/')[0] : null; - const origin_data = data.base_origin_data ? path.split('/')[1] : null; - - // remove trailing / from pathname - const locationPathname = location.pathname.endsWith('/') ? location.pathname.slice(0, -1) : location.pathname; - - const locationPathnameParts = locationPathname.split('/'); - - // construct urls for sample and origin data and add to processed data - - if (origin_data) { - const originDataUrl = locationPathnameParts.slice(0, -1).join('/'); - processedData.unshift({ label: 'Origin Data', value: {origin_data} }); +const DataFilesProjectFileListingMetadataAddon = ({ folderMetadata, metadata, path }) => { - } + const { loading } = useFileListing('FilesListing'); - if (sample) { - const sampleUrl = locationPathnameParts.slice(0, data.base_origin_data ? -2 : -1).join('/') - processedData.unshift({ label: 'Sample', value: {sample} }); - } + const getProjectMetadata = (metadata) => { + const formattedMetadata = { + created: formatDate(new Date(metadata.created)), + license: metadata.license ?? 'None', + } + if (metadata.doi) { + formattedMetadata.doi = metadata.doi; + } - // Divide processed data into chunks for two-column layout display - const chunkSize = Math.ceil(processedData.length / 2); - const chunks = []; - for (let i = 0; i < processedData.length; i += chunkSize) { - chunks.push(processedData.slice(i, i + chunkSize)); + return formattedMetadata; } - - const renderDataEntries = (entries) => ( - entries.map(({ label, value }, index) => ( -
- {label}: {value} -
- )) - ); - - // Render each data entry within its chunk for two-column layout - return ( -
- {chunks.map((chunk, index) => ( - - {renderDataEntries(chunk)} - - ))} -
- ); -} - -const DataFilesProjectFileListingMetadataAddon = ({ folderMetadata, metadata, path }) => { - - const { loading } = useFileListing('FilesListing'); return ( <> @@ -85,10 +30,13 @@ const DataFilesProjectFileListingMetadataAddon = ({ folderMetadata, metadata, pa folderMetadata ? ( <> {folderMetadata.description} - + ) : ( - metadata.description + <> + {metadata.description} + + ) )} diff --git a/client/src/components/_custom/drp/utils/DataDisplay/DataDisplay.jsx b/client/src/components/_custom/drp/utils/DataDisplay/DataDisplay.jsx new file mode 100644 index 0000000000..f8dfa3e998 --- /dev/null +++ b/client/src/components/_custom/drp/utils/DataDisplay/DataDisplay.jsx @@ -0,0 +1,89 @@ +import React from 'react'; +import PropTypes from 'prop-types'; +import { Section, SectionContent, LoadingSpinner } from '_common'; +import { useLocation, Link } from 'react-router-dom'; +import styles from './DataDisplay.module.scss'; +import { useFileListing } from 'hooks/datafiles'; + +const processSampleAndOriginData = (data, path) => { + // use the path to get sample and origin data names + const sample = data.sample ? path.split('/')[0] : null; + const origin_data = data.base_origin_data ? path.split('/')[1] : null; + + // remove trailing / from pathname + const locationPathname = location.pathname.endsWith('/') ? location.pathname.slice(0, -1) : location.pathname; + + const locationPathnameParts = locationPathname.split('/'); + + const sampleAndOriginMetadata = [] + + // construct urls for sample and origin data and add to processed data + if (sample) { + const sampleUrl = locationPathnameParts.slice(0, data.base_origin_data ? -2 : -1).join('/') + sampleAndOriginMetadata.push({ label: 'Sample', value: {sample} }); + } + + if (origin_data) { + const originDataUrl = locationPathnameParts.slice(0, -1).join('/'); + sampleAndOriginMetadata.push({ label: 'Origin Data', value: {origin_data} }); + + } + + return sampleAndOriginMetadata; +} + +const DataDisplay = ({ data, path, excludeKeys }) => { + + const location = useLocation(); + + // Function to format the dict key from snake_case to Label Case i.e. data_type -> Data Type + const formatLabel = (key) => + key.split('_') + .map(word => word.charAt(0).toUpperCase() + word.slice(1)) + .join(' '); + + //filter out empty values and unwanted keys + const processedData = Object.entries(data) + .filter(([key, value]) => value !== "" && !excludeKeys.includes(key)) + .map(([key, value]) => ({ + label: formatLabel(key), + value: typeof(value) === 'string' ? formatLabel(value) : value + })); + + + processedData.unshift(...processSampleAndOriginData(data, path)); + + // Divide processed data into chunks for two-column layout display + const chunkSize = Math.ceil(processedData.length / 2); + const chunks = []; + for (let i = 0; i < processedData.length; i += chunkSize) { + chunks.push(processedData.slice(i, i + chunkSize)); + } + + const renderDataEntries = (entries) => ( + entries.map(({ label, value }, index) => ( +
+ {label}: {value} +
+ )) + ); + + // Render each data entry within its chunk for two-column layout + return ( +
+ {chunks.map((chunk, index) => ( + + {renderDataEntries(chunk)} + + ))} +
+ ); + } + +DataDisplay.propTypes = { + data: PropTypes.object.isRequired, + path: PropTypes.string.isRequired, + excludeKeys: PropTypes.array +}; + +export default DataDisplay; \ No newline at end of file diff --git a/client/src/components/_custom/drp/utils/DataDisplay/DataDisplay.module.scss b/client/src/components/_custom/drp/utils/DataDisplay/DataDisplay.module.scss new file mode 100644 index 0000000000..fea9e890f0 --- /dev/null +++ b/client/src/components/_custom/drp/utils/DataDisplay/DataDisplay.module.scss @@ -0,0 +1,13 @@ +.metadata-section main { + padding-left: 0; +} + +.dataset-link { + composes: c-button from '../../../../../styles/components/c-button--new.css'; + composes: c-button--size-small from '../../../../../styles/components/c-button--new.css'; + composes: c-button__text from '../../../../../styles/components/c-button--new.css'; + composes: c-button--as-link from '../../../../../styles/components/c-button--new.css'; + font-size: 14px; + padding: 0; + margin-bottom: -3px; + } \ No newline at end of file diff --git a/server/portal/settings/settings_default.py b/server/portal/settings/settings_default.py index 0464d72b62..4bc4c9253f 100644 --- a/server/portal/settings/settings_default.py +++ b/server/portal/settings/settings_default.py @@ -190,7 +190,9 @@ "hasUserGuide": True, "hasCustomSagas": True, "hasCustomEndpoints": True, - "addons": ['DataFilesProjectFileListingAddon', 'DataFilesAddProjectModalAddon', 'DataFilesProjectEditDescriptionModalAddon', 'DataFilesProjectFileListingMetadataAddon', 'DataFilesProjectFileListingMetadataTitleAddon'], + "addons": ['DataFilesProjectFileListingAddon', 'DataFilesAddProjectModalAddon', + 'DataFilesProjectEditDescriptionModalAddon', 'DataFilesProjectFileListingMetadataAddon', + 'DataFilesProjectFileListingMetadataTitleAddon'], "allowedFileActions": ['trash', 'download'], "showDataFileType": True, "onboardingCompleteRedirect": '/workbench/', From 17d0b3511b8b5b8d80423f57df217b69877d2701 Mon Sep 17 00:00:00 2001 From: Shayan Khan Date: Thu, 9 May 2024 11:17:05 -0500 Subject: [PATCH 060/328] change project created date display format --- .../DataFilesProjectFileListingMetadataAddon.jsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx index 4ae6cb7098..9e5d6819e5 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx @@ -12,8 +12,11 @@ const DataFilesProjectFileListingMetadataAddon = ({ folderMetadata, metadata, pa const { loading } = useFileListing('FilesListing'); const getProjectMetadata = (metadata) => { + + const dateOptions = { month: 'long', day: 'numeric', year: 'numeric' } + const formattedMetadata = { - created: formatDate(new Date(metadata.created)), + created: new Date(metadata.created).toLocaleDateString('en-US', dateOptions), license: metadata.license ?? 'None', } From ec9943e718461dbfb70cfc41140630910d5a38d3 Mon Sep 17 00:00:00 2001 From: Shayan Khan Date: Thu, 9 May 2024 13:37:21 -0500 Subject: [PATCH 061/328] Metadata in the Preview Modal uses Expand component now --- .../DataFiles/DataFilesModals/DataFilesPreviewModal.jsx | 2 +- .../DataFilesPreviewModalAddon.jsx | 9 ++++++--- .../DataFilesPreviewModalAddon.module.scss | 6 +++--- server/portal/settings/settings_default.py | 4 +++- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesPreviewModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesPreviewModal.jsx index db79109586..b2b0dd79c4 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesPreviewModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesPreviewModal.jsx @@ -89,6 +89,7 @@ const DataFilesPreviewModal = () => { File Preview: {params.name} + {DataFilesPreviewModalAddon && params.metadata && !isLoading && } {(isLoading || (previewUsingHref && isFrameLoading)) && (
@@ -130,7 +131,6 @@ const DataFilesPreviewModal = () => {
)} - {DataFilesPreviewModalAddon && params.metadata && !isLoading && } ); diff --git a/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.jsx b/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.jsx index 035a8bfcd4..08756c8568 100644 --- a/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.jsx @@ -3,7 +3,7 @@ import { useQuery } from 'react-query'; import { fetchUtil } from 'utils/fetchUtil'; import { DynamicForm } from '_common/Form/DynamicForm'; import { Form, Formik } from 'formik'; -import { LoadingSpinner, Section, SectionHeader } from '_common'; +import { Expand, LoadingSpinner, Section, SectionHeader } from '_common'; import styles from './DataFilesPreviewModalAddon.module.scss'; import { useFileListing } from 'hooks/datafiles'; import { useDispatch, useSelector } from 'react-redux'; @@ -68,9 +68,11 @@ const DataFilesPreviewModalAddon = ({ metadata }) => { return ( !isLoading && form && <> +
- Metadata
{ )} - + } + /> ) diff --git a/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.module.scss b/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.module.scss index 80aaf5bbcb..c804ae2b43 100644 --- a/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.module.scss +++ b/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.module.scss @@ -1,6 +1,6 @@ -.section { - border: 1px solid black; - padding: 20px 0px; +.section main { + padding: 0; + margin: 0; } .listing-header { diff --git a/server/portal/settings/settings_default.py b/server/portal/settings/settings_default.py index df0957b0b0..8b5736e608 100644 --- a/server/portal/settings/settings_default.py +++ b/server/portal/settings/settings_default.py @@ -190,7 +190,9 @@ "hasUserGuide": True, "hasCustomSagas": True, "hasCustomEndpoints": True, - "addons": ['DataFilesProjectFileListingAddon', 'DataFilesAddProjectModalAddon', 'DataFilesProjectEditDescriptionModalAddon', 'DataFilesProjectFileListingMetadataAddon', 'DataFilesProjectFileListingMetadataTitleAddon', 'DataFilesUploadModalAddon', 'DataFilesPreviewModalAddon'], + "addons": ['DataFilesProjectFileListingAddon', 'DataFilesAddProjectModalAddon', 'DataFilesProjectEditDescriptionModalAddon', + 'DataFilesProjectFileListingMetadataAddon', 'DataFilesProjectFileListingMetadataTitleAddon', + 'DataFilesUploadModalAddon', 'DataFilesPreviewModalAddon'], "allowedFileActions": ['trash', 'download'], "showDataFileType": True, "onboardingCompleteRedirect": '/workbench/', From e1a8fc4a0eea9885ca33bca43a891bb0e1507741 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Mon, 13 May 2024 16:10:53 -0500 Subject: [PATCH 062/328] small fix for preview modal addon --- .../DataFilesPreviewModalAddon.jsx | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.jsx b/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.jsx index 08756c8568..cc8b50dc6a 100644 --- a/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.jsx @@ -42,10 +42,16 @@ const DataFilesPreviewModalAddon = ({ metadata }) => { return acc; }, {}); - const reloadPage = (updatedPath = "") => { - // using regex to replace last segment of pathname with optional parameter 'updatedPath' to navigate to a new path - const path = updatedPath ? location.pathname.replace(/\/[^\/]+\/?$/, '/' + updatedPath) : location.pathname; - history.push(path); + const reloadPage = (updatedPath = '') => { + // using regex to get the url up until the project name + let projectUrl = location.pathname.replace(/(\/projects\/[^/]+\/).*/, '$1'); + + if (projectUrl.endsWith('/')) { + projectUrl = projectUrl.slice(0, -1); + } + + const path = updatedPath ? `${projectUrl}/${updatedPath}` : `${projectUrl}`; + history.push(path); }; const handleSubmit = (values) => { From 2df3f85f64c047c2e9ae1630ff09a7e9f352b01d Mon Sep 17 00:00:00 2001 From: Shayan Khan Date: Wed, 15 May 2024 14:01:06 -0500 Subject: [PATCH 063/328] Added Wizard component from tup ui and related dependancies --- client/package-lock.json | 15 ++ client/package.json | 1 + .../_common/Wizard/Wizard.module.css | 45 ++++ .../src/components/_common/Wizard/Wizard.tsx | 194 ++++++++++++++++++ client/src/components/_common/Wizard/index.ts | 15 ++ 5 files changed, 270 insertions(+) create mode 100644 client/src/components/_common/Wizard/Wizard.module.css create mode 100644 client/src/components/_common/Wizard/Wizard.tsx create mode 100644 client/src/components/_common/Wizard/index.ts diff --git a/client/package-lock.json b/client/package-lock.json index 64adf42138..0e4a421924 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -29,6 +29,7 @@ "react-redux": "^7.2.5", "react-resize-detector": "^6.1.0", "react-router-dom": "^5.3.0", + "react-step-wizard": "^5.3.11", "react-table": "^7.0.5", "react-virtualized-auto-sizer": "^1.0.6", "react-window": "^1.8.6", @@ -13267,6 +13268,14 @@ "react": "^16.0.0 || ^17.0.0" } }, + "node_modules/react-step-wizard": { + "version": "5.3.11", + "resolved": "https://registry.npmjs.org/react-step-wizard/-/react-step-wizard-5.3.11.tgz", + "integrity": "sha512-TD3ocUdt4XWvisTNEiATh+cFuh1RK0LgvYqOResTIhLtFmdWnXzFVaNIZJ3qzW5Bm6yeotzZ4KAmRjmsqBXnWQ==", + "peerDependencies": { + "react": ">=15.0.1" + } + }, "node_modules/react-table": { "version": "7.7.0", "resolved": "https://registry.npmjs.org/react-table/-/react-table-7.7.0.tgz", @@ -24801,6 +24810,12 @@ "react-is": "^16.12.0 || ^17.0.0" } }, + "react-step-wizard": { + "version": "5.3.11", + "resolved": "https://registry.npmjs.org/react-step-wizard/-/react-step-wizard-5.3.11.tgz", + "integrity": "sha512-TD3ocUdt4XWvisTNEiATh+cFuh1RK0LgvYqOResTIhLtFmdWnXzFVaNIZJ3qzW5Bm6yeotzZ4KAmRjmsqBXnWQ==", + "requires": {} + }, "react-table": { "version": "7.7.0", "resolved": "https://registry.npmjs.org/react-table/-/react-table-7.7.0.tgz", diff --git a/client/package.json b/client/package.json index ffa7a39144..167e21536c 100644 --- a/client/package.json +++ b/client/package.json @@ -24,6 +24,7 @@ "react-redux": "^7.2.5", "react-resize-detector": "^6.1.0", "react-router-dom": "^5.3.0", + "react-step-wizard": "^5.3.11", "react-table": "^7.0.5", "react-virtualized-auto-sizer": "^1.0.6", "react-window": "^1.8.6", diff --git a/client/src/components/_common/Wizard/Wizard.module.css b/client/src/components/_common/Wizard/Wizard.module.css new file mode 100644 index 0000000000..3b63c16e6a --- /dev/null +++ b/client/src/components/_common/Wizard/Wizard.module.css @@ -0,0 +1,45 @@ +.container { + display: flex; +} + +.container > * { + flex: 50%; + background-color: var(--global-color-primary--x-light); + border: 1px solid var(--global-color-primary--light); + margin-right: 1em; + padding: 1em 1em 1em 1em; +} + +.step-summary { + padding-bottom: 1em; +} +.step-summary > .name { + display: flex; + justify-content: space-between; +} +.step-summary > .content { + font-size: smaller; + padding-left: 1em; +} +.step-summary > .edit { + padding: 0 0 0 0 !important; +} + +.step { + padding-top: 1em; +} + +.controls { + display: flex; + justify-content: flex-end; +} + +.controls > * { + margin-left: 0.25em; +} + +.submit { + border-bottom: 1px solid var(--global-color-primary--normal); + padding-bottom: 0.5em; + margin-bottom: 0.5em; +} diff --git a/client/src/components/_common/Wizard/Wizard.tsx b/client/src/components/_common/Wizard/Wizard.tsx new file mode 100644 index 0000000000..4ba2be1dc6 --- /dev/null +++ b/client/src/components/_common/Wizard/Wizard.tsx @@ -0,0 +1,194 @@ +import React, { useState, useContext, useCallback, useEffect } from 'react'; +import StepWizard, { StepWizardChildProps } from 'react-step-wizard'; +import { Button } from '_common'; +import { WizardStep } from '.'; +import { Formik, Form, useFormikContext } from 'formik'; +import styles from './Wizard.module.css'; + +export type WizardContextType = Partial; + +const WizardContext: React.Context = + React.createContext({}); + +export const useWizard = () => { + const props = useContext(WizardContext); + return props; +}; + +export const WizardNavigation: React.FC = () => { + const { currentStep, previousStep, totalSteps, nextStep, goToStep } = + useWizard(); + const { validateForm, handleSubmit } = useFormikContext(); + const onContinue = useCallback(async () => { + try { + const errors = await validateForm(); + if (!Object.keys(errors).length) { + handleSubmit && handleSubmit(); + nextStep && nextStep(); + } + } catch (e) { + console.error(e); + } + }, [validateForm, nextStep, handleSubmit]); + const onSkip = useCallback(async () => { + try { + const errors = await validateForm(); + if (!Object.keys(errors).length && goToStep && !!totalSteps) { + // Skip to End button doesn't appear to trigger handleSubmit, + // so it must be called explicitly + handleSubmit(); + goToStep(totalSteps); + } + } catch (e) { + console.error(e); + } + }, [validateForm, handleSubmit, goToStep, totalSteps]); + return ( +
+ {!!currentStep && currentStep > 1 && ( + + )} + {!!currentStep && !!totalSteps && currentStep < totalSteps && ( + <> + + + + )} +
+ ); +}; + +type WizardControlProps = { + steps: Array>; +} & Partial; + +function WizardSummary({ + steps, + ...stepWizardProps +}: WizardControlProps) { + const { goToNamedStep } = stepWizardProps; + const editCallback = useCallback( + (stepId: string) => goToNamedStep && goToNamedStep(stepId), + [goToNamedStep] + ); + return ( +
+

Summary

+ {steps.map((step) => ( +
+
+ {step.name} + +
+
{step.summary}
+
+ ))} +
+ ); +} + +type StepContainerProps = { + step: WizardStep; + formSubmit: (values: Partial) => void; +} & Partial; + +function StepContainer({ step, formSubmit }: StepContainerProps) { + const { validationSchema, initialValues, validate } = step; + return ( + +
+
+ {step.render} + +
+
+ + ); +} + +type WizardProps = { + steps: Array>; + memo?: any; + formSubmit: (values: Partial) => void; +}; + +function Wizard({ steps, memo, formSubmit }: WizardProps) { + const [stepWizardProps, setStepWizardProps] = useState< + Partial + >({}); + + const instanceCallback = useCallback( + (props: Partial) => { + setStepWizardProps({ + currentStep: 1, + totalSteps: steps.length, + ...props, + }); + }, + [setStepWizardProps, steps] + ); + + const stepChangeCallback = useCallback( + ({ activeStep }: { previousStep: number; activeStep: number }) => { + setStepWizardProps({ + ...stepWizardProps, + currentStep: activeStep, + }); + }, + [setStepWizardProps, stepWizardProps] + ); + + const { goToStep } = stepWizardProps; + + useEffect( + () => { + goToStep && goToStep(1); + }, + /* eslint-disable-next-line */ + [memo] + ); + + return ( + +
+ + {steps.map((step) => ( + + step={step} + key={`wizard-step-${step.id}`} + stepName={step.id} + formSubmit={formSubmit} + /> + ))} + + +
+ + ); +} + +export default Wizard; diff --git a/client/src/components/_common/Wizard/index.ts b/client/src/components/_common/Wizard/index.ts new file mode 100644 index 0000000000..7ad08920af --- /dev/null +++ b/client/src/components/_common/Wizard/index.ts @@ -0,0 +1,15 @@ +import React from 'react'; +import Wizard from './Wizard'; + +export type WizardStep = { + id: string; + name: string; + render: React.ReactNode; + summary: React.ReactNode; + validationSchema: any; + initialValues: Partial; + validate?: (values: Partial) => void; +}; + +export { useWizard, WizardNavigation } from './Wizard'; +export default Wizard; From 5081f8f16ad0b67735ba6fc1c8a9eba459c024d9 Mon Sep 17 00:00:00 2001 From: Shayan Khan Date: Thu, 16 May 2024 15:13:21 -0500 Subject: [PATCH 064/328] add endpoint to get drp tree view --- server/portal/apps/_custom/drp/views.py | 31 ++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/server/portal/apps/_custom/drp/views.py b/server/portal/apps/_custom/drp/views.py index 565857a8d4..9bb079b214 100644 --- a/server/portal/apps/_custom/drp/views.py +++ b/server/portal/apps/_custom/drp/views.py @@ -1,3 +1,4 @@ +import json from portal.views.base import BaseApiView from django.conf import settings from django.http import JsonResponse, HttpResponseForbidden @@ -22,4 +23,32 @@ def get(self, request): 'origin_data': list(origin_data) } - return JsonResponse(response_data) \ No newline at end of file + return JsonResponse(response_data) + +class DigitalRocksTreeView(BaseApiView): + + @staticmethod + def construct_tree(records, parent_id=None): + tree = {} + for record in records: + if record['parent'] == parent_id: + node_dict = { + "id": record['id'], + "name": record['name'], + "path": record['path'], + "metadata": record['metadata'], + "children": DigitalRocksTreeView.construct_tree(records, record['id']) + } + tree[record['id']] = node_dict + return [tree[node_id] for node_id in tree] + + def get(self, request): + + project_id = request.GET.get('project_id') + full_project_id = f'{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}' + + records = DataFilesMetadata.objects.filter(project_id=full_project_id).values('id', 'parent', 'name', 'path', 'metadata') + + tree = self.construct_tree(records) + + return JsonResponse(tree, safe=False) \ No newline at end of file From f188c892a9f938a34616f1779db047a9841fe13d Mon Sep 17 00:00:00 2001 From: Shayan Khan Date: Thu, 16 May 2024 15:13:44 -0500 Subject: [PATCH 065/328] add url pattern for new drp endpoint --- server/portal/apps/_custom/drp/urls.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/portal/apps/_custom/drp/urls.py b/server/portal/apps/_custom/drp/urls.py index a409819c5a..ef138f81df 100644 --- a/server/portal/apps/_custom/drp/urls.py +++ b/server/portal/apps/_custom/drp/urls.py @@ -3,9 +3,10 @@ :synopsis: Forms URLs """ from django.urls import re_path -from portal.apps._custom.drp.views import DigitalRocksSampleView +from portal.apps._custom.drp.views import DigitalRocksSampleView, DigitalRocksTreeView app_name = 'custom' urlpatterns = [ - re_path('', DigitalRocksSampleView.as_view(), name='drp'), + re_path('^$', DigitalRocksSampleView.as_view(), name='drp'), + re_path('tree/', DigitalRocksTreeView.as_view(), name='tree') ] From 570a85cb308281d67c04a79db247aae14ee47c2e Mon Sep 17 00:00:00 2001 From: Shayan Khan Date: Thu, 16 May 2024 16:45:24 -0500 Subject: [PATCH 066/328] Added initial rough publication flow using Wizard and MUI Tree --- client/package-lock.json | 52 ++- client/package.json | 1 + client/src/components/DataFiles/DataFiles.jsx | 24 +- .../DataFilesProjectFileListing.jsx | 2 +- .../src/components/_common/Wizard/Wizard.tsx | 2 +- .../DataFilesProjectFileListingAddon.jsx | 11 +- ...taFilesProjectFileListingAddon.module.scss | 5 + .../DataFilesProjectPublish.jsx | 324 ++++++++++++++++++ .../DataFilesProjectPublish.module.scss | 50 +++ .../drp/utils/DataDisplay/DataDisplay.jsx | 5 +- server/portal/settings/settings_default.py | 2 +- 11 files changed, 463 insertions(+), 15 deletions(-) create mode 100644 client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx create mode 100644 client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.module.scss diff --git a/client/package-lock.json b/client/package-lock.json index 0e4a421924..c51e1968e7 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -9,6 +9,7 @@ "version": "2.0.0", "dependencies": { "@material-ui/core": "^4.12.3", + "@material-ui/lab": "^4.0.0-alpha.61", "@niivue/niivue": "^0.32.0", "bowser": "^2.9.0", "cross-fetch": "^3.1.4", @@ -3156,6 +3157,33 @@ } } }, + "node_modules/@material-ui/lab": { + "version": "4.0.0-alpha.61", + "resolved": "https://registry.npmjs.org/@material-ui/lab/-/lab-4.0.0-alpha.61.tgz", + "integrity": "sha512-rSzm+XKiNUjKegj8bzt5+pygZeckNLOr+IjykH8sYdVk7dE9y2ZuUSofiMV2bJk3qU+JHwexmw+q0RyNZB9ugg==", + "deprecated": "Material UI v4 doesn't receive active development since September 2021. See the guide https://mui.com/material-ui/migration/migration-v4/ to upgrade to v5.", + "dependencies": { + "@babel/runtime": "^7.4.4", + "@material-ui/utils": "^4.11.3", + "clsx": "^1.0.4", + "prop-types": "^15.7.2", + "react-is": "^16.8.0 || ^17.0.0" + }, + "engines": { + "node": ">=8.0.0" + }, + "peerDependencies": { + "@material-ui/core": "^4.12.1", + "@types/react": "^16.8.6 || ^17.0.0", + "react": "^16.8.0 || ^17.0.0", + "react-dom": "^16.8.0 || ^17.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, "node_modules/@material-ui/styles": { "version": "4.11.4", "resolved": "https://registry.npmjs.org/@material-ui/styles/-/styles-4.11.4.tgz", @@ -3240,9 +3268,9 @@ } }, "node_modules/@material-ui/utils": { - "version": "4.11.2", - "resolved": "https://registry.npmjs.org/@material-ui/utils/-/utils-4.11.2.tgz", - "integrity": "sha512-Uul8w38u+PICe2Fg2pDKCaIG7kOyhowZ9vjiC1FsVwPABTW8vPPKfF6OvxRq3IiBaI1faOJmgdvMG7rMJARBhA==", + "version": "4.11.3", + "resolved": "https://registry.npmjs.org/@material-ui/utils/-/utils-4.11.3.tgz", + "integrity": "sha512-ZuQPV4rBK/V1j2dIkSSEcH5uT6AaHuKWFfotADHsC0wVL1NLd2WkFCm4ZZbX33iO4ydl6V0GPngKm8HZQ2oujg==", "dependencies": { "@babel/runtime": "^7.4.4", "prop-types": "^15.7.2", @@ -17460,6 +17488,18 @@ "react-transition-group": "^4.4.0" } }, + "@material-ui/lab": { + "version": "4.0.0-alpha.61", + "resolved": "https://registry.npmjs.org/@material-ui/lab/-/lab-4.0.0-alpha.61.tgz", + "integrity": "sha512-rSzm+XKiNUjKegj8bzt5+pygZeckNLOr+IjykH8sYdVk7dE9y2ZuUSofiMV2bJk3qU+JHwexmw+q0RyNZB9ugg==", + "requires": { + "@babel/runtime": "^7.4.4", + "@material-ui/utils": "^4.11.3", + "clsx": "^1.0.4", + "prop-types": "^15.7.2", + "react-is": "^16.8.0 || ^17.0.0" + } + }, "@material-ui/styles": { "version": "4.11.4", "resolved": "https://registry.npmjs.org/@material-ui/styles/-/styles-4.11.4.tgz", @@ -17501,9 +17541,9 @@ "requires": {} }, "@material-ui/utils": { - "version": "4.11.2", - "resolved": "https://registry.npmjs.org/@material-ui/utils/-/utils-4.11.2.tgz", - "integrity": "sha512-Uul8w38u+PICe2Fg2pDKCaIG7kOyhowZ9vjiC1FsVwPABTW8vPPKfF6OvxRq3IiBaI1faOJmgdvMG7rMJARBhA==", + "version": "4.11.3", + "resolved": "https://registry.npmjs.org/@material-ui/utils/-/utils-4.11.3.tgz", + "integrity": "sha512-ZuQPV4rBK/V1j2dIkSSEcH5uT6AaHuKWFfotADHsC0wVL1NLd2WkFCm4ZZbX33iO4ydl6V0GPngKm8HZQ2oujg==", "requires": { "@babel/runtime": "^7.4.4", "prop-types": "^15.7.2", diff --git a/client/package.json b/client/package.json index 167e21536c..d2fbd51a59 100644 --- a/client/package.json +++ b/client/package.json @@ -4,6 +4,7 @@ "private": true, "dependencies": { "@material-ui/core": "^4.12.3", + "@material-ui/lab": "^4.0.0-alpha.61", "@niivue/niivue": "^0.32.0", "bowser": "^2.9.0", "cross-fetch": "^3.1.4", diff --git a/client/src/components/DataFiles/DataFiles.jsx b/client/src/components/DataFiles/DataFiles.jsx index 7d67a05968..19df9d00f1 100644 --- a/client/src/components/DataFiles/DataFiles.jsx +++ b/client/src/components/DataFiles/DataFiles.jsx @@ -11,7 +11,7 @@ import { SectionMessage, LoadingSpinner, } from '_common'; -import { useFileListing, useSystems } from 'hooks/datafiles'; +import { useFileListing, useSystems, useAddonComponents } from 'hooks/datafiles'; import DataFilesToolbar from './DataFilesToolbar/DataFilesToolbar'; import DataFilesListing from './DataFilesListing/DataFilesListing'; import DataFilesSidebar from './DataFilesSidebar/DataFilesSidebar'; @@ -43,8 +43,30 @@ const DefaultSystemRedirect = () => { const DataFilesSwitch = React.memo(() => { const { path } = useRouteMatch(); + + const portalName = useSelector((state) => state.workbench.portalName); + + console.log('portalName', portalName) + + const { DataFilesProjectPublish } = useAddonComponents({ portalName }); + + console.log('DataFilesProjectPublish', DataFilesProjectPublish) + return ( + {DataFilesProjectPublish && + { + return ( + + + + ) + + }} + /> + } { diff --git a/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx b/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx index d3a0a4ec33..f2733450bf 100644 --- a/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx +++ b/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx @@ -125,7 +125,7 @@ const DataFilesProjectFileListing = ({ system, path }) => { {readOnlyTeam ? 'View' : 'Manage'} Team {DataFilesProjectFileListingAddon && ( - + )} } diff --git a/client/src/components/_common/Wizard/Wizard.tsx b/client/src/components/_common/Wizard/Wizard.tsx index 4ba2be1dc6..e8a5e5a3ab 100644 --- a/client/src/components/_common/Wizard/Wizard.tsx +++ b/client/src/components/_common/Wizard/Wizard.tsx @@ -185,7 +185,7 @@ function Wizard({ steps, memo, formSubmit }: WizardProps) { /> ))} - + {/* */} ); diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx index 95d71834cd..ecbc6df70a 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx @@ -3,11 +3,11 @@ import { Button } from '_common'; import { useDispatch, useSelector } from 'react-redux'; import styles from './DataFilesProjectFileListingAddon.module.scss'; import { useSelectedFiles } from 'hooks/datafiles'; -import { useQuery } from 'react-query'; -import { fetchUtil } from 'utils/fetchUtil'; import { createSampleModalHandler, createOriginDataModalHandler, createAnalysisDataModalHandler } from '../utils/datasetFormHandlers'; +import * as ROUTES from '../../../../constants/routes'; +import { Link } from 'react-router-dom'; -const DataFilesProjectFileListingAddon = () => { +const DataFilesProjectFileListingAddon = ({ system }) => { const dispatch = useDispatch(); const portalName = useSelector((state) => state.workbench.portalName); const { projectId } = useSelector((state) => state.projects.metadata); @@ -71,6 +71,11 @@ const DataFilesProjectFileListingAddon = () => { Add Analysis Dataset )} + | + + Request Publication + + ); }; diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.module.scss b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.module.scss index f834d322cd..c2c89cd3c5 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.module.scss +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.module.scss @@ -3,3 +3,8 @@ padding-left: 0.25em; padding-right: 0.25em; } + +.link { + font-weight: 500; + font-size: 12px; +} diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx new file mode 100644 index 0000000000..b4e03a10c6 --- /dev/null +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx @@ -0,0 +1,324 @@ +import React, { useEffect, useState } from 'react'; +import { + Button, + ShowMore, + LoadingSpinner, + SectionMessage, + SectionTableWrapper, + DescriptionList, + Section, + SectionContent, + Expand + } from '_common'; + import { + Link, useRouteMatch + } from 'react-router-dom'; +import { useDispatch, useSelector } from 'react-redux'; +import Wizard from '_common/Wizard'; +import styles from './DataFilesProjectPublish.module.scss' +import { fetchUtil } from 'utils/fetchUtil'; +import { TreeItem, TreeView } from '@material-ui/lab'; +import Icon from '_common/Icon'; +import * as ROUTES from '../../../../constants/routes'; +import { createAnalysisDataModalHandler, createOriginDataModalHandler, createSampleModalHandler } from '../utils/datasetFormHandlers'; +import DataDisplay from '../utils/DataDisplay/DataDisplay'; + + +const DataFilesProjectPublish = ({ system, path, api, scheme }) => { + + const portalName = 'drp'; + const projectId = 'CEPV3-DEV-1133'; + + const getTree = async (projectId, portalName) => { + const response = await fetchUtil({ + url: `api/${portalName.toLowerCase()}/tree`, + params: { + project_id: projectId, + }, + }); + + return response; + } + + const [tree, setTree] = useState([]); + const [expandedNodes, setExpandedNodes] = useState([]); + + const handleNodeToggle = (event, nodeIds) => { + // Update the list of expanded nodes + setExpandedNodes(nodeIds); + }; + + useEffect(async () => { + const response = await getTree(projectId, portalName); + setTree(response); + }, [projectId, portalName]) + + + const dispatch = useDispatch(); + + useEffect(() => { + dispatch({ + type: 'PROJECTS_GET_METADATA', + payload: system, + }); + }, [system]); + + const metadata = useSelector((state) => state.projects.metadata); + + const formSubmit = (values) => { + console.log("DataFilesProjectPublish: formSubmit: ", values); + } + + const onEdit = () => { + dispatch({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { operation: 'editproject', props: {} }, + }); + }; + + const handleSampleModal = createSampleModalHandler(dispatch); + const handleOriginDataModal = createOriginDataModalHandler(dispatch, projectId, portalName); + const handleAnalysisDataModal = createAnalysisDataModalHandler(dispatch, projectId, portalName); + + const onEditData = (node) => { + + const dataType = node.metadata.data_type; + // reconstruct editFile to mimic SelectedFile object + const editFile = { + "format": "folder", + "id" : node.path, + "metadata": node.metadata, + "name": node.metadata.name, + "system": system, + "path": node.path, + "type": "dir", + "_links": { + "self": { + "href": "tapis://" + node.path, + }, + } + }; + switch (dataType) { + case 'sample': + handleSampleModal('EDIT_SAMPLE_DATA', editFile); + break; + case 'origin_data': + handleOriginDataModal('EDIT_ORIGIN_DATASET', editFile); + break; + case 'analysis_data': + handleAnalysisDataModal('EDIT_ANALYSIS_DATASET', editFile); + break; + case 'file': + // Dispatch an action to toggle the modal for previewing the file + dispatch({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { + operation: 'preview', + props: { + api, + scheme, + system, + path: node.path, + name: node.metadata.name, // Assuming 'name' is from node.metadata + href: "tapis://" + node.path, + length: node.metadata.length, // Assuming 'length' is from node.metadata + metadata: node.metadata, + }, + }, + }); + break; + default: + break; + } + } + + const renderTree = (node) => ( + <> +
+
+ + + node.name + + //
+ } + classes={{label: styles['tree-label'], root: styles['tree-root'], content: styles['tree-content']}} + onLabelClick={() => handleNodeToggle} + > + {expandedNodes.includes(node.id) && ( +
+ +
+ + {node.metadata.description} + + +
+
+ )} + {Array.isArray(node.children) && node.children.map((child) => ( + renderTree(child) + ))} + + + + +
+ + + ); + + const getAllNodeIds = (nodes) => { + const ids = []; + nodes.forEach(node => { + ids.push(node.id); + if (Array.isArray(node.children)) { + ids.push(...getAllNodeIds(node.children)); + } + }); + return ids; + }; + + + console.log("DataFilesProjectPublish: metadata: ", metadata); + + const steps = [ + { + id: 'publication_instructions', + name: 'Publication Instructions', + render: ( + + Instructions + + } + > +
+

+ You are requesting to publish this project. By publishing your project, + it will be available to anyone to view and download the project data and metadata. + Please note: once a project is published it is no longer editable! +

+

+ You will begin the process of reviewing your data publication. + This publication represents your unique research. You are the person that + best knows your data and how to present it to the public. The system will + help you through the process. Please complete the form below to begin the publication process. +

+

+ Before publication, please corroborate with the main author of the project who else + should be added as author of this publication and the order in which authors should be added. +

+
+ + ), + initialValues: {} + }, + { + id: 'project_description', + name: 'Project Description', + render: ( + + Proofread Project + + } + headerActions={ +
+ <> + + +
+ } + > + {metadata.description} , + }} + direction={'vertical'} + /> + + ), + initialValues: {} + }, + { + id: 'project_structure', + name: 'Review Project Structure', + render: ( + + Review Project Structure + + } + headerActions={ +
+ <> + + +
+ } + > + {tree && tree.length > 0 && + + } + defaultExpandIcon={} + // expanded={getAllNodeIds(tree)} + onNodeToggle={handleNodeToggle} + > + {tree.map((node) => renderTree(node))} + + } + + ), + initialValues: {} + } + ] + + + // do the wizard now + + return ( + + Request Project Publication | {metadata.title} + + } + headerActions={ +
+ <> + Back to Project + + +
+ + } + > + + + + ); + +} + +export default DataFilesProjectPublish; \ No newline at end of file diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.module.scss b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.module.scss new file mode 100644 index 0000000000..f28337e956 --- /dev/null +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.module.scss @@ -0,0 +1,50 @@ +@import '../../../../styles/tools/mixins.scss'; + + +.root { + /* As a flex child */ + flex-grow: 1; + + /* WARNING: Mimicked on: History, Allocation, DataFiles, DataFilesProjectsList, DataFilesProjectFileListing, PublicData */ + padding-top: 1.75rem; /* ~28px (22.5px * design * 1.2 design-to-app ratio) */ + padding-left: 1.5em; /* ~24px (20px * design * 1.2 design-to-app ratio) */ + } + + .prj main { + margin-right: 0; + margin-bottom: 10px; + + } + + .title { + @include truncate-with-ellipsis; + max-width: 1400px; + font-size: 18px; + } + + .controls { + display: flex; + align-items: flex-start; + } + + .tree-label { + padding-top: 10px; + padding-bottom: 10px; + border: 1px solid var(--global-color-primary--light); + // border-radius: 10px; + + &:hover { + background-color: white; + } + } + + .description-div { + border: 1px solid var(--global-color-primary--light); + // border-top-style: none; + margin-left: 2px; + padding: 10px; + } + + .description { + font-size: 14px; + } \ No newline at end of file diff --git a/client/src/components/_custom/drp/utils/DataDisplay/DataDisplay.jsx b/client/src/components/_custom/drp/utils/DataDisplay/DataDisplay.jsx index f8dfa3e998..d79cfedfc2 100644 --- a/client/src/components/_custom/drp/utils/DataDisplay/DataDisplay.jsx +++ b/client/src/components/_custom/drp/utils/DataDisplay/DataDisplay.jsx @@ -50,8 +50,9 @@ const DataDisplay = ({ data, path, excludeKeys }) => { value: typeof(value) === 'string' ? formatLabel(value) : value })); - - processedData.unshift(...processSampleAndOriginData(data, path)); + if (path) { + processedData.unshift(...processSampleAndOriginData(data, path)); + } // Divide processed data into chunks for two-column layout display const chunkSize = Math.ceil(processedData.length / 2); diff --git a/server/portal/settings/settings_default.py b/server/portal/settings/settings_default.py index 8b5736e608..12de594f79 100644 --- a/server/portal/settings/settings_default.py +++ b/server/portal/settings/settings_default.py @@ -192,7 +192,7 @@ "hasCustomEndpoints": True, "addons": ['DataFilesProjectFileListingAddon', 'DataFilesAddProjectModalAddon', 'DataFilesProjectEditDescriptionModalAddon', 'DataFilesProjectFileListingMetadataAddon', 'DataFilesProjectFileListingMetadataTitleAddon', - 'DataFilesUploadModalAddon', 'DataFilesPreviewModalAddon'], + 'DataFilesUploadModalAddon', 'DataFilesPreviewModalAddon', 'DataFilesProjectPublish'], "allowedFileActions": ['trash', 'download'], "showDataFileType": True, "onboardingCompleteRedirect": '/workbench/', From fe507129713c9b4cb3f0d51df2fab465cf350d35 Mon Sep 17 00:00:00 2001 From: Shayan Khan Date: Fri, 17 May 2024 11:44:44 -0500 Subject: [PATCH 067/328] Added initial component for reordering authors --- .../utils/ReorderUserList/ReorderUserList.jsx | 48 +++++++++++++++++++ .../ReorderUserList.module.scss | 17 +++++++ 2 files changed, 65 insertions(+) create mode 100644 client/src/components/_custom/drp/utils/ReorderUserList/ReorderUserList.jsx create mode 100644 client/src/components/_custom/drp/utils/ReorderUserList/ReorderUserList.module.scss diff --git a/client/src/components/_custom/drp/utils/ReorderUserList/ReorderUserList.jsx b/client/src/components/_custom/drp/utils/ReorderUserList/ReorderUserList.jsx new file mode 100644 index 0000000000..84f7cc9caf --- /dev/null +++ b/client/src/components/_custom/drp/utils/ReorderUserList/ReorderUserList.jsx @@ -0,0 +1,48 @@ +import React, { useEffect, useState } from 'react'; +import { Button, Icon } from '_common'; +import styles from './ReorderUserList.module.scss' + +const ReorderUserList = ({ users }) => { + + const [reorderedUsers, setReorderedUsers] = useState([]); + + useEffect(() => { + const formattedUsers = users.map(user => user.user); + setReorderedUsers(formattedUsers); + }, [users]); + + + const moveUp = (index) => { + const reordered = [...reorderedUsers]; + const temp = reordered[index - 1]; + reordered[index - 1] = reordered[index]; + reordered[index] = temp; + setReorderedUsers(reordered); + }; + + const moveDown = (index) => { + const reordered = [...reorderedUsers]; + const temp = reordered[index + 1]; + reordered[index + 1] = reordered[index]; + reordered[index] = temp; + setReorderedUsers(reordered); + } + + console.log('reorderedUsers', reorderedUsers) + + return ( + <> + {reorderedUsers.map((user, index) => ( +
+ {user.last_name}, {user.first_name} +
+
+
+ ))} + + ); +}; + +export default ReorderUserList; \ No newline at end of file diff --git a/client/src/components/_custom/drp/utils/ReorderUserList/ReorderUserList.module.scss b/client/src/components/_custom/drp/utils/ReorderUserList/ReorderUserList.module.scss new file mode 100644 index 0000000000..6333431bcd --- /dev/null +++ b/client/src/components/_custom/drp/utils/ReorderUserList/ReorderUserList.module.scss @@ -0,0 +1,17 @@ +.user-div { + display: flex; + align-items: center; + margin-bottom: 10px; /* Adjust spacing between rows as needed */ + justify-content: flex-start; + width: 30%; + } + + .user-name { + flex-grow: 1; /* Take up available space */ + margin-right: 10px; /* Space between name and buttons */ + } + + .button-group { + display: flex; + gap: 10px; /* Space between buttons */ + } \ No newline at end of file From 8a18ddccca456b36f920ea9542e0fe9b5d070280 Mon Sep 17 00:00:00 2001 From: Shayan Khan Date: Fri, 17 May 2024 11:45:25 -0500 Subject: [PATCH 068/328] Refactored and created separate components for each wizard step --- client/src/components/DataFiles/DataFiles.jsx | 4 - .../DataFilesProjectPublish.jsx | 429 +++++------------- .../DataFilesProjectPublish.module.scss | 6 +- .../DataFilesProjectPublishWizard.module.scss | 53 +++ .../ProjectDescription.jsx | 43 ++ .../PublicationInstructions.jsx | 35 ++ .../ReviewAuthors.jsx | 20 + .../ReviewProjectStructure.jsx | 217 +++++++++ 8 files changed, 491 insertions(+), 316 deletions(-) create mode 100644 client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss create mode 100644 client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx create mode 100644 client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/PublicationInstructions.jsx create mode 100644 client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx create mode 100644 client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx diff --git a/client/src/components/DataFiles/DataFiles.jsx b/client/src/components/DataFiles/DataFiles.jsx index 19df9d00f1..001f55ec22 100644 --- a/client/src/components/DataFiles/DataFiles.jsx +++ b/client/src/components/DataFiles/DataFiles.jsx @@ -46,12 +46,8 @@ const DataFilesSwitch = React.memo(() => { const portalName = useSelector((state) => state.workbench.portalName); - console.log('portalName', portalName) - const { DataFilesProjectPublish } = useAddonComponents({ portalName }); - console.log('DataFilesProjectPublish', DataFilesProjectPublish) - return ( {DataFilesProjectPublish && diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx index b4e03a10c6..f6474b201e 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx @@ -1,324 +1,131 @@ import React, { useEffect, useState } from 'react'; import { - Button, - ShowMore, - LoadingSpinner, - SectionMessage, - SectionTableWrapper, - DescriptionList, - Section, - SectionContent, - Expand - } from '_common'; - import { - Link, useRouteMatch - } from 'react-router-dom'; + Button, + ShowMore, + LoadingSpinner, + SectionMessage, + SectionTableWrapper, + DescriptionList, + Section, + SectionContent, + Expand, +} from '_common'; +import { Link, useRouteMatch } from 'react-router-dom'; import { useDispatch, useSelector } from 'react-redux'; import Wizard from '_common/Wizard'; -import styles from './DataFilesProjectPublish.module.scss' +import styles from './DataFilesProjectPublish.module.scss'; import { fetchUtil } from 'utils/fetchUtil'; import { TreeItem, TreeView } from '@material-ui/lab'; import Icon from '_common/Icon'; import * as ROUTES from '../../../../constants/routes'; -import { createAnalysisDataModalHandler, createOriginDataModalHandler, createSampleModalHandler } from '../utils/datasetFormHandlers'; +import { + createAnalysisDataModalHandler, + createOriginDataModalHandler, + createSampleModalHandler, +} from '../utils/datasetFormHandlers'; import DataDisplay from '../utils/DataDisplay/DataDisplay'; - +import ReorderUserList from '../utils/ReorderUserList/ReorderUserList'; +import ProjectDescription from './DataFilesProjectPublishWizardSteps/ProjectDescription'; +import PublicationInstructions from './DataFilesProjectPublishWizardSteps/PublicationInstructions'; +import ReviewProjectStructure from './DataFilesProjectPublishWizardSteps/ReviewProjectStructure'; +import ReviewAuthors from './DataFilesProjectPublishWizardSteps/ReviewAuthors'; const DataFilesProjectPublish = ({ system, path, api, scheme }) => { - - const portalName = 'drp'; - const projectId = 'CEPV3-DEV-1133'; - - const getTree = async (projectId, portalName) => { - const response = await fetchUtil({ - url: `api/${portalName.toLowerCase()}/tree`, - params: { - project_id: projectId, - }, - }); - - return response; - } - - const [tree, setTree] = useState([]); - const [expandedNodes, setExpandedNodes] = useState([]); - - const handleNodeToggle = (event, nodeIds) => { - // Update the list of expanded nodes - setExpandedNodes(nodeIds); - }; - - useEffect(async () => { - const response = await getTree(projectId, portalName); - setTree(response); - }, [projectId, portalName]) - - - const dispatch = useDispatch(); - - useEffect(() => { - dispatch({ - type: 'PROJECTS_GET_METADATA', - payload: system, - }); - }, [system]); - - const metadata = useSelector((state) => state.projects.metadata); - - const formSubmit = (values) => { - console.log("DataFilesProjectPublish: formSubmit: ", values); - } - - const onEdit = () => { - dispatch({ - type: 'DATA_FILES_TOGGLE_MODAL', - payload: { operation: 'editproject', props: {} }, - }); - }; - - const handleSampleModal = createSampleModalHandler(dispatch); - const handleOriginDataModal = createOriginDataModalHandler(dispatch, projectId, portalName); - const handleAnalysisDataModal = createAnalysisDataModalHandler(dispatch, projectId, portalName); - - const onEditData = (node) => { - - const dataType = node.metadata.data_type; - // reconstruct editFile to mimic SelectedFile object - const editFile = { - "format": "folder", - "id" : node.path, - "metadata": node.metadata, - "name": node.metadata.name, - "system": system, - "path": node.path, - "type": "dir", - "_links": { - "self": { - "href": "tapis://" + node.path, - }, - } - }; - switch (dataType) { - case 'sample': - handleSampleModal('EDIT_SAMPLE_DATA', editFile); - break; - case 'origin_data': - handleOriginDataModal('EDIT_ORIGIN_DATASET', editFile); - break; - case 'analysis_data': - handleAnalysisDataModal('EDIT_ANALYSIS_DATASET', editFile); - break; - case 'file': - // Dispatch an action to toggle the modal for previewing the file - dispatch({ - type: 'DATA_FILES_TOGGLE_MODAL', - payload: { - operation: 'preview', - props: { - api, - scheme, - system, - path: node.path, - name: node.metadata.name, // Assuming 'name' is from node.metadata - href: "tapis://" + node.path, - length: node.metadata.length, // Assuming 'length' is from node.metadata - metadata: node.metadata, - }, - }, - }); - break; - default: - break; - } + const portalName = 'drp'; + const projectId = 'CEPV3-DEV-1133'; + + const getTree = async (projectId, portalName) => { + const response = await fetchUtil({ + url: `api/${portalName.toLowerCase()}/tree`, + params: { + project_id: projectId, + }, + }); + + return response; + }; + + const [tree, setTree] = useState([]); + + useEffect(async () => { + const response = await getTree(projectId, portalName); + setTree(response); + }, [projectId, portalName]); + + const dispatch = useDispatch(); + + useEffect(() => { + dispatch({ + type: 'PROJECTS_GET_METADATA', + payload: system, + }); + }, [system]); + + const metadata = useSelector((state) => state.projects.metadata); + + const formSubmit = (values) => { + console.log('DataFilesProjectPublish: formSubmit: ', values); + }; + + console.log('DataFilesProjectPublish: metadata: ', metadata); + + const wizardSteps = [ + { + id: 'publication_instructions', + name: 'Publication Instructions', + render: , + initialValues: {}, + }, + { + id: 'project_description', + name: 'Project Description', + render: , + initialValues: {}, + }, + { + id: 'project_structure', + name: 'Review Project Structure', + render: ( + + ), + initialValues: {}, + }, + { + id: 'project_authors', + name: 'Review Authors and Citations', + render: , + initialValues: {}, + }, + ]; + + // do the wizard now + + return ( + + Request Project Publication | {metadata.title} + } + headerActions={ +
+ <> + + Back to Project + + +
+ } + > + + + ); +}; - const renderTree = (node) => ( - <> -
-
- - - node.name - - //
- } - classes={{label: styles['tree-label'], root: styles['tree-root'], content: styles['tree-content']}} - onLabelClick={() => handleNodeToggle} - > - {expandedNodes.includes(node.id) && ( -
- -
- - {node.metadata.description} - - -
-
- )} - {Array.isArray(node.children) && node.children.map((child) => ( - renderTree(child) - ))} - - - - -
- - - ); - - const getAllNodeIds = (nodes) => { - const ids = []; - nodes.forEach(node => { - ids.push(node.id); - if (Array.isArray(node.children)) { - ids.push(...getAllNodeIds(node.children)); - } - }); - return ids; - }; - - - console.log("DataFilesProjectPublish: metadata: ", metadata); - - const steps = [ - { - id: 'publication_instructions', - name: 'Publication Instructions', - render: ( - - Instructions - - } - > -
-

- You are requesting to publish this project. By publishing your project, - it will be available to anyone to view and download the project data and metadata. - Please note: once a project is published it is no longer editable! -

-

- You will begin the process of reviewing your data publication. - This publication represents your unique research. You are the person that - best knows your data and how to present it to the public. The system will - help you through the process. Please complete the form below to begin the publication process. -

-

- Before publication, please corroborate with the main author of the project who else - should be added as author of this publication and the order in which authors should be added. -

-
- - ), - initialValues: {} - }, - { - id: 'project_description', - name: 'Project Description', - render: ( - - Proofread Project - - } - headerActions={ -
- <> - - -
- } - > - {metadata.description} , - }} - direction={'vertical'} - /> - - ), - initialValues: {} - }, - { - id: 'project_structure', - name: 'Review Project Structure', - render: ( - - Review Project Structure - - } - headerActions={ -
- <> - - -
- } - > - {tree && tree.length > 0 && - - } - defaultExpandIcon={} - // expanded={getAllNodeIds(tree)} - onNodeToggle={handleNodeToggle} - > - {tree.map((node) => renderTree(node))} - - } - - ), - initialValues: {} - } - ] - - - // do the wizard now - - return ( - - Request Project Publication | {metadata.title} - - } - headerActions={ -
- <> - Back to Project - - -
- - } - > - - - - ); - -} - -export default DataFilesProjectPublish; \ No newline at end of file +export default DataFilesProjectPublish; diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.module.scss b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.module.scss index f28337e956..8fc8930c1b 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.module.scss +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.module.scss @@ -46,5 +46,9 @@ } .description { - font-size: 14px; + font-size: 0.875rem; + } + + .section main { + padding-left: 1px; } \ No newline at end of file diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss new file mode 100644 index 0000000000..95ef99785d --- /dev/null +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss @@ -0,0 +1,53 @@ +@import '../../../../../styles/tools/mixins.scss'; + +.root { + /* As a flex child */ + flex-grow: 1; + + /* WARNING: Mimicked on: History, Allocation, DataFiles, DataFilesProjectsList, DataFilesProjectFileListing, PublicData */ + padding-top: 1.75rem; /* ~28px (22.5px * design * 1.2 design-to-app ratio) */ + padding-left: 1.5em; /* ~24px (20px * design * 1.2 design-to-app ratio) */ + } + + .prj main { + margin-right: 0; + margin-bottom: 10px; + + } + + .title { + @include truncate-with-ellipsis; + max-width: 1400px; + font-size: 18px; + } + + .controls { + display: flex; + align-items: flex-start; + } + + .tree-label { + padding-top: 10px; + padding-bottom: 10px; + border: 1px solid var(--global-color-primary--light); + // border-radius: 10px; + + &:hover { + background-color: white; + } + } + + .description-div { + border: 1px solid var(--global-color-primary--light); + // border-top-style: none; + margin-left: 2px; + padding: 10px; + } + + .description { + font-size: 0.875rem; + } + + .section main { + padding-left: 1px; + } diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx new file mode 100644 index 0000000000..f7dd67b81b --- /dev/null +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx @@ -0,0 +1,43 @@ +import React, { useEffect, useState } from 'react'; +import { + Button, + ShowMore, + SectionTableWrapper, + DescriptionList, +} from '_common'; +import styles from './DataFilesProjectPublishWizard.module.scss'; + +const ProjectDescription = ({ project }) => { + const onEdit = () => { + dispatch({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { operation: 'editproject', props: {} }, + }); + }; + + return ( + Proofread Project} + headerActions={ +
+ <> + + +
+ } + > + {project.description} , + }} + direction={'vertical'} + /> + + ); +}; + +export default ProjectDescription; diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/PublicationInstructions.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/PublicationInstructions.jsx new file mode 100644 index 0000000000..3af92f3b84 --- /dev/null +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/PublicationInstructions.jsx @@ -0,0 +1,35 @@ +import React from 'react'; +import { SectionTableWrapper, Section } from '_common'; +import styles from './DataFilesProjectPublishWizard.module.scss'; + +const PublicationInstructions = () => { + return ( + Publication Instructions} + > +
+

+ You are requesting to publish this project. By publishing your + project, it will be available to anyone to view and download the + project data and metadata. + Please note: once a project is published it is no longer + editable! +

+

+ You will begin the process of reviewing your data publication. This + publication represents your unique research. You are the person that + best knows your data and how to present it to the public. The system + will help you through the process. Please complete the form below to + begin the publication process. +

+

+ Before publication, please corroborate with the main author of the + project who else should be added as author of this publication and the + order in which authors should be added. +

+
+ + ); +}; + +export default PublicationInstructions; diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx new file mode 100644 index 0000000000..a29574df52 --- /dev/null +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx @@ -0,0 +1,20 @@ +import React, { useEffect, useState } from 'react'; +import { + Button, + ShowMore, + LoadingSpinner, + SectionMessage, + SectionTableWrapper, + DescriptionList, + Section, + SectionContent, + Expand, +} from '_common'; +import styles from './DataFilesProjectPublishWizard.module.scss'; +import ReorderUserList from '../../utils/ReorderUserList/ReorderUserList'; + +const ReviewAuthors = ({ project }) => { + return ; +}; + +export default ReviewAuthors; diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx new file mode 100644 index 0000000000..a9f3192454 --- /dev/null +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx @@ -0,0 +1,217 @@ +import React, { useEffect, useState } from 'react'; +import { + Button, + ShowMore, + LoadingSpinner, + SectionMessage, + SectionTableWrapper, + DescriptionList, + Section, + SectionContent, + Expand, + Icon, +} from '_common'; +import { TreeItem, TreeView } from '@material-ui/lab'; +import styles from './DataFilesProjectPublishWizard.module.scss'; +import DataDisplay from '../../utils/DataDisplay/DataDisplay'; +import { + createSampleModalHandler, + createOriginDataModalHandler, + createAnalysisDataModalHandler, +} from '../../utils/datasetFormHandlers'; +import { useDispatch, useSelector } from 'react-redux'; +import { useFileListing } from 'hooks/datafiles'; + +const ReviewProjectStructure = ({ projectTree }) => { + const dispatch = useDispatch(); + + const [expandedNodes, setExpandedNodes] = useState([]); + + const portalName = useSelector((state) => state.workbench.portalName); + const { projectId } = useSelector((state) => state.projects.metadata); + + const { params } = useFileListing('FilesListing'); + + console.log("PARAMS", params) + + const handleNodeToggle = (event, nodeIds) => { + // Update the list of expanded nodes + setExpandedNodes(nodeIds); + }; + + const handleSampleModal = createSampleModalHandler(dispatch); + const handleOriginDataModal = createOriginDataModalHandler( + dispatch, + projectId, + portalName + ); + const handleAnalysisDataModal = createAnalysisDataModalHandler( + dispatch, + projectId, + portalName + ); + + const onEdit = () => { + dispatch({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { operation: 'editproject', props: {} }, + }); + }; + + const onEditData = (node) => { + const dataType = node.metadata.data_type; + // reconstruct editFile to mimic SelectedFile object + const editFile = { + format: 'folder', + id: node.path, + metadata: node.metadata, + name: node.metadata.name, + system: params.system, + path: node.path, + type: 'dir', + _links: { + self: { + href: 'tapis://' + node.path, + }, + }, + }; + switch (dataType) { + case 'sample': + handleSampleModal('EDIT_SAMPLE_DATA', editFile); + break; + case 'origin_data': + handleOriginDataModal('EDIT_ORIGIN_DATASET', editFile); + break; + case 'analysis_data': + handleAnalysisDataModal('EDIT_ANALYSIS_DATASET', editFile); + break; + case 'file': + // Dispatch an action to toggle the modal for previewing the file + dispatch({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { + operation: 'preview', + props: { + api: params.api, + scheme: params.scheme, + system: params.system, + path: node.path, + name: node.metadata.name, // Assuming 'name' is from node.metadata + href: 'tapis://' + node.path, + length: node.metadata.length, // Assuming 'length' is from node.metadata + metadata: node.metadata, + }, + }, + }); + break; + default: + break; + } + }; + + const renderTree = (node) => ( + <> +
+
+ handleNodeToggle} + > + {expandedNodes.includes(node.id) && ( +
+ +
+ {node.metadata.description} + +
+
+ )} + {Array.isArray(node.children) && + node.children.map((child) => renderTree(child))} + +
+
+ + ); + + const getAllNodeIds = (nodes) => { + const ids = []; + nodes.forEach((node) => { + ids.push(node.id); + if (Array.isArray(node.children)) { + ids.push(...getAllNodeIds(node.children)); + } + }); + return ids; + }; + + return ( + + Review Data Structure and Description + + } + headerActions={ +
+ <> + + +
+ } + > +
+
+

+ Review the data tree structure to make sure that the relationships + between the data components are correct: +

+
    +
  • + Spell check the description and ensure it is clear and complete so + your project is understandable and searchable. +
    • +
  • Check if the data is rendered.
    • +
  • Review image metadata for rendering.
    • +
  • + Make sure the relationships between sample, origin data and + analysis data are correct in the tree and add if needed. +
    • +
+
+ + {projectTree && projectTree.length > 0 && ( + } + defaultExpandIcon={} + // expanded={getAllNodeIds(tree)} + onNodeToggle={handleNodeToggle} + > + {projectTree.map((node) => renderTree(node))} + + )} +
+ + ); +}; + +export default ReviewProjectStructure; From 28da085530ff7a00e19ef3120fb75415e584a441 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Fri, 17 May 2024 12:40:08 -0500 Subject: [PATCH 069/328] simplified dataset modal creating by adding a custom hook --- .../DataFilesProjectFileListingAddon.jsx | 21 ++- ...esProjectFileListingMetadataTitleAddon.jsx | 12 +- .../_custom/drp/utils/datasetFormHandlers.js | 112 ---------------- .../drp/utils/hooks/useDrpDatasetModals.js | 123 ++++++++++++++++++ 4 files changed, 136 insertions(+), 132 deletions(-) delete mode 100644 client/src/components/_custom/drp/utils/datasetFormHandlers.js create mode 100644 client/src/components/_custom/drp/utils/hooks/useDrpDatasetModals.js diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx index 95d71834cd..7a2bfc146d 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx @@ -3,19 +3,14 @@ import { Button } from '_common'; import { useDispatch, useSelector } from 'react-redux'; import styles from './DataFilesProjectFileListingAddon.module.scss'; import { useSelectedFiles } from 'hooks/datafiles'; -import { useQuery } from 'react-query'; -import { fetchUtil } from 'utils/fetchUtil'; -import { createSampleModalHandler, createOriginDataModalHandler, createAnalysisDataModalHandler } from '../utils/datasetFormHandlers'; +import useDrpDatasetModals from '../utils/hooks/useDrpDatasetModals'; const DataFilesProjectFileListingAddon = () => { - const dispatch = useDispatch(); const portalName = useSelector((state) => state.workbench.portalName); const { projectId } = useSelector((state) => state.projects.metadata); const { selectedFiles } = useSelectedFiles(); - const handleOpenSampleModal = createSampleModalHandler(dispatch); - const handleOriginDataModal = createOriginDataModalHandler(dispatch, projectId, portalName); - const handleAnalysisDataModal = createAnalysisDataModalHandler(dispatch, projectId, portalName); + const { createSampleModal, createOriginDataModal, createAnalysisDataModal } = useDrpDatasetModals(projectId, portalName); return ( <> @@ -24,12 +19,12 @@ const DataFilesProjectFileListingAddon = () => { selectedFiles[0].metadata['data_type'] === 'sample' ? ( ) : ( - )} @@ -39,7 +34,7 @@ const DataFilesProjectFileListingAddon = () => { @@ -58,7 +53,7 @@ const DataFilesProjectFileListingAddon = () => { diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx index d56cc95d3e..7284c9b58e 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx @@ -5,7 +5,7 @@ import styles from './DataFilesProjectFileListingMetadataTitleAddon.module.scss' import { Button, LoadingSpinner } from '_common'; import { fetchUtil } from 'utils/fetchUtil'; import { useFileListing } from 'hooks/datafiles'; -import { createSampleModalHandler, createOriginDataModalHandler, createAnalysisDataModalHandler } from '../utils/datasetFormHandlers'; +import useDrpDatasetModals from '../utils/hooks/useDrpDatasetModals'; const DataFilesProjectFileListingMetadataTitleAddon = ({ folderMetadata, metadata, system, path }) => { const dispatch = useDispatch(); @@ -14,9 +14,7 @@ const DataFilesProjectFileListingMetadataTitleAddon = ({ folderMetadata, metadat const { loading } = useFileListing('FilesListing'); - const handleSampleModal = createSampleModalHandler(dispatch); - const handleOriginDataModal = createOriginDataModalHandler(dispatch, projectId, portalName); - const handleAnalysisDataModal = createAnalysisDataModalHandler(dispatch, projectId, portalName); + const { createSampleModal, createOriginDataModal, createAnalysisDataModal } = useDrpDatasetModals(projectId, portalName) const onEditData = (dataType) => { const name = path.split('/').pop(); @@ -37,13 +35,13 @@ const DataFilesProjectFileListingMetadataTitleAddon = ({ folderMetadata, metadat }; switch (dataType) { case 'sample': - handleSampleModal('EDIT_SAMPLE_DATA', editFile); + createSampleModal('EDIT_SAMPLE_DATA', editFile); break; case 'origin_data': - handleOriginDataModal('EDIT_ORIGIN_DATASET', editFile); + createOriginDataModal('EDIT_ORIGIN_DATASET', editFile); break; case 'analysis_data': - handleAnalysisDataModal('EDIT_ANALYSIS_DATASET', editFile); + createAnalysisDataModal('EDIT_ANALYSIS_DATASET', editFile); break; default: break; diff --git a/client/src/components/_custom/drp/utils/datasetFormHandlers.js b/client/src/components/_custom/drp/utils/datasetFormHandlers.js deleted file mode 100644 index 6b426fee7d..0000000000 --- a/client/src/components/_custom/drp/utils/datasetFormHandlers.js +++ /dev/null @@ -1,112 +0,0 @@ -import { fetchUtil } from 'utils/fetchUtil'; - -const getFormFields = async (formName) => { - const response = await fetchUtil({ - url: 'api/forms', - params: { - form_name: formName, - }, - }); - return response; -}; - -const getDatasets = async (projectId, portalName, getOriginData = false) => { - const response = await fetchUtil({ - url: `api/${portalName.toLowerCase()}`, - params: { - project_id: projectId, - get_origin_data: getOriginData, - }, - }); - - return response; -}; - -export const createSampleModalHandler = - (dispatch) => - async (formName, selectedFile = null) => { - const form = await getFormFields(formName); - - dispatch({ - type: 'DATA_FILES_TOGGLE_MODAL', - payload: { - operation: 'dynamicform', - props: { form, selectedFile, formName }, - }, - }); - }; - -export const createOriginDataModalHandler = - (dispatch, projectId, portalName) => - async (formName, selectedFile = null) => { - const form = await getFormFields(formName); - const { samples } = await getDatasets(projectId, portalName); - - form.form_fields.map((field) => { - if (field.name === 'sample') { - field.options.push( - ...samples.map((sample) => { - return { - value: parseInt(sample.id), - label: sample.name, - }; - }) - ); - } - }); - - dispatch({ - type: 'DATA_FILES_TOGGLE_MODAL', - payload: { - operation: 'dynamicform', - props: { selectedFile, form, formName, additionalData: samples }, - }, - }); - }; - -export const createAnalysisDataModalHandler = - (dispatch, projectId, portalName) => - async (formName, selectedFile = null) => { - const form = await getFormFields(formName); - const { samples, origin_data: originDatasets } = await getDatasets( - projectId, - portalName, - true - ); - - form.form_fields.map((field) => { - if (field.name === 'sample') { - field.options.push( - ...samples.map((sample) => { - return { - value: parseInt(sample.id), - label: sample.name, - }; - }) - ); - } else if (field.name === 'base_origin_data') { - field.options.push( - ...originDatasets.map((originData) => { - return { - value: parseInt(originData.id), - label: originData.name, - dependentId: parseInt(originData.metadata.sample), - }; - }) - ); - } - }); - - dispatch({ - type: 'DATA_FILES_TOGGLE_MODAL', - payload: { - operation: 'dynamicform', - props: { - selectedFile, - form, - formName, - additionalData: { samples, originDatasets }, - }, - }, - }); - }; diff --git a/client/src/components/_custom/drp/utils/hooks/useDrpDatasetModals.js b/client/src/components/_custom/drp/utils/hooks/useDrpDatasetModals.js new file mode 100644 index 0000000000..3a989cbdc8 --- /dev/null +++ b/client/src/components/_custom/drp/utils/hooks/useDrpDatasetModals.js @@ -0,0 +1,123 @@ +import { useCallback } from 'react'; +import { useSelector, useDispatch } from 'react-redux'; +import { fetchUtil } from 'utils/fetchUtil'; + +const useDrpDatasetModals = (projectId, portalName) => { + const getFormFields = async (formName) => { + const response = await fetchUtil({ + url: 'api/forms', + params: { + form_name: formName, + }, + }); + return response; + }; + + const getDatasets = async (projectId, portalName, getOriginData = false) => { + const response = await fetchUtil({ + url: `api/${portalName.toLowerCase()}`, + params: { + project_id: projectId, + get_origin_data: getOriginData, + }, + }); + + return response; + }; + + const dispatch = useDispatch(); + + const createSampleModal = useCallback( + async (formName, selectedFile = null) => { + const form = await getFormFields(formName); + + dispatch({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { + operation: 'dynamicform', + props: { form, selectedFile, formName }, + }, + }); + }, + [dispatch] + ); + + const createOriginDataModal = useCallback( + async (formName, selectedFile = null) => { + const form = await getFormFields(formName); + const { samples } = await getDatasets(projectId, portalName); + + form.form_fields.map((field) => { + if (field.name === 'sample') { + field.options.push( + ...samples.map((sample) => { + return { + value: parseInt(sample.id), + label: sample.name, + }; + }) + ); + } + }); + + dispatch({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { + operation: 'dynamicform', + props: { selectedFile, form, formName, additionalData: samples }, + }, + }); + } + ); + + const createAnalysisDataModal = useCallback( + async (formName, selectedFile = null) => { + const form = await getFormFields(formName); + const { samples, origin_data: originDatasets } = await getDatasets( + projectId, + portalName, + true + ); + + form.form_fields.map((field) => { + if (field.name === 'sample') { + field.options.push( + ...samples.map((sample) => { + return { + value: parseInt(sample.id), + label: sample.name, + }; + }) + ); + } else if (field.name === 'base_origin_data') { + field.options.push( + ...originDatasets.map((originData) => { + return { + value: parseInt(originData.id), + label: originData.name, + dependentId: parseInt(originData.metadata.sample), + }; + }) + ); + } + }); + + dispatch({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { + operation: 'dynamicform', + props: { + selectedFile, + form, + formName, + additionalData: { samples, originDatasets }, + }, + }, + }); + } + ); + + return { createSampleModal, createOriginDataModal, createAnalysisDataModal }; +}; + +export default useDrpDatasetModals; From 9c3fd3dad9f912e688946e0664d2b291b723b887 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Fri, 17 May 2024 18:16:57 -0500 Subject: [PATCH 070/328] Added conditional reload of page after dataset edit, tweaked design --- .../DataFilesModals/DataFilesFormModal.jsx | 4 +- .../DataFilesPreviewModalAddon.jsx | 6 +- .../DataFilesProjectPublish.jsx | 85 +++++++++++-------- .../DataFilesProjectPublishWizard.module.scss | 5 ++ .../ProjectDescription.jsx | 3 + .../ReviewAuthors.jsx | 8 +- .../ReviewProjectStructure.jsx | 20 +++-- .../utils/ReorderUserList/ReorderUserList.jsx | 2 - .../drp/utils/hooks/useDrpDatasetModals.js | 7 +- client/src/redux/sagas/_custom/drp.sagas.js | 8 +- 10 files changed, 91 insertions(+), 57 deletions(-) diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx index b78fe52814..c929b94708 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx @@ -27,7 +27,7 @@ const DataFilesFormModal = () => { history.push(path); }; - const { form, selectedFile, formName, additionalData } = useSelector( + const { form, selectedFile, formName, additionalData, useReloadCallback } = useSelector( (state) => state.files.modalProps.dynamicform ); const isOpen = useSelector((state) => state.files.modals.dynamicform); @@ -64,7 +64,7 @@ const DataFilesFormModal = () => { payload: { params, values, - reloadPage, + reloadPage: useReloadCallback ? reloadPage : null, selectedFile, additionalData, }, diff --git a/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.jsx b/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.jsx index cc8b50dc6a..1315be5577 100644 --- a/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.jsx @@ -17,9 +17,9 @@ const DataFilesPreviewModalAddon = ({ metadata }) => { const { params } = useFileListing('FilesListing'); - const file = useSelector( + const { useReloadCallback, ...file } = useSelector( (state) => state.files.modalProps.preview - ); + ); const { data: form, isLoading } = useQuery('form_UPLOAD_FILE', () => fetchUtil({ @@ -65,7 +65,7 @@ const DataFilesPreviewModalAddon = ({ metadata }) => { payload: { params, values, - reloadPage, + reloadPage: useReloadCallback ? reloadPage : null, selectedFile: file }, }); diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx index 053719fa70..f92131785b 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx @@ -1,6 +1,6 @@ -import React, { useEffect, useState } from 'react'; -import { SectionTableWrapper } from '_common'; -import { Link, useRouteMatch } from 'react-router-dom'; +import React, { useCallback, useEffect, useState } from 'react'; +import { LoadingSpinner, SectionTableWrapper } from '_common'; +import { Link } from 'react-router-dom'; import { useDispatch, useSelector } from 'react-redux'; import Wizard from '_common/Wizard'; import styles from './DataFilesProjectPublish.module.scss'; @@ -11,7 +11,9 @@ import PublicationInstructions from './DataFilesProjectPublishWizardSteps/Public import ReviewProjectStructure from './DataFilesProjectPublishWizardSteps/ReviewProjectStructure'; import ReviewAuthors from './DataFilesProjectPublishWizardSteps/ReviewAuthors'; -const DataFilesProjectPublish = ({ system, path, api, scheme }) => { +const DataFilesProjectPublish = ({ system }) => { + + // FIX THIS const portalName = 'drp'; const projectId = 'CEPV3-DEV-1133'; @@ -28,11 +30,22 @@ const DataFilesProjectPublish = ({ system, path, api, scheme }) => { const [tree, setTree] = useState([]); - useEffect(async () => { + const { dynamicFormModal, previewModal } = useSelector((state) => ({ + dynamicFormModal: state.files.modals.dynamicform, + previewModal: state.files.modals.preview, + })); + + const fetchTree = useCallback(async () => { const response = await getTree(projectId, portalName); setTree(response); }, [projectId, portalName]); + useEffect(() => { + if (!dynamicFormModal || !previewModal) { + fetchTree(); + } + }, [projectId, portalName, dynamicFormModal, previewModal, fetchTree]); + const dispatch = useDispatch(); useEffect(() => { @@ -42,14 +55,12 @@ const DataFilesProjectPublish = ({ system, path, api, scheme }) => { }); }, [system]); - const metadata = useSelector((state) => state.projects.metadata); + const { metadata } = useSelector((state) => state.projects); const formSubmit = (values) => { - console.log('DataFilesProjectPublish: formSubmit: ', values); + // console.log('DataFilesProjectPublish: formSubmit: ', values); }; - console.log('DataFilesProjectPublish: metadata: ', metadata); - const wizardSteps = [ { id: 'publication_instructions', @@ -66,11 +77,7 @@ const DataFilesProjectPublish = ({ system, path, api, scheme }) => { { id: 'project_structure', name: 'Review Project Structure', - render: ( - - ), + render: , initialValues: {}, }, { @@ -84,28 +91,34 @@ const DataFilesProjectPublish = ({ system, path, api, scheme }) => { // do the wizard now return ( - - Request Project Publication | {metadata.title} - - } - headerActions={ -
- <> - - Back to Project - - -
- } - > - - + <> + {metadata.loading ? ( + + ) : ( + + Request Project Publication | {metadata.title} + + } + headerActions={ +
+ <> + + Back to Project + + +
+ } + > + + + )} + ); }; diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss index 95ef99785d..d553e666cd 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss @@ -51,3 +51,8 @@ .section main { padding-left: 1px; } + + .edit-button { + font-size: 0.875rem; + padding-bottom: 5px; + } \ No newline at end of file diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx index f7dd67b81b..42fb61339c 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx @@ -6,8 +6,11 @@ import { DescriptionList, } from '_common'; import styles from './DataFilesProjectPublishWizard.module.scss'; +import { useDispatch } from 'react-redux'; const ProjectDescription = ({ project }) => { + const dispatch = useDispatch(); + const onEdit = () => { dispatch({ type: 'DATA_FILES_TOGGLE_MODAL', diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx index a29574df52..97b4a4ea6d 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx @@ -14,7 +14,13 @@ import styles from './DataFilesProjectPublishWizard.module.scss'; import ReorderUserList from '../../utils/ReorderUserList/ReorderUserList'; const ReviewAuthors = ({ project }) => { - return ; + return ( + Review Authors and Citation} + > + + + ); }; export default ReviewAuthors; diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx index 82aaa36bba..556f6749db 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx @@ -28,14 +28,13 @@ const ReviewProjectStructure = ({ projectTree }) => { const { params } = useFileListing('FilesListing'); - console.log("PARAMS", params) - const handleNodeToggle = (event, nodeIds) => { // Update the list of expanded nodes setExpandedNodes(nodeIds); }; - const { createSampleModal, createOriginDataModal, createAnalysisDataModal } = useDrpDatasetModals(projectId, portalName) + const { createSampleModal, createOriginDataModal, createAnalysisDataModal } = + useDrpDatasetModals(projectId, portalName, false); const onEdit = () => { dispatch({ @@ -53,7 +52,7 @@ const ReviewProjectStructure = ({ projectTree }) => { metadata: node.metadata, name: node.metadata.name, system: params.system, - path: node.path, + path: node.path.split('/').slice(1).join('/'), type: 'dir', _links: { self: { @@ -81,11 +80,12 @@ const ReviewProjectStructure = ({ projectTree }) => { api: params.api, scheme: params.scheme, system: params.system, - path: node.path, - name: node.metadata.name, // Assuming 'name' is from node.metadata + path: node.path.split('/').slice(1).join('/'), + name: node.metadata.name, href: 'tapis://' + node.path, - length: node.metadata.length, // Assuming 'length' is from node.metadata + length: node.metadata.length, metadata: node.metadata, + useReloadCallback: false, }, }, }); @@ -112,7 +112,11 @@ const ReviewProjectStructure = ({ projectTree }) => { > {expandedNodes.includes(node.id) && (
-
diff --git a/client/src/components/_custom/drp/utils/ReorderUserList/ReorderUserList.jsx b/client/src/components/_custom/drp/utils/ReorderUserList/ReorderUserList.jsx index 84f7cc9caf..70140f3be2 100644 --- a/client/src/components/_custom/drp/utils/ReorderUserList/ReorderUserList.jsx +++ b/client/src/components/_custom/drp/utils/ReorderUserList/ReorderUserList.jsx @@ -28,8 +28,6 @@ const ReorderUserList = ({ users }) => { setReorderedUsers(reordered); } - console.log('reorderedUsers', reorderedUsers) - return ( <> {reorderedUsers.map((user, index) => ( diff --git a/client/src/components/_custom/drp/utils/hooks/useDrpDatasetModals.js b/client/src/components/_custom/drp/utils/hooks/useDrpDatasetModals.js index 3a989cbdc8..1d3f51fcbb 100644 --- a/client/src/components/_custom/drp/utils/hooks/useDrpDatasetModals.js +++ b/client/src/components/_custom/drp/utils/hooks/useDrpDatasetModals.js @@ -2,7 +2,7 @@ import { useCallback } from 'react'; import { useSelector, useDispatch } from 'react-redux'; import { fetchUtil } from 'utils/fetchUtil'; -const useDrpDatasetModals = (projectId, portalName) => { +const useDrpDatasetModals = (projectId, portalName, useReloadCallback = true) => { const getFormFields = async (formName) => { const response = await fetchUtil({ url: 'api/forms', @@ -35,7 +35,7 @@ const useDrpDatasetModals = (projectId, portalName) => { type: 'DATA_FILES_TOGGLE_MODAL', payload: { operation: 'dynamicform', - props: { form, selectedFile, formName }, + props: { form, selectedFile, formName, useReloadCallback }, }, }); }, @@ -64,7 +64,7 @@ const useDrpDatasetModals = (projectId, portalName) => { type: 'DATA_FILES_TOGGLE_MODAL', payload: { operation: 'dynamicform', - props: { selectedFile, form, formName, additionalData: samples }, + props: { selectedFile, form, formName, additionalData: samples, useReloadCallback }, }, }); } @@ -111,6 +111,7 @@ const useDrpDatasetModals = (projectId, portalName) => { form, formName, additionalData: { samples, originDatasets }, + useReloadCallback }, }, }); diff --git a/client/src/redux/sagas/_custom/drp.sagas.js b/client/src/redux/sagas/_custom/drp.sagas.js index 64c0320b1e..ffec09ebe1 100644 --- a/client/src/redux/sagas/_custom/drp.sagas.js +++ b/client/src/redux/sagas/_custom/drp.sagas.js @@ -34,7 +34,9 @@ function* executeOperation(isEdit, params, values, reloadCallback, file = null, ); } - yield call(reloadCallback, path); + if (reloadCallback) { + yield call(reloadCallback, path); + } yield put({ type: 'DATA_FILES_TOGGLE_MODAL', @@ -114,8 +116,10 @@ function* handleFile(action, isEdit) { data_type: 'file', } + const path = selectedFile.path.split('/').slice(0, -1).join('/') + yield call( - executeOperation, isEdit, params, metadata, reloadCallback, selectedFile, params.path + executeOperation, isEdit, params, metadata, reloadCallback, selectedFile, path ) } From c67db85112990556948f71064d269c48199a0c45 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Mon, 20 May 2024 12:18:38 -0500 Subject: [PATCH 071/328] format date and add keyword support for projects --- .../DataFilesProjectPublishWizardSteps/ProjectDescription.jsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx index 42fb61339c..0171f33f12 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx @@ -7,6 +7,7 @@ import { } from '_common'; import styles from './DataFilesProjectPublishWizard.module.scss'; import { useDispatch } from 'react-redux'; +import { formatDate } from 'utils/timeFormat'; const ProjectDescription = ({ project }) => { const dispatch = useDispatch(); @@ -34,8 +35,9 @@ const ProjectDescription = ({ project }) => { {project.description} , + Keywords: project.keywords ?? '', }} direction={'vertical'} /> From 358ee8c86c5c0a99d879f8efcb92369becc5689b Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Mon, 20 May 2024 14:10:17 -0500 Subject: [PATCH 072/328] added default metadata that is added to every uploaded file --- .../DataFilesUploadModalAddon.jsx | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/client/src/components/_custom/drp/DataFilesUploadModalAddon/DataFilesUploadModalAddon.jsx b/client/src/components/_custom/drp/DataFilesUploadModalAddon/DataFilesUploadModalAddon.jsx index 1149cfebe0..6fda57e506 100644 --- a/client/src/components/_custom/drp/DataFilesUploadModalAddon/DataFilesUploadModalAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesUploadModalAddon/DataFilesUploadModalAddon.jsx @@ -16,6 +16,14 @@ const DataFilesUploadModalAddon = ({ uploadedFiles, setUploadedFiles }) => { const standardImageType = /(\.|\/)(gif|jpe?g|png|tiff?)$/i; useEffect(() => { + + // if the uploaded files don't have basic metadata, add metadata + if (!uploadedFiles.every((file) => file.metadata)) { + setUploadedFiles(uploadedFiles.map(file => { + return {...file, metadata: { data_type: 'file' }} + })) + } + setUploadedFilesWithMetadata(uploadedFiles.filter((file) => !standardImageType.test(file.data.name))); }, [uploadedFiles]) From 487dca9a8a1ab65b44ee2487255dcdb4786dd691 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Tue, 21 May 2024 15:59:41 -0500 Subject: [PATCH 073/328] Added final wizard step, changed how wizard step components are imported --- .../DataFilesProjectPublish.jsx | 107 +++++++----------- .../DataFilesProjectPublishWizard.module.scss | 18 +++ .../ProjectDescription.jsx | 8 +- .../PublicationInstructions.jsx | 7 +- .../ReviewAuthors.jsx | 45 +++++++- .../ReviewProjectStructure.jsx | 7 +- .../SubmitPublicationRequest.jsx | 75 ++++++++++++ .../utils/ReorderUserList/ReorderUserList.jsx | 20 +--- 8 files changed, 203 insertions(+), 84 deletions(-) create mode 100644 client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/SubmitPublicationRequest.jsx diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx index f92131785b..d9d14da901 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx @@ -6,89 +6,68 @@ import Wizard from '_common/Wizard'; import styles from './DataFilesProjectPublish.module.scss'; import { fetchUtil } from 'utils/fetchUtil'; import * as ROUTES from '../../../../constants/routes'; -import ProjectDescription from './DataFilesProjectPublishWizardSteps/ProjectDescription'; -import PublicationInstructions from './DataFilesProjectPublishWizardSteps/PublicationInstructions'; -import ReviewProjectStructure from './DataFilesProjectPublishWizardSteps/ReviewProjectStructure'; -import ReviewAuthors from './DataFilesProjectPublishWizardSteps/ReviewAuthors'; +import { ProjectDescriptionStep } from './DataFilesProjectPublishWizardSteps/ProjectDescription'; +import { PublicationInstructionsStep } from './DataFilesProjectPublishWizardSteps/PublicationInstructions'; +import { ReviewProjectStructureStep } from './DataFilesProjectPublishWizardSteps/ReviewProjectStructure'; +import { ReviewAuthorsStep } from './DataFilesProjectPublishWizardSteps/ReviewAuthors'; +import { SubmitPublicationRequestStep } from './DataFilesProjectPublishWizardSteps/SubmitPublicationRequest'; const DataFilesProjectPublish = ({ system }) => { - - // FIX THIS - const portalName = 'drp'; - const projectId = 'CEPV3-DEV-1133'; - - const getTree = async (projectId, portalName) => { - const response = await fetchUtil({ - url: `api/${portalName.toLowerCase()}/tree`, - params: { - project_id: projectId, - }, - }); - - return response; - }; + const dispatch = useDispatch(); + const portalName = useSelector((state) => state.workbench.portalName); + const { projectId } = useSelector((state) => state.projects.metadata); + const [users, setUsers] = useState([]); const [tree, setTree] = useState([]); - const { dynamicFormModal, previewModal } = useSelector((state) => ({ + useEffect(() => { + dispatch({ + type: 'PROJECTS_GET_METADATA', + payload: system, + }); + }, [system]); + + const { dynamicFormModal, previewModal, metadata } = useSelector((state) => ({ dynamicFormModal: state.files.modals.dynamicform, previewModal: state.files.modals.preview, + metadata: state.projects.metadata, })); const fetchTree = useCallback(async () => { - const response = await getTree(projectId, portalName); - setTree(response); - }, [projectId, portalName]); + try { + const response = await fetchUtil({ + url: `api/${portalName.toLowerCase()}/tree`, + params: { + project_id: projectId, + }, + }); + setTree(response); + } catch (error) { + console.error('Error fetching tree data:', error); + setTree([]); + } + }, [portalName, projectId]); useEffect(() => { + // workaround to get updated data after modal closes if (!dynamicFormModal || !previewModal) { fetchTree(); } - }, [projectId, portalName, dynamicFormModal, previewModal, fetchTree]); - - const dispatch = useDispatch(); - - useEffect(() => { - dispatch({ - type: 'PROJECTS_GET_METADATA', - payload: system, - }); - }, [system]); - - const { metadata } = useSelector((state) => state.projects); - - const formSubmit = (values) => { - // console.log('DataFilesProjectPublish: formSubmit: ', values); - }; + }, [dynamicFormModal, previewModal, fetchTree]); const wizardSteps = [ - { - id: 'publication_instructions', - name: 'Publication Instructions', - render: , - initialValues: {}, - }, - { - id: 'project_description', - name: 'Project Description', - render: , - initialValues: {}, - }, - { - id: 'project_structure', - name: 'Review Project Structure', - render: , - initialValues: {}, - }, - { - id: 'project_authors', - name: 'Review Authors and Citations', - render: , - initialValues: {}, - }, + PublicationInstructionsStep(), + ProjectDescriptionStep({ project: metadata }), + ReviewProjectStructureStep({ projectTree: tree }), + ReviewAuthorsStep({ project: metadata, users, setUsers}), + SubmitPublicationRequestStep(), ]; - // do the wizard now + const formSubmit = (values) => { + // save ordered users to db. Add to project metadata? + console.log('DataFilesProjectPublish: formSubmit: ', values); + console.log('Ordered users', users) + }; return ( <> diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss index d553e666cd..965eb27a1d 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss @@ -55,4 +55,22 @@ .edit-button { font-size: 0.875rem; padding-bottom: 5px; + } + + .citation-box { + border: 1px solid var(--global-color-primary--light); + padding: 10px; + margin-bottom: 20px; + border-radius: 10px; + background-color: white; + } + + .required-text { + color: red; + } + + .submit-div { + display: flex; + justify-content: center; + margin-top: 20px; } \ No newline at end of file diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx index 0171f33f12..6cf5496ee7 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx @@ -38,6 +38,7 @@ const ProjectDescription = ({ project }) => { Created: formatDate(new Date(project.created)), Abstract: {project.description} , Keywords: project.keywords ?? '', + License: project.license ?? 'None', }} direction={'vertical'} /> @@ -45,4 +46,9 @@ const ProjectDescription = ({ project }) => { ); }; -export default ProjectDescription; +export const ProjectDescriptionStep = ({ project }) => ({ + id: 'project_description', + name: 'Project Description', + render: , + initialValues: {}, +}); diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/PublicationInstructions.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/PublicationInstructions.jsx index 3af92f3b84..5b2752b5bd 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/PublicationInstructions.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/PublicationInstructions.jsx @@ -32,4 +32,9 @@ const PublicationInstructions = () => { ); }; -export default PublicationInstructions; +export const PublicationInstructionsStep = () => ({ + id: 'publication_instructions', + name: 'Publication Instructions', + render: , + initialValues: {}, +}); diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx index 97b4a4ea6d..d2252b5189 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx @@ -13,14 +13,53 @@ import { import styles from './DataFilesProjectPublishWizard.module.scss'; import ReorderUserList from '../../utils/ReorderUserList/ReorderUserList'; -const ReviewAuthors = ({ project }) => { +const MLACitation = ({ project, users }) => { + let authorString; + + if (users.length === 1) { + authorString = `${users[0].last_name}, ${users[0].first_name}`; + } else if (users.length === 2) { + authorString = `${users[0].last_name}, ${users[0].first_name}, and ${users[1].last_name}, ${users[1].first_name}`; + } else { + authorString = `${users[0].last_name}, ${users[0].first_name}, et al`; + } + + const mlaText = `${authorString}. "${project.title}." Digital Rocks Portal `; + + return ( + <> +

MLA

+
+
{mlaText}
+
+ + ); +}; + +const ReviewAuthors = ({ project, users, setUsers }) => { + + useEffect(() => { + const formattedUsers = project.members.map((user) => user.user); + setUsers(formattedUsers); + }, [project]); + return ( Review Authors and Citation
} > - + {users.length > 0 && project && ( +
+ + +
+ )} ); }; -export default ReviewAuthors; +export const ReviewAuthorsStep = ({ project, users, setUsers }) => ({ + id: 'project_authors', + name: 'Review Authors and Citations', + render: , + initialValues: {}, +}); diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx index 556f6749db..3714033ea0 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx @@ -204,4 +204,9 @@ const ReviewProjectStructure = ({ projectTree }) => { ); }; -export default ReviewProjectStructure; +export const ReviewProjectStructureStep = ({ projectTree }) => ({ + id: 'project_structure', + name: 'Review Project Structure', + render: , + initialValues: {}, +}); diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/SubmitPublicationRequest.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/SubmitPublicationRequest.jsx new file mode 100644 index 0000000000..0ad70c7baa --- /dev/null +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/SubmitPublicationRequest.jsx @@ -0,0 +1,75 @@ +import React from 'react'; +import { Button, FormGroup, Input } from 'reactstrap'; +import { useFormikContext } from 'formik'; +import { SectionTableWrapper, Section } from '_common'; +import * as Yup from 'yup'; +import styles from './DataFilesProjectPublishWizard.module.scss'; + +const SubmitPublicationRequest = () => { + const { handleChange, handleBlur, values, isValid } = + useFormikContext(); + + return ( + Submit Publication Request
} + > + + + + {' '} + I have reviewed the information and confirm that it is correct. + * + + + + + + {' '} + I have reviewed related publications/ I do not have any related + publications.* + + +
+
+ If you have any doubts about the process please contact the data + curator Maria Esteva before submitting the data for publication. +
+
+ +
+
+ + ); +}; + +export const SubmitPublicationRequestStep = () => ({ + id: 'submit_publication_request', + name: 'Submit Publication Request', + render: , + initialValues: { + reviewInfo: false, + reviewRelatedPublications: false, + }, + validationSchema: Yup.object({ + reviewInfo: Yup.boolean().oneOf([true]), + reviewRelatedPublications: Yup.boolean().oneOf([true]), + }), +}); diff --git a/client/src/components/_custom/drp/utils/ReorderUserList/ReorderUserList.jsx b/client/src/components/_custom/drp/utils/ReorderUserList/ReorderUserList.jsx index 70140f3be2..0980d567a5 100644 --- a/client/src/components/_custom/drp/utils/ReorderUserList/ReorderUserList.jsx +++ b/client/src/components/_custom/drp/utils/ReorderUserList/ReorderUserList.jsx @@ -2,35 +2,27 @@ import React, { useEffect, useState } from 'react'; import { Button, Icon } from '_common'; import styles from './ReorderUserList.module.scss' -const ReorderUserList = ({ users }) => { - - const [reorderedUsers, setReorderedUsers] = useState([]); - - useEffect(() => { - const formattedUsers = users.map(user => user.user); - setReorderedUsers(formattedUsers); - }, [users]); - +const ReorderUserList = ({ users, setUsers }) => { const moveUp = (index) => { - const reordered = [...reorderedUsers]; + const reordered = [...users]; const temp = reordered[index - 1]; reordered[index - 1] = reordered[index]; reordered[index] = temp; - setReorderedUsers(reordered); + setUsers(reordered); }; const moveDown = (index) => { - const reordered = [...reorderedUsers]; + const reordered = [...users]; const temp = reordered[index + 1]; reordered[index + 1] = reordered[index]; reordered[index] = temp; - setReorderedUsers(reordered); + setUsers(reordered); } return ( <> - {reorderedUsers.map((user, index) => ( + {users.map((user, index) => (
{user.last_name}, {user.first_name}
From b3c56b785a4a41f3d3db87267d69bc390be667aa Mon Sep 17 00:00:00 2001 From: Shayan Khan Date: Thu, 23 May 2024 12:18:29 -0500 Subject: [PATCH 074/328] some cleanup and restructuring --- .../DataFilesProjectPublish.jsx | 5 ++- .../DataFilesProjectPublish.module.scss | 33 +------------------ .../DataFilesProjectPublishWizard.module.scss | 25 +++++++------- .../ReviewAuthors.jsx | 1 - .../ReviewProjectStructure.jsx | 14 +++++--- .../SubmitPublicationRequest.jsx | 25 +++++++++----- 6 files changed, 40 insertions(+), 63 deletions(-) diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx index d9d14da901..c37558289a 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx @@ -13,7 +13,6 @@ import { ReviewAuthorsStep } from './DataFilesProjectPublishWizardSteps/ReviewAu import { SubmitPublicationRequestStep } from './DataFilesProjectPublishWizardSteps/SubmitPublicationRequest'; const DataFilesProjectPublish = ({ system }) => { - const dispatch = useDispatch(); const portalName = useSelector((state) => state.workbench.portalName); const { projectId } = useSelector((state) => state.projects.metadata); @@ -59,14 +58,14 @@ const DataFilesProjectPublish = ({ system }) => { PublicationInstructionsStep(), ProjectDescriptionStep({ project: metadata }), ReviewProjectStructureStep({ projectTree: tree }), - ReviewAuthorsStep({ project: metadata, users, setUsers}), + ReviewAuthorsStep({ project: metadata, users, setUsers }), SubmitPublicationRequestStep(), ]; const formSubmit = (values) => { // save ordered users to db. Add to project metadata? console.log('DataFilesProjectPublish: formSubmit: ', values); - console.log('Ordered users', users) + console.log('Ordered users', users); }; return ( diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.module.scss b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.module.scss index 8fc8930c1b..b736edd364 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.module.scss +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.module.scss @@ -10,12 +10,6 @@ padding-left: 1.5em; /* ~24px (20px * design * 1.2 design-to-app ratio) */ } - .prj main { - margin-right: 0; - margin-bottom: 10px; - - } - .title { @include truncate-with-ellipsis; max-width: 1400px; @@ -26,29 +20,4 @@ display: flex; align-items: flex-start; } - - .tree-label { - padding-top: 10px; - padding-bottom: 10px; - border: 1px solid var(--global-color-primary--light); - // border-radius: 10px; - - &:hover { - background-color: white; - } - } - - .description-div { - border: 1px solid var(--global-color-primary--light); - // border-top-style: none; - margin-left: 2px; - padding: 10px; - } - - .description { - font-size: 0.875rem; - } - - .section main { - padding-left: 1px; - } \ No newline at end of file + \ No newline at end of file diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss index 965eb27a1d..37725abee1 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss @@ -9,12 +9,6 @@ padding-left: 1.5em; /* ~24px (20px * design * 1.2 design-to-app ratio) */ } - .prj main { - margin-right: 0; - margin-bottom: 10px; - - } - .title { @include truncate-with-ellipsis; max-width: 1400px; @@ -26,20 +20,23 @@ align-items: flex-start; } + .section-project-structure main { + margin-right: 0; + margin-bottom: 10px; + } + .tree-label { padding-top: 10px; padding-bottom: 10px; border: 1px solid var(--global-color-primary--light); - // border-radius: 10px; - &:hover { - background-color: white; - } + // &:hover { + // background-color: white; + // } } - .description-div { + .metadata-description-div { border: 1px solid var(--global-color-primary--light); - // border-top-style: none; margin-left: 2px; padding: 10px; } @@ -48,7 +45,7 @@ font-size: 0.875rem; } - .section main { + .description-section main { padding-left: 1px; } @@ -66,7 +63,7 @@ } .required-text { - color: red; + color: var(--global-color-danger--normal); } .submit-div { diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx index d2252b5189..d20aee0987 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx @@ -37,7 +37,6 @@ const MLACitation = ({ project, users }) => { }; const ReviewAuthors = ({ project, users, setUsers }) => { - useEffect(() => { const formattedUsers = project.members.map((user) => user.user); setUsers(formattedUsers); diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx index 3714033ea0..d741b35b0c 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx @@ -97,7 +97,10 @@ const ReviewProjectStructure = ({ projectTree }) => { const renderTree = (node) => ( <> -
+
{ label={node.name} classes={{ label: styles['tree-label'], - root: styles['tree-root'], - content: styles['tree-content'], }} onLabelClick={() => handleNodeToggle} > {expandedNodes.includes(node.id) && ( -
+
} > -
+

Review the data tree structure to make sure that the relationships diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/SubmitPublicationRequest.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/SubmitPublicationRequest.jsx index 0ad70c7baa..bc1a47b8f0 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/SubmitPublicationRequest.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/SubmitPublicationRequest.jsx @@ -1,13 +1,25 @@ -import React from 'react'; +import React, { useEffect, useState } from 'react'; import { Button, FormGroup, Input } from 'reactstrap'; import { useFormikContext } from 'formik'; import { SectionTableWrapper, Section } from '_common'; import * as Yup from 'yup'; import styles from './DataFilesProjectPublishWizard.module.scss'; +const validationSchema = Yup.object({ + reviewInfo: Yup.boolean().oneOf([true], 'Must be checked'), + reviewRelatedPublications: Yup.boolean().oneOf([true], 'Must be checked'), +}); + const SubmitPublicationRequest = () => { - const { handleChange, handleBlur, values, isValid } = - useFormikContext(); + const { handleChange, handleBlur, values } = useFormikContext(); + + const [submitDisabled, setSubmitDisabled] = useState(true); + + useEffect(() => { + validationSchema.isValid(values).then((valid) => { + setSubmitDisabled(!valid); + }); + }, [values]); return ( { curator Maria Esteva before submitting the data for publication.

-
@@ -68,8 +80,5 @@ export const SubmitPublicationRequestStep = () => ({ reviewInfo: false, reviewRelatedPublications: false, }, - validationSchema: Yup.object({ - reviewInfo: Yup.boolean().oneOf([true]), - reviewRelatedPublications: Yup.boolean().oneOf([true]), - }), + validationSchema, }); From 0fc416edb00c2dfe5d44b4abc1bf942c17df7a8a Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Thu, 23 May 2024 15:57:45 -0500 Subject: [PATCH 075/328] fix trash bug, remove not needed folders from drp tree view --- .../DataFilesProjectFileListingMetadataTitleAddon.jsx | 2 +- server/portal/apps/_custom/drp/views.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx index 7284c9b58e..acf9f6e7e9 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx @@ -59,7 +59,7 @@ const DataFilesProjectFileListingMetadataTitleAddon = ({ folderMetadata, metadat {loading ? ( ) : ( - folderMetadata ? ( + folderMetadata && folderMetadata.data_type ? ( <> {folderMetadata.name} diff --git a/server/portal/apps/_custom/drp/views.py b/server/portal/apps/_custom/drp/views.py index 9bb079b214..e66649efd5 100644 --- a/server/portal/apps/_custom/drp/views.py +++ b/server/portal/apps/_custom/drp/views.py @@ -44,10 +44,15 @@ def construct_tree(records, parent_id=None): def get(self, request): + metadata_data_types = ['sample', 'origin_data', 'analysis_data', 'file'] + project_id = request.GET.get('project_id') full_project_id = f'{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}' - records = DataFilesMetadata.objects.filter(project_id=full_project_id).values('id', 'parent', 'name', 'path', 'metadata') + records = DataFilesMetadata.objects.filter( + project_id=full_project_id, + metadata__data_type__in=metadata_data_types + ).values('id', 'parent', 'name', 'path', 'metadata') tree = self.construct_tree(records) From 6b88b23fe710b795c891e70fcffac097ef38023a Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Fri, 31 May 2024 11:45:03 -0500 Subject: [PATCH 076/328] publication modifications - Removed Skip to End button in publication wizard - Reworked Review Author step, user can add or remove authors to be displayed in a publication --- .../src/components/_common/Wizard/Wizard.tsx | 4 +- .../DataFilesProjectPublish.jsx | 10 ++- .../ReviewAuthors.jsx | 69 +++++++++++++++---- .../ProjectMembersList/ProjectMembersList.jsx | 26 +++++++ .../ProjectMembersList.module.scss | 18 +++++ .../utils/ReorderUserList/ReorderUserList.jsx | 12 +++- .../ReorderUserList.module.scss | 33 ++++----- 7 files changed, 134 insertions(+), 38 deletions(-) create mode 100644 client/src/components/_custom/drp/utils/ProjectMembersList/ProjectMembersList.jsx create mode 100644 client/src/components/_custom/drp/utils/ProjectMembersList/ProjectMembersList.module.scss diff --git a/client/src/components/_common/Wizard/Wizard.tsx b/client/src/components/_common/Wizard/Wizard.tsx index e8a5e5a3ab..cec40dea78 100644 --- a/client/src/components/_common/Wizard/Wizard.tsx +++ b/client/src/components/_common/Wizard/Wizard.tsx @@ -53,9 +53,9 @@ export const WizardNavigation: React.FC = () => { - + */} )}
diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx index c37558289a..3221ef188f 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx @@ -16,7 +16,7 @@ const DataFilesProjectPublish = ({ system }) => { const dispatch = useDispatch(); const portalName = useSelector((state) => state.workbench.portalName); const { projectId } = useSelector((state) => state.projects.metadata); - const [users, setUsers] = useState([]); + const [authors, setAuthors] = useState([]); const [tree, setTree] = useState([]); useEffect(() => { @@ -54,18 +54,22 @@ const DataFilesProjectPublish = ({ system }) => { } }, [dynamicFormModal, previewModal, fetchTree]); + const handleAuthorsUpdate = (authors) => { + setAuthors(authors); + } + const wizardSteps = [ PublicationInstructionsStep(), ProjectDescriptionStep({ project: metadata }), ReviewProjectStructureStep({ projectTree: tree }), - ReviewAuthorsStep({ project: metadata, users, setUsers }), + ReviewAuthorsStep({ project: metadata, onAuthorsUpdate: handleAuthorsUpdate}), SubmitPublicationRequestStep(), ]; const formSubmit = (values) => { // save ordered users to db. Add to project metadata? console.log('DataFilesProjectPublish: formSubmit: ', values); - console.log('Ordered users', users); + console.log('Ordered users', authors); }; return ( diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx index d20aee0987..3ddf730946 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx @@ -12,16 +12,17 @@ import { } from '_common'; import styles from './DataFilesProjectPublishWizard.module.scss'; import ReorderUserList from '../../utils/ReorderUserList/ReorderUserList'; +import ProjectMembersList from '../../utils/ProjectMembersList/ProjectMembersList'; -const MLACitation = ({ project, users }) => { +const MLACitation = ({ project, authors }) => { let authorString; - if (users.length === 1) { - authorString = `${users[0].last_name}, ${users[0].first_name}`; - } else if (users.length === 2) { - authorString = `${users[0].last_name}, ${users[0].first_name}, and ${users[1].last_name}, ${users[1].first_name}`; + if (authors.length === 1) { + authorString = `${authors[0].last_name}, ${authors[0].first_name}`; + } else if (authors.length === 2) { + authorString = `${authors[0].last_name}, ${authors[0].first_name}, and ${authors[1].last_name}, ${authors[1].first_name}`; } else { - authorString = `${users[0].last_name}, ${users[0].first_name}, et al`; + authorString = `${authors[0].last_name}, ${authors[0].first_name}, et al`; } const mlaText = `${authorString}. "${project.title}." Digital Rocks Portal `; @@ -36,29 +37,69 @@ const MLACitation = ({ project, users }) => { ); }; -const ReviewAuthors = ({ project, users, setUsers }) => { +const ReviewAuthors = ({ project, onAuthorsUpdate }) => { + const [authors, setAuthors] = useState([]); + const [members, setMembers] = useState([]); + useEffect(() => { - const formattedUsers = project.members.map((user) => user.user); - setUsers(formattedUsers); + const owners = project.members + .filter((user) => user.access === 'owner') + .map((user) => ({ ...user.user, isOwner: true })); + + const members = project.members + .filter( + (user) => + (user.access === 'read' || user.access === 'edit') && + !authors.includes(user.user) + ) + .map((user) => user.user); + + setAuthors(owners); + setMembers(members); + onAuthorsUpdate(owners); }, [project]); + const onReorder = (authors) => { + setAuthors(authors); + onAuthorsUpdate(authors); + }; + + const onAddAuthor = (member) => { + const newAuthors = [...authors, member]; + setAuthors(newAuthors); + setMembers((prevMembers) => prevMembers.filter((m) => m !== member)); + onAuthorsUpdate(newAuthors); + }; + + const onRemoveCoAuthor = (member) => { + const newAuthors = authors.filter((a) => a !== member); + setAuthors(newAuthors); + setMembers((prevMembers) => [...prevMembers, member]); + onAuthorsUpdate(newAuthors); + }; + return ( Review Authors and Citation
} > - {users.length > 0 && project && ( + {authors.length > 0 && project && (
- - + + +
)} ); }; -export const ReviewAuthorsStep = ({ project, users, setUsers }) => ({ +export const ReviewAuthorsStep = ({ project, onAuthorsUpdate }) => ({ id: 'project_authors', name: 'Review Authors and Citations', - render: , + render: , initialValues: {}, }); diff --git a/client/src/components/_custom/drp/utils/ProjectMembersList/ProjectMembersList.jsx b/client/src/components/_custom/drp/utils/ProjectMembersList/ProjectMembersList.jsx new file mode 100644 index 0000000000..591d05f1c1 --- /dev/null +++ b/client/src/components/_custom/drp/utils/ProjectMembersList/ProjectMembersList.jsx @@ -0,0 +1,26 @@ +import React, { useEffect, useState } from 'react'; +import { Button, Icon } from '_common'; +import styles from './ProjectMembersList.module.scss' + +const ProjectMembersList = ({ members, onAddCoAuthor }) => { + + return ( + <> +

+ Other Members +

+ {members.map((member, index) => ( +
+ {member.last_name}, {member.first_name} +
+ +
+
+ ))} + + ) +} + +export default ProjectMembersList; \ No newline at end of file diff --git a/client/src/components/_custom/drp/utils/ProjectMembersList/ProjectMembersList.module.scss b/client/src/components/_custom/drp/utils/ProjectMembersList/ProjectMembersList.module.scss new file mode 100644 index 0000000000..1f0add1a39 --- /dev/null +++ b/client/src/components/_custom/drp/utils/ProjectMembersList/ProjectMembersList.module.scss @@ -0,0 +1,18 @@ +.user-div { + display: flex; + align-items: center; + margin-bottom: 10px; + justify-content: flex-start; + width: 35%; + } + + .user-name { + flex-grow: 1; /* Take up available space */ + margin-right: 10px; /* Space between name and buttons */ + } + + .button-group { + display: flex; + gap: 15px; /* Space between buttons */ + min-width: 190px; + } \ No newline at end of file diff --git a/client/src/components/_custom/drp/utils/ReorderUserList/ReorderUserList.jsx b/client/src/components/_custom/drp/utils/ReorderUserList/ReorderUserList.jsx index 0980d567a5..63f23fbf7e 100644 --- a/client/src/components/_custom/drp/utils/ReorderUserList/ReorderUserList.jsx +++ b/client/src/components/_custom/drp/utils/ReorderUserList/ReorderUserList.jsx @@ -2,14 +2,14 @@ import React, { useEffect, useState } from 'react'; import { Button, Icon } from '_common'; import styles from './ReorderUserList.module.scss' -const ReorderUserList = ({ users, setUsers }) => { +const ReorderUserList = ({ users, onReorder, onRemoveAuthor }) => { const moveUp = (index) => { const reordered = [...users]; const temp = reordered[index - 1]; reordered[index - 1] = reordered[index]; reordered[index] = temp; - setUsers(reordered); + onReorder(reordered); }; const moveDown = (index) => { @@ -17,17 +17,23 @@ const ReorderUserList = ({ users, setUsers }) => { const temp = reordered[index + 1]; reordered[index + 1] = reordered[index]; reordered[index] = temp; - setUsers(reordered); + onReorder(reordered); } return ( <> +

Authors

{users.map((user, index) => (
{user.last_name}, {user.first_name}
+ )}
))} diff --git a/client/src/components/_custom/drp/utils/ReorderUserList/ReorderUserList.module.scss b/client/src/components/_custom/drp/utils/ReorderUserList/ReorderUserList.module.scss index 6333431bcd..a14e1a1284 100644 --- a/client/src/components/_custom/drp/utils/ReorderUserList/ReorderUserList.module.scss +++ b/client/src/components/_custom/drp/utils/ReorderUserList/ReorderUserList.module.scss @@ -1,17 +1,18 @@ .user-div { - display: flex; - align-items: center; - margin-bottom: 10px; /* Adjust spacing between rows as needed */ - justify-content: flex-start; - width: 30%; - } - - .user-name { - flex-grow: 1; /* Take up available space */ - margin-right: 10px; /* Space between name and buttons */ - } - - .button-group { - display: flex; - gap: 10px; /* Space between buttons */ - } \ No newline at end of file + display: flex; + align-items: center; + margin-bottom: 10px; + justify-content: flex-start; + width: 35%; +} + +.user-name { + flex-grow: 1; /* Take up available space */ + margin-right: 10px; /* Space between name and buttons */ +} + +.button-group { + display: flex; + gap: 15px; /* Space between buttons */ + min-width: 200px; +} \ No newline at end of file From 767ed9f880227ec62a5654c60af866c087197bea Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Fri, 31 May 2024 17:25:20 -0500 Subject: [PATCH 077/328] DynamicForm refactor, added dynamic form dependencies --- .../_common/Form/DynamicForm/DynamicForm.jsx | 100 ++++++++++++------ .../drp/utils/hooks/useDrpDatasetModals.js | 3 +- client/src/redux/sagas/_custom/drp.sagas.js | 2 +- server/portal/apps/_custom/drp/views.py | 2 +- 4 files changed, 71 insertions(+), 36 deletions(-) diff --git a/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx b/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx index 81155e29f7..a2e1b6729b 100644 --- a/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx +++ b/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx @@ -1,58 +1,92 @@ -import React, { useEffect, useState } from 'react'; +import React, { useEffect, useMemo, useState } from 'react'; import FormField from '../FormField'; import { Button } from '_common'; import { useFormikContext } from 'formik'; import { FormGroup, Input } from 'reactstrap'; import './DynamicForm.scss'; -const updateFormFieldsBasedOnDependency = (formFields, values, setFieldValue, modifiedField) => { - return formFields.map((field) => { +const DynamicForm = ({ initialFormFields, onChange }) => { + + const [formFields, setFormFields] = useState(initialFormFields); + const { setFieldValue, values, handleChange, handleBlur } = useFormikContext(); + + const handleFilterDependency = (field, values, setFieldValue, modifiedField) => { const { dependency } = field; - if (dependency) { - if (dependency.type === 'filter') { - const filteredOptions = field.options.filter(option => option.dependentId == values[dependency.name]); - const updatedOptions = [{ value: '', label: '' }, ...filteredOptions]; + const filteredOptions = field.options.filter(option => option.dependentId == values[dependency.name]); + const updatedOptions = [{ value: '', label: '' }, ...filteredOptions]; - // only update the field value if the modified field is the dependency field - if (modifiedField && modifiedField.name === dependency.name) { - setFieldValue(field.name, updatedOptions[0].value); - } + // Only update the field value if the modified field is the dependency field + if (modifiedField && modifiedField.name === dependency.name) { + setFieldValue(field.name, updatedOptions[0].value); + } - return { ...field, hidden: false, filteredOptions: updatedOptions, value: '' }; - } else if (dependency.type === 'visibility') { - if (dependency.value) { - return { ...field, hidden: field.dependency.value !== values[field.dependency.name] }; - } else { - return { ...field, hidden: !field.hidden }; - } - } + return { ...field, hidden: false, filteredOptions: updatedOptions, value: '' }; + } + + const handleVisibilityDependency = (field, values, setFieldValue, modifiedField) => { + + const { dependency } = field; + + // Stores the value of the dependency field that is currently entered in the form + let currentDependencyFieldValue; + + // If the dependency field is a nested field (e.g. sample.type) + if (dependency.name.includes('.')) { + const [dependencyName, nestedDependencyField] = dependency.name.split('.'); + const dependentFormField = formFields.find(field => field.name === dependencyName); + // converts both values to string to compare + const dependentField = dependentFormField?.options.find(option => `${option.value}` === `${values[dependencyName]}`); + + currentDependencyFieldValue = dependentField ? dependentField[nestedDependencyField] : null; + } else { + currentDependencyFieldValue = values[dependency.name]; } - return field; - }); -}; -const DynamicForm = ({ initialFormFields, onChange }) => { + const isHidden = Array.isArray(dependency.value) + ? !dependency.value.includes(currentDependencyFieldValue) + : dependency.value !== currentDependencyFieldValue; - const [formFields, setFormFields] = useState(initialFormFields); - // For file processing - const { setFieldValue, values, handleChange, handleBlur } = useFormikContext(); + // Resets the fields value when the field is hidden + if (isHidden) { + setFieldValue(field.name, ''); + } - // This function updates and filters any dependant fields. Field dependency is described in the form config file - const handleDependentFieldUpdate = (value, modifiedField) => { - const updatedFormFields = updateFormFieldsBasedOnDependency(formFields, { ...values, [modifiedField.name]: value }, setFieldValue, modifiedField); - setFormFields(updatedFormFields); - }; + return { ...field, hidden: isHidden }; + } + + const updateFormFieldsBasedOnDependency = useMemo(() => { + return (formFields, values, setFieldValue, modifiedField) => { + return formFields.map((field) => { + const { dependency } = field; + + if (dependency) { + if (dependency.type === 'filter') { + return handleFilterDependency(field, values, setFieldValue, modifiedField); + } else if (dependency.type === 'visibility') { + return handleVisibilityDependency(field, values, setFieldValue, modifiedField); + } + } + return field; + }); + }; + }, []); useEffect(() => { - const updatedFormFields = updateFormFieldsBasedOnDependency(formFields, values); + const updatedFormFields = updateFormFieldsBasedOnDependency(initialFormFields, values, setFieldValue); setFormFields(updatedFormFields); - }, []); + }, [updateFormFieldsBasedOnDependency, values]); useEffect(() => { onChange && onChange(formFields, values); }, [formFields, values]); + + // This function updates and filters any dependant fields. Field dependency is described in the form config file + const handleDependentFieldUpdate = (value, modifiedField) => { + const updatedFormFields = updateFormFieldsBasedOnDependency(formFields, { ...values, [modifiedField.name]: value }, setFieldValue, modifiedField); + setFormFields(updatedFormFields); + }; const renderFormField = (field) => { diff --git a/client/src/components/_custom/drp/utils/hooks/useDrpDatasetModals.js b/client/src/components/_custom/drp/utils/hooks/useDrpDatasetModals.js index 1d3f51fcbb..e4c5bf5133 100644 --- a/client/src/components/_custom/drp/utils/hooks/useDrpDatasetModals.js +++ b/client/src/components/_custom/drp/utils/hooks/useDrpDatasetModals.js @@ -52,6 +52,7 @@ const useDrpDatasetModals = (projectId, portalName, useReloadCallback = true) => field.options.push( ...samples.map((sample) => { return { + ...sample.metadata, value: parseInt(sample.id), label: sample.name, }; @@ -64,7 +65,7 @@ const useDrpDatasetModals = (projectId, portalName, useReloadCallback = true) => type: 'DATA_FILES_TOGGLE_MODAL', payload: { operation: 'dynamicform', - props: { selectedFile, form, formName, additionalData: samples, useReloadCallback }, + props: { selectedFile, form, formName, additionalData: { samples }, useReloadCallback }, }, }); } diff --git a/client/src/redux/sagas/_custom/drp.sagas.js b/client/src/redux/sagas/_custom/drp.sagas.js index ffec09ebe1..a1f540de36 100644 --- a/client/src/redux/sagas/_custom/drp.sagas.js +++ b/client/src/redux/sagas/_custom/drp.sagas.js @@ -62,7 +62,7 @@ function* handleSampleData(action, isEdit) { } function* handleOriginData(action, isEdit) { - const { params, values, reloadPage: reloadCallback, selectedFile, additionalData: samples } = action.payload; + const { params, values, reloadPage: reloadCallback, selectedFile, additionalData: { samples } } = action.payload; const sample = samples.find( (sample) => sample.id === parseInt(values.sample) diff --git a/server/portal/apps/_custom/drp/views.py b/server/portal/apps/_custom/drp/views.py index e66649efd5..3d28b67373 100644 --- a/server/portal/apps/_custom/drp/views.py +++ b/server/portal/apps/_custom/drp/views.py @@ -12,7 +12,7 @@ def get(self, request): full_project_id = f'{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}' - samples = DataFilesMetadata.objects.filter(project_id=full_project_id, metadata__data_type='sample').values('id', 'name', 'path') + samples = DataFilesMetadata.objects.filter(project_id=full_project_id, metadata__data_type='sample').values('id', 'name', 'path', 'metadata') origin_data = [] if get_origin_data == 'true': From 2767302c73a62579c09dee514c3fba799283b80c Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Wed, 5 Jun 2024 14:41:32 -0500 Subject: [PATCH 078/328] added pydantic validation and models, updated create workspace flow --- client/src/redux/sagas/_custom/drp.sagas.js | 9 +- server/poetry.lock | 159 ++++++++++++++++-- server/portal/apps/__init__.py | 12 ++ server/portal/apps/_custom/drp/models.py | 145 ++++++++++++++++ server/portal/apps/projects/views.py | 22 ++- .../shared_workspace_operations.py | 5 +- server/portal/libs/agave/operations.py | 22 ++- server/pyproject.toml | 1 + 8 files changed, 345 insertions(+), 30 deletions(-) create mode 100644 server/portal/apps/_custom/drp/models.py diff --git a/client/src/redux/sagas/_custom/drp.sagas.js b/client/src/redux/sagas/_custom/drp.sagas.js index a1f540de36..c2620489dd 100644 --- a/client/src/redux/sagas/_custom/drp.sagas.js +++ b/client/src/redux/sagas/_custom/drp.sagas.js @@ -10,6 +10,11 @@ import { useHistory, useLocation } from 'react-router-dom'; function* executeOperation(isEdit, params, values, reloadCallback, file = null, path = '') { + + const filteredValues = Object.fromEntries( + Object.entries(values).filter(([key, value]) => value !== "" && value !== null) + ) + if (file && isEdit) { yield call( updateMetadataUtil, @@ -20,7 +25,7 @@ function* executeOperation(isEdit, params, values, reloadCallback, file = null, '/' + path, file.name, values.name, - values + filteredValues ); } else { yield call( @@ -30,7 +35,7 @@ function* executeOperation(isEdit, params, values, reloadCallback, file = null, params.system, path, values.name, - values + filteredValues ); } diff --git a/server/poetry.lock b/server/poetry.lock index 7bf28a1440..35968740f2 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "amqp" @@ -14,6 +14,17 @@ files = [ [package.dependencies] vine = ">=5.0.0" +[[package]] +name = "annotated-types" +version = "0.7.0" +description = "Reusable constraint types to use with typing.Annotated" +optional = false +python-versions = ">=3.8" +files = [ + {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, + {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, +] + [[package]] name = "appnope" version = "0.1.3" @@ -1521,6 +1532,16 @@ files = [ {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, @@ -1925,8 +1946,6 @@ files = [ {file = "psycopg2-2.9.9-cp310-cp310-win_amd64.whl", hash = "sha256:426f9f29bde126913a20a96ff8ce7d73fd8a216cfb323b1f04da402d452853c3"}, {file = "psycopg2-2.9.9-cp311-cp311-win32.whl", hash = "sha256:ade01303ccf7ae12c356a5e10911c9e1c51136003a9a1d92f7aa9d010fb98372"}, {file = "psycopg2-2.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:121081ea2e76729acfb0673ff33755e8703d45e926e416cb59bae3a86c6a4981"}, - {file = "psycopg2-2.9.9-cp312-cp312-win32.whl", hash = "sha256:d735786acc7dd25815e89cc4ad529a43af779db2e25aa7c626de864127e5a024"}, - {file = "psycopg2-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:a7653d00b732afb6fc597e29c50ad28087dcb4fbfb28e86092277a559ae4e693"}, {file = "psycopg2-2.9.9-cp37-cp37m-win32.whl", hash = "sha256:5e0d98cade4f0e0304d7d6f25bbfbc5bd186e07b38eac65379309c4ca3193efa"}, {file = "psycopg2-2.9.9-cp37-cp37m-win_amd64.whl", hash = "sha256:7e2dacf8b009a1c1e843b5213a87f7c544b2b042476ed7755be813eaf4e8347a"}, {file = "psycopg2-2.9.9-cp38-cp38-win32.whl", hash = "sha256:ff432630e510709564c01dafdbe996cb552e0b9f3f065eb89bdce5bd31fabf4c"}, @@ -2049,6 +2068,116 @@ files = [ {file = "pycryptodome-3.19.1.tar.gz", hash = "sha256:8ae0dd1bcfada451c35f9e29a3e5db385caabc190f98e4a80ad02a61098fb776"}, ] +[[package]] +name = "pydantic" +version = "2.7.2" +description = "Data validation using Python type hints" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic-2.7.2-py3-none-any.whl", hash = "sha256:834ab954175f94e6e68258537dc49402c4a5e9d0409b9f1b86b7e934a8372de7"}, + {file = "pydantic-2.7.2.tar.gz", hash = "sha256:71b2945998f9c9b7919a45bde9a50397b289937d215ae141c1d0903ba7149fd7"}, +] + +[package.dependencies] +annotated-types = ">=0.4.0" +pydantic-core = "2.18.3" +typing-extensions = ">=4.6.1" + +[package.extras] +email = ["email-validator (>=2.0.0)"] + +[[package]] +name = "pydantic-core" +version = "2.18.3" +description = "Core functionality for Pydantic validation and serialization" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic_core-2.18.3-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:744697428fcdec6be5670460b578161d1ffe34743a5c15656be7ea82b008197c"}, + {file = "pydantic_core-2.18.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:37b40c05ced1ba4218b14986fe6f283d22e1ae2ff4c8e28881a70fb81fbfcda7"}, + {file = "pydantic_core-2.18.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:544a9a75622357076efb6b311983ff190fbfb3c12fc3a853122b34d3d358126c"}, + {file = "pydantic_core-2.18.3-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e2e253af04ceaebde8eb201eb3f3e3e7e390f2d275a88300d6a1959d710539e2"}, + {file = "pydantic_core-2.18.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:855ec66589c68aa367d989da5c4755bb74ee92ccad4fdb6af942c3612c067e34"}, + {file = "pydantic_core-2.18.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3d3e42bb54e7e9d72c13ce112e02eb1b3b55681ee948d748842171201a03a98a"}, + {file = "pydantic_core-2.18.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6ac9ffccc9d2e69d9fba841441d4259cb668ac180e51b30d3632cd7abca2b9b"}, + {file = "pydantic_core-2.18.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c56eca1686539fa0c9bda992e7bd6a37583f20083c37590413381acfc5f192d6"}, + {file = "pydantic_core-2.18.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:17954d784bf8abfc0ec2a633108207ebc4fa2df1a0e4c0c3ccbaa9bb01d2c426"}, + {file = "pydantic_core-2.18.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:98ed737567d8f2ecd54f7c8d4f8572ca7c7921ede93a2e52939416170d357812"}, + {file = "pydantic_core-2.18.3-cp310-none-win32.whl", hash = "sha256:9f9e04afebd3ed8c15d67a564ed0a34b54e52136c6d40d14c5547b238390e779"}, + {file = "pydantic_core-2.18.3-cp310-none-win_amd64.whl", hash = "sha256:45e4ffbae34f7ae30d0047697e724e534a7ec0a82ef9994b7913a412c21462a0"}, + {file = "pydantic_core-2.18.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:b9ebe8231726c49518b16b237b9fe0d7d361dd221302af511a83d4ada01183ab"}, + {file = "pydantic_core-2.18.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b8e20e15d18bf7dbb453be78a2d858f946f5cdf06c5072453dace00ab652e2b2"}, + {file = "pydantic_core-2.18.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c0d9ff283cd3459fa0bf9b0256a2b6f01ac1ff9ffb034e24457b9035f75587cb"}, + {file = "pydantic_core-2.18.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2f7ef5f0ebb77ba24c9970da18b771711edc5feaf00c10b18461e0f5f5949231"}, + {file = "pydantic_core-2.18.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:73038d66614d2e5cde30435b5afdced2b473b4c77d4ca3a8624dd3e41a9c19be"}, + {file = "pydantic_core-2.18.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6afd5c867a74c4d314c557b5ea9520183fadfbd1df4c2d6e09fd0d990ce412cd"}, + {file = "pydantic_core-2.18.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd7df92f28d351bb9f12470f4c533cf03d1b52ec5a6e5c58c65b183055a60106"}, + {file = "pydantic_core-2.18.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:80aea0ffeb1049336043d07799eace1c9602519fb3192916ff525b0287b2b1e4"}, + {file = "pydantic_core-2.18.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:aaee40f25bba38132e655ffa3d1998a6d576ba7cf81deff8bfa189fb43fd2bbe"}, + {file = "pydantic_core-2.18.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9128089da8f4fe73f7a91973895ebf2502539d627891a14034e45fb9e707e26d"}, + {file = "pydantic_core-2.18.3-cp311-none-win32.whl", hash = "sha256:fec02527e1e03257aa25b1a4dcbe697b40a22f1229f5d026503e8b7ff6d2eda7"}, + {file = "pydantic_core-2.18.3-cp311-none-win_amd64.whl", hash = "sha256:58ff8631dbab6c7c982e6425da8347108449321f61fe427c52ddfadd66642af7"}, + {file = "pydantic_core-2.18.3-cp311-none-win_arm64.whl", hash = "sha256:3fc1c7f67f34c6c2ef9c213e0f2a351797cda98249d9ca56a70ce4ebcaba45f4"}, + {file = "pydantic_core-2.18.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f0928cde2ae416a2d1ebe6dee324709c6f73e93494d8c7aea92df99aab1fc40f"}, + {file = "pydantic_core-2.18.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0bee9bb305a562f8b9271855afb6ce00223f545de3d68560b3c1649c7c5295e9"}, + {file = "pydantic_core-2.18.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e862823be114387257dacbfa7d78547165a85d7add33b446ca4f4fae92c7ff5c"}, + {file = "pydantic_core-2.18.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6a36f78674cbddc165abab0df961b5f96b14461d05feec5e1f78da58808b97e7"}, + {file = "pydantic_core-2.18.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ba905d184f62e7ddbb7a5a751d8a5c805463511c7b08d1aca4a3e8c11f2e5048"}, + {file = "pydantic_core-2.18.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7fdd362f6a586e681ff86550b2379e532fee63c52def1c666887956748eaa326"}, + {file = "pydantic_core-2.18.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:24b214b7ee3bd3b865e963dbed0f8bc5375f49449d70e8d407b567af3222aae4"}, + {file = "pydantic_core-2.18.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:691018785779766127f531674fa82bb368df5b36b461622b12e176c18e119022"}, + {file = "pydantic_core-2.18.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:60e4c625e6f7155d7d0dcac151edf5858102bc61bf959d04469ca6ee4e8381bd"}, + {file = "pydantic_core-2.18.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a4e651e47d981c1b701dcc74ab8fec5a60a5b004650416b4abbef13db23bc7be"}, + {file = "pydantic_core-2.18.3-cp312-none-win32.whl", hash = "sha256:ffecbb5edb7f5ffae13599aec33b735e9e4c7676ca1633c60f2c606beb17efc5"}, + {file = "pydantic_core-2.18.3-cp312-none-win_amd64.whl", hash = "sha256:2c8333f6e934733483c7eddffdb094c143b9463d2af7e6bd85ebcb2d4a1b82c6"}, + {file = "pydantic_core-2.18.3-cp312-none-win_arm64.whl", hash = "sha256:7a20dded653e516a4655f4c98e97ccafb13753987434fe7cf044aa25f5b7d417"}, + {file = "pydantic_core-2.18.3-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:eecf63195be644b0396f972c82598cd15693550f0ff236dcf7ab92e2eb6d3522"}, + {file = "pydantic_core-2.18.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2c44efdd3b6125419c28821590d7ec891c9cb0dff33a7a78d9d5c8b6f66b9702"}, + {file = "pydantic_core-2.18.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6e59fca51ffbdd1638b3856779342ed69bcecb8484c1d4b8bdb237d0eb5a45e2"}, + {file = "pydantic_core-2.18.3-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:70cf099197d6b98953468461d753563b28e73cf1eade2ffe069675d2657ed1d5"}, + {file = "pydantic_core-2.18.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:63081a49dddc6124754b32a3774331467bfc3d2bd5ff8f10df36a95602560361"}, + {file = "pydantic_core-2.18.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:370059b7883485c9edb9655355ff46d912f4b03b009d929220d9294c7fd9fd60"}, + {file = "pydantic_core-2.18.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a64faeedfd8254f05f5cf6fc755023a7e1606af3959cfc1a9285744cc711044"}, + {file = "pydantic_core-2.18.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:19d2e725de0f90d8671f89e420d36c3dd97639b98145e42fcc0e1f6d492a46dc"}, + {file = "pydantic_core-2.18.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:67bc078025d70ec5aefe6200ef094576c9d86bd36982df1301c758a9fff7d7f4"}, + {file = "pydantic_core-2.18.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:adf952c3f4100e203cbaf8e0c907c835d3e28f9041474e52b651761dc248a3c0"}, + {file = "pydantic_core-2.18.3-cp38-none-win32.whl", hash = "sha256:9a46795b1f3beb167eaee91736d5d17ac3a994bf2215a996aed825a45f897558"}, + {file = "pydantic_core-2.18.3-cp38-none-win_amd64.whl", hash = "sha256:200ad4e3133cb99ed82342a101a5abf3d924722e71cd581cc113fe828f727fbc"}, + {file = "pydantic_core-2.18.3-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:304378b7bf92206036c8ddd83a2ba7b7d1a5b425acafff637172a3aa72ad7083"}, + {file = "pydantic_core-2.18.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c826870b277143e701c9ccf34ebc33ddb4d072612683a044e7cce2d52f6c3fef"}, + {file = "pydantic_core-2.18.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e201935d282707394f3668380e41ccf25b5794d1b131cdd96b07f615a33ca4b1"}, + {file = "pydantic_core-2.18.3-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5560dda746c44b48bf82b3d191d74fe8efc5686a9ef18e69bdabccbbb9ad9442"}, + {file = "pydantic_core-2.18.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6b32c2a1f8032570842257e4c19288eba9a2bba4712af542327de9a1204faff8"}, + {file = "pydantic_core-2.18.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:929c24e9dea3990bc8bcd27c5f2d3916c0c86f5511d2caa69e0d5290115344a9"}, + {file = "pydantic_core-2.18.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e1a8376fef60790152564b0eab376b3e23dd6e54f29d84aad46f7b264ecca943"}, + {file = "pydantic_core-2.18.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:dccf3ef1400390ddd1fb55bf0632209d39140552d068ee5ac45553b556780e06"}, + {file = "pydantic_core-2.18.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:41dbdcb0c7252b58fa931fec47937edb422c9cb22528f41cb8963665c372caf6"}, + {file = "pydantic_core-2.18.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:666e45cf071669fde468886654742fa10b0e74cd0fa0430a46ba6056b24fb0af"}, + {file = "pydantic_core-2.18.3-cp39-none-win32.whl", hash = "sha256:f9c08cabff68704a1b4667d33f534d544b8a07b8e5d039c37067fceb18789e78"}, + {file = "pydantic_core-2.18.3-cp39-none-win_amd64.whl", hash = "sha256:4afa5f5973e8572b5c0dcb4e2d4fda7890e7cd63329bd5cc3263a25c92ef0026"}, + {file = "pydantic_core-2.18.3-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:77319771a026f7c7d29c6ebc623de889e9563b7087911b46fd06c044a12aa5e9"}, + {file = "pydantic_core-2.18.3-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:df11fa992e9f576473038510d66dd305bcd51d7dd508c163a8c8fe148454e059"}, + {file = "pydantic_core-2.18.3-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d531076bdfb65af593326ffd567e6ab3da145020dafb9187a1d131064a55f97c"}, + {file = "pydantic_core-2.18.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d33ce258e4e6e6038f2b9e8b8a631d17d017567db43483314993b3ca345dcbbb"}, + {file = "pydantic_core-2.18.3-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1f9cd7f5635b719939019be9bda47ecb56e165e51dd26c9a217a433e3d0d59a9"}, + {file = "pydantic_core-2.18.3-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:cd4a032bb65cc132cae1fe3e52877daecc2097965cd3914e44fbd12b00dae7c5"}, + {file = "pydantic_core-2.18.3-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:82f2718430098bcdf60402136c845e4126a189959d103900ebabb6774a5d9fdb"}, + {file = "pydantic_core-2.18.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:c0037a92cf0c580ed14e10953cdd26528e8796307bb8bb312dc65f71547df04d"}, + {file = "pydantic_core-2.18.3-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b95a0972fac2b1ff3c94629fc9081b16371dad870959f1408cc33b2f78ad347a"}, + {file = "pydantic_core-2.18.3-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:a62e437d687cc148381bdd5f51e3e81f5b20a735c55f690c5be94e05da2b0d5c"}, + {file = "pydantic_core-2.18.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b367a73a414bbb08507da102dc2cde0fa7afe57d09b3240ce82a16d608a7679c"}, + {file = "pydantic_core-2.18.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ecce4b2360aa3f008da3327d652e74a0e743908eac306198b47e1c58b03dd2b"}, + {file = "pydantic_core-2.18.3-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bd4435b8d83f0c9561a2a9585b1de78f1abb17cb0cef5f39bf6a4b47d19bafe3"}, + {file = "pydantic_core-2.18.3-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:616221a6d473c5b9aa83fa8982745441f6a4a62a66436be9445c65f241b86c94"}, + {file = "pydantic_core-2.18.3-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:7e6382ce89a92bc1d0c0c5edd51e931432202b9080dc921d8d003e616402efd1"}, + {file = "pydantic_core-2.18.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:ff58f379345603d940e461eae474b6bbb6dab66ed9a851ecd3cb3709bf4dcf6a"}, + {file = "pydantic_core-2.18.3.tar.gz", hash = "sha256:432e999088d85c8f36b9a3f769a8e2b57aabd817bbb729a90d1fe7f18f6f1f39"}, +] + +[package.dependencies] +typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" + [[package]] name = "pyflakes" version = "3.1.0" @@ -2330,7 +2459,6 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -2714,13 +2842,13 @@ tests = ["cython", "littleutils", "pygments", "pytest", "typeguard"] [[package]] name = "tapipy" -version = "1.4.1" +version = "1.6.3" description = "Python lib for interacting with an instance of the Tapis API Framework" optional = false python-versions = ">=3.7,<4.0" files = [ - {file = "tapipy-1.4.1-py3-none-any.whl", hash = "sha256:883ea9c95dd90f084aaf41168fd401324d2418a5b1356d7c8d5aebfd0d490b2d"}, - {file = "tapipy-1.4.1.tar.gz", hash = "sha256:50b7e1a700dfaf9e18723c15bdf371255f918bd15ced54657b7a95066b725f48"}, + {file = "tapipy-1.6.3-py3-none-any.whl", hash = "sha256:2166bc0ec7d16c45cf0c85a874a30d8d0a2721830d286cb9ff774ad060e733ad"}, + {file = "tapipy-1.6.3.tar.gz", hash = "sha256:edb5e6a57a92e030d3ac3db50e5b167a2014ff33a30c0f1d1e51f9cfed808eb9"}, ] [package.dependencies] @@ -2737,7 +2865,6 @@ pyyaml = ">=5.4" requests = ">=2.20.0,<3.0.0" setuptools = ">=21.0.0" six = ">=1.10,<2.0" -typing_extensions = "<4.6" urllib3 = ">=1.26.5,<2.0.0" [[package]] @@ -2840,13 +2967,13 @@ twisted = ["twisted (>=20.3.0)", "zope.interface (>=5.2.0)"] [[package]] name = "typing-extensions" -version = "4.5.0" -description = "Backported and Experimental Type Hints for Python 3.7+" +version = "4.12.1" +description = "Backported and Experimental Type Hints for Python 3.8+" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.5.0-py3-none-any.whl", hash = "sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4"}, - {file = "typing_extensions-4.5.0.tar.gz", hash = "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb"}, + {file = "typing_extensions-4.12.1-py3-none-any.whl", hash = "sha256:6024b58b69089e5a89c347397254e35f1bf02a907728ec7fee9bf0fe837d203a"}, + {file = "typing_extensions-4.12.1.tar.gz", hash = "sha256:915f5e35ff76f56588223f15fdd5938f9a1cf9195c0de25130c627e4d597f6d1"}, ] [[package]] @@ -2889,12 +3016,12 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "uwsgi" -version = "2.0.22" +version = "2.0.26" description = "The uWSGI server" optional = false python-versions = "*" files = [ - {file = "uwsgi-2.0.22.tar.gz", hash = "sha256:4cc4727258671ac5fa17ab422155e9aaef8a2008ebb86e4404b66deaae965db2"}, + {file = "uwsgi-2.0.26.tar.gz", hash = "sha256:86e6bfcd4dc20529665f5b7777193cdc48622fb2c59f0a7f1e3dc32b3882e7f9"}, ] [[package]] @@ -3037,4 +3164,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "96cf740d08cee6617dcfe4fc9d11532efbaf889fb844ab95d958d894d2e85364" +content-hash = "0658114906564d66f49af2a9e41bd71f4bc99af88f91c50686aa04fe41a366a7" diff --git a/server/portal/apps/__init__.py b/server/portal/apps/__init__.py index e69de29bb2..5863c33507 100644 --- a/server/portal/apps/__init__.py +++ b/server/portal/apps/__init__.py @@ -0,0 +1,12 @@ + +from portal.apps._custom.drp.models import DrpProjectMetadata, DrpSampleMetadata, DrpOriginDatasetMetadata, DrpAnalysisDatasetMetadata + + +SCHEMA_MAPPING = { + 'DRP': { + 'project': DrpProjectMetadata, + 'sample': DrpSampleMetadata, + 'origin_data': DrpOriginDatasetMetadata, + 'analysis_data': DrpAnalysisDatasetMetadata + } +} \ No newline at end of file diff --git a/server/portal/apps/_custom/drp/models.py b/server/portal/apps/_custom/drp/models.py new file mode 100644 index 0000000000..2db5dc58c2 --- /dev/null +++ b/server/portal/apps/_custom/drp/models.py @@ -0,0 +1,145 @@ +from pydantic import BaseModel, ConfigDict, model_validator +from typing import Optional, Literal + + +class DrpProjectRelatedDatasets(BaseModel): + """Model for DRP Project Related Datasets""" + + model_config = ConfigDict( + extra="forbid", + ) + + title: str + description: str = "" + link: str = "" + +class DrpProjectRelatedSoftware(BaseModel): + """Model for DRP Project Related Software""" + + model_config = ConfigDict( + extra="forbid", + ) + + title: str + description: str = "" + link: str = "" + +class DrpProjectRelatedPublications(BaseModel): + """Model for DRP Project Related Publications""" + + model_config = ConfigDict( + extra="forbid", + ) + + title: str + author: str + date_of_publication: str + publisher: str + abstract: Optional[str] = None + link: Optional[str] = None + + +class DrpProjectMetadata(BaseModel): + """Model for DRP Project Metadata""" + + model_config = ConfigDict( + extra="forbid", + ) + + title: str + description: str = "" + license: Optional[str] = None + doi: Optional[str] = None + related_datasets: list[DrpProjectRelatedDatasets] = [] + related_software: list[DrpProjectRelatedSoftware] = [] + related_publications: list[DrpProjectRelatedPublications] = [] + publication_date: Optional[str] = None + + +class DrpDatasetMetadata(BaseModel): + """Model for Base DRP Dataset Metadata""" + + model_config = ConfigDict( + extra="forbid", + ) + + name: str + description: str + data_type: Literal[ + "sample", + "origin_data", + "analysis_data", + "file" + ] + +class DrpSampleMetadata(DrpDatasetMetadata): + """Model for DRP Sample Metadata""" + + porous_media_type: Literal[ + "sandstone", + "soil", + "carbonate", + "greanite", + "beads", + "fibrous_media", + "coal", + "energy_storage", + "other" + ] + source: Literal[ + "natural", + "natural_extraterrestrial", + "artificial", + "computer_generated", + ] + collection_method: Optional[str] = None + onshore_offshore: Optional[Literal["onshore", "offshore"]] = None + depth: Optional[str] = None + water_depth: Optional[str] = None + geographic_origin: Optional[str] = None + procedure: Optional[str] = None + equipment: Optional[str] = None + algorithm_description: Optional[str] = None + grain_size_min: Optional[str] = None + grain_size_max: Optional[str] = None + grain_size_avg: Optional[str] = None + porosity: Optional[str] = None + geographical_location: Optional[str] = None + date_of_collection: Optional[str] = None + identifier: Optional[str] = None + + +class DrpOriginDatasetMetadata(DrpDatasetMetadata): + """Model for DRP Origin Dataset Metadata""" + + is_segmented: bool + voxel_x: Optional[float] = None + voxel_y: Optional[float] = None + voxel_z: Optional[float] = None + voxel_units: Optional[Literal[ + "nanometer", + "micrometer", + "millimeter", + "other" + ]] = None + sample: str + imaging_center: Optional[str] = None + imaging_equipment_and_model: Optional[str] = None + image_format: Optional[str] = None + image_dimensions: Optional[str] = None + image_byte_order: Optional[str] = None + dimensionality: Optional[str] = None + +class DrpAnalysisDatasetMetadata(BaseModel): + """Model for DRP Analysis Dataset Metadata""" + + is_segmented: bool + sample: str + base_origin_data: Optional[str] = None + analysis_type: Literal[ + "machine_learning", + "simulation", + "experimental", + "characterization", + "other" + ] diff --git a/server/portal/apps/projects/views.py b/server/portal/apps/projects/views.py index 1e8527c4d5..95f6c14f3b 100644 --- a/server/portal/apps/projects/views.py +++ b/server/portal/apps/projects/views.py @@ -16,15 +16,21 @@ from portal.apps.projects.workspace_operations.shared_workspace_operations import \ list_projects, get_project, create_shared_workspace, \ update_project, get_workspace_role, change_user_role, add_user_to_workspace, \ - remove_user, transfer_ownership + remove_user, transfer_ownership, increment_workspace_count from portal.apps.search.tasks import tapis_project_listing_indexer from portal.libs.elasticsearch.indexes import IndexedProject from elasticsearch_dsl import Q from portal.apps.projects.models.metadata import ProjectsMetadata from django.db import transaction +from portal.apps import SCHEMA_MAPPING LOGGER = logging.getLogger(__name__) +def validate_project_metadata(metadata): + portal_name = settings.PORTAL_NAMESPACE + schema = SCHEMA_MAPPING[portal_name]['project'] + validated_model = schema.model_validate(metadata) + return validated_model.model_dump(exclude_none=True) @method_decorator(agave_jwt_login, name='dispatch') @method_decorator(login_required, name='dispatch') @@ -112,16 +118,20 @@ def post(self, request): # pylint: disable=no-self-use description = data['description'] metadata = data['metadata'] - client = request.user.tapis_oauth.client - system_id = create_shared_workspace(client, title, request.user.username, description) + workspace_number = increment_workspace_count() + workspace_id = f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{settings.PORTAL_PROJECTS_ID_PREFIX}-{workspace_number}" if metadata is not None: + project_metadata = ProjectsMetadata( - project_id = system_id, - metadata = metadata + project_id = workspace_id, + metadata = validate_project_metadata(metadata) ) project_metadata.save() + + client = request.user.tapis_oauth.client + system_id = create_shared_workspace(client, title, request.user.username, description, workspace_number) return JsonResponse( { @@ -207,7 +217,7 @@ def patch( if metadata is not None: project_metadata = ProjectsMetadata.objects.get(project_id=f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}") - project_metadata.metadata = metadata + project_metadata.metadata = validate_project_metadata(metadata) project_metadata.save() client = request.user.tapis_oauth.client diff --git a/server/portal/apps/projects/workspace_operations/shared_workspace_operations.py b/server/portal/apps/projects/workspace_operations/shared_workspace_operations.py index 6537a61a05..edf0bd8045 100644 --- a/server/portal/apps/projects/workspace_operations/shared_workspace_operations.py +++ b/server/portal/apps/projects/workspace_operations/shared_workspace_operations.py @@ -116,12 +116,13 @@ def increment_workspace_count(force=None) -> int: ########################################## -def create_shared_workspace(client: Tapis, title: str, owner: str, description=""): +def create_shared_workspace(client: Tapis, title: str, owner: str, description="", predefind_workspace_number=None): """ Create a workspace system owned by user whose client is passed. """ service_client = service_account() - workspace_number = increment_workspace_count() + force = predefind_workspace_number if predefind_workspace_number else None + workspace_number = increment_workspace_count(force) workspace_id = f"{settings.PORTAL_PROJECTS_ID_PREFIX}-{workspace_number}" # Service client creates directory and gives owner write permissions diff --git a/server/portal/libs/agave/operations.py b/server/portal/libs/agave/operations.py index 7f9739f60c..2a8291daa4 100644 --- a/server/portal/libs/agave/operations.py +++ b/server/portal/libs/agave/operations.py @@ -13,6 +13,7 @@ from tapipy.errors import BaseTapyException from portal.apps.projects.models.metadata import ProjectsMetadata from portal.apps.datafiles.models import DataFilesMetadata +from portal.apps import SCHEMA_MAPPING logger = logging.getLogger(__name__) @@ -26,11 +27,13 @@ def get_datafile_metadata(system, path): @transaction.atomic def update_datafile_metadata(system, name, old_path, new_path, metadata): + + validated_metadata = validate_datafile_metadata({**metadata, 'name': name}) + files_metadata = DataFilesMetadata.objects.get(path=f'{system}/{old_path.strip("/")}') files_metadata.name = name files_metadata.path = f"{system}/{new_path.strip('/')}" - files_metadata.metadata = metadata - files_metadata.metadata['name'] = name + files_metadata.metadata = validated_metadata files_metadata.save() for child in DataFilesMetadata.objects.filter(parent=files_metadata.id): @@ -229,10 +232,12 @@ def mkdir(client, system, path, dir_name, metadata=None): project_instance = ProjectsMetadata.objects.get(project_id=system) + validated_metadata = validate_datafile_metadata(metadata) + files_metadata = DataFilesMetadata( name = dir_name, path = f'{system}/{path_input.strip("/")}', - metadata = metadata, + metadata = validated_metadata, project = project_instance ) @@ -504,10 +509,12 @@ def upload(client, system, path, uploaded_file, metadata=None): project_instance = ProjectsMetadata.objects.get(project_id=system) + validated_metadata = validate_datafile_metadata(metadata) + files_metadata = DataFilesMetadata( name = uploaded_file.name, path = f'{system}/{dest_path.strip("/")}', - metadata = metadata, + metadata = validated_metadata, project = project_instance ) @@ -616,3 +623,10 @@ def update_metadata(client, system, path, new_path, old_name, new_name, metadata new_name = ('/' + move_message).rsplit('/', 1)[1] update_datafile_metadata(system=system, name=new_name, old_path=path, new_path=f'{new_path.strip("/")}/{new_name}', metadata=metadata) + +def validate_datafile_metadata(metadata): + portal_name = settings.PORTAL_NAMESPACE + schema = SCHEMA_MAPPING[portal_name][metadata.get('data_type')] + validated_model = schema.model_validate(metadata) + + return validated_model.model_dump(exclude_none=True) \ No newline at end of file diff --git a/server/pyproject.toml b/server/pyproject.toml index 1f20058c5d..b3e9e8ca89 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -39,6 +39,7 @@ zeep = "^4.1.0" tapipy = "^1.3" gevent = "^23.9.1" pymemcache = "^4.0.0" +pydantic = "^2.5.0" [tool.poetry.group.dev.dependencies] pytest = "^7.3.1" From 264b36955039a90de0577c094c039a58232c4a62 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Wed, 5 Jun 2024 17:13:46 -0500 Subject: [PATCH 079/328] Added created and last_updated fields to Project and Datafile metadata --- server/portal/apps/_custom/drp/views.py | 2 +- ...4_datafilesmetadata_created_at_and_more.py | 24 +++++++++++++++++++ server/portal/apps/datafiles/models.py | 3 +++ ...05_projectsmetadata_created_at_and_more.py | 24 +++++++++++++++++++ .../portal/apps/projects/models/metadata.py | 3 +++ 5 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 server/portal/apps/datafiles/migrations/0004_datafilesmetadata_created_at_and_more.py create mode 100644 server/portal/apps/projects/migrations/0005_projectsmetadata_created_at_and_more.py diff --git a/server/portal/apps/_custom/drp/views.py b/server/portal/apps/_custom/drp/views.py index 3d28b67373..3cf66c909c 100644 --- a/server/portal/apps/_custom/drp/views.py +++ b/server/portal/apps/_custom/drp/views.py @@ -52,7 +52,7 @@ def get(self, request): records = DataFilesMetadata.objects.filter( project_id=full_project_id, metadata__data_type__in=metadata_data_types - ).values('id', 'parent', 'name', 'path', 'metadata') + ).order_by('created_at').values('id', 'parent', 'name', 'path', 'metadata', 'created_at') tree = self.construct_tree(records) diff --git a/server/portal/apps/datafiles/migrations/0004_datafilesmetadata_created_at_and_more.py b/server/portal/apps/datafiles/migrations/0004_datafilesmetadata_created_at_and_more.py new file mode 100644 index 0000000000..e76ad1866c --- /dev/null +++ b/server/portal/apps/datafiles/migrations/0004_datafilesmetadata_created_at_and_more.py @@ -0,0 +1,24 @@ +# Generated by Django 4.2.11 on 2024-06-05 20:39 + +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('datafiles', '0003_datafilesmetadata'), + ] + + operations = [ + migrations.AddField( + model_name='datafilesmetadata', + name='created_at', + field=models.DateTimeField(default=django.utils.timezone.now), + ), + migrations.AddField( + model_name='datafilesmetadata', + name='last_updated', + field=models.DateTimeField(auto_now=True), + ), + ] diff --git a/server/portal/apps/datafiles/models.py b/server/portal/apps/datafiles/models.py index 25110da69f..48b288798c 100644 --- a/server/portal/apps/datafiles/models.py +++ b/server/portal/apps/datafiles/models.py @@ -5,6 +5,7 @@ from django.db import models from django.db.models.signals import pre_save from django.dispatch import receiver +from django.utils import timezone class Link(models.Model): tapis_uri = models.TextField(primary_key=True) @@ -29,6 +30,8 @@ class DataFilesMetadata(models.Model): metadata = models.JSONField(default=dict, blank=True) path = models.CharField(max_length=1024, unique=True) project = models.ForeignKey('projects.ProjectsMetadata', on_delete=models.CASCADE, related_name='data_files') + created_at = models.DateTimeField(default=timezone.now) + last_updated = models.DateTimeField(auto_now=True) def __str__(self): return self.path diff --git a/server/portal/apps/projects/migrations/0005_projectsmetadata_created_at_and_more.py b/server/portal/apps/projects/migrations/0005_projectsmetadata_created_at_and_more.py new file mode 100644 index 0000000000..dba405d8e6 --- /dev/null +++ b/server/portal/apps/projects/migrations/0005_projectsmetadata_created_at_and_more.py @@ -0,0 +1,24 @@ +# Generated by Django 4.2.11 on 2024-06-05 20:39 + +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('projects', '0004_projectsmetadata'), + ] + + operations = [ + migrations.AddField( + model_name='projectsmetadata', + name='created_at', + field=models.DateTimeField(default=django.utils.timezone.now), + ), + migrations.AddField( + model_name='projectsmetadata', + name='last_updated', + field=models.DateTimeField(auto_now=True), + ), + ] diff --git a/server/portal/apps/projects/models/metadata.py b/server/portal/apps/projects/models/metadata.py index fca97dea88..feb8371585 100644 --- a/server/portal/apps/projects/models/metadata.py +++ b/server/portal/apps/projects/models/metadata.py @@ -6,6 +6,7 @@ import logging from django.conf import settings from django.db import models +from django.utils import timezone # pylint: disable=invalid-name logger = logging.getLogger(__name__) @@ -86,6 +87,8 @@ def to_dict(self): class ProjectsMetadata(models.Model): project_id = models.CharField(max_length=255, primary_key=True) metadata = models.JSONField(default=dict, null=True) + created_at = models.DateTimeField(default=timezone.now) + last_updated = models.DateTimeField(auto_now=True) def __str__(self): return self.project_id \ No newline at end of file From a9c8bc0c45bda6c9ef5d436adbbe463c69398c4b Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Thu, 6 Jun 2024 10:53:53 -0500 Subject: [PATCH 080/328] update data model --- server/portal/apps/_custom/drp/models.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/portal/apps/_custom/drp/models.py b/server/portal/apps/_custom/drp/models.py index 2db5dc58c2..2b02a39f6a 100644 --- a/server/portal/apps/_custom/drp/models.py +++ b/server/portal/apps/_custom/drp/models.py @@ -4,7 +4,7 @@ class DrpProjectRelatedDatasets(BaseModel): """Model for DRP Project Related Datasets""" - + model_config = ConfigDict( extra="forbid", ) @@ -100,10 +100,10 @@ class DrpSampleMetadata(DrpDatasetMetadata): procedure: Optional[str] = None equipment: Optional[str] = None algorithm_description: Optional[str] = None - grain_size_min: Optional[str] = None - grain_size_max: Optional[str] = None - grain_size_avg: Optional[str] = None - porosity: Optional[str] = None + grain_size_min: Optional[float] = None + grain_size_max: Optional[float] = None + grain_size_avg: Optional[float] = None + porosity: Optional[float] = None geographical_location: Optional[str] = None date_of_collection: Optional[str] = None identifier: Optional[str] = None From 36e2d1aa837ee03d37ec79f800022113a665ae98 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Fri, 7 Jun 2024 13:16:13 -0500 Subject: [PATCH 081/328] Added functionality to add related publications, datasets, and software - Can now add related publications, datasets, and software to a project - Modified DynamicForm to handle array type fields - Added support for array type fields when initializing a dynamic form - Updated Project Edit modal - Updated Expand component to allow for a default expanded state - Updated pydantic models for datasets --- .../DataFilesModals/DataFilesFormModal.jsx | 1 + ...ataFilesProjectEditDescription.module.scss | 5 + .../DataFilesProjectEditDescriptionModal.jsx | 36 +++---- .../DataFilesProjectFileListing.jsx | 2 +- .../src/components/_common/Expand/Expand.jsx | 4 +- .../_common/Form/DynamicForm/DynamicForm.jsx | 57 ++++++++++- .../Form/DynamicForm/DynamicForm.module.scss | 25 +++++ .../_common/Form/DynamicForm/DynamicForm.scss | 8 -- ...aFilesProjectEditDescriptionModalAddon.jsx | 11 ++- .../DataFilesProjectPublishWizard.module.scss | 8 ++ .../ProjectDescription.jsx | 99 +++++++++++++++++-- client/src/utils/dataKeyFormat.js | 9 ++ server/portal/apps/_custom/drp/models.py | 24 ++--- 13 files changed, 236 insertions(+), 53 deletions(-) create mode 100644 client/src/components/_common/Form/DynamicForm/DynamicForm.module.scss delete mode 100644 client/src/components/_common/Form/DynamicForm/DynamicForm.scss create mode 100644 client/src/utils/dataKeyFormat.js diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx index c929b94708..40e05bd2e8 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx @@ -33,6 +33,7 @@ const DataFilesFormModal = () => { const isOpen = useSelector((state) => state.files.modals.dynamicform); const { params } = useFileListing('FilesListing'); + // TODO: Add support for array type fields const initialValues = form?.form_fields.reduce((acc, field) => { let value = ''; if (field.optgroups) { diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesProjectEditDescription.module.scss b/client/src/components/DataFiles/DataFilesModals/DataFilesProjectEditDescription.module.scss index 6110805607..60c919e8cc 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesProjectEditDescription.module.scss +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesProjectEditDescription.module.scss @@ -11,3 +11,8 @@ .update-button { margin-left: auto; } + +.modal-body { + max-height: 70vh; + overflow: auto; +} diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesProjectEditDescriptionModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesProjectEditDescriptionModal.jsx index b0b7b09e0c..d11a505ee3 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesProjectEditDescriptionModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesProjectEditDescriptionModal.jsx @@ -4,7 +4,7 @@ import * as Yup from 'yup'; import { Formik, Form } from 'formik'; import FormField from '_common/Form/FormField'; import { Button, Message } from '_common'; -import { Modal, ModalHeader, ModalBody } from 'reactstrap'; +import { Modal, ModalHeader, ModalBody, ModalFooter } from 'reactstrap'; import styles from './DataFilesProjectEditDescription.module.scss'; import { useAddonComponents } from 'hooks/datafiles'; @@ -76,19 +76,19 @@ const DataFilesProjectEditDescriptionModal = () => { }); return ( - - - Edit Descriptions - - - - {({ isValid, dirty }) => ( -
- + + {({ isValid, dirty }) => ( + + + Edit Project + + + { className={styles['description-textarea']} /> {DataFilesProjectEditDescriptionModalAddon && } + +
{updatingError && ( @@ -132,10 +134,10 @@ const DataFilesProjectEditDescriptionModal = () => { Update Changes
- - )} + + + )} -
); }; diff --git a/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx b/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx index f2733450bf..9fd5258099 100644 --- a/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx +++ b/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx @@ -116,7 +116,7 @@ const DataFilesProjectFileListing = ({ system, path }) => { {canEditSystem ? ( <> | diff --git a/client/src/components/_common/Expand/Expand.jsx b/client/src/components/_common/Expand/Expand.jsx index 260ac982d9..68f4101d64 100644 --- a/client/src/components/_common/Expand/Expand.jsx +++ b/client/src/components/_common/Expand/Expand.jsx @@ -5,8 +5,8 @@ import Icon from '../Icon'; import './Expand.global.scss'; import styles from './Expand.module.scss'; -const Expand = ({ className, detail, message }) => { - const [isOpen, setIsOpen] = useState(false); +const Expand = ({ className, detail, message, isOpenDefault=false }) => { + const [isOpen, setIsOpen] = useState(isOpenDefault); const toggleCallback = useCallback(() => { setIsOpen(!isOpen); }, [isOpen, setIsOpen]); diff --git a/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx b/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx index a2e1b6729b..27cbd16c3f 100644 --- a/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx +++ b/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx @@ -1,9 +1,10 @@ import React, { useEffect, useMemo, useState } from 'react'; import FormField from '../FormField'; -import { Button } from '_common'; +import { Button, Expand } from '_common'; import { useFormikContext } from 'formik'; import { FormGroup, Input } from 'reactstrap'; -import './DynamicForm.scss'; +import { FieldArray } from 'formik'; +import styles from './DynamicForm.module.scss' const DynamicForm = ({ initialFormFields, onChange }) => { @@ -158,13 +159,59 @@ const DynamicForm = ({ initialFormFields, onChange }) => { ))} ); - + // uses FieldArray from formik to handle array fields. arrayHelpers from FieldArray is used to add and remove fields + case 'array': + return ( + <> + ( +
+
+

{field.label}

+
+ {values[field.name]?.map((_, index) => ( +
+ + {field.fields.map(subField => ( +
+ {renderFormField({ + ...subField, + name: `${field.name}[${index}].${subField.name}` + })} +
+ ))} + + + } + /> +
+ ))} + +
+ )} + /> + + + + ); case 'radio': return ( - + {field.options.map((option) => ( - + div:first-child { + padding-bottom: 20px; +} diff --git a/client/src/components/_common/Form/DynamicForm/DynamicForm.scss b/client/src/components/_common/Form/DynamicForm/DynamicForm.scss deleted file mode 100644 index 394eedd9fb..0000000000 --- a/client/src/components/_common/Form/DynamicForm/DynamicForm.scss +++ /dev/null @@ -1,8 +0,0 @@ -.bold-label { - font-weight: bold; -} - -.radio-input { - margin-left: 1rem; - margin-bottom: 0rem; -} diff --git a/client/src/components/_custom/drp/DataFilesProjectEditDescriptionModalAddon/DataFilesProjectEditDescriptionModalAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectEditDescriptionModalAddon/DataFilesProjectEditDescriptionModalAddon.jsx index 1be9a68047..75acb8b030 100644 --- a/client/src/components/_custom/drp/DataFilesProjectEditDescriptionModalAddon/DataFilesProjectEditDescriptionModalAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectEditDescriptionModalAddon/DataFilesProjectEditDescriptionModalAddon.jsx @@ -25,7 +25,16 @@ const DataFilesProjectEditDescriptionModalAddon = () => { if (!isLoading && form && metadata) { form.form_fields.forEach(field => { if (metadata.hasOwnProperty(field.name)) { - setFieldValue(field.name, metadata[field.name]) + // If the field is an array, we need to set the value for each subfield using index to access the correct value + if (field.type === 'array') { + metadata[field.name].forEach((item, index) => { + field.fields.forEach(subField => { + setFieldValue(`${field.name}[${index}].${subField.name}`, item[subField.name]) + }); + }) + } else { + setFieldValue(field.name, metadata[field.name]) + } } }); } diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss index 37725abee1..dd5b7a8114 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss @@ -20,6 +20,14 @@ align-items: flex-start; } + .project-expand-card { + padding-top: 20px; + } + + .project-expand-card > div:first-child { + padding-bottom: 20px; + } + .section-project-structure main { margin-right: 0; margin-bottom: 10px; diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx index 6cf5496ee7..9ac17ae817 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx @@ -4,13 +4,16 @@ import { ShowMore, SectionTableWrapper, DescriptionList, + Expand, } from '_common'; import styles from './DataFilesProjectPublishWizard.module.scss'; import { useDispatch } from 'react-redux'; import { formatDate } from 'utils/timeFormat'; +import { formatDataKey } from 'utils/dataKeyFormat'; const ProjectDescription = ({ project }) => { const dispatch = useDispatch(); + const [data, setData] = useState({}); const onEdit = () => { dispatch({ @@ -19,6 +22,94 @@ const ProjectDescription = ({ project }) => { }); }; + useEffect(() => { + let projectData = { + Title: project.title, + Created: formatDate(new Date(project.created)), + Abstract: {project.description} , + License: project.license ?? 'None', + } + + if (project.keywords) { + projectData['Keywords'] = project.keywords + } + + if (project.related_publications.length > 0) { + const relatedPublicationCards = project.related_publications.map((publication) => { + return ( + { + acc[formatDataKey(key)] = publication[key]; + return acc; + }, {}) + } + direction={'vertical'} + density={'compact'} + /> + } + /> + ) + }) + + projectData['Related Publications'] = relatedPublicationCards + } + + if (project.related_datasets.length > 0) { + const relatedDatasetCards = project.related_datasets.map((dataset) => { + return ( + { + acc[formatDataKey(key)] = dataset[key]; + return acc; + }, {}) + } + direction={'vertical'} + density={'compact'} + /> + } + /> + ) + }) + + projectData['Related Datasets'] = relatedDatasetCards + } + + if (project.related_software.length > 0) { + const relatedSoftwareCards = project.related_software.map((software) => { + return ( + { + acc[formatDataKey(key)] = software[key]; + return acc; + }, {}) + } + direction={'vertical'} + density={'compact'} + /> + } + /> + ) + }) + + projectData['Related Software'] = relatedSoftwareCards + } + + setData(projectData) + + }, [project]) + return ( Proofread Project
} @@ -33,13 +124,7 @@ const ProjectDescription = ({ project }) => { } > {project.description} , - Keywords: project.keywords ?? '', - License: project.license ?? 'None', - }} + data={data} direction={'vertical'} /> diff --git a/client/src/utils/dataKeyFormat.js b/client/src/utils/dataKeyFormat.js new file mode 100644 index 0000000000..4c04837fc3 --- /dev/null +++ b/client/src/utils/dataKeyFormat.js @@ -0,0 +1,9 @@ +/** + * Function to format the data_type value from snake_case to Label Case i.e. project_title -> Project Title + * @param {String} key + */ +export function formatDataKey(key) { + return key.split('_') + .map(word => word.charAt(0).toUpperCase() + word.slice(1)) + .join(' '); +} \ No newline at end of file diff --git a/server/portal/apps/_custom/drp/models.py b/server/portal/apps/_custom/drp/models.py index 2b02a39f6a..dbdff6bc0c 100644 --- a/server/portal/apps/_custom/drp/models.py +++ b/server/portal/apps/_custom/drp/models.py @@ -9,9 +9,9 @@ class DrpProjectRelatedDatasets(BaseModel): extra="forbid", ) - title: str - description: str = "" - link: str = "" + dataset_title: str + dataset_description: str = "" + dataset_link: str = "" class DrpProjectRelatedSoftware(BaseModel): """Model for DRP Project Related Software""" @@ -20,9 +20,9 @@ class DrpProjectRelatedSoftware(BaseModel): extra="forbid", ) - title: str - description: str = "" - link: str = "" + software_title: str + software_description: str = "" + software_link: str = "" class DrpProjectRelatedPublications(BaseModel): """Model for DRP Project Related Publications""" @@ -31,12 +31,12 @@ class DrpProjectRelatedPublications(BaseModel): extra="forbid", ) - title: str - author: str - date_of_publication: str - publisher: str - abstract: Optional[str] = None - link: Optional[str] = None + publication_title: str + publication_author: str + publication_date_of_publication: str + publication_publisher: str + publication_description: Optional[str] = None + publication_link: Optional[str] = None class DrpProjectMetadata(BaseModel): From 5937427a111f3c4729fcc11e2e53168abf51806c Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Fri, 7 Jun 2024 13:54:17 -0500 Subject: [PATCH 082/328] bug fix and model update --- .../ProjectDescription.jsx | 6 +++--- server/portal/apps/_custom/drp/models.py | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx index 9ac17ae817..c8a315125a 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx @@ -34,7 +34,7 @@ const ProjectDescription = ({ project }) => { projectData['Keywords'] = project.keywords } - if (project.related_publications.length > 0) { + if (project.related_publications?.length > 0) { const relatedPublicationCards = project.related_publications.map((publication) => { return ( { projectData['Related Publications'] = relatedPublicationCards } - if (project.related_datasets.length > 0) { + if (project.related_datasets?.length > 0) { const relatedDatasetCards = project.related_datasets.map((dataset) => { return ( { projectData['Related Datasets'] = relatedDatasetCards } - if (project.related_software.length > 0) { + if (project.related_software?.length > 0) { const relatedSoftwareCards = project.related_software.map((software) => { return ( Date: Fri, 7 Jun 2024 14:52:36 -0500 Subject: [PATCH 083/328] update drp pydantic models --- server/portal/apps/_custom/drp/models.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/server/portal/apps/_custom/drp/models.py b/server/portal/apps/_custom/drp/models.py index 8f55739727..3811aa5360 100644 --- a/server/portal/apps/_custom/drp/models.py +++ b/server/portal/apps/_custom/drp/models.py @@ -113,7 +113,7 @@ class DrpSampleMetadata(DrpDatasetMetadata): class DrpOriginDatasetMetadata(DrpDatasetMetadata): """Model for DRP Origin Dataset Metadata""" - is_segmented: bool + is_segmented: Literal["yes", "no"] voxel_x: Optional[float] = None voxel_y: Optional[float] = None voxel_z: Optional[float] = None @@ -123,24 +123,26 @@ class DrpOriginDatasetMetadata(DrpDatasetMetadata): "millimeter", "other" ]] = None - sample: str + sample: int imaging_center: Optional[str] = None imaging_equipment_and_model: Optional[str] = None image_format: Optional[str] = None image_dimensions: Optional[str] = None image_byte_order: Optional[str] = None dimensionality: Optional[str] = None + external_uri: Optional[str] = None -class DrpAnalysisDatasetMetadata(BaseModel): +class DrpAnalysisDatasetMetadata(DrpDatasetMetadata): """Model for DRP Analysis Dataset Metadata""" - is_segmented: bool - sample: str - base_origin_data: Optional[str] = None - analysis_type: Literal[ + is_segmented: Literal["yes", "no"] + sample: int + base_origin_data: Optional[int] = None + dataset_type: Literal[ "machine_learning", "simulation", "experimental", "characterization", "other" ] + external_uri: Optional[str] = None From 3b59004bfbd1892e6e4ab750e2d938c1e307c477 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Thu, 13 Jun 2024 13:21:46 -0500 Subject: [PATCH 084/328] file upload bug fix --- server/portal/apps/__init__.py | 5 +++-- server/portal/apps/_custom/drp/models.py | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/server/portal/apps/__init__.py b/server/portal/apps/__init__.py index 5863c33507..acd1809b38 100644 --- a/server/portal/apps/__init__.py +++ b/server/portal/apps/__init__.py @@ -1,5 +1,5 @@ -from portal.apps._custom.drp.models import DrpProjectMetadata, DrpSampleMetadata, DrpOriginDatasetMetadata, DrpAnalysisDatasetMetadata +from portal.apps._custom.drp.models import DrpProjectMetadata, DrpSampleMetadata, DrpOriginDatasetMetadata, DrpAnalysisDatasetMetadata, DrpFileMetadata SCHEMA_MAPPING = { @@ -7,6 +7,7 @@ 'project': DrpProjectMetadata, 'sample': DrpSampleMetadata, 'origin_data': DrpOriginDatasetMetadata, - 'analysis_data': DrpAnalysisDatasetMetadata + 'analysis_data': DrpAnalysisDatasetMetadata, + 'file': DrpFileMetadata } } \ No newline at end of file diff --git a/server/portal/apps/_custom/drp/models.py b/server/portal/apps/_custom/drp/models.py index 3811aa5360..1cb253c492 100644 --- a/server/portal/apps/_custom/drp/models.py +++ b/server/portal/apps/_custom/drp/models.py @@ -141,8 +141,29 @@ class DrpAnalysisDatasetMetadata(DrpDatasetMetadata): dataset_type: Literal[ "machine_learning", "simulation", + "geometric_analysis", "experimental", "characterization", "other" ] external_uri: Optional[str] = None + +class DrpFileMetadata(BaseModel): + """Model for DRP File Metadata""" + + model_config = ConfigDict( + extra="forbid", + ) + + data_type: Literal['file'] + name: Optional[str] = None + image_type: Optional[Literal[ + '8_bit', '16_bit_signed', '16_bit_unsigned', '32_bit_signed', '32_bit_unsigned', '32_bit_real', '64_bit_real', + '24_bit_rgb', '24_bit_rgb_planar', '24_bit_bgr', '24_bit_integer', '32_bit_argb', '32_bit_abgr', '1_bit_bitmap', + ]] = None + height: Optional[int] = None + width: Optional[int] = None + number_of_images: Optional[int] = None + offset_to_first_image: Optional[int] = None + gap_between_images: Optional[int] = None + byte_order: Optional[Literal['big_endian', 'little_endian']] = None From 73360395177bbfe27d4d7b05bf88a6f02c34103c Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Tue, 2 Jul 2024 13:55:52 -0500 Subject: [PATCH 085/328] misc fixes based on testing session - fixed sample pdydantic model missing some fields which caused errors during sample creation - Added back the Data Files Toolbar buttons for file interactions. Added custom flag and logic that determine when to enable and disable the buttons - Improved logic when reloading page after editing metadata - Added metadata creation/saving logic when copying files - streamlined operations.py by removing some duplicate code --- .../DataFilesToolbar/DataFilesToolbar.jsx | 37 ++++++++----- .../DataFilesToolbar/customFilePermissions.js | 24 ++++++++ client/src/redux/sagas/_custom/drp.sagas.js | 5 +- client/src/redux/sagas/datafiles.sagas.js | 5 +- client/src/utils/filePermissions.js | 16 +++--- server/portal/apps/_custom/drp/models.py | 6 +- server/portal/libs/agave/operations.py | 55 +++++++++---------- server/portal/settings/settings_default.py | 2 +- 8 files changed, 93 insertions(+), 57 deletions(-) create mode 100644 client/src/components/_custom/drp/utils/DataFilesToolbar/customFilePermissions.js diff --git a/client/src/components/DataFiles/DataFilesToolbar/DataFilesToolbar.jsx b/client/src/components/DataFiles/DataFilesToolbar/DataFilesToolbar.jsx index c61ab411da..d106b3df2a 100644 --- a/client/src/components/DataFiles/DataFilesToolbar/DataFilesToolbar.jsx +++ b/client/src/components/DataFiles/DataFilesToolbar/DataFilesToolbar.jsx @@ -1,4 +1,4 @@ -import React, { useCallback } from 'react'; +import React, { useCallback, useEffect, useState } from 'react'; import { useSelector, useDispatch, shallowEqual } from 'react-redux'; import { useHistory, useLocation } from 'react-router-dom'; import PropTypes from 'prop-types'; @@ -65,7 +65,20 @@ const DataFilesToolbar = ({ scheme, api }) => { const { projectId } = useSelector((state) => state.projects.metadata); - const { allowedFileActions } = useSelector((state) => state.workbench.config); + // defaults to return true if no custom permission check is provided + const [customPermissionCheck, setCustomPermissionCheck] = useState(() => () => true); + const { hasCustomDataFilesToolbarChecks } = useSelector((state) => state.workbench.config) + const { portalName } = useSelector((state) => state.workbench); + + useEffect(() => { + // dynamically import custom permission check function if it exists + if (hasCustomDataFilesToolbarChecks && portalName) { + import(`../../_custom/${portalName}/utils/DataFilesToolbar/customFilePermissions.js`) + .then((module) => { + setCustomPermissionCheck(() => module.default); + }) + } + }, [hasCustomDataFilesToolbarChecks, portalName]) const authenticatedUser = useSelector( (state) => state.authenticatedUser.user.username @@ -103,15 +116,13 @@ const DataFilesToolbar = ({ scheme, api }) => { const modifiableUserData = api === 'tapis' && scheme !== 'public' && scheme !== 'community'; - const showCopy = allowedFileActions.includes('copy'); - - const showMove = modifiableUserData && allowedFileActions.includes('move'); + const showMove = modifiableUserData; - const showRename = modifiableUserData && allowedFileActions.includes('rename'); + const showRename = modifiableUserData; - const showTrash = modifiableUserData && allowedFileActions.includes('trash'); + const showTrash = modifiableUserData - const showDownload = api === 'tapis' && allowedFileActions.includes('download'); + const showDownload = api === 'tapis' const showMakeLink = useSelector( (state) => @@ -119,14 +130,14 @@ const DataFilesToolbar = ({ scheme, api }) => { state.workbench.config.makeLink && api === 'tapis' && (scheme === 'private' || scheme === 'projects') - ) && allowedFileActions.includes('link'); + ) const showCompress = !!useSelector( - (state) => state.workbench.config.extractApp && modifiableUserData && allowedFileActions.includes('compress') + (state) => state.workbench.config.extractApp && modifiableUserData ); const showExtract = !!useSelector( - (state) => state.workbench.config.compressApp && modifiableUserData && allowedFileActions.includes('extract') + (state) => state.workbench.config.compressApp && modifiableUserData ); const showMakePublic = useSelector( @@ -202,7 +213,7 @@ const DataFilesToolbar = ({ scheme, api }) => { }); }; - const permissionParams = { files: selectedFiles, scheme, api }; + const permissionParams = { files: selectedFiles, scheme, api, customPermissionCheck }; const canDownload = getFilePermissions('download', permissionParams); const areMultipleFilesOrFolderSelected = getFilePermissions( 'areMultipleFilesOrFolderSelected', @@ -253,14 +264,12 @@ const DataFilesToolbar = ({ scheme, api }) => { disabled={!canMove} /> )} - {showCopy && ( - )} {showDownload && ( { + + const protectedDataTypes = [ + 'sample', + 'origin_data', + 'analysis_data' + ] + + switch(operation) { + case 'copy': + case 'move': + case 'download': + case 'extract': + case 'compress': + case 'areMultipleFilesOrFolderSelected': + return files.every((file) => !protectedDataTypes.includes(file.metadata['data_type'])); + default: + return true; + } + +} + +export default customFilePermissions; \ No newline at end of file diff --git a/client/src/redux/sagas/_custom/drp.sagas.js b/client/src/redux/sagas/_custom/drp.sagas.js index c2620489dd..93b1412267 100644 --- a/client/src/redux/sagas/_custom/drp.sagas.js +++ b/client/src/redux/sagas/_custom/drp.sagas.js @@ -10,7 +10,7 @@ import { useHistory, useLocation } from 'react-router-dom'; function* executeOperation(isEdit, params, values, reloadCallback, file = null, path = '') { - + // filter out empty values from the metadata const filteredValues = Object.fromEntries( Object.entries(values).filter(([key, value]) => value !== "" && value !== null) ) @@ -40,7 +40,8 @@ function* executeOperation(isEdit, params, values, reloadCallback, file = null, } if (reloadCallback) { - yield call(reloadCallback, path); + const reloadPath = isEdit && file.path === params.path ? file.path : path; + yield call(reloadCallback, reloadPath); } yield put({ diff --git a/client/src/redux/sagas/datafiles.sagas.js b/client/src/redux/sagas/datafiles.sagas.js index d1f49432f6..19f8b31e58 100644 --- a/client/src/redux/sagas/datafiles.sagas.js +++ b/client/src/redux/sagas/datafiles.sagas.js @@ -374,7 +374,8 @@ export async function copyFileUtil( destApi, destSystem, destPath, - destPathName + destPathName, + metadata ) { let url, body; if (api === destApi) { @@ -387,6 +388,7 @@ export async function copyFileUtil( file_name: filename, filetype, dest_path_name: destPathName, + metadata: metadata ?? null }; } else { url = removeDuplicateSlashes(`/api/datafiles/transfer/${filetype}/`); @@ -437,6 +439,7 @@ export function* copyFile(src, dest, index) { dest.system, dest.path, dest.name, + src.metadata, index ); yield put({ diff --git a/client/src/utils/filePermissions.js b/client/src/utils/filePermissions.js index 1a5912d26f..49526b49e7 100644 --- a/client/src/utils/filePermissions.js +++ b/client/src/utils/filePermissions.js @@ -6,7 +6,7 @@ * @param {String} api * @returns {Boolean} */ -export default function getFilePermissions(name, { files, scheme, api }) { +export default function getFilePermissions(name, { files, scheme, api, customPermissionCheck }) { const protectedFiles = [ '.bash_profile', '.bashrc', @@ -25,28 +25,28 @@ export default function getFilePermissions(name, { files, scheme, api }) { switch (name) { case 'rename': return ( - !isProtected && files.length === 1 && isPrivate && api !== 'googledrive' + !isProtected && files.length === 1 && isPrivate && api !== 'googledrive' && customPermissionCheck(name, files) ); case 'download': return ( files.length === 1 && files[0].format !== 'folder' && - api !== 'googledrive' + api !== 'googledrive' && customPermissionCheck(name, files) ); case 'areMultipleFilesOrFolderSelected': return ( (files.length > 1 || files.some((file) => file.format === 'folder')) && - api !== 'googledrive' + api !== 'googledrive' && customPermissionCheck(name, files) ); case 'extract': - return files.length === 1 && isArchive && isPrivate && api === 'tapis'; + return files.length === 1 && isArchive && isPrivate && api === 'tapis' && customPermissionCheck(name, files); case 'compress': - return !isArchive && files.length > 0 && isPrivate && api === 'tapis'; + return !isArchive && files.length > 0 && isPrivate && api === 'tapis' && customPermissionCheck(name, files); case 'copy': - return files.length > 0; + return files.length > 0 && customPermissionCheck(name, files); case 'move': return ( - !isProtected && files.length > 0 && isPrivate && api !== 'googledrive' + !isProtected && files.length > 0 && isPrivate && api !== 'googledrive' && customPermissionCheck(name, files) ); case 'trash': return ( diff --git a/server/portal/apps/_custom/drp/models.py b/server/portal/apps/_custom/drp/models.py index 1cb253c492..2483471f70 100644 --- a/server/portal/apps/_custom/drp/models.py +++ b/server/portal/apps/_custom/drp/models.py @@ -80,13 +80,16 @@ class DrpSampleMetadata(DrpDatasetMetadata): "sandstone", "soil", "carbonate", - "greanite", + "granite", "beads", "fibrous_media", "coal", "energy_storage", "other" ] + + porous_media_other_description: Optional[str] = None + source: Literal[ "natural", "natural_extraterrestrial", @@ -108,6 +111,7 @@ class DrpSampleMetadata(DrpDatasetMetadata): geographical_location: Optional[str] = None date_of_collection: Optional[str] = None identifier: Optional[str] = None + location: Optional[str] = None class DrpOriginDatasetMetadata(DrpDatasetMetadata): diff --git a/server/portal/libs/agave/operations.py b/server/portal/libs/agave/operations.py index 2a8291daa4..0c54b99223 100644 --- a/server/portal/libs/agave/operations.py +++ b/server/portal/libs/agave/operations.py @@ -14,6 +14,7 @@ from portal.apps.projects.models.metadata import ProjectsMetadata from portal.apps.datafiles.models import DataFilesMetadata from portal.apps import SCHEMA_MAPPING +import json logger = logging.getLogger(__name__) @@ -39,6 +40,23 @@ def update_datafile_metadata(system, name, old_path, new_path, metadata): for child in DataFilesMetadata.objects.filter(parent=files_metadata.id): update_datafile_metadata(system=system, name=child.name, old_path=child.path.split('/', 1)[1], new_path=f"{new_path}/{child.name}", metadata=child.metadata) +@transaction.atomic +def create_datafile_metadata(system, path, name, metadata): + + project_instance = ProjectsMetadata.objects.get(project_id=system) + + validated_metadata = validate_datafile_metadata(metadata) + + files_metadata = DataFilesMetadata( + name = name, + path = path, + metadata = validated_metadata, + project = project_instance + ) + + files_metadata.save() + print(f'File Metadata for path {path} saved successfully') + def listing(client, system, path, offset=0, limit=100, *args, **kwargs): """ @@ -229,20 +247,7 @@ def mkdir(client, system, path, dir_name, metadata=None): path_input = str(Path(path) / Path(dir_name)) if metadata is not None: - - project_instance = ProjectsMetadata.objects.get(project_id=system) - - validated_metadata = validate_datafile_metadata(metadata) - - files_metadata = DataFilesMetadata( - name = dir_name, - path = f'{system}/{path_input.strip("/")}', - metadata = validated_metadata, - project = project_instance - ) - - files_metadata.save() - print(f'File Metadata for path {path_input} saved successfully') + create_datafile_metadata(system, f'{system}/{path_input.strip("/")}', dir_name, metadata) client.files.mkdir(systemId=system, path=path_input) @@ -332,8 +337,8 @@ def move(client, src_system, src_path, dest_system, dest_path, file_name=None, m return move_result - -def copy(client, src_system, src_path, dest_system, dest_path, file_name=None, +@transaction.atomic +def copy(client, src_system, src_path, dest_system, dest_path, file_name=None, metadata=None, *args, **kwargs): """Copies the current file to the provided destination path. @@ -365,6 +370,9 @@ def copy(client, src_system, src_path, dest_system, dest_path, file_name=None, dest_path_full = os.path.join(dest_path.strip('/'), file_name) + if metadata is not None: + create_datafile_metadata(dest_system, f'{dest_system}/{dest_path_full.strip("/")}', file_name, metadata) + if src_system == dest_system: copy_result = client.files.moveCopy(systemId=src_system, path=src_path, @@ -506,20 +514,7 @@ def upload(client, system, path, uploaded_file, metadata=None): dest_path = os.path.join(path.strip('/'), uploaded_file.name) if metadata is not None: - - project_instance = ProjectsMetadata.objects.get(project_id=system) - - validated_metadata = validate_datafile_metadata(metadata) - - files_metadata = DataFilesMetadata( - name = uploaded_file.name, - path = f'{system}/{dest_path.strip("/")}', - metadata = validated_metadata, - project = project_instance - ) - - files_metadata.save() - print(f'File Metadata for path {dest_path} saved successfully') + create_datafile_metadata(system, f'{system}/{dest_path.strip("/")}', uploaded_file.name, metadata) response_json = client.files.insert(systemId=system, path=dest_path, file=uploaded_file) tapis_indexer.apply_async(kwargs={'access_token': client.access_token.access_token, diff --git a/server/portal/settings/settings_default.py b/server/portal/settings/settings_default.py index 12de594f79..45b9e733ab 100644 --- a/server/portal/settings/settings_default.py +++ b/server/portal/settings/settings_default.py @@ -190,10 +190,10 @@ "hasUserGuide": True, "hasCustomSagas": True, "hasCustomEndpoints": True, + "hasCustomDataFilesToolbarChecks": True, "addons": ['DataFilesProjectFileListingAddon', 'DataFilesAddProjectModalAddon', 'DataFilesProjectEditDescriptionModalAddon', 'DataFilesProjectFileListingMetadataAddon', 'DataFilesProjectFileListingMetadataTitleAddon', 'DataFilesUploadModalAddon', 'DataFilesPreviewModalAddon', 'DataFilesProjectPublish'], - "allowedFileActions": ['trash', 'download'], "showDataFileType": True, "onboardingCompleteRedirect": '/workbench/', "noPHISystem": "", From c95f70095ae0706a65b1f006385731367ca648db Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Wed, 3 Jul 2024 10:36:43 -0500 Subject: [PATCH 086/328] drp metadata display is now sorted based on model keys --- server/portal/apps/_custom/drp/models.py | 20 ++++++++++---------- server/portal/apps/datafiles/models.py | 17 +++++++++++++++++ server/portal/libs/agave/operations.py | 5 ++--- 3 files changed, 29 insertions(+), 13 deletions(-) diff --git a/server/portal/apps/_custom/drp/models.py b/server/portal/apps/_custom/drp/models.py index 2483471f70..8bf214bcad 100644 --- a/server/portal/apps/_custom/drp/models.py +++ b/server/portal/apps/_custom/drp/models.py @@ -111,13 +111,19 @@ class DrpSampleMetadata(DrpDatasetMetadata): geographical_location: Optional[str] = None date_of_collection: Optional[str] = None identifier: Optional[str] = None - location: Optional[str] = None + location: Optional[str] = None # TODO_DRP: Remove in new model class DrpOriginDatasetMetadata(DrpDatasetMetadata): """Model for DRP Origin Dataset Metadata""" is_segmented: Literal["yes", "no"] + sample: int + imaging_center: Optional[str] = None + imaging_equipment_and_model: Optional[str] = None + image_format: Optional[str] = None + image_dimensions: Optional[str] = None + image_byte_order: Optional[str] = None voxel_x: Optional[float] = None voxel_y: Optional[float] = None voxel_z: Optional[float] = None @@ -127,21 +133,13 @@ class DrpOriginDatasetMetadata(DrpDatasetMetadata): "millimeter", "other" ]] = None - sample: int - imaging_center: Optional[str] = None - imaging_equipment_and_model: Optional[str] = None - image_format: Optional[str] = None - image_dimensions: Optional[str] = None - image_byte_order: Optional[str] = None dimensionality: Optional[str] = None - external_uri: Optional[str] = None + external_uri: Optional[str] = None # TODO_DRP: Remove in new model class DrpAnalysisDatasetMetadata(DrpDatasetMetadata): """Model for DRP Analysis Dataset Metadata""" is_segmented: Literal["yes", "no"] - sample: int - base_origin_data: Optional[int] = None dataset_type: Literal[ "machine_learning", "simulation", @@ -151,6 +149,8 @@ class DrpAnalysisDatasetMetadata(DrpDatasetMetadata): "other" ] external_uri: Optional[str] = None + sample: int + base_origin_data: Optional[int] = None class DrpFileMetadata(BaseModel): """Model for DRP File Metadata""" diff --git a/server/portal/apps/datafiles/models.py b/server/portal/apps/datafiles/models.py index 48b288798c..65d906c267 100644 --- a/server/portal/apps/datafiles/models.py +++ b/server/portal/apps/datafiles/models.py @@ -6,6 +6,8 @@ from django.db.models.signals import pre_save from django.dispatch import receiver from django.utils import timezone +from django.conf import settings +from portal.apps import SCHEMA_MAPPING class Link(models.Model): tapis_uri = models.TextField(primary_key=True) @@ -33,6 +35,21 @@ class DataFilesMetadata(models.Model): created_at = models.DateTimeField(default=timezone.now) last_updated = models.DateTimeField(auto_now=True) + def get_metadata(self): + """ + Transforms the metadata in the order defined in the pydantic model + """ + portal_name = settings.PORTAL_NAMESPACE + schema = SCHEMA_MAPPING[portal_name][self.metadata.get('data_type')] + + if not schema: + return self.metadata + + model_keys = list(schema.model_fields.keys()) + ordered_metadata = {k: self.metadata.get(k) for k in model_keys if k in self.metadata} + + return ordered_metadata + def __str__(self): return self.path diff --git a/server/portal/libs/agave/operations.py b/server/portal/libs/agave/operations.py index 0c54b99223..f0379e1da6 100644 --- a/server/portal/libs/agave/operations.py +++ b/server/portal/libs/agave/operations.py @@ -14,15 +14,14 @@ from portal.apps.projects.models.metadata import ProjectsMetadata from portal.apps.datafiles.models import DataFilesMetadata from portal.apps import SCHEMA_MAPPING -import json logger = logging.getLogger(__name__) def get_datafile_metadata(system, path): try: - metadata_record = DataFilesMetadata.objects.get(path=f'{system}/{path.strip("/")}') - return metadata_record.metadata + datafile_metadata = DataFilesMetadata.objects.get(path=f'{system}/{path.strip("/")}') + return datafile_metadata.get_metadata() except: return None From 478d8a15baa38146f962ed0a6253f3966b541956 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Wed, 3 Jul 2024 13:20:23 -0500 Subject: [PATCH 087/328] simplified tree construction and ordering metadata --- server/portal/apps/_custom/drp/views.py | 20 ++++++++++---------- server/portal/apps/datafiles/models.py | 11 ++++++----- server/portal/libs/agave/operations.py | 2 +- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/server/portal/apps/_custom/drp/views.py b/server/portal/apps/_custom/drp/views.py index 3cf66c909c..4a00773e19 100644 --- a/server/portal/apps/_custom/drp/views.py +++ b/server/portal/apps/_custom/drp/views.py @@ -29,18 +29,18 @@ class DigitalRocksTreeView(BaseApiView): @staticmethod def construct_tree(records, parent_id=None): - tree = {} + tree = [] for record in records: - if record['parent'] == parent_id: + if record.parent_id == parent_id: node_dict = { - "id": record['id'], - "name": record['name'], - "path": record['path'], - "metadata": record['metadata'], - "children": DigitalRocksTreeView.construct_tree(records, record['id']) + "id": record.id, + "name": record.name, + "path": record.path, + "metadata": record.ordered_metadata, + "children": DigitalRocksTreeView.construct_tree(records, record.id) } - tree[record['id']] = node_dict - return [tree[node_id] for node_id in tree] + tree.append(node_dict) + return tree def get(self, request): @@ -52,7 +52,7 @@ def get(self, request): records = DataFilesMetadata.objects.filter( project_id=full_project_id, metadata__data_type__in=metadata_data_types - ).order_by('created_at').values('id', 'parent', 'name', 'path', 'metadata', 'created_at') + ).order_by('created_at') tree = self.construct_tree(records) diff --git a/server/portal/apps/datafiles/models.py b/server/portal/apps/datafiles/models.py index 65d906c267..4b9f64a43b 100644 --- a/server/portal/apps/datafiles/models.py +++ b/server/portal/apps/datafiles/models.py @@ -8,6 +8,7 @@ from django.utils import timezone from django.conf import settings from portal.apps import SCHEMA_MAPPING +from collections import OrderedDict class Link(models.Model): tapis_uri = models.TextField(primary_key=True) @@ -35,12 +36,12 @@ class DataFilesMetadata(models.Model): created_at = models.DateTimeField(default=timezone.now) last_updated = models.DateTimeField(auto_now=True) - def get_metadata(self): - """ - Transforms the metadata in the order defined in the pydantic model - """ + @property + def ordered_metadata(self): + """Return the metadata in the order defined in the pydantic model""" + portal_name = settings.PORTAL_NAMESPACE - schema = SCHEMA_MAPPING[portal_name][self.metadata.get('data_type')] + schema = SCHEMA_MAPPING[portal_name][self.metadata.get('data_type')] if not schema: return self.metadata diff --git a/server/portal/libs/agave/operations.py b/server/portal/libs/agave/operations.py index f0379e1da6..43d297971e 100644 --- a/server/portal/libs/agave/operations.py +++ b/server/portal/libs/agave/operations.py @@ -21,7 +21,7 @@ def get_datafile_metadata(system, path): try: datafile_metadata = DataFilesMetadata.objects.get(path=f'{system}/{path.strip("/")}') - return datafile_metadata.get_metadata() + return datafile_metadata.ordered_metadata except: return None From 98d708c9ddc09c5dda0f7bbefb249e3ebf9a1c98 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Wed, 3 Jul 2024 13:21:21 -0500 Subject: [PATCH 088/328] added data type indicator to tree representation --- .../DataFilesProjectPublishWizard.module.scss | 17 +++++++++++++++++ .../ReviewProjectStructure.jsx | 14 +++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss index dd5b7a8114..054bc556cb 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss @@ -62,6 +62,23 @@ padding-bottom: 5px; } + .node-name-div { + display: flex; + justify-content: space-between; + align-items: center; + } + + .data-type-box { + display: inline-block; + background-color: #808080; /* Light grey background */ + color: #fff; /* Dark text color */ + padding: 2px 8px; /* Small top/bottom padding, larger left/right padding */ + border-radius: 12px; /* Rounded corners */ + font-size: 0.875rem; /* Smaller font size */ + margin: 8px 8px; + font-weight: bold; + } + .citation-box { border: 1px solid var(--global-color-primary--light); padding: 10px; diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx index d741b35b0c..522fc6aef3 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx @@ -95,6 +95,11 @@ const ReviewProjectStructure = ({ projectTree }) => { } }; + const formatDatatype = (data_type) => + data_type.split('_') + .map(word => word.charAt(0).toUpperCase() + word.slice(1)) + .join(' '); + const renderTree = (node) => ( <>
{ + {node.name} + + {formatDatatype(node.metadata.data_type)} + +
+ } classes={{ label: styles['tree-label'], }} From 6cad34d34a3c21ef78987f9cda7965a874043e83 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Fri, 19 Jul 2024 12:55:24 -0500 Subject: [PATCH 089/328] Update customFilePermission import --- .../DataFilesToolbar/DataFilesToolbar.jsx | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/client/src/components/DataFiles/DataFilesToolbar/DataFilesToolbar.jsx b/client/src/components/DataFiles/DataFilesToolbar/DataFilesToolbar.jsx index d106b3df2a..d43e1d517f 100644 --- a/client/src/components/DataFiles/DataFilesToolbar/DataFilesToolbar.jsx +++ b/client/src/components/DataFiles/DataFilesToolbar/DataFilesToolbar.jsx @@ -72,12 +72,22 @@ const DataFilesToolbar = ({ scheme, api }) => { useEffect(() => { // dynamically import custom permission check function if it exists + + const loadCustomPermissions = async () => { + try { + const module = await import( + `../../_custom/${portalName.toLowerCase()}/utils/DataFilesToolbar/customFilePermissions.js` + ); + setCustomPermissionCheck(() => module.default); + } catch (error) { + console.error('Error loading custom permission check:', error); + } + } + if (hasCustomDataFilesToolbarChecks && portalName) { - import(`../../_custom/${portalName}/utils/DataFilesToolbar/customFilePermissions.js`) - .then((module) => { - setCustomPermissionCheck(() => module.default); - }) + loadCustomPermissions(); } + }, [hasCustomDataFilesToolbarChecks, portalName]) const authenticatedUser = useSelector( From 90de6d7c906b59ef9009c0117b48f388a5b396b5 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Tue, 30 Jul 2024 16:20:43 -0500 Subject: [PATCH 090/328] small fix --- .../drp/utils/DataFilesToolbar/customFilePermissions.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/client/src/components/_custom/drp/utils/DataFilesToolbar/customFilePermissions.js b/client/src/components/_custom/drp/utils/DataFilesToolbar/customFilePermissions.js index ab8ecf252f..1fc0ad10b2 100644 --- a/client/src/components/_custom/drp/utils/DataFilesToolbar/customFilePermissions.js +++ b/client/src/components/_custom/drp/utils/DataFilesToolbar/customFilePermissions.js @@ -14,7 +14,13 @@ const customFilePermissions = (operation, files) => { case 'extract': case 'compress': case 'areMultipleFilesOrFolderSelected': - return files.every((file) => !protectedDataTypes.includes(file.metadata['data_type'])); + return files.every((file) => { + if (!file.metadata) { + return true; + } + + return !protectedDataTypes.includes(file.metadata['data_type']) + }); default: return true; } From 4ffd4fbbc114ecec75d0adcbef85bcb24a0d4d24 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Tue, 30 Jul 2024 17:25:10 -0500 Subject: [PATCH 091/328] Added view for 'view-only' members on a project - Removes buttons that allow for modification of datasets for view-only members --- .../DataFilesProjectFileListingAddon.jsx | 137 ++++++++++-------- ...esProjectFileListingMetadataTitleAddon.jsx | 29 +++- .../ProjectDescription.jsx | 6 + 3 files changed, 109 insertions(+), 63 deletions(-) diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx index 04bcb9bc62..fad27a86ea 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx @@ -14,65 +14,88 @@ const DataFilesProjectFileListingAddon = ({ system }) => { const { createSampleModal, createOriginDataModal, createAnalysisDataModal } = useDrpDatasetModals(projectId, portalName); + const { canEditDataset, canRequestPublication } = useSelector( + (state) => + state.projects.metadata.members + .filter((member) => + member.user + ? member.user.username === state.authenticatedUser.user.username + : { access: null } + ) + .map((currentUser) => { + return { + canEditDataset: currentUser.access === 'owner' || currentUser.access === 'edit', + canRequestPublication: currentUser.access === 'owner' + } + })[0] + ); + return ( <> - | - {selectedFiles.length == 1 && selectedFiles[0]?.metadata && - selectedFiles[0].metadata['data_type'] === 'sample' ? ( - - ) : ( - - )} - | - {selectedFiles.length == 1 && selectedFiles[0]?.metadata && - selectedFiles[0].metadata['data_type'] === 'origin_data' ? ( - - ) : ( - - )} - | - {selectedFiles.length == 1 && selectedFiles[0]?.metadata && - selectedFiles[0].metadata['data_type'] === 'analysis_data' ? ( - - ) : ( - - )} - | - - Request Publication - - + {canEditDataset && ( + <> + | + {selectedFiles.length == 1 && selectedFiles[0]?.metadata && + selectedFiles[0].metadata['data_type'] === 'sample' ? ( + + ) : ( + + )} + | + {selectedFiles.length == 1 && selectedFiles[0]?.metadata && + selectedFiles[0].metadata['data_type'] === 'origin_data' ? ( + + ) : ( + + )} + | + {selectedFiles.length == 1 && selectedFiles[0]?.metadata && + selectedFiles[0].metadata['data_type'] === 'analysis_data' ? ( + + ) : ( + + )} + + )} + {canRequestPublication && ( + <> + | + + Request Publication + + + )} ); }; diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx index acf9f6e7e9..da6be45a11 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx @@ -14,6 +14,21 @@ const DataFilesProjectFileListingMetadataTitleAddon = ({ folderMetadata, metadat const { loading } = useFileListing('FilesListing'); + const { canEditDataset } = useSelector( + (state) => + state.projects.metadata.members + .filter((member) => + member.user + ? member.user.username === state.authenticatedUser.user.username + : { access: null } + ) + .map((currentUser) => { + return { + canEditDataset: currentUser.access === 'owner' || currentUser.access === 'edit', + } + })[0] + ); + const { createSampleModal, createOriginDataModal, createAnalysisDataModal } = useDrpDatasetModals(projectId, portalName) const onEditData = (dataType) => { @@ -65,12 +80,14 @@ const DataFilesProjectFileListingMetadataTitleAddon = ({ folderMetadata, metadat {formatDatatype(folderMetadata.data_type)} - + {canEditDataset && ( + + )} ) : ( metadata.title diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx index c8a315125a..861148c088 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx @@ -56,6 +56,8 @@ const ProjectDescription = ({ project }) => { }) projectData['Related Publications'] = relatedPublicationCards + } else { + projectData['Related Publications'] = 'None' } if (project.related_datasets?.length > 0) { @@ -80,6 +82,8 @@ const ProjectDescription = ({ project }) => { }) projectData['Related Datasets'] = relatedDatasetCards + } else { + projectData['Related Datasets'] = 'None' } if (project.related_software?.length > 0) { @@ -104,6 +108,8 @@ const ProjectDescription = ({ project }) => { }) projectData['Related Software'] = relatedSoftwareCards + } else { + projectData['Related Software'] = 'None' } setData(projectData) From 15ea63cfa3c979b778e47d63506a227331184669 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Wed, 31 Jul 2024 14:12:04 -0500 Subject: [PATCH 092/328] fix for modal redirect after editing dataset --- .../DataFiles/DataFilesModals/DataFilesFormModal.jsx | 2 +- client/src/redux/sagas/_custom/drp.sagas.js | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx index 40e05bd2e8..b6e9de7cb6 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx @@ -24,7 +24,7 @@ const DataFilesFormModal = () => { } const path = updatedPath ? `${projectUrl}/${updatedPath}` : `${projectUrl}`; - history.push(path); + history.replace(path); }; const { form, selectedFile, formName, additionalData, useReloadCallback } = useSelector( diff --git a/client/src/redux/sagas/_custom/drp.sagas.js b/client/src/redux/sagas/_custom/drp.sagas.js index 93b1412267..88faa9b894 100644 --- a/client/src/redux/sagas/_custom/drp.sagas.js +++ b/client/src/redux/sagas/_custom/drp.sagas.js @@ -40,7 +40,14 @@ function* executeOperation(isEdit, params, values, reloadCallback, file = null, } if (reloadCallback) { - const reloadPath = isEdit && file.path === params.path ? file.path : path; + let reloadPath = ''; + + if (isEdit && file.path === params.path) { + reloadPath = (file.name !== values.name) ? file.path.replace(file.name, values.name) : file.path; + } else { + reloadPath = path; + } + yield call(reloadCallback, reloadPath); } From 18aad1a5e895c7f54a02dfff329a1dffc8f04804 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Mon, 12 Aug 2024 16:45:51 -0500 Subject: [PATCH 093/328] Added publication request process - Added frontend util for making a publication request to the new endpoint - Added a publication request endpoint that accepts POST requests - Added flow for publication request which involves adding service account with correct permissions, creating a new review system, transferring files and metadata to the review system, and adding reviewers --- .../DataFilesProjectPublish.jsx | 9 +- .../DataFilesProjectPublishWizard.module.scss | 4 + .../SubmitPublicationRequest.jsx | 22 ++- .../src/redux/reducers/projects.reducers.js | 30 ++++ client/src/redux/sagas/projects.sagas.js | 31 ++++ server/portal/apps/_custom/drp/models.py | 4 + server/portal/apps/datafiles/models.py | 2 +- .../portal/apps/projects/models/metadata.py | 26 +++- server/portal/apps/projects/tasks.py | 138 ++++++++++++++++++ server/portal/apps/projects/views.py | 5 +- .../shared_workspace_operations.py | 40 ++++- server/portal/apps/publications/__init__.py | 0 server/portal/apps/publications/apps.py | 5 + .../publications/migrations/0001_initial.py | 32 ++++ .../apps/publications/migrations/__init__.py | 0 server/portal/apps/publications/models.py | 32 ++++ server/portal/apps/publications/urls.py | 9 ++ server/portal/apps/publications/views.py | 68 +++++++++ server/portal/settings/settings.py | 6 + server/portal/urls.py | 1 + 20 files changed, 451 insertions(+), 13 deletions(-) create mode 100644 server/portal/apps/projects/tasks.py create mode 100644 server/portal/apps/publications/__init__.py create mode 100644 server/portal/apps/publications/apps.py create mode 100644 server/portal/apps/publications/migrations/0001_initial.py create mode 100644 server/portal/apps/publications/migrations/__init__.py create mode 100644 server/portal/apps/publications/models.py create mode 100644 server/portal/apps/publications/urls.py create mode 100644 server/portal/apps/publications/views.py diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx index 3221ef188f..1390018048 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx @@ -67,9 +67,12 @@ const DataFilesProjectPublish = ({ system }) => { ]; const formSubmit = (values) => { - // save ordered users to db. Add to project metadata? - console.log('DataFilesProjectPublish: formSubmit: ', values); - console.log('Ordered users', authors); + if (Object.keys(values).length > 0) { + dispatch({ + type: 'PROJECTS_CREATE_PUBLICATION_REQUEST', + payload: data + }); + } }; return ( diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss index 054bc556cb..de4338aa53 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss @@ -15,6 +15,10 @@ font-size: 18px; } + .submit-button { + min-width: 200px; + } + .controls { display: flex; align-items: flex-start; diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/SubmitPublicationRequest.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/SubmitPublicationRequest.jsx index bc1a47b8f0..c502ec8b30 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/SubmitPublicationRequest.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/SubmitPublicationRequest.jsx @@ -1,9 +1,10 @@ import React, { useEffect, useState } from 'react'; -import { Button, FormGroup, Input } from 'reactstrap'; +import { FormGroup, Input } from 'reactstrap'; import { useFormikContext } from 'formik'; -import { SectionTableWrapper, Section } from '_common'; +import { SectionTableWrapper, Section, Button } from '_common'; import * as Yup from 'yup'; import styles from './DataFilesProjectPublishWizard.module.scss'; +import { useSelector } from 'react-redux'; const validationSchema = Yup.object({ reviewInfo: Yup.boolean().oneOf([true], 'Must be checked'), @@ -11,10 +12,23 @@ const validationSchema = Yup.object({ }); const SubmitPublicationRequest = () => { - const { handleChange, handleBlur, values } = useFormikContext(); + const { handleChange, handleBlur, values, submitForm } = useFormikContext(); const [submitDisabled, setSubmitDisabled] = useState(true); + const { loading, error } = useSelector((state) => { + if ( + state.projects.operation && + state.projects.operation.name === 'publicationRequest' + ) { + return state.projects.operation; + } + return { + loading: false, + error: false, + }; + }); + useEffect(() => { validationSchema.isValid(values).then((valid) => { setSubmitDisabled(!valid); @@ -63,7 +77,7 @@ const SubmitPublicationRequest = () => { curator Maria Esteva before submitting the data for publication.
-
diff --git a/client/src/redux/reducers/projects.reducers.js b/client/src/redux/reducers/projects.reducers.js index a50ef39316..7f10496c33 100644 --- a/client/src/redux/reducers/projects.reducers.js +++ b/client/src/redux/reducers/projects.reducers.js @@ -219,6 +219,36 @@ export default function projects(state = initialState, action) { ...state, operation: initialState.operation, }; + case 'PROJECTS_CREATE_PUBLICATION_REQUEST_STARTED': + return { + ...state, + operation: { + name: 'publicationRequest', + loading: true, + error: null, + result: null, + }, + }; + case 'PROJECTS_CREATE_PUBLICATION_REQUEST_SUCCESS': + return { + ...state, + operation: { + name: 'publicationRequest', + loading: false, + error: null, + result: action.payload, + }, + }; + case 'PROJECTS_CREATE_PUBLICATION_REQUEST_FAILED': + return { + ...state, + operation: { + name: 'publicationRequest', + loading: false, + error: action.payload, + result: null, + }, + }; default: return state; } diff --git a/client/src/redux/sagas/projects.sagas.js b/client/src/redux/sagas/projects.sagas.js index aee76e2de6..281c414df6 100644 --- a/client/src/redux/sagas/projects.sagas.js +++ b/client/src/redux/sagas/projects.sagas.js @@ -185,6 +185,36 @@ export function* setTitleDescription(action) { } } +export async function createPublicationRequestUtil(data) { + const result = await fetchUtil({ + url: `/api/publications/publication-request/`, + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(data), + }); + return result.response; +} + +export function* createPublicationRequest(action) { + yield put({ + type: 'PROJECTS_CREATE_PUBLICATION_REQUEST_STARTED', + }); + try { + const result = yield call(createPublicationRequestUtil, action.payload); + yield put({ + type: 'PROJECTS_CREATE_PUBLICATION_REQUEST_SUCCESS', + payload: result, + }); + } catch (error) { + yield put({ + type: 'PROJECTS_CREATE_PUBLICATION_REQUEST_FAILED', + payload: error, + }); + } +} + export function* watchProjects() { yield takeLatest('PROJECTS_GET_LISTING', getProjectsListing); yield takeLatest('PROJECTS_SHOW_SHARED_WORKSPACES', showSharedWorkspaces); @@ -192,4 +222,5 @@ export function* watchProjects() { yield takeLatest('PROJECTS_GET_METADATA', getMetadata); yield takeLatest('PROJECTS_SET_MEMBER', setMember); yield takeLatest('PROJECTS_SET_TITLE_DESCRIPTION', setTitleDescription); + yield takeLatest('PROJECTS_CREATE_PUBLICATION_REQUEST', createPublicationRequest); } diff --git a/server/portal/apps/_custom/drp/models.py b/server/portal/apps/_custom/drp/models.py index 8bf214bcad..1266d5336b 100644 --- a/server/portal/apps/_custom/drp/models.py +++ b/server/portal/apps/_custom/drp/models.py @@ -1,6 +1,9 @@ from pydantic import BaseModel, ConfigDict, model_validator from typing import Optional, Literal +""" +Pydantic models for DRP Metadata +""" class DrpProjectRelatedDatasets(BaseModel): """Model for DRP Project Related Datasets""" @@ -55,6 +58,7 @@ class DrpProjectMetadata(BaseModel): related_software: list[DrpProjectRelatedSoftware] = [] related_publications: list[DrpProjectRelatedPublications] = [] publication_date: Optional[str] = None + authors: list[str] = [] class DrpDatasetMetadata(BaseModel): diff --git a/server/portal/apps/datafiles/models.py b/server/portal/apps/datafiles/models.py index 4b9f64a43b..bc622edf17 100644 --- a/server/portal/apps/datafiles/models.py +++ b/server/portal/apps/datafiles/models.py @@ -63,4 +63,4 @@ def set_or_udpate_parent(sender, instance, **kwargs): parent_folder = sender.objects.get(path=parent_path) instance.parent = parent_folder else: - instance.parent = None + instance.parent = None diff --git a/server/portal/apps/projects/models/metadata.py b/server/portal/apps/projects/models/metadata.py index feb8371585..eb520039c9 100644 --- a/server/portal/apps/projects/models/metadata.py +++ b/server/portal/apps/projects/models/metadata.py @@ -91,4 +91,28 @@ class ProjectsMetadata(models.Model): last_updated = models.DateTimeField(auto_now=True) def __str__(self): - return self.project_id \ No newline at end of file + return self.project_id + + def get_metadata(self): + reviews_as_review_project = self.publication_reviews.all() + reviews_as_source_project = self.source_publication_reviews.all() + + # Format the reviews into a list of dictionaries + reviews_data = [ + { + 'id': review.id, + 'status': review.status, + 'comments': review.comments, + 'reviewers': [user.username for user in review.reviewers.all()], + 'created_at': review.created_at, + 'last_updated': review.last_updated + } + for review in (reviews_as_review_project | reviews_as_source_project) + ] + + # Add the reviews data to the metadata + updated_metadata = self.metadata.copy() if self.metadata else {} + if reviews_data: + updated_metadata['publication_requests'] = reviews_data + + return updated_metadata diff --git a/server/portal/apps/projects/tasks.py b/server/portal/apps/projects/tasks.py new file mode 100644 index 0000000000..282e06b637 --- /dev/null +++ b/server/portal/apps/projects/tasks.py @@ -0,0 +1,138 @@ +import logging +from django.conf import settings +from celery import shared_task +from django.db import transaction +from portal.apps.projects.models.metadata import ProjectsMetadata +from portal.apps.publications.models import PublicationRequest +from portal.apps.datafiles.models import DataFilesMetadata +from portal.apps import SCHEMA_MAPPING +from portal.libs.agave.utils import user_account, service_account +from django.contrib.auth import get_user_model + +logger = logging.getLogger(__name__) + +@shared_task(bind=True, max_retries=3, queue='default') +def copy_files_and_metadata(self, user_access_token, source_workspace_id, review_workspace_id, source_system_id, review_system_id): + + logger.info(f'Starting copy task for system {source_system_id} to system {review_system_id}') + + with transaction.atomic(): + # do a system file listing at the root using the client. This will be the list of files to copy + + user_client = user_account(user_access_token) + service_client = service_account() + + source_system_files = user_client.files.listFiles(systemId=source_system_id, path='/') + source_system_files_metadata = DataFilesMetadata.objects.filter(project_id=source_system_id).order_by('created_at') + review_project = ProjectsMetadata.objects.get(project_id=review_system_id) + + # create new metadata records for each file in the listing + for metadata in source_system_files_metadata: + review_system_file_metadata = DataFilesMetadata( + name = metadata.name, + path = f'{review_system_id}/{metadata.path.split("/", 1)[-1]}', + metadata = metadata.metadata, + project = review_project + ) + + review_system_file_metadata.save() + + # create transfer object for tapis file copy + transfer_elements = [ + { + 'sourceURI': file.url, + 'destinationURI': f'tapis://{review_system_id}/{file.path}' + } + for file in source_system_files + ] + + transfer = service_client.files.createTransferTask(elements=transfer_elements) + + copy_result = { + 'uuid': transfer.uuid, + 'status': transfer.status + } + + logger.info(f'Created transfer task {transfer.uuid} for system {source_system_id} to system {review_system_id}') + + post_file_transfer.apply_async(kwargs={ + 'user_access_token': user_access_token, + 'source_workspace_id': source_workspace_id, + 'review_workspace_id': review_workspace_id, + 'source_system_id': source_system_id, + 'review_system_id': review_system_id, + 'transfer_task_id': transfer.uuid + }, countdown=30) + + +@shared_task(bind=True, max_retries=3, queue='default') +def post_file_transfer(self, user_access_token, source_workspace_id, review_workspace_id, source_system_id, review_system_id, transfer_task_id): + + logger.info(f'Starting post transfer task for transfer id {transfer_task_id} for system {source_system_id} to system {review_system_id}') + + user_client = user_account(user_access_token) + service_client = service_account() + portal_admin_username = settings.PORTAL_ADMIN_USERNAME + publication_reviewers = settings.PUBLICATION_REVIEWERS + + transfer_complete = False + transfer_status = 'PENDING' + + while not transfer_complete: + # check the status of the transfer task + transfer_details = service_client.files.getTransferTask(transferTaskId=transfer_task_id) + transfer_status = transfer_details.status + + if transfer_status in ['COMPLETED', 'FAILED']: + transfer_complete = True + else: + # Schedule this task to run again after 30 seconds if the transfer is still pending + self.apply_async(kwargs={ + 'user_access_token': user_access_token, + 'source_workspace_id': source_workspace_id, + 'review_workspace_id': review_workspace_id, + 'source_system_id': source_system_id, + 'review_system_id': review_system_id, + 'transfer_task_id': transfer_task_id + }, countdown=30) + + return + + if transfer_status == 'COMPLETED': + from portal.apps.projects.workspace_operations.shared_workspace_operations import add_user_to_workspace + + with transaction.atomic(): + # create a publication review object + review_project = ProjectsMetadata.objects.get(project_id=review_system_id) + source_project = ProjectsMetadata.objects.get(project_id=source_system_id) + + publication_request = PublicationRequest( + review_project = review_project, + source_project = source_project, + ) + + publication_request.save() + + for reviewer in publication_reviewers: + user = get_user_model().objects.get(username=reviewer) + publication_request.reviewers.add(user) + + publication_request.save() + + logger.info(f'Created publication review for system {review_system_id}') + + # remove the service account from the source workspace + user_client.systems.unShareSystem(systemId=source_system_id, users=[portal_admin_username]) + user_client.systems.revokeUserPerms(systemId=source_system_id, + userName=portal_admin_username, + permissions=["READ", "MODIFY", "EXECUTE"]) + user_client.files.deletePermissions(systemId=source_system_id, + username=portal_admin_username, + path="/") + logger.info(f'Removed service account from workspace {source_workspace_id}') + + # add the reviewers to the review workspace + for reviewer in publication_reviewers: + add_user_to_workspace(service_client, review_workspace_id, reviewer, "reader") + logger.info(f'Added reviewer {reviewer} to review system {review_system_id}') + diff --git a/server/portal/apps/projects/views.py b/server/portal/apps/projects/views.py index 95f6c14f3b..eb9b86f1fd 100644 --- a/server/portal/apps/projects/views.py +++ b/server/portal/apps/projects/views.py @@ -167,8 +167,9 @@ def get(self, request, project_id=None, system_id=None): prj = get_project(request.user.tapis_oauth.client, project_id) try: - project_metadata = ProjectsMetadata.objects.get(project_id=f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}") - prj.update(project_metadata.metadata) + project = ProjectsMetadata.objects.get(project_id=f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}") + project.metadata = project.get_metadata() + prj.update(project.metadata) except: pass diff --git a/server/portal/apps/projects/workspace_operations/shared_workspace_operations.py b/server/portal/apps/projects/workspace_operations/shared_workspace_operations.py index edf0bd8045..21bae71e37 100644 --- a/server/portal/apps/projects/workspace_operations/shared_workspace_operations.py +++ b/server/portal/apps/projects/workspace_operations/shared_workspace_operations.py @@ -1,10 +1,10 @@ # from portal.utils.encryption import createKeyPair from portal.libs.agave.utils import service_account from tapipy.tapis import Tapis +from django.db import transaction from django.conf import settings from django.contrib.auth import get_user_model -# from portal.apps.onboarding.steps.system_access_v3 import create_system_credentials, register_public_key - +from portal.apps.projects.models.metadata import ProjectsMetadata import logging logger = logging.getLogger(__name__) @@ -338,3 +338,39 @@ def get_workspace_role(client, workspace_id, username): return 'GUEST' return None + +@transaction.atomic +def create_publication_review_shared_workspace(client, source_workspace_id: str, source_system_id: str, review_workspace_id: str, + review_system_id: str, title: str, description=""): + + portal_admin_username = settings.PORTAL_ADMIN_USERNAME + service_client = service_account() + + # add admin to the source workspace to allow for file copying + resp = add_user_to_workspace(client, source_workspace_id, portal_admin_username) + + source_project = ProjectsMetadata.objects.get(project_id=source_system_id) + + # Create new Project entry for the review system + review_project = ProjectsMetadata( + project_id = review_system_id, + metadata = { + **source_project.metadata, + "is_review_project": True, + } + ) + + review_project.save() + + create_workspace_dir(review_workspace_id) + + set_workspace_acls(service_client, + settings.PORTAL_PROJECTS_ROOT_SYSTEM_NAME, + review_workspace_id, + portal_admin_username, + "add", + "writer") + + system_id = create_workspace_system(service_client, review_workspace_id, title, description) + + return system_id diff --git a/server/portal/apps/publications/__init__.py b/server/portal/apps/publications/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/server/portal/apps/publications/apps.py b/server/portal/apps/publications/apps.py new file mode 100644 index 0000000000..dc5c042507 --- /dev/null +++ b/server/portal/apps/publications/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class PublicationsConfig(AppConfig): + name = 'portal.apps.publications' \ No newline at end of file diff --git a/server/portal/apps/publications/migrations/0001_initial.py b/server/portal/apps/publications/migrations/0001_initial.py new file mode 100644 index 0000000000..f49eabb54d --- /dev/null +++ b/server/portal/apps/publications/migrations/0001_initial.py @@ -0,0 +1,32 @@ +# Generated by Django 4.2.11 on 2024-08-12 21:37 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('projects', '0005_projectsmetadata_created_at_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='PublicationRequest', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('status', models.CharField(choices=[('PENDING', 'Pending'), ('APPROVED', 'Approved'), ('REJECTED', 'Rejected')], default='PENDING', max_length=255)), + ('comments', models.TextField(blank=True, null=True)), + ('created_at', models.DateTimeField(default=django.utils.timezone.now)), + ('last_updated', models.DateTimeField(auto_now=True)), + ('review_project', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='publication_reviews', to='projects.projectsmetadata')), + ('reviewers', models.ManyToManyField(related_name='publication_reviewers', to=settings.AUTH_USER_MODEL)), + ('source_project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='source_publication_reviews', to='projects.projectsmetadata')), + ], + ), + ] diff --git a/server/portal/apps/publications/migrations/__init__.py b/server/portal/apps/publications/migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/server/portal/apps/publications/models.py b/server/portal/apps/publications/models.py new file mode 100644 index 0000000000..0f0ac5dfd6 --- /dev/null +++ b/server/portal/apps/publications/models.py @@ -0,0 +1,32 @@ +"""Publications model. + +.. :module:: portal.apps.publications.models + :synopsis: Metadata model for publications. +""" +import logging +from django.conf import settings +from django.db import models +from django.utils import timezone + +# pylint: disable=invalid-name +logger = logging.getLogger(__name__) +# pylint: enable=invalid-name + + +class PublicationRequest(models.Model): + + class Status(models.TextChoices): + PENDING = 'PENDING' + APPROVED = 'APPROVED' + REJECTED = 'REJECTED' + + review_project = models.ForeignKey('projects.ProjectsMetadata', related_name='publication_reviews', on_delete=models.SET_NULL, null=True) + source_project = models.ForeignKey('projects.ProjectsMetadata', related_name='source_publication_reviews', on_delete=models.CASCADE) + reviewers = models.ManyToManyField(settings.AUTH_USER_MODEL, related_name='publication_reviewers') + status = models.CharField(max_length=255, choices=Status.choices, default=Status.PENDING) + comments = models.TextField(blank=True, null=True) + created_at = models.DateTimeField(default=timezone.now) + last_updated = models.DateTimeField(auto_now=True) + + def __str__(self): + return f'Review for {self.project.project_id}' \ No newline at end of file diff --git a/server/portal/apps/publications/urls.py b/server/portal/apps/publications/urls.py new file mode 100644 index 0000000000..f12a14c0f5 --- /dev/null +++ b/server/portal/apps/publications/urls.py @@ -0,0 +1,9 @@ +"""Publications API Urls +""" +from portal.apps.publications import views +from django.urls import path + +app_name = 'publications' +urlpatterns = [ + path('publication-request/', views.PublicationRequestView.as_view(), name='publication_request'), +] \ No newline at end of file diff --git a/server/portal/apps/publications/views.py b/server/portal/apps/publications/views.py new file mode 100644 index 0000000000..5b784c026f --- /dev/null +++ b/server/portal/apps/publications/views.py @@ -0,0 +1,68 @@ +"""Publication views. + +.. :module:: apps.publications.views + :synopsis: Views to handle Publications +""" +import json +import logging +from django.contrib.auth.decorators import login_required +from django.conf import settings +from django.http import JsonResponse +from django.utils.decorators import method_decorator +from portal.exceptions.api import ApiException +from portal.views.base import BaseApiView +from portal.apps.projects.workspace_operations.shared_workspace_operations import create_publication_review_shared_workspace +from portal.apps.projects.models.metadata import ProjectsMetadata +from django.db import transaction +from portal.apps.projects.tasks import copy_files_and_metadata +from portal.apps.notifications.models import Notification +from django.http import HttpResponse + + +LOGGER = logging.getLogger(__name__) + +class PublicationRequestView(BaseApiView): + + @method_decorator(login_required, name='dispatch') + def post(self, request): + + data = json.loads(request.body) + + client = request.user.tapis_oauth.client + + source_workspace_id = data['projectId'] + review_workspace_id = f"{source_workspace_id}-REVIEW" + source_system_id = f'{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{source_workspace_id}' + review_system_id = f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{review_workspace_id}" + + with transaction.atomic(): + # Update authors for the source project + source_project = ProjectsMetadata.objects.get(project_id=source_system_id) + source_project.metadata['authors'] = data['authors'] + source_project.save() + + system_id = create_publication_review_shared_workspace(client, source_workspace_id, source_system_id, review_workspace_id, + review_system_id, data['title'], data['description']) + + + # Start task to copy files and metadata + copy_files_and_metadata.apply_async(kwargs={ + 'user_access_token': client.access_token.access_token, + 'source_workspace_id': source_workspace_id, + 'review_workspace_id': review_workspace_id, + 'source_system_id': source_system_id, + 'review_system_id': review_system_id + }) + + # Create notification + event_data = { + Notification.EVENT_TYPE: 'default', + Notification.STATUS: Notification.INFO, + Notification.USER: request.user.username, + Notification.MESSAGE: f'{source_workspace_id} submitted for review', + } + + with transaction.atomic(): + Notification.objects.create(**event_data) + + return HttpResponse('OK') \ No newline at end of file diff --git a/server/portal/settings/settings.py b/server/portal/settings/settings.py index 74e0ebb8b7..68d8de50f5 100644 --- a/server/portal/settings/settings.py +++ b/server/portal/settings/settings.py @@ -110,6 +110,7 @@ 'portal.apps.site_search', 'portal.apps.jupyter_mounts', 'portal.apps.portal_messages', + 'portal.apps.publications', ] MIDDLEWARE = [ @@ -735,3 +736,8 @@ """ if os.path.isfile(os.path.join(BASE_DIR, 'settings', 'settings_local.py')): from .settings_local import * # noqa: F403, F401 + +""" +SETTINGS: PUBLICATIONS +""" +PUBLICATION_REVIEWERS = getattr(settings_secret, '_PUBLICATION_REVIEWERS', []) \ No newline at end of file diff --git a/server/portal/urls.py b/server/portal/urls.py index 5dbe668176..6089aa18ee 100644 --- a/server/portal/urls.py +++ b/server/portal/urls.py @@ -90,6 +90,7 @@ path('api/projects/', include('portal.apps.projects.urls', namespace='projects')), path('api/site-search/', include('portal.apps.site_search.api.urls', namespace='site_search_api')), path('api/forms/', include('portal.apps.forms.urls', namespace='forms')), + path('api/publications/', include('portal.apps.publications.urls', namespace='publications')), # webhooks path('webhooks/', include('portal.apps.webhooks.urls', namespace='webhooks')), From 722ab336ae6cc77e643a06e7faa3911fb5c69abb Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Tue, 13 Aug 2024 17:10:19 -0500 Subject: [PATCH 094/328] fix form submit --- .../drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx index 1390018048..eec294ec50 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx @@ -67,6 +67,11 @@ const DataFilesProjectPublish = ({ system }) => { ]; const formSubmit = (values) => { + const data = { + ...metadata, + authors: authors + } + if (Object.keys(values).length > 0) { dispatch({ type: 'PROJECTS_CREATE_PUBLICATION_REQUEST', From 42c44315e954b9bbcd6e694dd39b536d602959f3 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Thu, 15 Aug 2024 11:52:08 -0500 Subject: [PATCH 095/328] bug fixes and additions - Showing loading indicators on drp form submissions - file preview only shows metadata for supported file types - added extra error handling --- .../_common/Form/DynamicForm/DynamicForm.jsx | 16 ++- .../DataFilesPreviewModalAddon.jsx | 33 +++--- .../DataFilesProjectPublish.jsx | 24 +++-- .../src/redux/reducers/datafiles.reducers.js | 3 + client/src/redux/sagas/_custom/drp.sagas.js | 100 +++++++++++------- 5 files changed, 108 insertions(+), 68 deletions(-) diff --git a/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx b/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx index 27cbd16c3f..8e51738ecb 100644 --- a/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx +++ b/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx @@ -1,10 +1,11 @@ import React, { useEffect, useMemo, useState } from 'react'; import FormField from '../FormField'; -import { Button, Expand } from '_common'; +import { Button, Expand, InlineMessage } from '_common'; import { useFormikContext } from 'formik'; import { FormGroup, Input } from 'reactstrap'; import { FieldArray } from 'formik'; import styles from './DynamicForm.module.scss' +import { useSelector } from 'react-redux'; const DynamicForm = ({ initialFormFields, onChange }) => { @@ -12,6 +13,8 @@ const DynamicForm = ({ initialFormFields, onChange }) => { const [formFields, setFormFields] = useState(initialFormFields); const { setFieldValue, values, handleChange, handleBlur } = useFormikContext(); + const status = useSelector((state) => state.files.operationStatus.dynamicform); + const handleFilterDependency = (field, values, setFieldValue, modifiedField) => { const { dependency } = field; @@ -243,9 +246,14 @@ const DynamicForm = ({ initialFormFields, onChange }) => { ); case 'submit': return ( - + <> + {status === 'ERROR' && ( + An error has occurred + )} + + ); } }; diff --git a/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.jsx b/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.jsx index 1315be5577..26aaaf2d14 100644 --- a/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.jsx @@ -15,9 +15,24 @@ const DataFilesPreviewModalAddon = ({ metadata }) => { const history = useHistory(); const location = useLocation(); + // regex from old digitalrocks portal + const standardImageType = /(\.|\/)(gif|jpe?g|png|tiff?)$/i; + + const status = useSelector((state) => state.files.operationStatus.dynamicform); + + useEffect(() => { + if (status === 'SUCCESS') { + dispatch({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { operation: 'preview', props: {} }, + }); + } + }, [status, dispatch]); + + const { params } = useFileListing('FilesListing'); - const { useReloadCallback, ...file } = useSelector( + const { ...file } = useSelector( (state) => state.files.modalProps.preview ); @@ -42,16 +57,8 @@ const DataFilesPreviewModalAddon = ({ metadata }) => { return acc; }, {}); - const reloadPage = (updatedPath = '') => { - // using regex to get the url up until the project name - let projectUrl = location.pathname.replace(/(\/projects\/[^/]+\/).*/, '$1'); - - if (projectUrl.endsWith('/')) { - projectUrl = projectUrl.slice(0, -1); - } - - const path = updatedPath ? `${projectUrl}/${updatedPath}` : `${projectUrl}`; - history.push(path); + const reloadPage = () => { + history.replace(location.pathname); }; const handleSubmit = (values) => { @@ -65,14 +72,14 @@ const DataFilesPreviewModalAddon = ({ metadata }) => { payload: { params, values, - reloadPage: useReloadCallback ? reloadPage : null, + reloadPage, selectedFile: file }, }); }; return ( - !isLoading && form && + !isLoading && form && !standardImageType.test(file.name) && <> { })); const fetchTree = useCallback(async () => { - try { - const response = await fetchUtil({ - url: `api/${portalName.toLowerCase()}/tree`, - params: { - project_id: projectId, - }, - }); - setTree(response); - } catch (error) { - console.error('Error fetching tree data:', error); - setTree([]); + if (projectId) { + try { + const response = await fetchUtil({ + url: `api/${portalName.toLowerCase()}/tree`, + params: { + project_id: projectId, + }, + }); + setTree(response); + } catch (error) { + console.error('Error fetching tree data:', error); + setTree([]); + } } }, [portalName, projectId]); diff --git a/client/src/redux/reducers/datafiles.reducers.js b/client/src/redux/reducers/datafiles.reducers.js index 791dd700f7..306a15ca42 100644 --- a/client/src/redux/reducers/datafiles.reducers.js +++ b/client/src/redux/reducers/datafiles.reducers.js @@ -1,3 +1,5 @@ +import { dynamic } from "redux-saga-test-plan/providers"; + export const initialSystemState = { storage: { configuration: [], @@ -112,6 +114,7 @@ export const initialFilesState = { error: null, loading: false, }, + dynamicform: {}, }, loadingScroll: { FilesListing: false, diff --git a/client/src/redux/sagas/_custom/drp.sagas.js b/client/src/redux/sagas/_custom/drp.sagas.js index 88faa9b894..5542d9d7cd 100644 --- a/client/src/redux/sagas/_custom/drp.sagas.js +++ b/client/src/redux/sagas/_custom/drp.sagas.js @@ -10,55 +10,75 @@ import { useHistory, useLocation } from 'react-router-dom'; function* executeOperation(isEdit, params, values, reloadCallback, file = null, path = '') { + + yield put({ + type: 'DATA_FILES_SET_OPERATION_STATUS', + payload: { status: 'RUNNING', operation: 'dynamicform' }, + }); + // filter out empty values from the metadata const filteredValues = Object.fromEntries( Object.entries(values).filter(([key, value]) => value !== "" && value !== null) ) - if (file && isEdit) { - yield call( - updateMetadataUtil, - params.api, - params.scheme, - params.system, - '/' + file.path, - '/' + path, - file.name, - values.name, - filteredValues - ); - } else { - yield call( - mkdirUtil, - params.api, - params.scheme, - params.system, - path, - values.name, - filteredValues - ); - } - - if (reloadCallback) { - let reloadPath = ''; - - if (isEdit && file.path === params.path) { - reloadPath = (file.name !== values.name) ? file.path.replace(file.name, values.name) : file.path; + try { + if (file && isEdit) { + yield call( + updateMetadataUtil, + params.api, + params.scheme, + params.system, + '/' + file.path, + '/' + path, + file.name, + values.name || file.name, + filteredValues + ); } else { - reloadPath = path; + yield call( + mkdirUtil, + params.api, + params.scheme, + params.system, + path, + values.name, + filteredValues + ); } - yield call(reloadCallback, reloadPath); - } - - yield put({ - type: 'DATA_FILES_TOGGLE_MODAL', - payload: { - operation: 'dynamicform', - props: {}, - }, - }); + if (reloadCallback) { + + const newPath = isEdit && file.path === params.path ? `${path}/${file.path.split('/').pop()}` : path; + + const reloadPath = (isEdit && file.name !== values.name) ? newPath.replace(file.name, values.name) : newPath; + + yield call(reloadCallback, reloadPath); + } + yield put({ + type: 'DATA_FILES_SET_OPERATION_STATUS', + payload: { status: 'SUCCESS', operation: 'dynamicform' }, + }); + + yield put({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { + operation: 'dynamicform', + props: {}, + }, + }); + + yield put({ + type: 'DATA_FILES_SET_OPERATION_STATUS', + payload: { status: {}, operation: 'dynamicform' }, + }); + + } catch (e) { + yield put({ + type: 'DATA_FILES_SET_OPERATION_STATUS', + payload: { status: 'ERROR', operation: 'dynamicform' }, + }); + } } function* handleSampleData(action, isEdit) { From f840679c2b503c795702b839aa807ab2673ea7e4 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Fri, 16 Aug 2024 11:35:15 -0500 Subject: [PATCH 096/328] fix formatting --- .../DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.jsx | 1 - client/src/redux/reducers/datafiles.reducers.js | 1 - 2 files changed, 2 deletions(-) diff --git a/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.jsx b/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.jsx index 26aaaf2d14..b439c64771 100644 --- a/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.jsx @@ -29,7 +29,6 @@ const DataFilesPreviewModalAddon = ({ metadata }) => { } }, [status, dispatch]); - const { params } = useFileListing('FilesListing'); const { ...file } = useSelector( diff --git a/client/src/redux/reducers/datafiles.reducers.js b/client/src/redux/reducers/datafiles.reducers.js index 306a15ca42..067164268c 100644 --- a/client/src/redux/reducers/datafiles.reducers.js +++ b/client/src/redux/reducers/datafiles.reducers.js @@ -1,4 +1,3 @@ -import { dynamic } from "redux-saga-test-plan/providers"; export const initialSystemState = { storage: { From 2a500a225741f3c3945d20ea36b7f28960d1b6f7 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Fri, 16 Aug 2024 11:35:50 -0500 Subject: [PATCH 097/328] formatting fix --- client/src/redux/reducers/datafiles.reducers.js | 1 - 1 file changed, 1 deletion(-) diff --git a/client/src/redux/reducers/datafiles.reducers.js b/client/src/redux/reducers/datafiles.reducers.js index 067164268c..ce28aebbed 100644 --- a/client/src/redux/reducers/datafiles.reducers.js +++ b/client/src/redux/reducers/datafiles.reducers.js @@ -1,4 +1,3 @@ - export const initialSystemState = { storage: { configuration: [], From e5ae770b5f91e44ed6edbc4bbbde5178b2f8bfb5 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Fri, 16 Aug 2024 12:29:59 -0500 Subject: [PATCH 098/328] Added initial publication review flow for project reviewers --- client/src/components/DataFiles/DataFiles.jsx | 18 +++- .../DataFilesProjectFileListingAddon.jsx | 55 ++++++++--- ...taFilesProjectFileListingAddon.module.scss | 1 + .../DataFilesProjectPublishWizard.module.scss | 11 ++- .../ProjectDescription.jsx | 32 +++++-- .../ReviewAuthors.jsx | 32 +++++-- .../ReviewProjectStructure.jsx | 46 +++++++--- .../SubmitPublicationReview.jsx | 66 ++++++++++++++ .../DataFilesProjectReview.jsx | 91 +++++++++++++++++++ .../DataFilesProjectReview.module.scss | 23 +++++ server/portal/settings/settings_default.py | 4 +- 11 files changed, 325 insertions(+), 54 deletions(-) create mode 100644 client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/SubmitPublicationReview.jsx create mode 100644 client/src/components/_custom/drp/DataFilesProjectReview/DataFilesProjectReview.jsx create mode 100644 client/src/components/_custom/drp/DataFilesProjectReview/DataFilesProjectReview.module.scss diff --git a/client/src/components/DataFiles/DataFiles.jsx b/client/src/components/DataFiles/DataFiles.jsx index 001f55ec22..e7737dc039 100644 --- a/client/src/components/DataFiles/DataFiles.jsx +++ b/client/src/components/DataFiles/DataFiles.jsx @@ -46,7 +46,7 @@ const DataFilesSwitch = React.memo(() => { const portalName = useSelector((state) => state.workbench.portalName); - const { DataFilesProjectPublish } = useAddonComponents({ portalName }); + const { DataFilesProjectPublish, DataFilesProjectReview } = useAddonComponents({ portalName }); return ( @@ -56,10 +56,22 @@ const DataFilesSwitch = React.memo(() => { render={({ match: { params } }) => { return ( - + + + ) + }} + /> + } + { + DataFilesProjectReview && + { + return ( + + ) - }} /> } diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx index fad27a86ea..e4cca2f37a 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx @@ -14,21 +14,37 @@ const DataFilesProjectFileListingAddon = ({ system }) => { const { createSampleModal, createOriginDataModal, createAnalysisDataModal } = useDrpDatasetModals(projectId, portalName); - const { canEditDataset, canRequestPublication } = useSelector( - (state) => - state.projects.metadata.members - .filter((member) => - member.user - ? member.user.username === state.authenticatedUser.user.username - : { access: null } - ) - .map((currentUser) => { - return { - canEditDataset: currentUser.access === 'owner' || currentUser.access === 'edit', - canRequestPublication: currentUser.access === 'owner' - } - })[0] - ); + const { canEditDataset, canRequestPublication, canReviewPublication } = useSelector((state) => { + const { members } = state.projects.metadata; + const { username } = state.authenticatedUser.user; + const currentUser = members.find((member) => member.user?.username === username); + + if (!currentUser) { + return { + canEditDataset: false, + canRequestPublication: false, + canReviewPublication: false, + }; + } + + const { access } = currentUser; + const { is_review_project, publication_requests } = state.projects.metadata; + + let canReviewPublication = false; + + if (is_review_project && publication_requests.length > 0) { + const pendingRequest = publication_requests.find((request) => request.status === 'PENDING'); + if (pendingRequest) { + canReviewPublication = pendingRequest.reviewers.includes(username); + } + } + + return { + canEditDataset: access === 'owner' || access === 'edit', + canRequestPublication: access === 'owner', + canReviewPublication, + }; + }); return ( <> @@ -96,6 +112,15 @@ const DataFilesProjectFileListingAddon = ({ system }) => { )} + {canReviewPublication && ( + <> + | + + Review Publication Request + + + )} + ); }; diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.module.scss b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.module.scss index c2c89cd3c5..9e4b49e50f 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.module.scss +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.module.scss @@ -7,4 +7,5 @@ .link { font-weight: 500; font-size: 12px; + padding-top: 0.15em; } diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss index de4338aa53..e029fa7ee8 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss @@ -15,10 +15,6 @@ font-size: 18px; } - .submit-button { - min-width: 200px; - } - .controls { display: flex; align-items: flex-start; @@ -99,4 +95,9 @@ display: flex; justify-content: center; margin-top: 20px; - } \ No newline at end of file + } + + .submit-button { + min-width: 200px; + margin: 20px; + } diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx index 861148c088..671f78e0a7 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx @@ -7,7 +7,7 @@ import { Expand, } from '_common'; import styles from './DataFilesProjectPublishWizard.module.scss'; -import { useDispatch } from 'react-redux'; +import { useDispatch, useSelector } from 'react-redux'; import { formatDate } from 'utils/timeFormat'; import { formatDataKey } from 'utils/dataKeyFormat'; @@ -15,6 +15,18 @@ const ProjectDescription = ({ project }) => { const dispatch = useDispatch(); const [data, setData] = useState({}); + const canEdit = useSelector((state) => { + const { members } = state.projects.metadata; + const { username } = state.authenticatedUser.user; + const currentUser = members.find((member) => member.user?.username === username); + + if (!currentUser) { + return false; + } + + return currentUser.access === 'owner' || currentUser.access === 'edit'; + }); + const onEdit = () => { dispatch({ type: 'DATA_FILES_TOGGLE_MODAL', @@ -120,13 +132,17 @@ const ProjectDescription = ({ project }) => { Proofread Project} headerActions={ -
- <> - - -
+ <> + {canEdit && ( +
+ <> + + +
+ )} + } > { let authorString; @@ -41,8 +42,21 @@ const ReviewAuthors = ({ project, onAuthorsUpdate }) => { const [authors, setAuthors] = useState([]); const [members, setMembers] = useState([]); + + const canEdit = useSelector((state) => { + const { members } = state.projects.metadata; + const { username } = state.authenticatedUser.user; + const currentUser = members.find((member) => member.user?.username === username); + + if (!currentUser) { + return false; + } + + return currentUser.access === 'owner' || currentUser.access === 'edit'; + }); + useEffect(() => { - const owners = project.members + const owners = project.authors ?? project.members .filter((user) => user.access === 'owner') .map((user) => ({ ...user.user, isOwner: true })); @@ -85,12 +99,16 @@ const ReviewAuthors = ({ project, onAuthorsUpdate }) => { {authors.length > 0 && project && (
- - + {canEdit && ( + <> + + + + )} +
)} diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx index 522fc6aef3..fbce2624a4 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx @@ -36,6 +36,18 @@ const ReviewProjectStructure = ({ projectTree }) => { const { createSampleModal, createOriginDataModal, createAnalysisDataModal } = useDrpDatasetModals(projectId, portalName, false); + const canEdit = useSelector((state) => { + const { members } = state.projects.metadata; + const { username } = state.authenticatedUser.user; + const currentUser = members.find((member) => member.user?.username === username); + + if (!currentUser) { + return false; + } + + return currentUser.access === 'owner' || currentUser.access === 'edit'; + }); + const onEdit = () => { dispatch({ type: 'DATA_FILES_TOGGLE_MODAL', @@ -125,13 +137,15 @@ const ReviewProjectStructure = ({ projectTree }) => { > {expandedNodes.includes(node.id) && (
- + {(canEdit || node.metadata.data_type === 'file') && ( + + )}
{node.metadata.description} {
} headerActions={ -
- <> - - -
+ <> + {canEdit && ( +
+ <> + + +
+ )} + } >
{ + const { handleChange, handleBlur, values, submitForm } = useFormikContext(); + + const [submitDisabled, setSubmitDisabled] = useState(true); + + const { loading, error } = useSelector((state) => { + if ( + state.projects.operation && + state.projects.operation.name === 'publicationRequest' + ) { + return state.projects.operation; + } + return { + loading: false, + error: false, + }; + }); + + useEffect(() => { + validationSchema.isValid(values).then((valid) => { + setSubmitDisabled(!valid); + }); + }, [values]); + + return ( + Confirm Publication Review
} + > +
+
+ If you have any doubts about the process please contact the data + curator Maria Esteva before submitting the data for publication. +
+
+ + +
+
+ + ); +}; + +export const SubmitPublicationReviewStep = () => ({ + id: 'submit_publication_review', + name: 'Submit Publication Review', + render: , + initialValues: {}, + validationSchema, +}); diff --git a/client/src/components/_custom/drp/DataFilesProjectReview/DataFilesProjectReview.jsx b/client/src/components/_custom/drp/DataFilesProjectReview/DataFilesProjectReview.jsx new file mode 100644 index 0000000000..20a2c0bbba --- /dev/null +++ b/client/src/components/_custom/drp/DataFilesProjectReview/DataFilesProjectReview.jsx @@ -0,0 +1,91 @@ +import React, { useCallback, useEffect, useState } from 'react'; +import { LoadingSpinner, SectionTableWrapper } from '_common'; +import { Link } from 'react-router-dom'; +import { useDispatch, useSelector } from 'react-redux'; +import Wizard from '_common/Wizard'; +import styles from './DataFilesProjectReview.module.scss'; +import { fetchUtil } from 'utils/fetchUtil'; +import * as ROUTES from '../../../../constants/routes'; +import { ProjectDescriptionStep } from '../DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription'; +import { ReviewProjectStructureStep } from '../DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure'; +import { ReviewAuthorsStep } from '../DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors'; +import { SubmitPublicationReviewStep } from '../DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/SubmitPublicationReview'; + +const DataFilesProjectReview = ({ system }) => { + const dispatch = useDispatch(); + const portalName = useSelector((state) => state.workbench.portalName); + const { projectId } = useSelector((state) => state.projects.metadata); + const [tree, setTree] = useState([]); + + useEffect(() => { + dispatch({ + type: 'PROJECTS_GET_METADATA', + payload: system, + }); + }, [system]); + + const { metadata } = useSelector((state) => state.projects); + + const fetchTree = useCallback(async () => { + try { + const response = await fetchUtil({ + url: `api/${portalName.toLowerCase()}/tree`, + params: { + project_id: projectId, + }, + }); + setTree(response); + } catch (error) { + console.error('Error fetching tree data:', error); + setTree([]); + } + }, [portalName, projectId]); + + useEffect(() => { + fetchTree(); + }, [portalName, projectId, fetchTree]); + + const wizardSteps = [ + ProjectDescriptionStep({ project: metadata }), + ReviewProjectStructureStep({ projectTree: tree }), + ReviewAuthorsStep({ project: metadata, onAuthorsUpdate: () => {}}), + SubmitPublicationReviewStep(), + ] + + const formSubmit = (values) => { + console.log(values); + } + + return ( + <> + {metadata.loading ? ( + + ) : ( + + Review Publication Request | {metadata.title} + + } + headerActions={ +
+ <> + + Back to Project + + +
+ } + > + + + )} + + ) +} + +export default DataFilesProjectReview; \ No newline at end of file diff --git a/client/src/components/_custom/drp/DataFilesProjectReview/DataFilesProjectReview.module.scss b/client/src/components/_custom/drp/DataFilesProjectReview/DataFilesProjectReview.module.scss new file mode 100644 index 0000000000..b736edd364 --- /dev/null +++ b/client/src/components/_custom/drp/DataFilesProjectReview/DataFilesProjectReview.module.scss @@ -0,0 +1,23 @@ +@import '../../../../styles/tools/mixins.scss'; + + +.root { + /* As a flex child */ + flex-grow: 1; + + /* WARNING: Mimicked on: History, Allocation, DataFiles, DataFilesProjectsList, DataFilesProjectFileListing, PublicData */ + padding-top: 1.75rem; /* ~28px (22.5px * design * 1.2 design-to-app ratio) */ + padding-left: 1.5em; /* ~24px (20px * design * 1.2 design-to-app ratio) */ + } + + .title { + @include truncate-with-ellipsis; + max-width: 1400px; + font-size: 18px; + } + + .controls { + display: flex; + align-items: flex-start; + } + \ No newline at end of file diff --git a/server/portal/settings/settings_default.py b/server/portal/settings/settings_default.py index 45b9e733ab..0520a33712 100644 --- a/server/portal/settings/settings_default.py +++ b/server/portal/settings/settings_default.py @@ -176,7 +176,7 @@ """ _WORKBENCH_SETTINGS = { "debug": _DEBUG, - "makeLink": True, + "makeLink": False, "viewPath": True, "compressApp": 'compress', "extractApp": 'extract', @@ -193,7 +193,7 @@ "hasCustomDataFilesToolbarChecks": True, "addons": ['DataFilesProjectFileListingAddon', 'DataFilesAddProjectModalAddon', 'DataFilesProjectEditDescriptionModalAddon', 'DataFilesProjectFileListingMetadataAddon', 'DataFilesProjectFileListingMetadataTitleAddon', - 'DataFilesUploadModalAddon', 'DataFilesPreviewModalAddon', 'DataFilesProjectPublish'], + 'DataFilesUploadModalAddon', 'DataFilesPreviewModalAddon', 'DataFilesProjectPublish', 'DataFilesProjectReview'], "showDataFileType": True, "onboardingCompleteRedirect": '/workbench/', "noPHISystem": "", From 525b77340fb235d92df48cb198b3d5e16b37da59 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Fri, 16 Aug 2024 13:42:07 -0500 Subject: [PATCH 099/328] bug fixes --- .../DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx | 2 +- .../drp/DataFilesProjectReview/DataFilesProjectReview.jsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx index 25a90679e6..0042778999 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx @@ -56,7 +56,7 @@ const ReviewAuthors = ({ project, onAuthorsUpdate }) => { }); useEffect(() => { - const owners = project.authors ?? project.members + const owners = project.authors?.length > 0 ? project.authors : project.members .filter((user) => user.access === 'owner') .map((user) => ({ ...user.user, isOwner: true })); diff --git a/client/src/components/_custom/drp/DataFilesProjectReview/DataFilesProjectReview.jsx b/client/src/components/_custom/drp/DataFilesProjectReview/DataFilesProjectReview.jsx index 20a2c0bbba..784e88a9d3 100644 --- a/client/src/components/_custom/drp/DataFilesProjectReview/DataFilesProjectReview.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectReview/DataFilesProjectReview.jsx @@ -53,7 +53,7 @@ const DataFilesProjectReview = ({ system }) => { ] const formSubmit = (values) => { - console.log(values); + } return ( From 5a122fb5f0a5aa157d81b74a7f6c9d553f4af238 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Fri, 16 Aug 2024 17:17:45 -0500 Subject: [PATCH 100/328] fix for preview modal metadata --- .../ReviewProjectStructure.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx index 522fc6aef3..eb4314be74 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx @@ -81,7 +81,7 @@ const ReviewProjectStructure = ({ projectTree }) => { scheme: params.scheme, system: params.system, path: node.path.split('/').slice(1).join('/'), - name: node.metadata.name, + name: node.name, href: 'tapis://' + node.path, length: node.metadata.length, metadata: node.metadata, From be3db60ec5aa3fa12bd8a37527cf48e6b9f5c1c2 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Mon, 19 Aug 2024 16:17:28 -0500 Subject: [PATCH 101/328] improved form validation in edit project modal --- .../DataFilesProjectEditDescriptionModal.jsx | 26 +++++------ ...aFilesProjectEditDescriptionModalAddon.jsx | 43 +++++++++++++++++-- 2 files changed, 54 insertions(+), 15 deletions(-) diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesProjectEditDescriptionModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesProjectEditDescriptionModal.jsx index d11a505ee3..58543012b4 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesProjectEditDescriptionModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesProjectEditDescriptionModal.jsx @@ -1,4 +1,4 @@ -import React, { useCallback, useMemo } from 'react'; +import React, { useCallback, useMemo, useState } from 'react'; import { useDispatch, useSelector } from 'react-redux'; import * as Yup from 'yup'; import { Formik, Form } from 'formik'; @@ -64,16 +64,18 @@ const DataFilesProjectEditDescriptionModal = () => { [projectId, dispatch] ); - const validationSchema = Yup.object().shape({ - title: Yup.string() - .min(3, 'Title must be at least 3 characters') - .max(150, 'Title must be at most 150 characters') - .required('Please enter a title.'), - description: Yup.string().max( - 800, - 'Description must be at most 800 characters' - ), - }); + const [validationSchema, setValidationSchema] = useState( + Yup.object().shape({ + title: Yup.string() + .min(3, 'Title must be at least 3 characters') + .max(150, 'Title must be at most 150 characters') + .required('Please enter a title.'), + description: Yup.string().max( + 800, + 'Description must be at most 800 characters' + ), + }) + ); return ( @@ -114,7 +116,7 @@ const DataFilesProjectEditDescriptionModal = () => { type="textarea" className={styles['description-textarea']} /> - {DataFilesProjectEditDescriptionModalAddon && } + {DataFilesProjectEditDescriptionModalAddon && }
diff --git a/client/src/components/_custom/drp/DataFilesProjectEditDescriptionModalAddon/DataFilesProjectEditDescriptionModalAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectEditDescriptionModalAddon/DataFilesProjectEditDescriptionModalAddon.jsx index 75acb8b030..b62f974d8b 100644 --- a/client/src/components/_custom/drp/DataFilesProjectEditDescriptionModalAddon/DataFilesProjectEditDescriptionModalAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectEditDescriptionModalAddon/DataFilesProjectEditDescriptionModalAddon.jsx @@ -4,9 +4,9 @@ import { fetchUtil } from 'utils/fetchUtil'; import { DynamicForm } from '_common/Form/DynamicForm'; import { useSelector } from 'react-redux'; import { useFormikContext } from 'formik' +import * as Yup from 'yup'; - -const DataFilesProjectEditDescriptionModalAddon = () => { +const DataFilesProjectEditDescriptionModalAddon = ({ setValidationSchema }) => { const { setFieldValue } = useFormikContext(); @@ -40,10 +40,47 @@ const DataFilesProjectEditDescriptionModalAddon = () => { } }, [form]) + const onFormChange = (formFields, values) => { + + let schema = {} + + Object.keys(values).forEach(key => { + const field = formFields.find((f) => f.name === key); + + if (field) { + if (field.type === 'array') { + schema[key] = Yup.array().of( + Yup.object().shape( + field.fields.reduce((acc, subField) => { + if (subField.validation?.required) { + acc[subField.name] = Yup.string().required( + `${subField.label} is required` + ); + } + return acc; + }, {}) + ) + ); + } else { + schema[key] = Yup.string().required( + `${field.label} is required` + ); + } + } + }) + + setValidationSchema((prevSchema) => { + return Yup.object().shape({ + ...prevSchema?.fields, + ...schema + }) + }) + } + return (
{!isLoading && form && - + onFormChange(formFields, values)}/> }
); From 4d8ca683ad443cb23830d11df209902753ac0690 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Mon, 19 Aug 2024 16:20:24 -0500 Subject: [PATCH 102/328] some linting --- ...aFilesProjectEditDescriptionModalAddon.jsx | 52 ++++++++++--------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/client/src/components/_custom/drp/DataFilesProjectEditDescriptionModalAddon/DataFilesProjectEditDescriptionModalAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectEditDescriptionModalAddon/DataFilesProjectEditDescriptionModalAddon.jsx index b62f974d8b..e5c926dcab 100644 --- a/client/src/components/_custom/drp/DataFilesProjectEditDescriptionModalAddon/DataFilesProjectEditDescriptionModalAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectEditDescriptionModalAddon/DataFilesProjectEditDescriptionModalAddon.jsx @@ -3,13 +3,12 @@ import { useQuery } from 'react-query'; import { fetchUtil } from 'utils/fetchUtil'; import { DynamicForm } from '_common/Form/DynamicForm'; import { useSelector } from 'react-redux'; -import { useFormikContext } from 'formik' +import { useFormikContext } from 'formik'; import * as Yup from 'yup'; const DataFilesProjectEditDescriptionModalAddon = ({ setValidationSchema }) => { - const { setFieldValue } = useFormikContext(); - + const { data: form, isLoading } = useQuery('form_EDIT_PROJECT', () => fetchUtil({ url: 'api/forms', @@ -19,32 +18,34 @@ const DataFilesProjectEditDescriptionModalAddon = ({ setValidationSchema }) => { }) ); - const { metadata } = useSelector(state => state.projects) + const { metadata } = useSelector((state) => state.projects); useEffect(() => { if (!isLoading && form && metadata) { - form.form_fields.forEach(field => { + form.form_fields.forEach((field) => { if (metadata.hasOwnProperty(field.name)) { // If the field is an array, we need to set the value for each subfield using index to access the correct value if (field.type === 'array') { metadata[field.name].forEach((item, index) => { - field.fields.forEach(subField => { - setFieldValue(`${field.name}[${index}].${subField.name}`, item[subField.name]) + field.fields.forEach((subField) => { + setFieldValue( + `${field.name}[${index}].${subField.name}`, + item[subField.name] + ); }); - }) + }); } else { - setFieldValue(field.name, metadata[field.name]) + setFieldValue(field.name, metadata[field.name]); } } }); } - }, [form]) + }, [form]); const onFormChange = (formFields, values) => { + let schema = {}; - let schema = {} - - Object.keys(values).forEach(key => { + Object.keys(values).forEach((key) => { const field = formFields.find((f) => f.name === key); if (field) { @@ -62,28 +63,29 @@ const DataFilesProjectEditDescriptionModalAddon = ({ setValidationSchema }) => { ) ); } else { - schema[key] = Yup.string().required( - `${field.label} is required` - ); + schema[key] = Yup.string().required(`${field.label} is required`); } } - }) + }); setValidationSchema((prevSchema) => { return Yup.object().shape({ ...prevSchema?.fields, - ...schema - }) - }) - } + ...schema, + }); + }); + }; return (
- {!isLoading && form && - onFormChange(formFields, values)}/> - } + {!isLoading && form && ( + onFormChange(formFields, values)} + /> + )}
); }; -export default DataFilesProjectEditDescriptionModalAddon; \ No newline at end of file +export default DataFilesProjectEditDescriptionModalAddon; From acb599252da7eac92bf1a87b1433999577ba51a2 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Tue, 27 Aug 2024 17:20:39 -0500 Subject: [PATCH 103/328] Added View Publication Request functionality --- .../DataFilesModals/DataFilesModals.jsx | 2 + .../DataFilesPublicationRequestModal.jsx | 69 +++++++++++++++++++ ...taFilesPublicationRequestModal.module.scss | 44 ++++++++++++ .../DataFilesProjectFileListingAddon.jsx | 22 +++++- .../src/redux/reducers/datafiles.reducers.js | 2 + .../portal/apps/projects/models/metadata.py | 10 ++- server/portal/apps/projects/tasks.py | 8 ++- server/portal/apps/publications/views.py | 15 ++++ 8 files changed, 168 insertions(+), 4 deletions(-) create mode 100644 client/src/components/DataFiles/DataFilesModals/DataFilesPublicationRequestModal.jsx create mode 100644 client/src/components/DataFiles/DataFilesModals/DataFilesPublicationRequestModal.module.scss diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesModals.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesModals.jsx index 8d2083c18b..8399ea17fd 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesModals.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesModals.jsx @@ -18,6 +18,7 @@ import DataFilesMakePublicModal from './DataFilesMakePublicModal'; import DataFilesDownloadMessageModal from './DataFilesDownloadMessageModal'; import './DataFilesModals.scss'; import DataFilesFormModal from './DataFilesFormModal'; +import DataFilesPublicationRequestModal from './DataFilesPublicationRequestModal'; export default function DataFilesModals() { return ( @@ -40,6 +41,7 @@ export default function DataFilesModals() { + ); } diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesPublicationRequestModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesPublicationRequestModal.jsx new file mode 100644 index 0000000000..979b5f65b9 --- /dev/null +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesPublicationRequestModal.jsx @@ -0,0 +1,69 @@ +import React, { useState, useEffect, useCallback } from 'react'; +import { useDispatch, useSelector, shallowEqual } from 'react-redux'; +import { DescriptionList } from '_common'; +import { Modal, ModalHeader, ModalBody } from 'reactstrap'; +import styles from './DataFilesPublicationRequestModal.module.scss'; +import { formatDateTime } from 'utils/timeFormat'; + +const DataFilesPublicationRequestModal = () => { + const dispatch = useDispatch(); + const [data, setData] = useState([]); + + const isOpen = useSelector((state) => state.files.modals.publicationRequest); + const { publicationRequests } = + useSelector((state) => state.files.modalProps.publicationRequest) || []; + + useEffect(() => { + const data = {}; + + publicationRequests?.forEach((request, index) => { + const publicationRequestDataObj = { + Status: request.status, + Reviewers: request.reviewers.reduce((acc, reviewer, index) => { + return ( + acc + + (index > 0 ? ', ' : '') + + `${reviewer.first_name} ${reviewer.last_name}` + ); + }, ''), + Submitted: formatDateTime(new Date(request.created_at)), + }; + + const heading = `Publication Request ${index + 1}`; + + data[heading] = ; + }); + + setData(data); + }, [publicationRequests]); + + const toggle = useCallback(() => { + dispatch({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { operation: 'publicationRequest', props: {} }, + }); + }, []); + + return ( + <> + + + Publication Requests + + + + + + + ); +}; + +export default DataFilesPublicationRequestModal; diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesPublicationRequestModal.module.scss b/client/src/components/DataFiles/DataFilesModals/DataFilesPublicationRequestModal.module.scss new file mode 100644 index 0000000000..929c10e1ec --- /dev/null +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesPublicationRequestModal.module.scss @@ -0,0 +1,44 @@ +.panel-content { + width: 100%; + overflow-y: scroll; + + /* Cross-browser solution to padding ignored by overflow (in spec-compliant Firefox) */ + /* SEE: https://stackoverflow.com/a/38997047/11817077 */ + padding-bottom: 0; + &::after { + content: ''; + display: block; + height: var(--padding); + } +} + +dl.panel-content { + --buffer-horz: 12px; /* ~10px design * 1.2 design-to-app ratio */ + --buffer-vert: 10px; /* gut feel based loosely on random space from design */ + --border: var(--global-border-width--normal) solid + var(--global-color-primary--light); +} + +dl.panel-content > dt, +dl.panel-content > dd { + padding-left: var(--buffer-horz); + padding-right: var(--buffer-horz); + padding-top: var(--buffer-vert); +} + +dl.panel-content > dt { + border-top: var(--border); +} +dl.panel-content > dt:first-of-type { + border-top: none; +} + +dl.panel-content > dt:nth-of-type(even), +dl.panel-content > dd:nth-of-type(even) { + background-color: var(--global-color-primary--x-light); +} + +/* Remove the colon from top-level labels */ +dl.panel-content > dt::after { + display: none; +} diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx index e4cca2f37a..7750cd4b5d 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx @@ -10,10 +10,20 @@ import * as ROUTES from '../../../../constants/routes'; const DataFilesProjectFileListingAddon = ({ system }) => { const portalName = useSelector((state) => state.workbench.portalName); const { projectId } = useSelector((state) => state.projects.metadata); + const { metadata } = useSelector((state) => state.projects); const { selectedFiles } = useSelectedFiles(); + const dispatch = useDispatch(); + const { createSampleModal, createOriginDataModal, createAnalysisDataModal } = useDrpDatasetModals(projectId, portalName); + const createPublicationRequestModal = () => { + dispatch({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { operation: 'publicationRequest', props: { publicationRequests: metadata?.publication_requests }}, + }); + }; + const { canEditDataset, canRequestPublication, canReviewPublication } = useSelector((state) => { const { members } = state.projects.metadata; const { username } = state.authenticatedUser.user; @@ -120,7 +130,17 @@ const DataFilesProjectFileListingAddon = ({ system }) => { )} - + {metadata?.publication_requests?.length > 0 && ( + <> + | + + + )} ); }; diff --git a/client/src/redux/reducers/datafiles.reducers.js b/client/src/redux/reducers/datafiles.reducers.js index ce28aebbed..6d19a74432 100644 --- a/client/src/redux/reducers/datafiles.reducers.js +++ b/client/src/redux/reducers/datafiles.reducers.js @@ -164,6 +164,7 @@ export const initialFilesState = { makePublic: false, downloadMessage: false, dynamicform: false, + publicationRequest: false, }, modalProps: { preview: {}, @@ -178,6 +179,7 @@ export const initialFilesState = { makePublic: {}, downloadMessage: {}, dynamicform: {}, + publicationRequest: {}, }, refs: {}, preview: { diff --git a/server/portal/apps/projects/models/metadata.py b/server/portal/apps/projects/models/metadata.py index eb520039c9..4d7e137674 100644 --- a/server/portal/apps/projects/models/metadata.py +++ b/server/portal/apps/projects/models/metadata.py @@ -7,6 +7,7 @@ from django.conf import settings from django.db import models from django.utils import timezone +from django.forms.models import model_to_dict # pylint: disable=invalid-name logger = logging.getLogger(__name__) @@ -103,7 +104,14 @@ def get_metadata(self): 'id': review.id, 'status': review.status, 'comments': review.comments, - 'reviewers': [user.username for user in review.reviewers.all()], + 'reviewers': [ + { + 'username': user.username, + 'email': user.email, + 'first_name': user.first_name, + 'last_name': user.last_name, + } + for user in review.reviewers.all()], 'created_at': review.created_at, 'last_updated': review.last_updated } diff --git a/server/portal/apps/projects/tasks.py b/server/portal/apps/projects/tasks.py index 282e06b637..2ff7900c14 100644 --- a/server/portal/apps/projects/tasks.py +++ b/server/portal/apps/projects/tasks.py @@ -8,6 +8,7 @@ from portal.apps import SCHEMA_MAPPING from portal.libs.agave.utils import user_account, service_account from django.contrib.auth import get_user_model +from django.core.exceptions import ObjectDoesNotExist logger = logging.getLogger(__name__) @@ -114,8 +115,11 @@ def post_file_transfer(self, user_access_token, source_workspace_id, review_work publication_request.save() for reviewer in publication_reviewers: - user = get_user_model().objects.get(username=reviewer) - publication_request.reviewers.add(user) + try: + user = get_user_model().objects.get(username=reviewer) + publication_request.reviewers.add(user) + except ObjectDoesNotExist: + continue publication_request.save() diff --git a/server/portal/apps/publications/views.py b/server/portal/apps/publications/views.py index 5b784c026f..5d87f65b04 100644 --- a/server/portal/apps/publications/views.py +++ b/server/portal/apps/publications/views.py @@ -17,11 +17,26 @@ from portal.apps.projects.tasks import copy_files_and_metadata from portal.apps.notifications.models import Notification from django.http import HttpResponse +from portal.apps.publications.models import PublicationRequest LOGGER = logging.getLogger(__name__) class PublicationRequestView(BaseApiView): + + def get(self, request): + project_id = request.GET.get('project_id') + + if project_id: + try: + project = ProjectsMetadata.objects.get(project_id=project_id) + publication_requests = PublicationRequest.objects.filter(source_project=project).values() + except ProjectsMetadata.DoesNotExist: + raise ApiException(f'Project {project_id} not found', status=404) + + return JsonResponse(list(publication_requests), safe=False) + + return JsonResponse([]) @method_decorator(login_required, name='dispatch') def post(self, request): From 8113b4a27a04820f44c7f554d4888742430f6ea4 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Tue, 27 Aug 2024 17:28:38 -0500 Subject: [PATCH 104/328] fix for Review Publication Request button visibility --- .../DataFilesProjectFileListingAddon.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx index 7750cd4b5d..43dc50a090 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx @@ -45,7 +45,7 @@ const DataFilesProjectFileListingAddon = ({ system }) => { if (is_review_project && publication_requests.length > 0) { const pendingRequest = publication_requests.find((request) => request.status === 'PENDING'); if (pendingRequest) { - canReviewPublication = pendingRequest.reviewers.includes(username); + canReviewPublication = pendingRequest.reviewers.some((reviewer) => reviewer.username === username); } } From 0416927c22e20358e26ebb6f2c3eec74ca3218ea Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Tue, 27 Aug 2024 17:49:06 -0500 Subject: [PATCH 105/328] prevent submission of pub request if one is already pending --- .../DataFilesProjectFileListingAddon.jsx | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx index 43dc50a090..be00eddd77 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx @@ -41,17 +41,20 @@ const DataFilesProjectFileListingAddon = ({ system }) => { const { is_review_project, publication_requests } = state.projects.metadata; let canReviewPublication = false; + let canRequestPublication = access === 'owner'; - if (is_review_project && publication_requests.length > 0) { + if (publication_requests.length > 0) { const pendingRequest = publication_requests.find((request) => request.status === 'PENDING'); + if (pendingRequest) { - canReviewPublication = pendingRequest.reviewers.some((reviewer) => reviewer.username === username); + canRequestPublication = false; // Prevent requesting publication if there is a pending request + canReviewPublication = is_review_project && pendingRequest.reviewers.some((reviewer) => reviewer.username === username); } } return { canEditDataset: access === 'owner' || access === 'edit', - canRequestPublication: access === 'owner', + canRequestPublication, canReviewPublication, }; }); From d551bbb92e8b7243e095d96df4287f4b757d3076 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Wed, 28 Aug 2024 16:19:23 -0500 Subject: [PATCH 106/328] prevent access to request publication if pending review --- .../DataFilesProjectPublish.jsx | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx index 3fbfd130f8..74d1137f0f 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx @@ -1,6 +1,6 @@ import React, { useCallback, useEffect, useState } from 'react'; import { LoadingSpinner, SectionTableWrapper } from '_common'; -import { Link } from 'react-router-dom'; +import { Link, useHistory, useLocation } from 'react-router-dom'; import { useDispatch, useSelector } from 'react-redux'; import Wizard from '_common/Wizard'; import styles from './DataFilesProjectPublish.module.scss'; @@ -14,8 +14,10 @@ import { SubmitPublicationRequestStep } from './DataFilesProjectPublishWizardSte const DataFilesProjectPublish = ({ system }) => { const dispatch = useDispatch(); + const history = useHistory(); + const location = useLocation(); const portalName = useSelector((state) => state.workbench.portalName); - const { projectId } = useSelector((state) => state.projects.metadata); + const { projectId, publication_requests } = useSelector((state) => state.projects.metadata); const [authors, setAuthors] = useState([]); const [tree, setTree] = useState([]); @@ -49,6 +51,14 @@ const DataFilesProjectPublish = ({ system }) => { } }, [portalName, projectId]); + useEffect(() => { + // Check if there is any PENDING publication request + if (publication_requests?.some((request) => request.status === 'PENDING')) { + // Navigate back to the previous location + history.replace(location.state?.from || `${ROUTES.WORKBENCH}${ROUTES.DATA}`); + } + }, [publication_requests, history]); + useEffect(() => { // workaround to get updated data after modal closes if (!dynamicFormModal || !previewModal) { From ef5c4a7f12d031ffcff31be57c774b8ca93eaca0 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Fri, 30 Aug 2024 14:38:19 -0500 Subject: [PATCH 107/328] remove unused import --- server/portal/apps/projects/models/metadata.py | 1 - 1 file changed, 1 deletion(-) diff --git a/server/portal/apps/projects/models/metadata.py b/server/portal/apps/projects/models/metadata.py index 4d7e137674..a94218f450 100644 --- a/server/portal/apps/projects/models/metadata.py +++ b/server/portal/apps/projects/models/metadata.py @@ -7,7 +7,6 @@ from django.conf import settings from django.db import models from django.utils import timezone -from django.forms.models import model_to_dict # pylint: disable=invalid-name logger = logging.getLogger(__name__) From 2fff522cbff48628685e6ef0ffb9ce43961ab591 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Fri, 30 Aug 2024 15:17:36 -0500 Subject: [PATCH 108/328] bug fix --- .../DataFilesProjectFileListingAddon.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx index be00eddd77..b8f3f495b0 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx @@ -43,7 +43,7 @@ const DataFilesProjectFileListingAddon = ({ system }) => { let canReviewPublication = false; let canRequestPublication = access === 'owner'; - if (publication_requests.length > 0) { + if (publication_requests?.length > 0) { const pendingRequest = publication_requests.find((request) => request.status === 'PENDING'); if (pendingRequest) { From 5d4fa56b03a330682c85360fe97b233b2adc2b5c Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Mon, 30 Sep 2024 18:19:26 -0500 Subject: [PATCH 109/328] improved implementation of post file transfer task --- server/portal/apps/projects/tasks.py | 122 ++++++++++++++------------- 1 file changed, 64 insertions(+), 58 deletions(-) diff --git a/server/portal/apps/projects/tasks.py b/server/portal/apps/projects/tasks.py index 2ff7900c14..6848802cef 100644 --- a/server/portal/apps/projects/tasks.py +++ b/server/portal/apps/projects/tasks.py @@ -71,72 +71,78 @@ def post_file_transfer(self, user_access_token, source_workspace_id, review_work logger.info(f'Starting post transfer task for transfer id {transfer_task_id} for system {source_system_id} to system {review_system_id}') - user_client = user_account(user_access_token) - service_client = service_account() - portal_admin_username = settings.PORTAL_ADMIN_USERNAME - publication_reviewers = settings.PUBLICATION_REVIEWERS - - transfer_complete = False - transfer_status = 'PENDING' + try: + user_client = user_account(user_access_token) + service_client = service_account() + portal_admin_username = settings.PORTAL_ADMIN_USERNAME + publication_reviewers = settings.PUBLICATION_REVIEWERS - while not transfer_complete: - # check the status of the transfer task + # Check the transfer status transfer_details = service_client.files.getTransferTask(transferTaskId=transfer_task_id) transfer_status = transfer_details.status - if transfer_status in ['COMPLETED', 'FAILED']: - transfer_complete = True - else: - # Schedule this task to run again after 30 seconds if the transfer is still pending - self.apply_async(kwargs={ - 'user_access_token': user_access_token, + # Check if the transfer is still pending + if transfer_status in ['PENDING', 'IN_PROGRESS']: + # Reschedule the task to check again after 30 seconds + logger.info(f'Transfer {transfer_task_id} is still pending, retrying in 30 seconds.') + self.apply_async( + kwargs={ + 'user_access_token': user_access_token, 'source_workspace_id': source_workspace_id, 'review_workspace_id': review_workspace_id, - 'source_system_id': source_system_id, + 'source_system_id': source_system_id, 'review_system_id': review_system_id, 'transfer_task_id': transfer_task_id - }, countdown=30) - + }, + countdown=30 + ) return + elif transfer_status == 'COMPLETED': + logger.info(f'Transfer {transfer_task_id} completed successfully for system {source_system_id} to system {review_system_id}') + from portal.apps.projects.workspace_operations.shared_workspace_operations import add_user_to_workspace + + with transaction.atomic(): + # create a publication review object + review_project = ProjectsMetadata.objects.get(project_id=review_system_id) + source_project = ProjectsMetadata.objects.get(project_id=source_system_id) + + publication_request = PublicationRequest( + review_project = review_project, + source_project = source_project, + ) + + publication_request.save() + + for reviewer in publication_reviewers: + try: + user = get_user_model().objects.get(username=reviewer) + publication_request.reviewers.add(user) + except ObjectDoesNotExist: + continue + + publication_request.save() + + logger.info(f'Created publication review for system {review_system_id}') + + # remove the service account from the source workspace + user_client.systems.unShareSystem(systemId=source_system_id, users=[portal_admin_username]) + user_client.systems.revokeUserPerms(systemId=source_system_id, + userName=portal_admin_username, + permissions=["READ", "MODIFY", "EXECUTE"]) + user_client.files.deletePermissions(systemId=source_system_id, + username=portal_admin_username, + path="/") + logger.info(f'Removed service account from workspace {source_workspace_id}') + + # add the reviewers to the review workspace + for reviewer in publication_reviewers: + add_user_to_workspace(service_client, review_workspace_id, reviewer, "reader") + logger.info(f'Added reviewer {reviewer} to review system {review_system_id}') + else: + logger.error(f'Error processing transfer {transfer_task_id} for system {source_system_id} to system {review_system_id}: Transfer status is {transfer_status}') + raise Exception(f'Transfer {transfer_task_id} failed with status {transfer_status}') - if transfer_status == 'COMPLETED': - from portal.apps.projects.workspace_operations.shared_workspace_operations import add_user_to_workspace - - with transaction.atomic(): - # create a publication review object - review_project = ProjectsMetadata.objects.get(project_id=review_system_id) - source_project = ProjectsMetadata.objects.get(project_id=source_system_id) - - publication_request = PublicationRequest( - review_project = review_project, - source_project = source_project, - ) - - publication_request.save() - - for reviewer in publication_reviewers: - try: - user = get_user_model().objects.get(username=reviewer) - publication_request.reviewers.add(user) - except ObjectDoesNotExist: - continue - - publication_request.save() - - logger.info(f'Created publication review for system {review_system_id}') - - # remove the service account from the source workspace - user_client.systems.unShareSystem(systemId=source_system_id, users=[portal_admin_username]) - user_client.systems.revokeUserPerms(systemId=source_system_id, - userName=portal_admin_username, - permissions=["READ", "MODIFY", "EXECUTE"]) - user_client.files.deletePermissions(systemId=source_system_id, - username=portal_admin_username, - path="/") - logger.info(f'Removed service account from workspace {source_workspace_id}') - - # add the reviewers to the review workspace - for reviewer in publication_reviewers: - add_user_to_workspace(service_client, review_workspace_id, reviewer, "reader") - logger.info(f'Added reviewer {reviewer} to review system {review_system_id}') + except Exception as e: + logger.error(f'Error processing transfer {transfer_task_id} for system {source_system_id} to system {review_system_id}: {e}') + self.retry(exc=e, countdown=30) From a5607b05c3a04a79da6c00ab9ca39a44c2c5b879 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Fri, 4 Oct 2024 18:27:49 -0500 Subject: [PATCH 110/328] WIP integrating entity logic from DS --- .../drp/utils/hooks/useDrpDatasetModals.js | 16 +- client/src/redux/sagas/_custom/drp.sagas.js | 68 ++++--- client/src/redux/sagas/projects.sagas.js | 35 ++++ server/poetry.lock | 22 ++- server/portal/apps/__init__.py | 14 +- server/portal/apps/_custom/drp/constants.py | 12 ++ server/portal/apps/_custom/drp/models.py | 26 ++- server/portal/apps/_custom/drp/views.py | 11 +- server/portal/apps/datafiles/models.py | 2 +- .../management/commands/migrate-projects.py | 4 +- .../commands/migrate-projects_unit_test.py | 10 +- ...e_projectmetadata_legacyprojectmetadata.py | 17 ++ .../0007_projectmetadata_and_more.py | 47 +++++ .../portal/apps/projects/models/__init__.py | 4 +- server/portal/apps/projects/models/base.py | 6 +- .../portal/apps/projects/models/metadata.py | 2 +- .../apps/projects/models/project_metadata.py | 184 ++++++++++++++++++ .../portal/apps/projects/models/unit_test.py | 16 +- .../apps/projects/schema_models/base.py | 28 +++ server/portal/apps/projects/serializers.py | 4 +- server/portal/apps/projects/unit_test.py | 14 +- server/portal/apps/projects/urls.py | 1 + server/portal/apps/projects/views.py | 116 +++++++++-- .../workspace_operations/graph_operations.py | 141 ++++++++++++++ .../project_meta_operations.py | 184 ++++++++++++++++++ .../shared_workspace_migration.py | 6 +- server/portal/apps/search/tasks.py | 4 +- server/portal/apps/signals/receivers.py | 4 +- server/portal/libs/agave/operations.py | 58 ++++-- server/pyproject.toml | 1 + 30 files changed, 941 insertions(+), 116 deletions(-) create mode 100644 server/portal/apps/_custom/drp/constants.py create mode 100644 server/portal/apps/projects/migrations/0006_rename_projectmetadata_legacyprojectmetadata.py create mode 100644 server/portal/apps/projects/migrations/0007_projectmetadata_and_more.py create mode 100644 server/portal/apps/projects/models/project_metadata.py create mode 100644 server/portal/apps/projects/schema_models/base.py create mode 100644 server/portal/apps/projects/workspace_operations/graph_operations.py create mode 100644 server/portal/apps/projects/workspace_operations/project_meta_operations.py diff --git a/client/src/components/_custom/drp/utils/hooks/useDrpDatasetModals.js b/client/src/components/_custom/drp/utils/hooks/useDrpDatasetModals.js index e4c5bf5133..5334f4029c 100644 --- a/client/src/components/_custom/drp/utils/hooks/useDrpDatasetModals.js +++ b/client/src/components/_custom/drp/utils/hooks/useDrpDatasetModals.js @@ -52,9 +52,9 @@ const useDrpDatasetModals = (projectId, portalName, useReloadCallback = true) => field.options.push( ...samples.map((sample) => { return { - ...sample.metadata, - value: parseInt(sample.id), - label: sample.name, + ...sample.value, + value: sample.uuid, + label: sample.value.name, }; }) ); @@ -85,8 +85,8 @@ const useDrpDatasetModals = (projectId, portalName, useReloadCallback = true) => field.options.push( ...samples.map((sample) => { return { - value: parseInt(sample.id), - label: sample.name, + value: sample.uuid, + label: sample.value.name, }; }) ); @@ -94,9 +94,9 @@ const useDrpDatasetModals = (projectId, portalName, useReloadCallback = true) => field.options.push( ...originDatasets.map((originData) => { return { - value: parseInt(originData.id), - label: originData.name, - dependentId: parseInt(originData.metadata.sample), + value: originData.uuid, + label: originData.value.name, + dependentId: originData.value.sample, }; }) ); diff --git a/client/src/redux/sagas/_custom/drp.sagas.js b/client/src/redux/sagas/_custom/drp.sagas.js index 5542d9d7cd..1b888a8efd 100644 --- a/client/src/redux/sagas/_custom/drp.sagas.js +++ b/client/src/redux/sagas/_custom/drp.sagas.js @@ -7,6 +7,7 @@ import { mkdirUtil,updateMetadataUtil, } from '../datafiles.sagas'; import { useHistory, useLocation } from 'react-router-dom'; +import { createEntityUtil, patchEntityUtil } from '../projects.sagas'; function* executeOperation(isEdit, params, values, reloadCallback, file = null, path = '') { @@ -24,26 +25,42 @@ function* executeOperation(isEdit, params, values, reloadCallback, file = null, try { if (file && isEdit) { yield call( - updateMetadataUtil, - params.api, - params.scheme, + patchEntityUtil, + filteredValues.data_type, params.system, - '/' + file.path, - '/' + path, - file.name, - values.name || file.name, - filteredValues - ); - } else { + file.path, + filteredValues, + file.uuid + ) + // yield call( + // updateMetadataUtil, + // params.api, + // params.scheme, + // params.system, + // '/' + file.path, + // '/' + path, + // file.name, + // values.name || file.name, + // filteredValues + // ); + } else { yield call( - mkdirUtil, - params.api, - params.scheme, + createEntityUtil, + filteredValues.data_type, params.system, - path, - values.name, + path || "/", filteredValues - ); + ) + + // yield call( + // mkdirUtil, + // params.api, + // params.scheme, + // params.system, + // path, + // values.name, + // filteredValues + // ); } if (reloadCallback) { @@ -98,17 +115,19 @@ function* handleOriginData(action, isEdit) { const { params, values, reloadPage: reloadCallback, selectedFile, additionalData: { samples } } = action.payload; const sample = samples.find( - (sample) => sample.id === parseInt(values.sample) + (sample) => sample.uuid === values.sample ); const metadata = { ...values, data_type: 'origin_data', - sample: sample.id, + sample: sample.uuid, } // get the path without system name - const path = sample.path.split('/').slice(1).join('/'); + // const path = sample.value.path.split('/').slice(1).join('/'); + const path = sample.value.name; // PROBLEM + yield call( executeOperation, isEdit, params, metadata, reloadCallback, selectedFile, path ) @@ -118,21 +137,20 @@ function* handleAnalysisData(action, isEdit) { const { params, values, reloadPage: reloadCallback, selectedFile, additionalData: { samples, originDatasets } } = action.payload; const sample = samples.find( - (sample) => sample.id === parseInt(values.sample) + (sample) => sample.uuid === values.sample ); const originData = originDatasets.find( - (originData) => originData.id === parseInt(values.base_origin_data) + (originData) => originData.uuid === values.base_origin_data ); - let path = originData ? originData.path : sample.path; - path = path.split('/').slice(1).join('/') + let path = originData ? `${sample.value.name}/${originData.value.name}` : sample.value.name; const metadata = { ...values, data_type: 'analysis_data', - sample: sample.id, - base_origin_data: originData ? originData.id : '' + sample: sample.uuid, + base_origin_data: originData ? originData.uuid : '' } yield call( diff --git a/client/src/redux/sagas/projects.sagas.js b/client/src/redux/sagas/projects.sagas.js index 281c414df6..484b70f7fc 100644 --- a/client/src/redux/sagas/projects.sagas.js +++ b/client/src/redux/sagas/projects.sagas.js @@ -215,6 +215,41 @@ export function* createPublicationRequest(action) { } } +export async function createEntityUtil(entityType, projectId, path, data) { + const result = await fetchUtil({ + url: `/api/projects/${projectId}/entities/create`, + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + name: entityType, + value: data, + path: path + }), + }); + + return result.response; +} + +export async function patchEntityUtil(entityType, projectId, path, data, entityUuid) { + const result = await fetchUtil({ + url: `/api/projects/${projectId}/entities/create`, + method: 'PATCH', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + name: entityType, + value: data, + uuid: entityUuid ?? null, + path: path + }), + }); + + return result.response; +} + export function* watchProjects() { yield takeLatest('PROJECTS_GET_LISTING', getProjectsListing); yield takeLatest('PROJECTS_SHOW_SHARED_WORKSPACES', showSharedWorkspaces); diff --git a/server/poetry.lock b/server/poetry.lock index 35968740f2..26a4291597 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "amqp" @@ -1691,6 +1691,24 @@ files = [ {file = "msgpack-1.0.7.tar.gz", hash = "sha256:572efc93db7a4d27e404501975ca6d2d9775705c2d922390d878fcf768d92c87"}, ] +[[package]] +name = "networkx" +version = "3.3" +description = "Python package for creating and manipulating graphs and networks" +optional = false +python-versions = ">=3.10" +files = [ + {file = "networkx-3.3-py3-none-any.whl", hash = "sha256:28575580c6ebdaf4505b22c6256a2b9de86b316dc63ba9e93abde3d78dfdbcf2"}, + {file = "networkx-3.3.tar.gz", hash = "sha256:0c127d8b2f4865f59ae9cb8aafcd60b5c70f3241ebd66f7defad7c4ab90126c9"}, +] + +[package.extras] +default = ["matplotlib (>=3.6)", "numpy (>=1.23)", "pandas (>=1.4)", "scipy (>=1.9,!=1.11.0,!=1.11.1)"] +developer = ["changelist (==0.5)", "mypy (>=1.1)", "pre-commit (>=3.2)", "rtoml"] +doc = ["myst-nb (>=1.0)", "numpydoc (>=1.7)", "pillow (>=9.4)", "pydata-sphinx-theme (>=0.14)", "sphinx (>=7)", "sphinx-gallery (>=0.14)", "texext (>=0.6.7)"] +extra = ["lxml (>=4.6)", "pydot (>=2.0)", "pygraphviz (>=1.12)", "sympy (>=1.10)"] +test = ["pytest (>=7.2)", "pytest-cov (>=4.0)"] + [[package]] name = "oauthlib" version = "3.2.2" @@ -3164,4 +3182,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "0658114906564d66f49af2a9e41bd71f4bc99af88f91c50686aa04fe41a366a7" +content-hash = "94ef98debf8e3a9036ba58eed8f145577a9183ae4156267fb1483834f7f01e16" diff --git a/server/portal/apps/__init__.py b/server/portal/apps/__init__.py index acd1809b38..1bd39af027 100644 --- a/server/portal/apps/__init__.py +++ b/server/portal/apps/__init__.py @@ -1,13 +1,11 @@ from portal.apps._custom.drp.models import DrpProjectMetadata, DrpSampleMetadata, DrpOriginDatasetMetadata, DrpAnalysisDatasetMetadata, DrpFileMetadata - +from portal.apps._custom.drp import constants SCHEMA_MAPPING = { - 'DRP': { - 'project': DrpProjectMetadata, - 'sample': DrpSampleMetadata, - 'origin_data': DrpOriginDatasetMetadata, - 'analysis_data': DrpAnalysisDatasetMetadata, - 'file': DrpFileMetadata - } + constants.PROJECT: DrpProjectMetadata, + constants.SAMPLE: DrpSampleMetadata, + constants.ORIGIN_DATA: DrpOriginDatasetMetadata, + constants.ANALYSIS_DATA: DrpAnalysisDatasetMetadata, + constants.FILE: DrpFileMetadata } \ No newline at end of file diff --git a/server/portal/apps/_custom/drp/constants.py b/server/portal/apps/_custom/drp/constants.py new file mode 100644 index 0000000000..481aae583d --- /dev/null +++ b/server/portal/apps/_custom/drp/constants.py @@ -0,0 +1,12 @@ +# pylint: disable=line-too-long +"""Mapping of possible metadata names.""" + +PROJECT = "drp.project" +PROJECT_GRAPH = "drp.project.graph" + +SAMPLE = "drp.project.sample" +DIGITAL_DATASET = "drp.project.digital_dataset" +ANALYSIS_DATA = "drp.project.analysis_dataset" +ORIGIN_DATA = "drp.project.origin_data" + +FILE = "drp.project.file" \ No newline at end of file diff --git a/server/portal/apps/_custom/drp/models.py b/server/portal/apps/_custom/drp/models.py index 1266d5336b..c1b0362aa1 100644 --- a/server/portal/apps/_custom/drp/models.py +++ b/server/portal/apps/_custom/drp/models.py @@ -1,10 +1,27 @@ from pydantic import BaseModel, ConfigDict, model_validator from typing import Optional, Literal +from pydantic.alias_generators import to_camel +from functools import partial """ Pydantic models for DRP Metadata """ +class DrpMetadataModel(BaseModel): + model_config = ConfigDict( + alias_generator=to_camel, + populate_by_name=True, + from_attributes=True, + extra="forbid", + coerce_numbers_to_str=True, + ) + + def model_dump(self, *args, **kwargs): + # default by_alias to true for camelCase serialization + return partial(super().model_dump, by_alias=True, exclude_none=True)( + *args, **kwargs + ) + class DrpProjectRelatedDatasets(BaseModel): """Model for DRP Project Related Datasets""" @@ -42,13 +59,14 @@ class DrpProjectRelatedPublications(BaseModel): publication_link: Optional[str] = None -class DrpProjectMetadata(BaseModel): +class DrpProjectMetadata(DrpMetadataModel): """Model for DRP Project Metadata""" model_config = ConfigDict( extra="forbid", ) + project_id: str title: str description: str = "" license: Optional[str] = None @@ -122,7 +140,7 @@ class DrpOriginDatasetMetadata(DrpDatasetMetadata): """Model for DRP Origin Dataset Metadata""" is_segmented: Literal["yes", "no"] - sample: int + sample: str imaging_center: Optional[str] = None imaging_equipment_and_model: Optional[str] = None image_format: Optional[str] = None @@ -153,8 +171,8 @@ class DrpAnalysisDatasetMetadata(DrpDatasetMetadata): "other" ] external_uri: Optional[str] = None - sample: int - base_origin_data: Optional[int] = None + sample: str + base_origin_data: Optional[str] = None class DrpFileMetadata(BaseModel): """Model for DRP File Metadata""" diff --git a/server/portal/apps/_custom/drp/views.py b/server/portal/apps/_custom/drp/views.py index 4a00773e19..ffd35f5208 100644 --- a/server/portal/apps/_custom/drp/views.py +++ b/server/portal/apps/_custom/drp/views.py @@ -3,6 +3,8 @@ from django.conf import settings from django.http import JsonResponse, HttpResponseForbidden from portal.apps.datafiles.models import DataFilesMetadata +from portal.apps.projects.models.project_metadata import ProjectMetadata +from portal.apps._custom.drp import constants class DigitalRocksSampleView(BaseApiView): @@ -12,11 +14,16 @@ def get(self, request): full_project_id = f'{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}' - samples = DataFilesMetadata.objects.filter(project_id=full_project_id, metadata__data_type='sample').values('id', 'name', 'path', 'metadata') + # samples = DataFilesMetadata.objects.filter(project_id=full_project_id, metadata__data_type='sample').values('id', 'name', 'path', 'metadata') + + samples = ProjectMetadata.objects.filter(base_project__value__projectId=full_project_id, name=constants.SAMPLE).values('uuid', 'name', 'value') + origin_data = [] if get_origin_data == 'true': - origin_data = DataFilesMetadata.objects.filter(project_id=full_project_id, metadata__data_type='origin_data').values('id', 'name', 'path', 'metadata') + # origin_data = DataFilesMetadata.objects.filter(project_id=full_project_id, metadata__data_type='origin_data').values('id', 'name', 'path', 'metadata') + + origin_data = ProjectMetadata.objects.filter(base_project__value__projectId=full_project_id, name=constants.ORIGIN_DATA).values('uuid', 'name', 'value') response_data = { 'samples': list(samples), diff --git a/server/portal/apps/datafiles/models.py b/server/portal/apps/datafiles/models.py index bc622edf17..ac0db0c7ee 100644 --- a/server/portal/apps/datafiles/models.py +++ b/server/portal/apps/datafiles/models.py @@ -41,7 +41,7 @@ def ordered_metadata(self): """Return the metadata in the order defined in the pydantic model""" portal_name = settings.PORTAL_NAMESPACE - schema = SCHEMA_MAPPING[portal_name][self.metadata.get('data_type')] + schema = SCHEMA_MAPPING[self.metadata.get('data_type')] if not schema: return self.metadata diff --git a/server/portal/apps/projects/management/commands/migrate-projects.py b/server/portal/apps/projects/management/commands/migrate-projects.py index 61ba861f87..43af459c93 100644 --- a/server/portal/apps/projects/management/commands/migrate-projects.py +++ b/server/portal/apps/projects/management/commands/migrate-projects.py @@ -3,7 +3,7 @@ from django.core.management.base import BaseCommand from portal.libs.agave.utils import service_account from portal.apps.projects.models.base import Project -from portal.apps.projects.models.metadata import ProjectMetadata +from portal.apps.projects.models.metadata import LegacyProjectMetadata from portal.apps.search.tasks import index_project from django.contrib.auth import get_user_model import logging @@ -24,7 +24,7 @@ class Command(BaseCommand): def handle(self, *args, **options): """Handle command.""" agc = service_account() - metadatas = ProjectMetadata.objects.all() + metadatas = LegacyProjectMetadata.objects.all() for meta in metadatas: if not meta.pi: try: diff --git a/server/portal/apps/projects/management/commands/migrate-projects_unit_test.py b/server/portal/apps/projects/management/commands/migrate-projects_unit_test.py index 5e08742767..ce991d8ff6 100644 --- a/server/portal/apps/projects/management/commands/migrate-projects_unit_test.py +++ b/server/portal/apps/projects/management/commands/migrate-projects_unit_test.py @@ -1,5 +1,5 @@ from portal.apps.projects.models.base import Project -from portal.apps.projects.models.metadata import ProjectMetadata +from portal.apps.projects.models.metadata import LegacyProjectMetadata from django.core import management from django.db.models import signals import pytest @@ -10,12 +10,12 @@ @pytest.fixture(autouse=True) def disconnect_signal(): - yield signals.post_save.disconnect(sender=ProjectMetadata, dispatch_uid="index_project") + yield signals.post_save.disconnect(sender=LegacyProjectMetadata, dispatch_uid="index_project") @pytest.fixture def ownerless_project(django_db_reset_sequences): - meta = ProjectMetadata.objects.create( + meta = LegacyProjectMetadata.objects.create( title="project", project_id="CEP-1", description=None, @@ -56,7 +56,7 @@ def test_migrate_projects(regular_user, mock_project_metadata, mock_project_stor 'username': 'ADMIN' } management.call_command("migrate-projects") - assert ProjectMetadata.objects.all()[0].pi.username == regular_user.username + assert LegacyProjectMetadata.objects.all()[0].pi.username == regular_user.username assert mock_index_project.apply_async.called @@ -68,4 +68,4 @@ def test_migrate_projects_wrong_admins(regular_user, mock_project_metadata, mock 'username2': 'ADMIN' } management.call_command("migrate-projects") - assert ProjectMetadata.objects.all()[0].pi is None + assert LegacyProjectMetadata.objects.all()[0].pi is None diff --git a/server/portal/apps/projects/migrations/0006_rename_projectmetadata_legacyprojectmetadata.py b/server/portal/apps/projects/migrations/0006_rename_projectmetadata_legacyprojectmetadata.py new file mode 100644 index 0000000000..0075f48b7f --- /dev/null +++ b/server/portal/apps/projects/migrations/0006_rename_projectmetadata_legacyprojectmetadata.py @@ -0,0 +1,17 @@ +# Generated by Django 4.2.11 on 2024-09-16 20:37 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('projects', '0005_projectsmetadata_created_at_and_more'), + ] + + operations = [ + migrations.RenameModel( + old_name='ProjectMetadata', + new_name='LegacyProjectMetadata', + ), + ] diff --git a/server/portal/apps/projects/migrations/0007_projectmetadata_and_more.py b/server/portal/apps/projects/migrations/0007_projectmetadata_and_more.py new file mode 100644 index 0000000000..232f5449be --- /dev/null +++ b/server/portal/apps/projects/migrations/0007_projectmetadata_and_more.py @@ -0,0 +1,47 @@ +# Generated by Django 4.2.11 on 2024-09-16 20:40 + +from django.conf import settings +import django.core.serializers.json +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone +import portal.apps.projects.models.project_metadata + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('projects', '0006_rename_projectmetadata_legacyprojectmetadata'), + ] + + operations = [ + migrations.CreateModel( + name='ProjectMetadata', + fields=[ + ('uuid', models.CharField(default=portal.apps.projects.models.project_metadata.uuid_pk, editable=False, max_length=100, primary_key=True, serialize=False)), + ('name', models.CharField(help_text="Metadata namespace, e.g. 'designsafe.project'", max_length=100, validators=[django.core.validators.MinLengthValidator(1)])), + ('value', models.JSONField(encoder=django.core.serializers.json.DjangoJSONEncoder, help_text='JSON document containing file metadata, including title/description')), + ('created', models.DateTimeField(default=django.utils.timezone.now)), + ('last_updated', models.DateTimeField(auto_now=True)), + ('base_project', models.ForeignKey(help_text='Base project containing this entity.For top-level project metadata, this is `self`.', on_delete=django.db.models.deletion.CASCADE, to='projects.projectmetadata')), + ('users', models.ManyToManyField(help_text='Users who have access to a project.', related_name='projects', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'indexes': [models.Index(models.F('value__projectId'), name='value_project_id'), models.Index(fields=['name'], name='projects_pr_name_5382e9_idx')], + }, + ), + migrations.AddConstraint( + model_name='projectmetadata', + constraint=models.UniqueConstraint(models.F('value__projectId'), condition=models.Q(('name', 'drp.project')), name='unique_id_per_project'), + ), + migrations.AddConstraint( + model_name='projectmetadata', + constraint=models.UniqueConstraint(condition=models.Q(('name', 'drp.project.graph')), fields=('base_project_id',), name='unique_graph_per_project'), + ), + migrations.AddConstraint( + model_name='projectmetadata', + constraint=models.CheckConstraint(check=models.Q(('name', 'drp.project'), ('value__projectId__isnull', True), _negated=True), name='base_projectId_not_null'), + ), + ] diff --git a/server/portal/apps/projects/models/__init__.py b/server/portal/apps/projects/models/__init__.py index f7faf51f9e..0b11b024bf 100644 --- a/server/portal/apps/projects/models/__init__.py +++ b/server/portal/apps/projects/models/__init__.py @@ -1,6 +1,6 @@ """Models""" from portal.apps.projects.models.base import Project, ProjectId -from portal.apps.projects.models.metadata import ProjectMetadata +from portal.apps.projects.models.metadata import LegacyProjectMetadata -__all__ = ['Project', 'ProjectId', 'ProjectMetadata'] +__all__ = ['Project', 'ProjectId', 'LegacyProjectMetadata'] diff --git a/server/portal/apps/projects/models/base.py b/server/portal/apps/projects/models/base.py index 7d055d0d58..2c70b74ab8 100644 --- a/server/portal/apps/projects/models/base.py +++ b/server/portal/apps/projects/models/base.py @@ -23,7 +23,7 @@ # from portal.libs.agave.models.systems.storage import StorageSystem # from portal.libs.agave.serializers import BaseAgaveSystemSerializer from portal.apps.projects import utils as ProjectsUtils -from portal.apps.projects.models.metadata import ProjectMetadata +from portal.apps.projects.models.metadata import LegacyProjectMetadata from portal.apps.projects.exceptions import NotAuthorizedError from portal.apps.accounts.models import SSHKeys @@ -156,7 +156,7 @@ def _get_metadata(self): while creating a metadata record. """ try: - meta = ProjectMetadata.objects.get(project_id=self.project_id) + meta = LegacyProjectMetadata.objects.get(project_id=self.project_id) except ObjectDoesNotExist: meta = self._create_metadata(self.title, self.project_id) @@ -248,7 +248,7 @@ def _create_metadata(title, project_id, owner=None): if owner: defaults['owner'] = owner - (meta, created) = ProjectMetadata.objects.get_or_create( + (meta, created) = LegacyProjectMetadata.objects.get_or_create( project_id=project_id, defaults=defaults ) diff --git a/server/portal/apps/projects/models/metadata.py b/server/portal/apps/projects/models/metadata.py index a94218f450..7a9b2c9350 100644 --- a/server/portal/apps/projects/models/metadata.py +++ b/server/portal/apps/projects/models/metadata.py @@ -76,7 +76,7 @@ def __str__(self): ) -class ProjectMetadata(AbstractProjectMetadata): +class LegacyProjectMetadata(AbstractProjectMetadata): """Project Metadata""" def to_dict(self): diff --git a/server/portal/apps/projects/models/project_metadata.py b/server/portal/apps/projects/models/project_metadata.py new file mode 100644 index 0000000000..0d3cbf1b10 --- /dev/null +++ b/server/portal/apps/projects/models/project_metadata.py @@ -0,0 +1,184 @@ +"""Models for representing project metadata""" +import uuid +from django.utils import timezone +from django.db import models +from django.core.validators import MinLengthValidator +from django.core.serializers.json import DjangoJSONEncoder +from django.contrib.auth import get_user_model +from django.conf import settings +from django.dispatch import receiver +from portal.apps._custom.drp import constants +from portal.apps import SCHEMA_MAPPING +from django.db.models.signals import pre_save + +user_model = get_user_model() + + +def uuid_pk(): + """Generate a string UUID for use as a primary key.""" + return str(uuid.uuid4()) + + +class ProjectMetadata(models.Model): + """This model represents project metadata. Each entity has a foreign-key relation + to the base project metadata (the base project relates to itself). + + Some useful operations: + - list all projects for a user: + `.projects.filter(name="designsafe.project")` + - Get all entities with a given project ID: + `ProjectMetadata.objects.filter(base_project__value__projectId="PRJ-1234")` + + """ + + uuid = models.CharField( + max_length=100, primary_key=True, default=uuid_pk, editable=False + ) + name = models.CharField( + max_length=100, + validators=[MinLengthValidator(1)], + help_text="Metadata namespace, e.g. 'designsafe.project'", + ) + value = models.JSONField( + encoder=DjangoJSONEncoder, + help_text=( + "JSON document containing file metadata, including title/description" + ), + ) + + users = models.ManyToManyField( + to=user_model, + related_name="projects", + help_text="Users who have access to a project." + ) + base_project = models.ForeignKey( + "self", + on_delete=models.CASCADE, + help_text=( + "Base project containing this entity." + "For top-level project metadata, this is `self`." + ), + ) + created = models.DateTimeField(default=timezone.now) + last_updated = models.DateTimeField(auto_now=True) + + @property + def project_id(self) -> str: + """Convenience method for retrieving project IDs.""" + return self.base_project.value.get("projectId") + + @property + def project_graph(self): + """Convenience method for returning the project graph metadata""" + return self.__class__.objects.get( + name=constants.PROJECT_GRAPH, base_project=self.base_project + ) + + @property + def ordered_metadata(self): + """Return the metadata in the order defined in the pydantic model""" + schema = SCHEMA_MAPPING[self.name] + + if not schema: + return self.value + + model_keys = list(schema.model_fields.keys()) + ordered_metadata = {k: self.value.get(k) for k in model_keys if k in self.value} + + return ordered_metadata + + @classmethod + def get_project_by_id(cls, project_id: str): + """Return base project metadata matching a project ID value.""" + return cls.objects.get(name=constants.PROJECT, value__projectId=project_id) + + @classmethod + def get_entities_by_project_id(cls, project_id: str): + """Return an iterable of all metadata objects for a given project ID.""" + return cls.objects.filter(base_project__value__projectId=project_id) + + @classmethod + def get_entity_by_project_id_and_path(cls, project_id: str, path: str): + """Return a single metadata object for a given project ID and path""" + try: + return cls.objects.get(base_project__value__projectId=project_id, value__path=path) + except cls.DoesNotExist: + return None + + def to_dict(self): + """dict representation.""" + return { + "uuid": self.uuid, + "name": self.name, + "value": self.value, + "created": self.created, + "lastUpdated": self.last_updated + } + + def sync_users(self): + """Sync associated users with the user list in the metadata.""" + if self.name != constants.PROJECT: + return + prj_users = [user.get("username") for user in self.value.get("users", [])] + project_user_query = models.Q(username__in=prj_users) + admin_user_query = models.Q(username__in=settings.PROJECT_ADMIN_USERS) + self.users.set(user_model.objects.filter(project_user_query | admin_user_query)) + + def save(self, *args, **kwargs): + if self.name == constants.PROJECT: + self.base_project = self + super().save(*args, **kwargs) + + class Meta: + # Create indexes on name and project ID, since these will be used for lookups. + indexes = [ + models.Index(models.F("value__projectId"), name="value_project_id"), + models.Index(fields=["name"]), + ] + + constraints = [ + # Each project has a unique ID + models.UniqueConstraint( + models.F("value__projectId"), + condition=models.Q(name=constants.PROJECT), + name="unique_id_per_project", + ), + # A project can have at most 1 graph associated to it. + models.UniqueConstraint( + fields=["base_project_id"], + condition=models.Q(name=constants.PROJECT_GRAPH), + name="unique_graph_per_project", + ), + # Top-level project metadata cannot be saved without a project ID. + models.CheckConstraint( + check=~models.Q(name=constants.PROJECT, value__projectId__isnull=True), + name="base_projectId_not_null", + ), + ] + +# @receiver(pre_save, sender=ProjectMetadata) +# def set_or_update_parent_uuid(sender, instance, **kwargs): +# # Assuming 'folder_path' is stored in instance.value +# current_path = instance.value.get('path') + +# # If new record or the 'folder_path' has changed +# if not instance.pk or current_path != sender.objects.get(pk=instance.pk).value.get('path'): +# parent_path = current_path.rstrip('/').rsplit('/', 1)[0] + '/' # Extract the parent path + +# if len(parent_path.split('/')) > 1: +# # Attempt to find the parent entity +# parent_entity = sender.objects.get(value__folder_path=parent_path) +# parent_uuid = parent_entity.uuid +# else: +# parent_uuid = None + + +# # Update 'parent_uuid' in the instance's 'value' field +# instance.value['parent_uuid'] = parent_uuid + + +# @receiver(models.signals.post_save, sender=ProjectMetadata) +# def handle_save(instance: ProjectMetadata, **_): +# """After saving a project, update the associated users so that listings can +# be performed.""" +# instance.sync_users() diff --git a/server/portal/apps/projects/models/unit_test.py b/server/portal/apps/projects/models/unit_test.py index 25fe28eb72..a17cf48d7b 100644 --- a/server/portal/apps/projects/models/unit_test.py +++ b/server/portal/apps/projects/models/unit_test.py @@ -4,7 +4,7 @@ :synopsis: Projects app unit tests. """ -from portal.apps.projects.models.metadata import ProjectMetadata +from portal.apps.projects.models.metadata import LegacyProjectMetadata from portal.apps.projects.models.base import Project from portal.apps.projects.models.utils import get_latest_project_storage # TODOv3: deprecate with projects @@ -34,7 +34,7 @@ def test_create_metadata(mock_owner, mock_project_save_signal): 'title': 'Project Title', 'owner': mock_owner } - (meta, result) = ProjectMetadata.objects.get_or_create( + (meta, result) = LegacyProjectMetadata.objects.get_or_create( project_id=project_id, defaults=defaults ) @@ -52,7 +52,7 @@ def test_metadata_str(mock_owner, mock_project_save_signal): defaults = { 'title': 'Project Title', } - meta = ProjectMetadata.objects.get_or_create( + meta = LegacyProjectMetadata.objects.get_or_create( project_id=project_id, defaults=defaults ) @@ -63,7 +63,7 @@ def test_metadata_str(mock_owner, mock_project_save_signal): @pytest.mark.skip(reason="TODOv3: deprecate with projects") def test_project_create(mock_owner, portal_project, agave_client, mock_project_save_signal): Project.create(agave_client, "my_project", "mock_project_id", mock_owner) - assert ProjectMetadata.objects.all().count() == 1 + assert LegacyProjectMetadata.objects.all().count() == 1 @pytest.mark.skip(reason="TODOv3: deprecate with projects") @@ -71,7 +71,7 @@ def test_project_create_dir_failure(mock_owner, portal_project, agave_client, mo portal_project._create_dir.side_effect = Exception() with pytest.raises(Exception): Project.create(agave_client, "my_project", "mock_project_id", mock_owner) - assert ProjectMetadata.objects.all().count() == 0 + assert LegacyProjectMetadata.objects.all().count() == 0 @pytest.mark.skip(reason="TODOv3: deprecate with projects") @@ -79,7 +79,7 @@ def test_project_create_storage_failure(mock_owner, portal_project, agave_client portal_project._create_storage.side_effect = Exception() with pytest.raises(Exception): Project.create(agave_client, "my_project", "mock_project_id", mock_owner) - assert ProjectMetadata.objects.all().count() == 0 + assert LegacyProjectMetadata.objects.all().count() == 0 @pytest.mark.skip(reason="TODOv3: deprecate with projects") @@ -95,8 +95,8 @@ def test_metadata_create_on_project_load(agave_client, mock_owner, mock_project_ # 'PRJ-123', # storage=sys # ) - assert ProjectMetadata.objects.all().count() == 1 - assert ProjectMetadata.objects.last().pi == mock_owner + assert LegacyProjectMetadata.objects.all().count() == 1 + assert LegacyProjectMetadata.objects.last().pi == mock_owner @pytest.mark.skip(reason="TODOv3: deprecate with projects") diff --git a/server/portal/apps/projects/schema_models/base.py b/server/portal/apps/projects/schema_models/base.py new file mode 100644 index 0000000000..4c83c72b77 --- /dev/null +++ b/server/portal/apps/projects/schema_models/base.py @@ -0,0 +1,28 @@ +"""Utiity models used in multiple field types""" + +from datetime import datetime +from functools import partial +from typing import Annotated, Literal, Optional +from pydantic import AliasChoices, BaseModel, BeforeValidator, ConfigDict, Field +from pydantic.alias_generators import to_camel +from django.contrib.auth import get_user_model +from pytas.http import TASClient + +class FileObj(BaseModel): + """Model for associated files""" + + system: str + name: str + path: str + type: Literal["file", "dir"] + length: Optional[int] = None + last_modified: Optional[str] = None + uuid: Optional[str] = None + +class PartialEntityWithFiles(BaseModel): + """Model for representing an entity with associated files.""" + + model_config = ConfigDict(extra="ignore") + + # file_tags: list[FileTag] = [] + file_objs: list[FileObj] = [] diff --git a/server/portal/apps/projects/serializers.py b/server/portal/apps/projects/serializers.py index 422563b082..8559efc089 100644 --- a/server/portal/apps/projects/serializers.py +++ b/server/portal/apps/projects/serializers.py @@ -7,7 +7,7 @@ import datetime import json from django.contrib.auth import get_user_model -from portal.apps.projects.models.metadata import ProjectMetadata +from portal.apps.projects.models.metadata import LegacyProjectMetadata from portal.libs.agave.utils import to_camel_case LOGGER = logging.getLogger(__name__) @@ -34,7 +34,7 @@ class MetadataJSONSerializer(json.JSONEncoder): def default(self, obj): # pylint: disable=method-hidden, arguments-differ """Default.""" - if isinstance(obj, ProjectMetadata): + if isinstance(obj, LegacyProjectMetadata): ret = {} for field in obj._meta.fields: if not field.serialize: diff --git a/server/portal/apps/projects/unit_test.py b/server/portal/apps/projects/unit_test.py index e6f207cd3a..61753f1106 100644 --- a/server/portal/apps/projects/unit_test.py +++ b/server/portal/apps/projects/unit_test.py @@ -5,7 +5,7 @@ """ from mock import MagicMock from portal.apps.projects.models.base import Project -from portal.apps.projects.models.metadata import ProjectMetadata +from portal.apps.projects.models.metadata import LegacyProjectMetadata from portal.apps.projects.exceptions import NotAuthorizedError import pytest import logging @@ -47,8 +47,8 @@ def test_project_init(mock_tapis_client, mock_storage_system, project_model, moc ) ) - assert ProjectMetadata.objects.all().count() == 1 - assert ProjectMetadata.objects.get(project_id='PRJ-123', title='my title') + assert LegacyProjectMetadata.objects.all().count() == 1 + assert LegacyProjectMetadata.objects.get(project_id='PRJ-123', title='my title') @pytest.mark.skip(reason="TODOv3: update with new Shared Workspaces operations") @@ -60,8 +60,8 @@ def test_project_create(mock_owner, mock_tapis_client, service_account, mock_sto name='PRJ-123', description='Test Title', site='test') - assert ProjectMetadata.objects.all().count() == 1 - assert ProjectMetadata.objects.get(project_id='PRJ-123', title='Test Title') + assert LegacyProjectMetadata.objects.all().count() == 1 + assert LegacyProjectMetadata.objects.get(project_id='PRJ-123', title='Test Title') assert prj._ac == mock_tapis_client assert prj.storage.storage.port == 22 @@ -204,7 +204,7 @@ def test_add_pi_unauthorized(mock_owner, django_user_model, mock_tapis_client, m def test_create_metadata(mock_owner, mock_tapis_client, mock_storage_system, project_model, mock_signal): # Test creating metadata with no owner project_model._create_metadata("Project Title", "PRJ-123") - assert ProjectMetadata.objects.get(project_id="PRJ-123").owner is None + assert LegacyProjectMetadata.objects.get(project_id="PRJ-123").owner is None project_model._create_metadata("Project Title 2", "PRJ-124", mock_owner) - assert ProjectMetadata.objects.get(project_id="PRJ-124").owner == mock_owner + assert LegacyProjectMetadata.objects.get(project_id="PRJ-124").owner == mock_owner diff --git a/server/portal/apps/projects/urls.py b/server/portal/apps/projects/urls.py index 542df1510e..a01904be18 100644 --- a/server/portal/apps/projects/urls.py +++ b/server/portal/apps/projects/urls.py @@ -10,5 +10,6 @@ path('/project-role//', views.get_project_role), path('/system-role//', views.get_system_role), path('/', views.ProjectInstanceApiView.as_view(), name='project'), + path('/entities/create', views.ProjectEntityView.as_view()), path('', views.ProjectsApiView.as_view(), name='projects_api') ] diff --git a/server/portal/apps/projects/views.py b/server/portal/apps/projects/views.py index eb9b86f1fd..72ac01c741 100644 --- a/server/portal/apps/projects/views.py +++ b/server/portal/apps/projects/views.py @@ -5,6 +5,7 @@ """ import json import logging +from django.http import HttpRequest, JsonResponse from django.contrib.auth.decorators import login_required from django.conf import settings from django.http import JsonResponse @@ -21,8 +22,16 @@ from portal.libs.elasticsearch.indexes import IndexedProject from elasticsearch_dsl import Q from portal.apps.projects.models.metadata import ProjectsMetadata +from portal.apps.projects.models.project_metadata import ProjectMetadata from django.db import transaction from portal.apps import SCHEMA_MAPPING +from django.db import models +from portal.apps.projects.workspace_operations.project_meta_operations import create_entity_metadata, create_project_metdata, patch_metadata +from portal.libs.agave.operations import mkdir +from pathlib import Path +from portal.apps._custom.drp import constants +from portal.apps.projects.workspace_operations.graph_operations import add_node_to_project, initialize_project_graph, get_node_from_path, update_node_in_project +import networkx as nx LOGGER = logging.getLogger(__name__) @@ -122,13 +131,15 @@ def post(self, request): # pylint: disable=no-self-use workspace_id = f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{settings.PORTAL_PROJECTS_ID_PREFIX}-{workspace_number}" if metadata is not None: - - project_metadata = ProjectsMetadata( - project_id = workspace_id, - metadata = validate_project_metadata(metadata) - ) - - project_metadata.save() + metadata["projectId"] = workspace_id + project_meta = create_project_metdata(metadata) + initialize_project_graph(project_meta.project_id) + # project_metadata = ProjectsMetadata( + # project_id = workspace_id, + # metadata = validate_project_metadata(metadata) + # ) + + # project_metadata.save() client = request.user.tapis_oauth.client system_id = create_shared_workspace(client, title, request.user.username, description, workspace_number) @@ -166,10 +177,20 @@ def get(self, request, project_id=None, system_id=None): prj = get_project(request.user.tapis_oauth.client, project_id) + print(f"PRJ {prj}") + try: - project = ProjectsMetadata.objects.get(project_id=f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}") - project.metadata = project.get_metadata() - prj.update(project.metadata) + + print("HERE") + + project = ProjectMetadata.objects.get(models.Q(value__projectId=f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}")) + + # print(f"Test Project: {test_project}") + + # project = ProjectsMetadata.objects.get(project_id=f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}") + # project.metadata = project.get_metadata() + prj.update(project.value) + prj["projectId"] = project_id except: pass @@ -217,8 +238,10 @@ def patch( metadata = data['metadata'] if metadata is not None: - project_metadata = ProjectsMetadata.objects.get(project_id=f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}") - project_metadata.metadata = validate_project_metadata(metadata) + project_metadata = ProjectMetadata.objects.get(models.Q(value__projectId=f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}")) + project_metadata.value = validate_project_metadata(metadata) + # project_metadata = ProjectsMetadata.objects.get(project_id=f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}") + # project_metadata.metadata = validate_project_metadata(metadata) project_metadata.save() client = request.user.tapis_oauth.client @@ -362,3 +385,72 @@ def get_system_role(request, project_id, username): role = get_workspace_role(client, project_id, username) return JsonResponse({'username': username, 'role': role}) + +class ProjectEntityView(BaseApiView): + + def patch(self, request: HttpRequest, project_id: str): + + client = request.user.tapis_oauth.client + + if not request.user.is_authenticated: + raise ApiException("Unauthenticated user", status=401) + + req_body = json.loads(request.body) + value = req_body.get("value", {}) + entity_uuid = req_body.get("uuid", "") + path = req_body.get("path", "") + # path = str(Path(value['path'].rstrip('/'))) if value.get('path') else '' + + print(f'VALUE {value}') + print(f'PATH {path}') + print(f'ENTITY UUID {entity_uuid}') + + try: + patch_metadata(client, project_id, value, entity_uuid, path) + except Exception as exc: + raise ApiException("Error updating entity metadata", status=500) from exc + + return JsonResponse({"result": "OK"}) + + + def post(self, request: HttpRequest, project_id: str): + """Add a new entity to a project""" + + client = request.user.tapis_oauth.client + if not request.user.is_authenticated: + raise ApiException("Unauthenticated user", status=401) + + print(f'project ID {project_id}') + + try: + project: ProjectMetadata = ProjectMetadata.objects.get( + models.Q(uuid=project_id) | models.Q(value__projectId=project_id) + ) + except ProjectMetadata.DoesNotExist as exc: + raise ApiException( + "User does not have access to the requested project", status=403 + ) from exc + + portal = settings.PORTAL_NAMESPACE.lower() + req_body = json.loads(request.body) + value = req_body.get("value", {}) + name = req_body.get("name", "") + path = req_body.get("path", "") + + new_meta = create_entity_metadata(project.project_id, getattr(constants, name.upper()), { + **value, + # "path": str(Path(value['path'].rstrip('/')) / Path(value['name'])) if value.get('path') else '' + }) + + parent_node = get_node_from_path(project.project_id, path) + + add_node_to_project(project_id, parent_node['id'], new_meta.uuid, name, value['name']) + + if (value and path): + mkdir(client, project_id, path, value['name']) + + # FOR CREATING GRAPH + # if name in ALLOWED_RELATIONS[constants.PROJECT]: + # add_node_to_project(project_id, "NODE_ROOT", new_meta.uuid, name) + return JsonResponse({"result": "OK"}) + diff --git a/server/portal/apps/projects/workspace_operations/graph_operations.py b/server/portal/apps/projects/workspace_operations/graph_operations.py new file mode 100644 index 0000000000..7f4dfbe823 --- /dev/null +++ b/server/portal/apps/projects/workspace_operations/graph_operations.py @@ -0,0 +1,141 @@ +import networkx as nx +from django.db import transaction +import uuid +import copy + +from portal.apps._custom.drp import constants +from portal.apps.projects.models.project_metadata import ProjectMetadata + + +def _get_next_child_order(graph: nx.DiGraph, parent_node: str) -> int: + child_nodes = graph.successors(parent_node) + max_order = max((graph.nodes[child]["order"] for child in child_nodes), default=-1) + return int(max_order) + 1 + +def _add_node_to_graph( + graph: nx.DiGraph, parent_node_id: str, meta_uuid: str, name: str, label: str +) -> tuple[nx.DiGraph, str | None]: + """Add a node with data to a graph, and return the graph.""" + if not graph.has_node(parent_node_id): + raise nx.exception.NodeNotFound + + # no-op if metadata with this UUID is already associated. + if meta_uuid in ( + graph.nodes[node]["uuid"] for node in graph.successors(parent_node_id) + ): + return (graph, None) + + _graph: nx.DiGraph = copy.deepcopy(graph) + order = _get_next_child_order(_graph, parent_node_id) + child_node_id = f"NODE_{name}_{uuid.uuid4()}" + _graph.add_node(child_node_id, uuid=meta_uuid, name=name, order=order, label=label) + _graph.add_edge(parent_node_id, child_node_id) + return (_graph, child_node_id) + +def initialize_project_graph(project_id: str): + """ + Initialize the entity graph in a default state for a project. For type Other, the + default graph has an "empty" root node that contains the base entity as its child. + This is to allow multiple versions to be published as siblings in the graph. + Otherwise, the graph is initialized as a single node pointing to the project root. + This method should be called when creating a new project AND when changing a + project's type. + """ + project_model = ProjectMetadata.get_project_by_id(project_id) + project_graph = nx.DiGraph() + + root_node_id = "NODE_ROOT" + project_type = project_model.value.get("projectType", None) + base_node_data = { + "uuid": project_model.uuid, + "name": project_model.name, + "projectType": project_type, + "order": 0, + "label": project_model.value.get("title") + } + + if project_type == "other": + # type Other projects have a "null" parent node above the project root, to + # support multiple versions. + project_graph.add_node( + root_node_id, **{"uuid": None, "name": None, "projectType": "other"} + ) + base_node_id = f"NODE_project_{uuid.uuid4()}" + project_graph.add_node(base_node_id, **base_node_data) + project_graph.add_edge(root_node_id, base_node_id) + else: + project_graph.add_node(root_node_id, **base_node_data) + + graph_model_value = nx.node_link_data(project_graph) + res, _ = ProjectMetadata.objects.update_or_create( + name=constants.PROJECT_GRAPH, + base_project=project_model, + defaults={"value": graph_model_value}, + ) + return res + +def traverse_graph(project_graph, root_node, path_components): + current_node = root_node + for component in path_components: + found = False + for successor in project_graph.successors(current_node): + name = project_graph.nodes[successor]['label'] + if name == component: + current_node = successor + found = True + break + if not found: + raise Exception(f"Component '{component}' not found under node '{project_graph.nodes[current_node]['label']}'.") + return {"id": current_node, **project_graph.nodes[current_node]} + +def get_node_from_path(project_id: str, path: str) -> str: + """Return the node ID for the parent of a node with the given path.""" + + graph_model = ProjectMetadata.objects.get( + name=constants.PROJECT_GRAPH, base_project__value__projectId=project_id + ) + project_graph = nx.node_link_graph(graph_model.value) + + path_parts = path.strip("/").split("/") + + if len(path_parts) == 0 or path_parts[0] == "": + return {"id": "NODE_ROOT"} + + node = traverse_graph(project_graph, "NODE_ROOT", path_parts) + + print('returned node', node) + + return node + raise nx.exception.NodeNotFound + +def update_node_in_project(project_id: str, node_id: str, value: dict): + """Update the database entry for a project graph to update a node.""" + with transaction.atomic(): + graph_model = ProjectMetadata.objects.select_for_update().get( + name=constants.PROJECT_GRAPH, base_project__value__projectId=project_id + ) + project_graph = nx.node_link_graph(graph_model.value) + + if not project_graph.has_node(node_id): + raise nx.exception.NodeNotFound + + project_graph.nodes[node_id]["value"] = value + graph_model.value = nx.node_link_data(project_graph) + graph_model.save() + +def add_node_to_project(project_id: str, parent_node: str, meta_uuid: str, name: str, label: str): + """Update the database entry for a project graph to add a node.""" + # Lock the project graph's tale row to prevent conflicting updates. + with transaction.atomic(): + graph_model = ProjectMetadata.objects.select_for_update().get( + name=constants.PROJECT_GRAPH, base_project__value__projectId=project_id + ) + project_graph = nx.node_link_graph(graph_model.value) + + (updated_graph, new_node_id) = _add_node_to_graph( + project_graph, parent_node, meta_uuid, name, label + ) + + graph_model.value = nx.node_link_data(updated_graph) + graph_model.save() + return new_node_id \ No newline at end of file diff --git a/server/portal/apps/projects/workspace_operations/project_meta_operations.py b/server/portal/apps/projects/workspace_operations/project_meta_operations.py new file mode 100644 index 0000000000..f50caa9e89 --- /dev/null +++ b/server/portal/apps/projects/workspace_operations/project_meta_operations.py @@ -0,0 +1,184 @@ +import operator +from django.db import models, transaction +from portal.apps import SCHEMA_MAPPING +from portal.apps._custom.drp import constants +from django.conf import settings +from portal.apps.projects.models.project_metadata import ProjectMetadata +from portal.apps.projects.schema_models.base import ( + FileObj, + PartialEntityWithFiles +) +from portal.apps.projects.workspace_operations.graph_operations import get_node_from_path, update_node_in_project + +portal = settings.PORTAL_NAMESPACE.lower() + +def create_project_metdata(value): + """Create a project metadata object in the database.""" + schema_model = SCHEMA_MAPPING[constants.PROJECT] + validated_model = schema_model.model_validate(value) + + project_db_model = ProjectMetadata( + name=constants.PROJECT, value=validated_model.model_dump() + ) + project_db_model.save() + return project_db_model + +def create_entity_metadata(project_id, name, value): + """Create entity metadata associated with an existing project.""" + base_project = ProjectMetadata.get_project_by_id(project_id) + schema_model = SCHEMA_MAPPING[name] + validated_model = schema_model.model_validate(value) + + entity_db_model = ProjectMetadata( + name=name, value=validated_model.model_dump(exclude_none=True), base_project=base_project + ) + entity_db_model.save() + return entity_db_model + +def get_entity(project_id, path): + """Retrieve metadata for a specific entity.""" + try: + node = get_node_from_path(project_id, path) + + if not node or node['id'] == 'NODE_ROOT': + return None + entity = ProjectMetadata.objects.get(uuid=node['uuid']) + # entity = ProjectMetadata.get_entity_by_project_id_and_path(project_id, path) + return entity + except ProjectMetadata.DoesNotExist: + return None + +@transaction.atomic +def patch_metadata(client, project_id, value, uuid=None, path=None): + """Update an entity's `value` attribute. This method patches the metadata + so that only fields in the payload are overwritten.""" + node = get_node_from_path(project_id, path) + + if uuid: + entity = ProjectMetadata.objects.get(uuid=uuid) + elif project_id and path: + entity = ProjectMetadata.objects.get(uuid=node['uuid']) + else: + raise ValueError("Either 'uuid' or both 'project_id' and 'path' must be provided.") + + print('entity', entity) + + current_name = entity.value.get('name') + current_path = entity.value.get('path') + + new_name = value['name'] + new_path = path + + print('current_name', current_name) + print('current_path', current_path) + print('new_name', new_name) + print('new_path', new_path) + + # # If the name or path has changed, move the entity to the new location. + if current_name != new_name or current_path != new_path: + + from portal.libs.agave.operations import move + + move_result = move(client, project_id, current_path, project_id, new_path, new_name) + move_message = move_result['message'].split('DestinationPath: ', 1)[1] + new_name = ('/' + move_message).rsplit('/', 1)[1] + value['name'] = new_name + update_node_in_project(project_id, node['id'], value) + + + schema_model = SCHEMA_MAPPING[entity.name] + + patched_metadata = {**entity.value, **value} + validated_model = schema_model.model_validate(patched_metadata) + entity.value = validated_model.model_dump(exclude_none=True) + entity.save() + + return entity + +def delete_entity(uuid: str): + """Delete a non-root entity.""" + entity = ProjectMetadata.objects.get(uuid=uuid) + if entity.name in (constants.PROJECT, constants.PROJECT_GRAPH): + raise ValueError("Cannot delete a top-level project or graph object.") + entity.delete() + + return "OK" + +def clear_entities(project_id): + """Delete all entities except the project root and graph. Used when changing project + type, so that file associations don't get stuck on unreachable entities. + """ + + ProjectMetadata.get_entities_by_project_id(project_id).filter( + ~models.Q(name__in=[constants.PROJECT, constants.PROJECT_GRAPH]) + ).delete() + + return "OK" + +def _merge_file_objs( + prev_file_objs: list[FileObj], new_file_objs: list[FileObj] +) -> list[FileObj]: + """Combine two arrays of FileObj models, overwriting the first if there are conflicts.""" + new_file_paths = [f.path for f in new_file_objs] + deduped_file_objs = [fo for fo in prev_file_objs if fo.path not in new_file_paths] + + return sorted( + [*deduped_file_objs, *new_file_objs], key=operator.attrgetter("name", "path") + ) + +def _filter_file_objs( + prev_file_objs: list[FileObj], paths_to_remove: list[str] +) -> list[FileObj]: + return sorted( + [fo for fo in prev_file_objs if fo.path not in paths_to_remove], + key=operator.attrgetter("name", "path"), + ) + +def add_file_associations(uuid: str, new_file_objs: list[FileObj]): + """Associate one or more file objects to an entity.""" + # Use atomic transaction here to prevent multiple calls from clobbering each other + with transaction.atomic(): + entity = ProjectMetadata.objects.select_for_update().get(uuid=uuid) + entity_file_model = PartialEntityWithFiles.model_validate(entity.value) + + merged_file_objs = _merge_file_objs(entity_file_model.file_objs, new_file_objs) + entity.value["fileObjs"] = [f.model_dump() for f in merged_file_objs] + + entity.save() + return entity + +def set_file_associations(uuid: str, new_file_objs: list[FileObj]): + """Replace the file associations for an entity with the specified set.""" + # Use atomic transaction here to prevent multiple calls from clobbering each other + with transaction.atomic(): + entity = ProjectMetadata.objects.select_for_update().get(uuid=uuid) + entity.value["fileObjs"] = [f.model_dump() for f in new_file_objs] + entity.save() + return entity + +def remove_file_associations(uuid: str, file_paths: list[str]): + """Remove file associations from an entity by their paths.""" + with transaction.atomic(): + entity = ProjectMetadata.objects.select_for_update().get(uuid=uuid) + entity_file_model = PartialEntityWithFiles.model_validate(entity.value) + + filtered_file_objs = _filter_file_objs(entity_file_model.file_objs, file_paths) + entity.value["fileObjs"] = [f.model_dump() for f in filtered_file_objs] + + # Remove tags associated with these entity/file path combinations. + tagged_paths = [] + for path in file_paths: + tagged_paths += [ + t["path"] + for t in entity.value.get("fileTags", []) + if t["path"].startswith(path) + ] + entity.value["fileTags"] = [ + t + for t in entity.value.get("fileTags", []) + if not (t["path"] in tagged_paths) + ] + entity.save() + return entity + + diff --git a/server/portal/apps/projects/workspace_operations/shared_workspace_migration.py b/server/portal/apps/projects/workspace_operations/shared_workspace_migration.py index 58cd57e3f4..6710748c60 100644 --- a/server/portal/apps/projects/workspace_operations/shared_workspace_migration.py +++ b/server/portal/apps/projects/workspace_operations/shared_workspace_migration.py @@ -3,7 +3,7 @@ """ import requests from django.conf import settings -from portal.apps.projects.models import ProjectMetadata +from portal.apps.projects.models import LegacyProjectMetadata from portal.apps.projects.workspace_operations.shared_workspace_operations import create_workspace_system, add_user_to_workspace from portal.libs.agave.utils import service_account @@ -36,7 +36,7 @@ def migrate_project(project_id): client = service_account() system_id = f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}" try: - v2_project = ProjectMetadata.objects.get(project_id=project_id) + v2_project = LegacyProjectMetadata.objects.get(project_id=project_id) except MultipleObjectsReturned: print('FAILURE: more than 1 project with this ID') return @@ -92,5 +92,5 @@ def migrate_project(project_id): def migrate_all_projects(): - for prj in ProjectMetadata.objects.all(): + for prj in LegacyProjectMetadata.objects.all(): migrate_project(prj.project_id) diff --git a/server/portal/apps/search/tasks.py b/server/portal/apps/search/tasks.py index 61bf00a2ea..87e5d656b5 100644 --- a/server/portal/apps/search/tasks.py +++ b/server/portal/apps/search/tasks.py @@ -5,7 +5,7 @@ from portal.libs.agave.utils import user_account, service_account from portal.libs.elasticsearch.utils import index_listing, index_project_listing from portal.apps.users.utils import get_tas_allocations -from portal.apps.projects.models.metadata import ProjectMetadata +from portal.apps.projects.models.metadata import LegacyProjectMetadata from portal.libs.elasticsearch.docs.base import (IndexedAllocation, IndexedProject) from portal.libs.elasticsearch.utils import get_sha256_hash @@ -73,7 +73,7 @@ def index_allocations(self, username): @shared_task(bind=True, max_retries=3, queue='indexing') def index_project(self, project_id): - project = ProjectMetadata.objects.get(project_id=project_id) + project = LegacyProjectMetadata.objects.get(project_id=project_id) project_dict = project.to_dict() project_doc = IndexedProject(**project_dict) project_doc.meta.id = project_id diff --git a/server/portal/apps/signals/receivers.py b/server/portal/apps/signals/receivers.py index db00823744..b48151d362 100644 --- a/server/portal/apps/signals/receivers.py +++ b/server/portal/apps/signals/receivers.py @@ -3,7 +3,7 @@ from django.db.models.signals import post_save from portal.apps.notifications.models import Notification from portal.apps.onboarding.models import SetupEvent -from portal.apps.projects.models.metadata import ProjectMetadata +from portal.apps.projects.models.metadata import LegacyProjectMetadata from portal.apps.search.tasks import index_project from django.contrib.auth import get_user_model import logging @@ -66,7 +66,7 @@ def send_notification_ws(sender, instance, created, **kwargs): return -@receiver(post_save, sender=ProjectMetadata, dispatch_uid='index_project') +@receiver(post_save, sender=LegacyProjectMetadata, dispatch_uid='index_project') def index_project_on_save(sender, instance, created, **kwargs): index_project.apply_async(args=[instance.project_id]) diff --git a/server/portal/libs/agave/operations.py b/server/portal/libs/agave/operations.py index 43d297971e..a45f026ab6 100644 --- a/server/portal/libs/agave/operations.py +++ b/server/portal/libs/agave/operations.py @@ -14,6 +14,7 @@ from portal.apps.projects.models.metadata import ProjectsMetadata from portal.apps.datafiles.models import DataFilesMetadata from portal.apps import SCHEMA_MAPPING +from portal.apps.projects.workspace_operations.project_meta_operations import get_entity logger = logging.getLogger(__name__) @@ -85,32 +86,55 @@ def listing(client, system, path, offset=0, limit=100, *args, **kwargs): offset=int(offset), limit=int(limit)) + print(f"Listing for {system}/{path} returned {(raw_listing)}") - folder_metadata = get_datafile_metadata(system=system, path=path) + folder_metadata = get_entity(system, path) try: # Convert file objects to dicts for serialization. - listing = list(map(lambda f: { - 'system': system, - 'type': 'dir' if f.type == 'dir' else 'file', - 'format': 'folder' if f.type == 'dir' else 'raw', - 'mimeType': f.mimeType, - 'path': f.path, - 'name': f.name, - 'length': f.size, - 'lastModified': f.lastModified, - '_links': { - 'self': {'href': f.url} - }, - 'metadata': get_datafile_metadata(system=system, path=f.path) - }, raw_listing)) + + listing = [] + + for f in raw_listing: + entity = get_entity(system, f.path) + + listing.append({ + 'uuid': entity.to_dict().get('uuid') if entity else None, + 'system': system, + 'type': 'dir' if f.type == 'dir' else 'file', + 'format': 'folder' if f.type == 'dir' else 'raw', + 'mimeType': f.mimeType, + 'path': f.path, + 'name': f.name, + 'length': f.size, + 'lastModified': f.lastModified, + '_links': { + 'self': {'href': f.url} + }, + 'metadata': entity.ordered_metadata if entity else None + }) + + # listing = list(map(lambda f: { + # 'system': system, + # 'type': 'dir' if f.type == 'dir' else 'file', + # 'format': 'folder' if f.type == 'dir' else 'raw', + # 'mimeType': f.mimeType, + # 'path': f.path, + # 'name': f.name, + # 'length': f.size, + # 'lastModified': f.lastModified, + # '_links': { + # 'self': {'href': f.url} + # }, + # 'metadata': get_entity_metadata(system, f.path) + # }, raw_listing)) except IndexError: # Return [] if the listing is empty. listing = [] # Update Elasticsearch after each listing. tapis_listing_indexer.delay(listing) - return {'listing': listing, 'reachedEnd': len(listing) < int(limit), 'folder_metadata': folder_metadata} + return {'listing': listing, 'reachedEnd': len(listing) < int(limit), 'folder_metadata': folder_metadata.ordered_metadata if folder_metadata else None} def iterate_listing(client, system, path, limit=100): @@ -369,7 +393,7 @@ def copy(client, src_system, src_path, dest_system, dest_path, file_name=None, m dest_path_full = os.path.join(dest_path.strip('/'), file_name) - if metadata is not None: + if metadata is not None: create_datafile_metadata(dest_system, f'{dest_system}/{dest_path_full.strip("/")}', file_name, metadata) if src_system == dest_system: diff --git a/server/pyproject.toml b/server/pyproject.toml index b3e9e8ca89..5b0ee19982 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -40,6 +40,7 @@ tapipy = "^1.3" gevent = "^23.9.1" pymemcache = "^4.0.0" pydantic = "^2.5.0" +networkx = "^3.2.1" [tool.poetry.group.dev.dependencies] pytest = "^7.3.1" From bf1dae7b10f35fea6048fa4cc6751912cbdb03ae Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Mon, 14 Oct 2024 15:07:04 -0500 Subject: [PATCH 111/328] update poetry.lock --- server/poetry.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/poetry.lock b/server/poetry.lock index e551d7dfa6..dc4ff15be0 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -3130,4 +3130,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "c90f57361d9d8076a37997b3771c0fb9acc6070cb14b2489d2ba2c635a068902" +content-hash = "3c14a4107b26468da3fa96e04c4a72e73863b93fc83e98fee5adda24e0d5a4c4" From 96c8cfcee39b318078d3ea4c0c1cf2ce2b4ef208 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Fri, 18 Oct 2024 11:02:01 -0500 Subject: [PATCH 112/328] added fileObj implementation, entity creation and patching fully working --- .../DataFilesModals/DataFilesPreviewModal.jsx | 2 +- .../DataFilesModals/DataFilesUploadModal.jsx | 2 +- ...taFilesProjectFileListingMetadataAddon.jsx | 2 +- .../ReviewProjectStructure.jsx | 1 + client/src/redux/sagas/_custom/drp.sagas.js | 4 +- client/src/redux/sagas/projects.sagas.js | 5 +- server/poetry.lock | 2 +- server/portal/apps/_custom/drp/models.py | 33 ++- .../apps/projects/models/project_metadata.py | 46 ++++- .../apps/projects/schema_models/base.py | 18 +- server/portal/apps/projects/views.py | 65 +++--- .../workspace_operations/graph_operations.py | 47 ++++- .../project_meta_operations.py | 192 +++++++++++++----- server/portal/libs/agave/operations.py | 56 +++-- 14 files changed, 330 insertions(+), 145 deletions(-) diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesPreviewModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesPreviewModal.jsx index b2b0dd79c4..6233850454 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesPreviewModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesPreviewModal.jsx @@ -89,7 +89,7 @@ const DataFilesPreviewModal = () => { File Preview: {params.name} - {DataFilesPreviewModalAddon && params.metadata && !isLoading && } + {DataFilesPreviewModalAddon && !isLoading && } {(isLoading || (previewUsingHref && isFrameLoading)) && (
diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesUploadModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesUploadModal.jsx index bf0d3a627c..5b5c1f300a 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesUploadModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesUploadModal.jsx @@ -145,7 +145,7 @@ const DataFilesUploadModal = ({ className, layout }) => { />
)} - {DataFilesUploadModalAddon && ( + {DataFilesUploadModalAddon && params.scheme === 'projects' && ( { diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx index 51e8ba444e..3e3a0e596f 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx @@ -155,6 +155,7 @@ const ReviewProjectStructure = ({ projectTree }) => { 'data_type', 'sample', 'base_origin_data', + 'file_objs', ]} />
diff --git a/client/src/redux/sagas/_custom/drp.sagas.js b/client/src/redux/sagas/_custom/drp.sagas.js index 1b888a8efd..8803db9587 100644 --- a/client/src/redux/sagas/_custom/drp.sagas.js +++ b/client/src/redux/sagas/_custom/drp.sagas.js @@ -24,11 +24,13 @@ function* executeOperation(isEdit, params, values, reloadCallback, file = null, try { if (file && isEdit) { + yield call( patchEntityUtil, filteredValues.data_type, params.system, - file.path, + '/' + file.path, + '/' + path, filteredValues, file.uuid ) diff --git a/client/src/redux/sagas/projects.sagas.js b/client/src/redux/sagas/projects.sagas.js index 484b70f7fc..87f2a58759 100644 --- a/client/src/redux/sagas/projects.sagas.js +++ b/client/src/redux/sagas/projects.sagas.js @@ -232,7 +232,7 @@ export async function createEntityUtil(entityType, projectId, path, data) { return result.response; } -export async function patchEntityUtil(entityType, projectId, path, data, entityUuid) { +export async function patchEntityUtil(entityType, projectId, path, updatedPath, data, entityUuid) { const result = await fetchUtil({ url: `/api/projects/${projectId}/entities/create`, method: 'PATCH', @@ -243,7 +243,8 @@ export async function patchEntityUtil(entityType, projectId, path, data, entityU name: entityType, value: data, uuid: entityUuid ?? null, - path: path + path: path, + updatedPath: updatedPath }), }); diff --git a/server/poetry.lock b/server/poetry.lock index 9cf367e441..3cfa8a3d0c 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -3148,4 +3148,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "3c14a4107b26468da3fa96e04c4a72e73863b93fc83e98fee5adda24e0d5a4c4" +content-hash = "3fad1a8073278577998125ec05cb516de5b4ed9973b4c267fe40d53854c1613b" diff --git a/server/portal/apps/_custom/drp/models.py b/server/portal/apps/_custom/drp/models.py index c1b0362aa1..a057a3ddf6 100644 --- a/server/portal/apps/_custom/drp/models.py +++ b/server/portal/apps/_custom/drp/models.py @@ -22,7 +22,26 @@ def model_dump(self, *args, **kwargs): *args, **kwargs ) -class DrpProjectRelatedDatasets(BaseModel): +class FileObj(DrpMetadataModel): + """Model for associated files""" + + system: str + name: str + path: str + type: Literal["file", "dir"] + length: Optional[int] = None + last_modified: Optional[str] = None + uuid: Optional[str] = None + +class PartialEntityWithFiles(DrpMetadataModel): + """Model for representing an entity with associated files.""" + + model_config = ConfigDict(extra="ignore") + + # file_tags: list[FileTag] = [] + file_objs: list[FileObj] = [] + +class DrpProjectRelatedDatasets(DrpMetadataModel): """Model for DRP Project Related Datasets""" model_config = ConfigDict( @@ -33,7 +52,7 @@ class DrpProjectRelatedDatasets(BaseModel): dataset_description: str = "" dataset_link: str = "" -class DrpProjectRelatedSoftware(BaseModel): +class DrpProjectRelatedSoftware(DrpMetadataModel): """Model for DRP Project Related Software""" model_config = ConfigDict( @@ -44,7 +63,7 @@ class DrpProjectRelatedSoftware(BaseModel): software_description: str = "" software_link: str = "" -class DrpProjectRelatedPublications(BaseModel): +class DrpProjectRelatedPublications(DrpMetadataModel): """Model for DRP Project Related Publications""" model_config = ConfigDict( @@ -77,9 +96,9 @@ class DrpProjectMetadata(DrpMetadataModel): related_publications: list[DrpProjectRelatedPublications] = [] publication_date: Optional[str] = None authors: list[str] = [] + file_objs: list[FileObj] = [] - -class DrpDatasetMetadata(BaseModel): +class DrpDatasetMetadata(DrpMetadataModel): """Model for Base DRP Dataset Metadata""" model_config = ConfigDict( @@ -94,6 +113,7 @@ class DrpDatasetMetadata(BaseModel): "analysis_data", "file" ] + file_objs: list[FileObj] = [] class DrpSampleMetadata(DrpDatasetMetadata): """Model for DRP Sample Metadata""" @@ -135,7 +155,6 @@ class DrpSampleMetadata(DrpDatasetMetadata): identifier: Optional[str] = None location: Optional[str] = None # TODO_DRP: Remove in new model - class DrpOriginDatasetMetadata(DrpDatasetMetadata): """Model for DRP Origin Dataset Metadata""" @@ -174,7 +193,7 @@ class DrpAnalysisDatasetMetadata(DrpDatasetMetadata): sample: str base_origin_data: Optional[str] = None -class DrpFileMetadata(BaseModel): +class DrpFileMetadata(DrpMetadataModel): """Model for DRP File Metadata""" model_config = ConfigDict( diff --git a/server/portal/apps/projects/models/project_metadata.py b/server/portal/apps/projects/models/project_metadata.py index 0d3cbf1b10..ad6c0f3fdb 100644 --- a/server/portal/apps/projects/models/project_metadata.py +++ b/server/portal/apps/projects/models/project_metadata.py @@ -10,9 +10,13 @@ from portal.apps._custom.drp import constants from portal.apps import SCHEMA_MAPPING from django.db.models.signals import pre_save +from typing import get_args, List user_model = get_user_model() +def snake_to_camel(snake_str): + components = snake_str.split('_') + return components[0] + ''.join(x.title() for x in components[1:]) def uuid_pk(): """Generate a string UUID for use as a primary key.""" @@ -75,15 +79,43 @@ def project_graph(self): ) @property - def ordered_metadata(self): - """Return the metadata in the order defined in the pydantic model""" - schema = SCHEMA_MAPPING[self.name] + def ordered_value(self): + """ + Return the metadata in the order defined in the Pydantic model. + Also converts camelCase keys to snake_case. This is a temporary workaround until fields in settings_forms.py can be updated to use camelCase. + """ + schema = SCHEMA_MAPPING.get(self.name) if not schema: - return self.value - - model_keys = list(schema.model_fields.keys()) - ordered_metadata = {k: self.value.get(k) for k in model_keys if k in self.value} + return self.value # Return the field value directly if no schema is found + + ordered_metadata = {} + + # Iterate through the model fields to preserve the order + for field in schema.model_fields.keys(): + camel_field = snake_to_camel(field) + field_value = self.value.get(camel_field) + + # Skip the field if there is no value (None or not present) + if field_value is None: + continue + + # if the fiels is a list, then we need to get the model of the list and process it + if isinstance(field_value, list): + field_annotation = schema.model_fields[field].annotation + item_type = get_args(field_annotation)[0] if get_args(field_annotation) else None # returns the model class of the list + + # Check if the item type is a Pydantic model + if item_type and hasattr(item_type, "model_fields"): + # Re-order each item in the list if it's a list of Pydantic models + ordered_metadata[field] = [ + {k: item.get(snake_to_camel(k)) for k in item_type.model_fields.keys()} + for item in field_value + ] + else: + ordered_metadata[field] = field_value + else: + ordered_metadata[field] = field_value return ordered_metadata diff --git a/server/portal/apps/projects/schema_models/base.py b/server/portal/apps/projects/schema_models/base.py index 4c83c72b77..95776c1bb6 100644 --- a/server/portal/apps/projects/schema_models/base.py +++ b/server/portal/apps/projects/schema_models/base.py @@ -7,22 +7,6 @@ from pydantic.alias_generators import to_camel from django.contrib.auth import get_user_model from pytas.http import TASClient +from portal.apps._custom.drp.models import DrpMetadataModel -class FileObj(BaseModel): - """Model for associated files""" - system: str - name: str - path: str - type: Literal["file", "dir"] - length: Optional[int] = None - last_modified: Optional[str] = None - uuid: Optional[str] = None - -class PartialEntityWithFiles(BaseModel): - """Model for representing an entity with associated files.""" - - model_config = ConfigDict(extra="ignore") - - # file_tags: list[FileTag] = [] - file_objs: list[FileObj] = [] diff --git a/server/portal/apps/projects/views.py b/server/portal/apps/projects/views.py index 72ac01c741..9ee186243e 100644 --- a/server/portal/apps/projects/views.py +++ b/server/portal/apps/projects/views.py @@ -26,18 +26,17 @@ from django.db import transaction from portal.apps import SCHEMA_MAPPING from django.db import models -from portal.apps.projects.workspace_operations.project_meta_operations import create_entity_metadata, create_project_metdata, patch_metadata +from portal.apps.projects.workspace_operations.project_meta_operations import create_entity_metadata, create_project_metdata, patch_file_obj_entity, patch_entity, patch_project_entity from portal.libs.agave.operations import mkdir from pathlib import Path from portal.apps._custom.drp import constants from portal.apps.projects.workspace_operations.graph_operations import add_node_to_project, initialize_project_graph, get_node_from_path, update_node_in_project -import networkx as nx LOGGER = logging.getLogger(__name__) def validate_project_metadata(metadata): portal_name = settings.PORTAL_NAMESPACE - schema = SCHEMA_MAPPING[portal_name]['project'] + schema = SCHEMA_MAPPING[constants.PROJECT] validated_model = schema.model_validate(metadata) return validated_model.model_dump(exclude_none=True) @@ -177,19 +176,9 @@ def get(self, request, project_id=None, system_id=None): prj = get_project(request.user.tapis_oauth.client, project_id) - print(f"PRJ {prj}") - try: - - print("HERE") - project = ProjectMetadata.objects.get(models.Q(value__projectId=f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}")) - - # print(f"Test Project: {test_project}") - - # project = ProjectsMetadata.objects.get(project_id=f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}") - # project.metadata = project.get_metadata() - prj.update(project.value) + prj.update(project.ordered_value) prj["projectId"] = project_id except: pass @@ -236,15 +225,12 @@ def patch( """ data = json.loads(request.body) metadata = data['metadata'] + project_id_full = f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}" + client = request.user.tapis_oauth.client - if metadata is not None: - project_metadata = ProjectMetadata.objects.get(models.Q(value__projectId=f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}")) - project_metadata.value = validate_project_metadata(metadata) - # project_metadata = ProjectsMetadata.objects.get(project_id=f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}") - # project_metadata.metadata = validate_project_metadata(metadata) - project_metadata.save() + if metadata is not None: + patch_project_entity(project_id_full, metadata) - client = request.user.tapis_oauth.client workspace_def = update_project(client, project_id, data['title'], data['description']) if metadata is not None: @@ -257,7 +243,6 @@ def patch( } ) - @method_decorator(agave_jwt_login, name='dispatch') @method_decorator(login_required, name='dispatch') class ProjectMembersApiView(BaseApiView): @@ -399,16 +384,18 @@ def patch(self, request: HttpRequest, project_id: str): value = req_body.get("value", {}) entity_uuid = req_body.get("uuid", "") path = req_body.get("path", "") - # path = str(Path(value['path'].rstrip('/'))) if value.get('path') else '' - - print(f'VALUE {value}') - print(f'PATH {path}') - print(f'ENTITY UUID {entity_uuid}') + updated_path = req_body.get("updatedPath", "") - try: - patch_metadata(client, project_id, value, entity_uuid, path) - except Exception as exc: - raise ApiException("Error updating entity metadata", status=500) from exc + if value['data_type'] == 'file': + try: + patch_file_obj_entity(client, project_id, value, path) + except Exception as exc: + raise ApiException("Error updating file metadata", status=500) from exc + else: + try: + patch_entity(client, project_id, value, entity_uuid, path, updated_path) + except Exception as exc: + raise ApiException("Error updating entity metadata", status=500) from exc return JsonResponse({"result": "OK"}) @@ -420,8 +407,6 @@ def post(self, request: HttpRequest, project_id: str): if not request.user.is_authenticated: raise ApiException("Unauthenticated user", status=401) - print(f'project ID {project_id}') - try: project: ProjectMetadata = ProjectMetadata.objects.get( models.Q(uuid=project_id) | models.Q(value__projectId=project_id) @@ -431,26 +416,22 @@ def post(self, request: HttpRequest, project_id: str): "User does not have access to the requested project", status=403 ) from exc - portal = settings.PORTAL_NAMESPACE.lower() req_body = json.loads(request.body) value = req_body.get("value", {}) name = req_body.get("name", "") path = req_body.get("path", "") - new_meta = create_entity_metadata(project.project_id, getattr(constants, name.upper()), { + new_meta = create_entity_metadata(project_id, getattr(constants, name.upper()), { **value, - # "path": str(Path(value['path'].rstrip('/')) / Path(value['name'])) if value.get('path') else '' }) - parent_node = get_node_from_path(project.project_id, path) - - add_node_to_project(project_id, parent_node['id'], new_meta.uuid, name, value['name']) + # FOR CREATING GRAPH + parent_node = get_node_from_path(project_id, path) + add_node_to_project(project_id, parent_node['id'], new_meta.uuid, new_meta.name, value['name']) + # FOR CREATING DATA FILE FOLDER if (value and path): mkdir(client, project_id, path, value['name']) - # FOR CREATING GRAPH - # if name in ALLOWED_RELATIONS[constants.PROJECT]: - # add_node_to_project(project_id, "NODE_ROOT", new_meta.uuid, name) return JsonResponse({"result": "OK"}) diff --git a/server/portal/apps/projects/workspace_operations/graph_operations.py b/server/portal/apps/projects/workspace_operations/graph_operations.py index 7f4dfbe823..013b72413c 100644 --- a/server/portal/apps/projects/workspace_operations/graph_operations.py +++ b/server/portal/apps/projects/workspace_operations/graph_operations.py @@ -1,3 +1,4 @@ +from typing import Any, Dict import networkx as nx from django.db import transaction import uuid @@ -85,10 +86,10 @@ def traverse_graph(project_graph, root_node, path_components): found = True break if not found: - raise Exception(f"Component '{component}' not found under node '{project_graph.nodes[current_node]['label']}'.") + return None return {"id": current_node, **project_graph.nodes[current_node]} -def get_node_from_path(project_id: str, path: str) -> str: +def get_node_from_path(project_id: str, path: str) -> Dict[str, Any]: """Return the node ID for the parent of a node with the given path.""" graph_model = ProjectMetadata.objects.get( @@ -103,12 +104,34 @@ def get_node_from_path(project_id: str, path: str) -> str: node = traverse_graph(project_graph, "NODE_ROOT", path_parts) - print('returned node', node) + return node + +def get_root_node(project_id: str) -> Dict[str, Any]: + """Return the root node for a project graph.""" + graph_model = ProjectMetadata.objects.get( + name=constants.PROJECT_GRAPH, base_project__value__projectId=project_id + ) + project_graph = nx.node_link_graph(graph_model.value) + return {"id": "NODE_ROOT", **project_graph.nodes["NODE_ROOT"]} + +def get_file_association_from_path(project_id: str, path: str) -> Dict[str, Any]: + """Return the node ID for the parent of a node with the given path.""" + + graph_model = ProjectMetadata.objects.get( + name=constants.PROJECT_GRAPH, base_project__value__projectId=project_id + ) + project_graph = nx.node_link_graph(graph_model.value) + + path_parts = path.strip("/").split("/") + + if len(path_parts) == 0 or path_parts[0] == "": + return {"id": "NODE_ROOT"} + + node = traverse_graph(project_graph, "NODE_ROOT", path_parts) return node - raise nx.exception.NodeNotFound -def update_node_in_project(project_id: str, node_id: str, value: dict): +def update_node_in_project(project_id: str, node_id: str, new_parent: str = None, new_name: str = None): """Update the database entry for a project graph to update a node.""" with transaction.atomic(): graph_model = ProjectMetadata.objects.select_for_update().get( @@ -119,7 +142,19 @@ def update_node_in_project(project_id: str, node_id: str, value: dict): if not project_graph.has_node(node_id): raise nx.exception.NodeNotFound - project_graph.nodes[node_id]["value"] = value + if new_parent: + # Remove the node from the graph and re-add it under the new parent. + parent_node = new_parent + if not project_graph.has_node(parent_node): + raise nx.exception.NodeNotFound + project_graph.remove_edge( + next(project_graph.predecessors(node_id)), node_id + ) + project_graph.add_edge(parent_node, node_id) + + if new_name: + project_graph.nodes[node_id]["label"] = new_name + graph_model.value = nx.node_link_data(project_graph) graph_model.save() diff --git a/server/portal/apps/projects/workspace_operations/project_meta_operations.py b/server/portal/apps/projects/workspace_operations/project_meta_operations.py index f50caa9e89..4f228f504f 100644 --- a/server/portal/apps/projects/workspace_operations/project_meta_operations.py +++ b/server/portal/apps/projects/workspace_operations/project_meta_operations.py @@ -4,11 +4,9 @@ from portal.apps._custom.drp import constants from django.conf import settings from portal.apps.projects.models.project_metadata import ProjectMetadata -from portal.apps.projects.schema_models.base import ( - FileObj, - PartialEntityWithFiles -) -from portal.apps.projects.workspace_operations.graph_operations import get_node_from_path, update_node_in_project +from portal.apps._custom.drp.models import PartialEntityWithFiles, FileObj +from portal.apps.projects.workspace_operations.graph_operations import get_node_from_path, get_root_node, update_node_in_project +from pathlib import Path portal = settings.PORTAL_NAMESPACE.lower() @@ -48,47 +46,96 @@ def get_entity(project_id, path): except ProjectMetadata.DoesNotExist: return None +def get_file_association_entity(project_id, path): + try: + parent_path = str(Path(path).parent) + + parent_node = _get_valid_node(project_id, parent_path) + + parent_entity = ProjectMetadata.objects.get(uuid=parent_node['uuid']) + file_objs = parent_entity.value.get('fileObjs', []) + file_obj = next((f for f in file_objs if f['path'] == path), None) + if file_obj: + entity = ProjectMetadata.objects.get(uuid=file_obj['uuid']) + return entity + else: + return None + except: + return None + +@transaction.atomic +def patch_project_entity(project_id, value): + """Update a project's `value` attribute. This method patches the metadata + so that only fields in the payload are overwritten.""" + + entity = ProjectMetadata.get_project_by_id(project_id) + schema_model = SCHEMA_MAPPING[entity.name] + + patched_metadata = {**value, 'projectId': project_id, 'fileObjs': entity.value.get('fileObjs', [])} + + update_node_in_project(project_id, 'NODE_ROOT', None, value.get('title')) + + validated_model = schema_model.model_validate(patched_metadata) + entity.value = validated_model.model_dump(exclude_none=True) + entity.save() + return entity + @transaction.atomic -def patch_metadata(client, project_id, value, uuid=None, path=None): +def patch_entity(client, project_id, value, uuid=None, path=None, updated_path=None): """Update an entity's `value` attribute. This method patches the metadata so that only fields in the payload are overwritten.""" - node = get_node_from_path(project_id, path) - if uuid: - entity = ProjectMetadata.objects.get(uuid=uuid) - elif project_id and path: - entity = ProjectMetadata.objects.get(uuid=node['uuid']) - else: - raise ValueError("Either 'uuid' or both 'project_id' and 'path' must be provided.") - - print('entity', entity) + node = get_node_from_path(project_id, path) if path else get_root_node(project_id) + entity = ProjectMetadata.objects.get(uuid=node['uuid']) current_name = entity.value.get('name') - current_path = entity.value.get('path') + current_path_full = path - new_name = value['name'] - new_path = path + new_name = value.get('name') + new_path = updated_path - print('current_name', current_name) - print('current_path', current_path) - print('new_name', new_name) - print('new_path', new_path) - # # If the name or path has changed, move the entity to the new location. - if current_name != new_name or current_path != new_path: + if all([current_name, new_name, new_path, current_path_full]) and ( + current_name != new_name or updated_path != str(Path(current_path_full).parent) + ): + new_name = _move_entity(client, project_id, current_path_full, new_path, new_name, value) - from portal.libs.agave.operations import move - - move_result = move(client, project_id, current_path, project_id, new_path, new_name) - move_message = move_result['message'].split('DestinationPath: ', 1)[1] - new_name = ('/' + move_message).rsplit('/', 1)[1] - value['name'] = new_name - update_node_in_project(project_id, node['id'], value) + if new_path: + parent_node = get_node_from_path(project_id, new_path) + update_node_in_project(project_id, node['id'], parent_node['id'], new_name) + else: + update_node_in_project(project_id, node['id'], None, new_name) + schema_model = SCHEMA_MAPPING[entity.name] + + patched_metadata = {**value, 'fileObjs': entity.value.get('fileObjs', [])} + validated_model = schema_model.model_validate(patched_metadata) + entity.value = validated_model.model_dump(exclude_none=True) + entity.save() + + return entity + +@transaction.atomic +def patch_file_obj_entity(client, project_id, value, path): + """Update an entity's `value` attribute""" + + parent_path = str(Path(path).parent) + + parent_node = _get_valid_node(project_id, parent_path) + + parent_entity = ProjectMetadata.objects.get(uuid=parent_node['uuid']) + file_objs = parent_entity.value.get('fileObjs', []) + file_obj = next((f for f in file_objs if f['path'] == path.strip('/')), None) + + if not file_obj: + return None + + entity = ProjectMetadata.objects.get(uuid=file_obj['uuid']) schema_model = SCHEMA_MAPPING[entity.name] - patched_metadata = {**entity.value, **value} + patched_metadata = {**value} + validated_model = schema_model.model_validate(patched_metadata) entity.value = validated_model.model_dump(exclude_none=True) entity.save() @@ -104,6 +151,17 @@ def delete_entity(uuid: str): return "OK" +def _move_entity(client, project_id, current_path, new_path, new_name, value): + """Handle moving an entity to a new location if the name or path changes.""" + from portal.libs.agave.operations import move + + move_result = move(client, project_id, current_path, project_id, new_path, new_name) + move_message = move_result['message'].split('DestinationPath: ', 1)[1] + new_name = ('/' + move_message).rsplit('/', 1)[1] + + value['name'] = new_name + return new_name + def clear_entities(project_id): """Delete all entities except the project root and graph. Used when changing project type, so that file associations don't get stuck on unreachable entities. @@ -140,7 +198,6 @@ def add_file_associations(uuid: str, new_file_objs: list[FileObj]): with transaction.atomic(): entity = ProjectMetadata.objects.select_for_update().get(uuid=uuid) entity_file_model = PartialEntityWithFiles.model_validate(entity.value) - merged_file_objs = _merge_file_objs(entity_file_model.file_objs, new_file_objs) entity.value["fileObjs"] = [f.model_dump() for f in merged_file_objs] @@ -165,20 +222,63 @@ def remove_file_associations(uuid: str, file_paths: list[str]): filtered_file_objs = _filter_file_objs(entity_file_model.file_objs, file_paths) entity.value["fileObjs"] = [f.model_dump() for f in filtered_file_objs] - # Remove tags associated with these entity/file path combinations. - tagged_paths = [] - for path in file_paths: - tagged_paths += [ - t["path"] - for t in entity.value.get("fileTags", []) - if t["path"].startswith(path) - ] - entity.value["fileTags"] = [ - t - for t in entity.value.get("fileTags", []) - if not (t["path"] in tagged_paths) - ] entity.save() return entity +def create_file_entity(project_id: str, value: dict, uploaded_file, path: str): + + new_meta = create_entity_metadata(project_id, getattr(constants, value.get('data_type').upper()), { + **value, + }) + + parent_node = get_node_from_path(project_id, path) + + file_obj = FileObj( + system=project_id, + name=uploaded_file.name, + path=f'{path.strip("/")}/{uploaded_file.name}', + type='file', + length=uploaded_file.size, + uuid=new_meta.uuid + ) + + if parent_node and parent_node['id'] != 'NODE_ROOT': + add_file_associations(parent_node['uuid'], [file_obj]) + else: + # Add file association to root node if no parent node/entity exists + root_node = get_root_node(project_id) + add_file_associations(root_node['uuid'], [file_obj]) + +def _get_valid_node(project_id, path): + node = get_node_from_path(project_id, path) + return node if node and node['id'] != 'NODE_ROOT' else get_root_node(project_id) + +@transaction.atomic +def patch_file_association(project_id, value, source_path_full, dest_path_full, new_name, operation): + + source_parent_path = str(Path(source_path_full).parent) + dest_parent_path = str(Path(dest_path_full).parent) + source_node = _get_valid_node(project_id, source_parent_path) + dest_node = _get_valid_node(project_id, dest_parent_path) + + source_entity = ProjectMetadata.objects.get(uuid=source_node['uuid']) + dest_entity = ProjectMetadata.objects.get(uuid=dest_node['uuid']) + + file_obj_dict = next( + (f for f in source_entity.value.get('fileObjs', []) if f['path'] == source_path_full.strip('/')), None + ) + + if not file_obj_dict: + return + + file_obj_dict['name'] = new_name + file_obj_dict['path'] = dest_path_full.strip('/') + file_obj = FileObj(**file_obj_dict) + if operation == 'move': + remove_file_associations(source_entity.uuid, [source_path_full]) + add_file_associations(dest_entity.uuid, [file_obj]) + elif operation == 'copy': + new_meta = create_entity_metadata(project_id, getattr(constants, value.get('data_type').upper()), {**value}) + file_obj.uuid = new_meta.uuid + add_file_associations(dest_entity.uuid, [file_obj]) \ No newline at end of file diff --git a/server/portal/libs/agave/operations.py b/server/portal/libs/agave/operations.py index a45f026ab6..f23844f991 100644 --- a/server/portal/libs/agave/operations.py +++ b/server/portal/libs/agave/operations.py @@ -10,11 +10,16 @@ from portal.libs.agave.utils import text_preview, get_file_size, increment_file_name from portal.libs.agave.filter_mapping import filter_mapping from pathlib import Path +from portal.apps._custom.drp.models import FileObj from tapipy.errors import BaseTapyException from portal.apps.projects.models.metadata import ProjectsMetadata from portal.apps.datafiles.models import DataFilesMetadata from portal.apps import SCHEMA_MAPPING -from portal.apps.projects.workspace_operations.project_meta_operations import get_entity +from portal.apps.projects.workspace_operations.project_meta_operations import (add_file_associations, create_entity_metadata, get_entity, + get_file_association_entity, patch_file_association) +from portal.apps._custom.drp import constants +from portal.apps.projects.workspace_operations.graph_operations import add_node_to_project, get_root_node, get_node_from_path + logger = logging.getLogger(__name__) @@ -85,9 +90,7 @@ def listing(client, system, path, offset=0, limit=100, *args, **kwargs): path=path, offset=int(offset), limit=int(limit)) - - print(f"Listing for {system}/{path} returned {(raw_listing)}") - + folder_metadata = get_entity(system, path) try: @@ -96,7 +99,8 @@ def listing(client, system, path, offset=0, limit=100, *args, **kwargs): listing = [] for f in raw_listing: - entity = get_entity(system, f.path) + + entity = get_entity(system, f.path) if f.type == 'dir' else get_file_association_entity(system, f.path) listing.append({ 'uuid': entity.to_dict().get('uuid') if entity else None, @@ -111,7 +115,7 @@ def listing(client, system, path, offset=0, limit=100, *args, **kwargs): '_links': { 'self': {'href': f.url} }, - 'metadata': entity.ordered_metadata if entity else None + 'metadata': entity.ordered_value if entity else None }) # listing = list(map(lambda f: { @@ -134,7 +138,7 @@ def listing(client, system, path, offset=0, limit=100, *args, **kwargs): # Update Elasticsearch after each listing. tapis_listing_indexer.delay(listing) - return {'listing': listing, 'reachedEnd': len(listing) < int(limit), 'folder_metadata': folder_metadata.ordered_metadata if folder_metadata else None} + return {'listing': listing, 'reachedEnd': len(listing) < int(limit), 'folder_metadata': folder_metadata.ordered_value if folder_metadata else None} def iterate_listing(client, system, path, limit=100): @@ -270,7 +274,10 @@ def mkdir(client, system, path, dir_name, metadata=None): path_input = str(Path(path) / Path(dir_name)) if metadata is not None: - create_datafile_metadata(system, f'{system}/{path_input.strip("/")}', dir_name, metadata) + new_meta = create_entity_metadata(system, 'drp.project.trash', { + **metadata, + }) + add_node_to_project(system, 'NODE_ROOT', new_meta.uuid, new_meta.name, dir_name) client.files.mkdir(systemId=system, path=path_input) @@ -324,7 +331,8 @@ def move(client, src_system, src_path, dest_system, dest_path, file_name=None, m dest_path_full = os.path.join(dest_path.strip('/'), file_name) if metadata is not None: - update_datafile_metadata(system=dest_system, name=file_name, old_path=src_path.strip("/"), new_path=dest_path_full.strip("/"), metadata=metadata) + if (metadata.get('data_type') == 'file'): + patch_file_association(src_system, metadata, src_path, dest_path_full, file_name, 'move') if src_system == dest_system: move_result = client.files.moveCopy(systemId=src_system, @@ -394,7 +402,8 @@ def copy(client, src_system, src_path, dest_system, dest_path, file_name=None, m dest_path_full = os.path.join(dest_path.strip('/'), file_name) if metadata is not None: - create_datafile_metadata(dest_system, f'{dest_system}/{dest_path_full.strip("/")}', file_name, metadata) + if (metadata.get('data_type') == 'file'): + patch_file_association(src_system, metadata, src_path, dest_path_full, file_name, 'copy') if src_system == dest_system: copy_result = client.files.moveCopy(systemId=src_system, @@ -513,7 +522,7 @@ def trash(client, system, path, homeDir, metadata=None): return resp - +@transaction.atomic def upload(client, system, path, uploaded_file, metadata=None): """Upload a file. Params @@ -536,8 +545,29 @@ def upload(client, system, path, uploaded_file, metadata=None): dest_path = os.path.join(path.strip('/'), uploaded_file.name) - if metadata is not None: - create_datafile_metadata(system, f'{system}/{dest_path.strip("/")}', uploaded_file.name, metadata) + if metadata is not None and getattr(constants, metadata.get('data_type').upper(), None): + + new_meta = create_entity_metadata(system, getattr(constants, metadata.get('data_type').upper()), { + **metadata, + }) + + parent_node = get_node_from_path(system, path) + + file_obj = FileObj( + system=system, + name=uploaded_file.name, + path=dest_path, + type='file', + length=uploaded_file.size, + uuid=new_meta.uuid + ) + + if parent_node and parent_node['id'] != 'NODE_ROOT': + add_file_associations(parent_node['uuid'], [file_obj]) + else: + # Add file association to root node if no parent node/entity exists + root_node = get_root_node(system) + add_file_associations(root_node['uuid'], [file_obj]) response_json = client.files.insert(systemId=system, path=dest_path, file=uploaded_file) tapis_indexer.apply_async(kwargs={'access_token': client.access_token.access_token, From 81723efe4811610f022a9d5cebbac7168720e8dd Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Mon, 21 Oct 2024 12:54:42 -0500 Subject: [PATCH 113/328] updated tree implementation to use new metadata --- .../ReviewProjectStructure.jsx | 57 +++++++++---------- server/portal/apps/_custom/drp/views.py | 43 +++++++++++--- .../project_meta_operations.py | 2 +- 3 files changed, 63 insertions(+), 39 deletions(-) diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx index 3e3a0e596f..9cdb163fc1 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx @@ -28,6 +28,12 @@ const ReviewProjectStructure = ({ projectTree }) => { const { params } = useFileListing('FilesListing'); + useEffect(() => { + if (projectTree && projectTree.length > 0) { + setExpandedNodes([projectTree[0].uuid]); + } + }, [projectTree]); + const handleNodeToggle = (event, nodeIds) => { // Update the list of expanded nodes setExpandedNodes(nodeIds); @@ -59,18 +65,13 @@ const ReviewProjectStructure = ({ projectTree }) => { const dataType = node.metadata.data_type; // reconstruct editFile to mimic SelectedFile object const editFile = { - format: 'folder', - id: node.path, + id: node.uuid, + uuid: node.uuid, metadata: node.metadata, name: node.metadata.name, system: params.system, - path: node.path.split('/').slice(1).join('/'), type: 'dir', - _links: { - self: { - href: 'tapis://' + node.path, - }, - }, + }; switch (dataType) { case 'sample': @@ -92,10 +93,10 @@ const ReviewProjectStructure = ({ projectTree }) => { api: params.api, scheme: params.scheme, system: params.system, - path: node.path.split('/').slice(1).join('/'), + path: node.path, name: node.name, - href: 'tapis://' + node.path, - length: node.metadata.length, + href: `tapis://${params.system}/${node.path}`, + length: node.length, metadata: node.metadata, useReloadCallback: false, }, @@ -112,7 +113,8 @@ const ReviewProjectStructure = ({ projectTree }) => { .map(word => word.charAt(0).toUpperCase() + word.slice(1)) .join(' '); - const renderTree = (node) => ( + const renderTree = (node) => { + return ( <>
{ >
- {node.name} - - {formatDatatype(node.metadata.data_type)} + {node.label ?? node.name} + {node.metadata.data_type && ( + + {formatDatatype(node.metadata.data_type)} + )}
} classes={{ @@ -135,7 +139,7 @@ const ReviewProjectStructure = ({ projectTree }) => { }} onLabelClick={() => handleNodeToggle} > - {expandedNodes.includes(node.id) && ( + {expandedNodes.includes(node.uuid) && node.id !== 'NODE_ROOT' && (
{(canEdit || node.metadata.data_type === 'file') && (
)} + {Array.isArray(node.fileObjs) && + node.fileObjs.map((fileObj) => renderTree(fileObj))} {Array.isArray(node.children) && node.children.map((child) => renderTree(child))}
- ); - - const getAllNodeIds = (nodes) => { - const ids = []; - nodes.forEach((node) => { - ids.push(node.id); - if (Array.isArray(node.children)) { - ids.push(...getAllNodeIds(node.children)); - } - }); - return ids; - }; + )}; return ( { } defaultExpandIcon={} - // expanded={getAllNodeIds(tree)} + expanded={expandedNodes} onNodeToggle={handleNodeToggle} > {projectTree.map((node) => renderTree(node))} diff --git a/server/portal/apps/_custom/drp/views.py b/server/portal/apps/_custom/drp/views.py index ffd35f5208..8cb223e888 100644 --- a/server/portal/apps/_custom/drp/views.py +++ b/server/portal/apps/_custom/drp/views.py @@ -5,7 +5,7 @@ from portal.apps.datafiles.models import DataFilesMetadata from portal.apps.projects.models.project_metadata import ProjectMetadata from portal.apps._custom.drp import constants - +import networkx as nx class DigitalRocksSampleView(BaseApiView): def get(self, request): @@ -48,6 +48,10 @@ def construct_tree(records, parent_id=None): } tree.append(node_dict) return tree + + @staticmethod + def _get_entity(uuid): + return ProjectMetadata.objects.get(uuid=uuid) def get(self, request): @@ -56,11 +60,36 @@ def get(self, request): project_id = request.GET.get('project_id') full_project_id = f'{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}' - records = DataFilesMetadata.objects.filter( - project_id=full_project_id, - metadata__data_type__in=metadata_data_types - ).order_by('created_at') + graph_model = ProjectMetadata.objects.get( + name=constants.PROJECT_GRAPH, base_project__value__projectId=full_project_id + ) + + graph = nx.node_link_graph(graph_model.value) + + for node in graph.nodes: + node_uuid = graph.nodes[node].get("uuid") + + entity = self._get_entity(node_uuid) + metadata = entity.ordered_value - tree = self.construct_tree(records) + file_objs = entity.value.get('fileObjs', []) + + file_objs_dict = [] + + for file_obj in file_objs: + file_obj_entity = self._get_entity(file_obj.get('uuid')) + file_obj_metadata = file_obj_entity.ordered_value + + file_objs_dict.append({ + **file_obj, + 'metadata': file_obj_metadata + }) + + graph.nodes[node]['metadata'] = metadata + graph.nodes[node]['fileObjs'] = file_objs_dict + + tree = nx.tree_data(graph, "NODE_ROOT") + + return JsonResponse([tree], safe=False) + - return JsonResponse(tree, safe=False) \ No newline at end of file diff --git a/server/portal/apps/projects/workspace_operations/project_meta_operations.py b/server/portal/apps/projects/workspace_operations/project_meta_operations.py index 4f228f504f..3fd46f7e6a 100644 --- a/server/portal/apps/projects/workspace_operations/project_meta_operations.py +++ b/server/portal/apps/projects/workspace_operations/project_meta_operations.py @@ -86,7 +86,7 @@ def patch_entity(client, project_id, value, uuid=None, path=None, updated_path=N so that only fields in the payload are overwritten.""" node = get_node_from_path(project_id, path) if path else get_root_node(project_id) - entity = ProjectMetadata.objects.get(uuid=node['uuid']) + entity = ProjectMetadata.objects.get(uuid=uuid if uuid else node['uuid']) current_name = entity.value.get('name') current_path_full = path From 2742ce49c656c46d9f197e1260c36847fa71d8ee Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Thu, 24 Oct 2024 13:03:54 -0500 Subject: [PATCH 114/328] Updated implementation - Updated implementation of fileObj value - Review pipeline fully functional with tree view --- .../DataFilesProjectFileListing.jsx | 6 + .../ReviewProjectStructure.jsx | 10 +- .../DataFilesProjectReview.jsx | 20 +- .../src/redux/reducers/projects.reducers.js | 35 +++ client/src/redux/sagas/projects.sagas.js | 27 ++ server/portal/apps/_custom/drp/models.py | 46 ++-- server/portal/apps/_custom/drp/views.py | 31 ++- server/portal/apps/projects/tasks.py | 239 ++++++++++-------- server/portal/apps/projects/views.py | 6 +- .../workspace_operations/graph_operations.py | 17 -- .../project_meta_operations.py | 131 ++++++++-- .../project_publish_operations.py | 0 .../shared_workspace_operations.py | 49 ++-- ...licationrequest_review_project_and_more.py | 25 ++ server/portal/apps/publications/models.py | 7 +- server/portal/apps/publications/urls.py | 1 + server/portal/apps/publications/views.py | 52 +++- server/portal/libs/agave/operations.py | 51 ++-- server/portal/settings/settings.py | 9 + 19 files changed, 489 insertions(+), 273 deletions(-) create mode 100644 server/portal/apps/projects/workspace_operations/project_publish_operations.py create mode 100644 server/portal/apps/publications/migrations/0002_alter_publicationrequest_review_project_and_more.py diff --git a/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx b/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx index 9fd5258099..b3f8c65087 100644 --- a/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx +++ b/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx @@ -28,6 +28,12 @@ const DataFilesProjectFileListing = ({ system, path }) => { type: 'PROJECTS_GET_METADATA', payload: system, }); + + dispatch({ + type: 'PROJECTS_GET_PUBLICATION_REQUESTS', + payload: system, + }) + }, [system]); useEffect(() => { diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx index 9cdb163fc1..863f9f938d 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx @@ -30,7 +30,7 @@ const ReviewProjectStructure = ({ projectTree }) => { useEffect(() => { if (projectTree && projectTree.length > 0) { - setExpandedNodes([projectTree[0].uuid]); + setExpandedNodes([projectTree[0].id]); } }, [projectTree]); @@ -65,7 +65,7 @@ const ReviewProjectStructure = ({ projectTree }) => { const dataType = node.metadata.data_type; // reconstruct editFile to mimic SelectedFile object const editFile = { - id: node.uuid, + id: node.id, uuid: node.uuid, metadata: node.metadata, name: node.metadata.name, @@ -122,8 +122,8 @@ const ReviewProjectStructure = ({ projectTree }) => { >
{node.label ?? node.name} @@ -139,7 +139,7 @@ const ReviewProjectStructure = ({ projectTree }) => { }} onLabelClick={() => handleNodeToggle} > - {expandedNodes.includes(node.uuid) && node.id !== 'NODE_ROOT' && ( + {expandedNodes.includes(node.id) && node.id !== 'NODE_ROOT' && (
{(canEdit || node.metadata.data_type === 'file') && ( ) : ( )} | diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx index bb45c8a327..88ca11d6b3 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx @@ -5,7 +5,7 @@ import { useFileListing } from 'hooks/datafiles'; import DataDisplay from '../utils/DataDisplay/DataDisplay'; import { formatDate } from 'utils/timeFormat'; -const excludeKeys = ['name', 'description', 'data_type', 'sample', 'base_origin_data', 'file_objs']; +const excludeKeys = ['name', 'description', 'data_type', 'sample', 'digital_dataset', 'file_objs']; const DataFilesProjectFileListingMetadataAddon = ({ folderMetadata, metadata, path }) => { diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx index da6be45a11..d760641639 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx @@ -52,7 +52,7 @@ const DataFilesProjectFileListingMetadataTitleAddon = ({ folderMetadata, metadat case 'sample': createSampleModal('EDIT_SAMPLE_DATA', editFile); break; - case 'origin_data': + case 'digital_dataset': createOriginDataModal('EDIT_ORIGIN_DATASET', editFile); break; case 'analysis_data': diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx index 863f9f938d..70dd83f978 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx @@ -32,7 +32,7 @@ const ReviewProjectStructure = ({ projectTree }) => { if (projectTree && projectTree.length > 0) { setExpandedNodes([projectTree[0].id]); } - }, [projectTree]); + }, []); const handleNodeToggle = (event, nodeIds) => { // Update the list of expanded nodes @@ -71,13 +71,13 @@ const ReviewProjectStructure = ({ projectTree }) => { name: node.metadata.name, system: params.system, type: 'dir', - + path: node.path, }; switch (dataType) { case 'sample': createSampleModal('EDIT_SAMPLE_DATA', editFile); break; - case 'origin_data': + case 'digital_dataset': createOriginDataModal('EDIT_ORIGIN_DATASET', editFile); break; case 'analysis_data': @@ -113,8 +113,7 @@ const ReviewProjectStructure = ({ projectTree }) => { .map(word => word.charAt(0).toUpperCase() + word.slice(1)) .join(' '); - const renderTree = (node) => { - return ( + const renderTree = (node) => ( <>
{ 'description', 'data_type', 'sample', - 'base_origin_data', + 'digital_dataset', 'file_objs', ]} /> @@ -173,8 +172,8 @@ const ReviewProjectStructure = ({ projectTree }) => {
- )}; - + ); + return ( { // use the path to get sample and origin data names const sample = data.sample ? path.split('/')[0] : null; - const origin_data = data.base_origin_data ? path.split('/')[1] : null; + const origin_data = data.digital_dataset ? path.split('/')[1] : null; // remove trailing / from pathname const locationPathname = location.pathname.endsWith('/') ? location.pathname.slice(0, -1) : location.pathname; @@ -19,7 +19,7 @@ const processSampleAndOriginData = (data, path) => { // construct urls for sample and origin data and add to processed data if (sample) { - const sampleUrl = locationPathnameParts.slice(0, data.base_origin_data ? -2 : -1).join('/') + const sampleUrl = locationPathnameParts.slice(0, data.digital_dataset ? -2 : -1).join('/') sampleAndOriginMetadata.push({ label: 'Sample', value: {sample} }); } diff --git a/client/src/components/_custom/drp/utils/DataFilesToolbar/customFilePermissions.js b/client/src/components/_custom/drp/utils/DataFilesToolbar/customFilePermissions.js index 1fc0ad10b2..0b25a0ebd9 100644 --- a/client/src/components/_custom/drp/utils/DataFilesToolbar/customFilePermissions.js +++ b/client/src/components/_custom/drp/utils/DataFilesToolbar/customFilePermissions.js @@ -3,7 +3,7 @@ const customFilePermissions = (operation, files) => { const protectedDataTypes = [ 'sample', - 'origin_data', + 'digital_dataset', 'analysis_data' ] diff --git a/client/src/redux/sagas/_custom/drp.sagas.js b/client/src/redux/sagas/_custom/drp.sagas.js index 8803db9587..ffef67d869 100644 --- a/client/src/redux/sagas/_custom/drp.sagas.js +++ b/client/src/redux/sagas/_custom/drp.sagas.js @@ -24,12 +24,11 @@ function* executeOperation(isEdit, params, values, reloadCallback, file = null, try { if (file && isEdit) { - yield call( patchEntityUtil, filteredValues.data_type, params.system, - '/' + file.path, + file.path ? '/' + file.path : '', '/' + path, filteredValues, file.uuid @@ -122,7 +121,7 @@ function* handleOriginData(action, isEdit) { const metadata = { ...values, - data_type: 'origin_data', + data_type: 'digital_dataset', sample: sample.uuid, } @@ -152,7 +151,7 @@ function* handleAnalysisData(action, isEdit) { ...values, data_type: 'analysis_data', sample: sample.uuid, - base_origin_data: originData ? originData.uuid : '' + // base_origin_data: originData ? originData.uuid : '' } yield call( diff --git a/server/portal/apps/__init__.py b/server/portal/apps/__init__.py index 1bd39af027..36e0a49af2 100644 --- a/server/portal/apps/__init__.py +++ b/server/portal/apps/__init__.py @@ -6,6 +6,7 @@ constants.PROJECT: DrpProjectMetadata, constants.SAMPLE: DrpSampleMetadata, constants.ORIGIN_DATA: DrpOriginDatasetMetadata, + constants.DIGITAL_DATASET: DrpOriginDatasetMetadata, constants.ANALYSIS_DATA: DrpAnalysisDatasetMetadata, constants.FILE: DrpFileMetadata } \ No newline at end of file diff --git a/server/portal/apps/_custom/drp/models.py b/server/portal/apps/_custom/drp/models.py index ad521feda7..8f6b599bef 100644 --- a/server/portal/apps/_custom/drp/models.py +++ b/server/portal/apps/_custom/drp/models.py @@ -130,6 +130,7 @@ class DrpDatasetMetadata(DrpMetadataModel): data_type: Literal[ "sample", "origin_data", + "digital_dataset", "analysis_data", "file" ] @@ -211,4 +212,5 @@ class DrpAnalysisDatasetMetadata(DrpDatasetMetadata): ] external_uri: Optional[str] = None sample: str - base_origin_data: Optional[str] = None + # base_origin_data: Optional[str] = None + digital_dataset: Optional[str] = None diff --git a/server/portal/apps/_custom/drp/views.py b/server/portal/apps/_custom/drp/views.py index 36d8ffa39b..8b784c9017 100644 --- a/server/portal/apps/_custom/drp/views.py +++ b/server/portal/apps/_custom/drp/views.py @@ -6,6 +6,7 @@ from portal.apps.projects.models.project_metadata import ProjectMetadata from portal.apps._custom.drp import constants import networkx as nx +from networkx import shortest_path from portal.apps.projects.workspace_operations.project_meta_operations import get_ordered_value class DigitalRocksSampleView(BaseApiView): @@ -24,7 +25,7 @@ def get(self, request): if get_origin_data == 'true': # origin_data = DataFilesMetadata.objects.filter(project_id=full_project_id, metadata__data_type='origin_data').values('id', 'name', 'path', 'metadata') - origin_data = ProjectMetadata.objects.filter(base_project__value__projectId=full_project_id, name=constants.ORIGIN_DATA).values('uuid', 'name', 'value') + origin_data = ProjectMetadata.objects.filter(base_project__value__projectId=full_project_id, name=constants.DIGITAL_DATASET).values('uuid', 'name', 'value') response_data = { 'samples': list(samples), @@ -65,6 +66,8 @@ def get(self, request): graph = nx.node_link_graph(graph_model.value) + trash_node_id = None + for node_id in graph.nodes: trash_node = graph.nodes[node_id].get('name') == settings.TAPIS_DEFAULT_TRASH_NAME if trash_node: @@ -80,6 +83,13 @@ def get(self, request): node = graph.nodes[node_id] + # Get the path from NODE_ROOT to the current node + if nx.has_path(graph, 'NODE_ROOT', node_id): + path_nodes = shortest_path(graph, 'NODE_ROOT', node_id)[1:] + node['path'] = '/'.join(graph.nodes[parent]['label'] for parent in path_nodes if 'label' in graph.nodes[parent]) + else: + node['path'] = "" + if node.get('value'): metadata = get_ordered_value(node['name'], node['value']) file_objs = node['value'].get('fileObjs', []) diff --git a/server/portal/apps/projects/views.py b/server/portal/apps/projects/views.py index 8488a70555..bb18faea7d 100644 --- a/server/portal/apps/projects/views.py +++ b/server/portal/apps/projects/views.py @@ -395,8 +395,8 @@ def patch(self, request: HttpRequest, project_id: str): raise ApiException("Error updating file metadata", status=500) from exc else: try: - new_name = move_entity(client, project_id, path, updated_path, value) - patch_entity_and_node(project_id, value, path, updated_path, new_name) + new_name = move_entity(client, project_id, path, updated_path, value, entity_uuid) + patch_entity_and_node(project_id, value, path, updated_path, new_name, entity_uuid) except Exception as exc: raise ApiException("Error updating entity metadata", status=500) from exc diff --git a/server/portal/apps/projects/workspace_operations/graph_operations.py b/server/portal/apps/projects/workspace_operations/graph_operations.py index b62ce20c05..e1df18c823 100644 --- a/server/portal/apps/projects/workspace_operations/graph_operations.py +++ b/server/portal/apps/projects/workspace_operations/graph_operations.py @@ -125,7 +125,7 @@ def update_node_in_project(project_id: str, node_id: str, new_parent: str = None if not project_graph.has_node(node_id): raise nx.exception.NodeNotFound - if new_parent: + if new_parent and new_parent != node_id: # Remove the node from the graph and re-add it under the new parent. parent_node = new_parent if not project_graph.has_node(parent_node): @@ -156,4 +156,16 @@ def add_node_to_project(project_id: str, parent_node: str, meta_uuid: str, name: graph_model.value = nx.node_link_data(updated_graph) graph_model.save() - return new_node_id \ No newline at end of file + return new_node_id + +def get_node_from_uuid(project_id: str, uuid: str): + """Get a node from the project graph using its UUID.""" + graph_model = ProjectMetadata.objects.get( + name=constants.PROJECT_GRAPH, base_project__value__projectId=project_id + ) + project_graph = nx.node_link_graph(graph_model.value) + + for node_id in project_graph.nodes: + if project_graph.nodes[node_id]["uuid"] == uuid: + return {"id": node_id, **project_graph.nodes[node_id]} + return None \ No newline at end of file diff --git a/server/portal/apps/projects/workspace_operations/project_meta_operations.py b/server/portal/apps/projects/workspace_operations/project_meta_operations.py index e3971a96c0..a6cea1d2f0 100644 --- a/server/portal/apps/projects/workspace_operations/project_meta_operations.py +++ b/server/portal/apps/projects/workspace_operations/project_meta_operations.py @@ -10,7 +10,7 @@ from portal.apps._custom.drp import constants from portal.apps.projects.models.project_metadata import ProjectMetadata from portal.apps._custom.drp.models import PartialEntityWithFiles, FileObj -from portal.apps.projects.workspace_operations.graph_operations import get_node_from_path, get_root_node, update_node_in_project +from portal.apps.projects.workspace_operations.graph_operations import get_node_from_path, get_node_from_uuid, get_root_node, update_node_in_project portal = settings.PORTAL_NAMESPACE.lower() @@ -192,13 +192,19 @@ def patch_file_obj_entity(client, project_id, value, path): return entity @transaction.atomic -def patch_entity_and_node(project_id, value, path, new_path, new_name): +def patch_entity_and_node(project_id, value, path, new_path, new_name, uuid=None): """Perform an operation on an entity.""" new_path_full = os.path.join(new_path.strip('/'), new_name) - source_node = get_node_from_path(project_id, path) - entity = ProjectMetadata.objects.get(uuid=source_node['uuid']) + if (path): + source_node = get_node_from_path(project_id, path) + elif (not path and uuid): + source_node = get_node_from_uuid(project_id, uuid) + else: + raise ValueError("Invalid parameters: path or uuid must be provided.") + + entity = ProjectMetadata.objects.get(uuid=uuid if uuid else source_node['uuid']) new_parent_node = get_node_from_path(project_id, new_path) From dcee1cbfeb3a2e81223be2289196387eb81898d0 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Tue, 29 Oct 2024 17:22:05 -0500 Subject: [PATCH 118/328] Added support for multiple projects in data files --- client/src/components/DataFiles/DataFiles.jsx | 44 ++++++++++++------- .../DataFilesListing/DataFilesListing.jsx | 3 +- .../DataFilesListingCells.jsx | 4 +- .../DataFilesAddProjectModal.jsx | 8 +++- .../DataFilesModals/DataFilesCopyModal.jsx | 13 +++--- .../DataFilesModals/DataFilesFormModal.jsx | 4 +- .../DataFilesProjectFileListing.jsx | 5 ++- .../DataFilesProjectsList.jsx | 17 +++++-- .../DataFilesSidebar/DataFilesSidebar.jsx | 28 ++++++++---- client/src/components/Workbench/Workbench.jsx | 1 - .../DataFilesProjectFileListingAddon.jsx | 6 +-- .../DataFilesProjectPublish.jsx | 4 +- .../DataFilesProjectReview.jsx | 4 +- .../src/redux/reducers/projects.reducers.js | 1 + client/src/redux/sagas/projects.sagas.js | 9 ++-- server/portal/apps/projects/urls.py | 1 + server/portal/apps/projects/views.py | 6 +-- .../shared_workspace_operations.py | 14 +++++- server/portal/settings/settings_default.py | 15 ++++++- 19 files changed, 127 insertions(+), 60 deletions(-) diff --git a/client/src/components/DataFiles/DataFiles.jsx b/client/src/components/DataFiles/DataFiles.jsx index e7737dc039..d8f930059a 100644 --- a/client/src/components/DataFiles/DataFiles.jsx +++ b/client/src/components/DataFiles/DataFiles.jsx @@ -30,13 +30,16 @@ const DefaultSystemRedirect = () => { useEffect(() => { if (systems.length === 0) return; const defaultSystem = systems[0]; - history.push( - `/workbench/data/${defaultSystem.api}/${defaultSystem.scheme}/${ - defaultSystem.scheme === 'projects' - ? '' - : `${defaultSystem.system}${defaultSystem.homeDir || ''}/` - }` - ); + + let path = `/workbench/data/${defaultSystem.api}/${defaultSystem.scheme}`; + + if (defaultSystem.scheme === 'projects') { + path += defaultSystem.system ? `/${defaultSystem.system}` : '/'; + } else { + path += `/${defaultSystem.system}${defaultSystem.homeDir || ''}/`; + } + + history.push(path); }, [systems]); return <>; }; @@ -52,11 +55,11 @@ const DataFilesSwitch = React.memo(() => { {DataFilesProjectPublish && { return ( - + ) }} @@ -65,21 +68,33 @@ const DataFilesSwitch = React.memo(() => { { DataFilesProjectReview && { return ( - + ) }} /> } + { + return ( + + ); + }} + /> { return ( @@ -100,10 +115,7 @@ const DataFilesSwitch = React.memo(() => { ); }} - /> - - - + /> diff --git a/client/src/components/DataFiles/DataFilesListing/DataFilesListing.jsx b/client/src/components/DataFiles/DataFilesListing/DataFilesListing.jsx index 925324e672..7686252c4b 100644 --- a/client/src/components/DataFiles/DataFilesListing/DataFilesListing.jsx +++ b/client/src/components/DataFiles/DataFilesListing/DataFilesListing.jsx @@ -38,7 +38,7 @@ const fileTypes = [ '3D Visualization', ]; -const DataFilesListing = ({ api, scheme, system, path, isPublic }) => { +const DataFilesListing = ({ api, scheme, system, path, isPublic, rootSystem }) => { // Redux hooks const location = useLocation(); const systems = useSelector( @@ -85,6 +85,7 @@ const DataFilesListing = ({ api, scheme, system, path, isPublic }) => { ({ row }) => { return ( { + ({ system, path, name, format, api, scheme, href, isPublic, length, metadata, rootSystem }) => { const dispatch = useDispatch(); const previewCallback = (e) => { e.stopPropagation(); @@ -75,7 +75,7 @@ export const FileNavCell = React.memo( { shallowEqual ); - const sharedWorkspacesDisplayName = systems.find((e) => e.scheme === 'projects')?.name; + const system = systems.find((s) => s.scheme === 'projects' && s.defaultProject == true); + + const sharedWorkspacesDisplayName = system?.name; + const rootSystem = system?.system; + const toggle = () => { dispatch({ @@ -62,7 +66,7 @@ const DataFilesAddProjectModal = () => { const onCreate = (system) => { toggle(); - history.push(`${match.path}/tapis/projects/${system}`); + history.push(`${match.path}/tapis/projects/${rootSystem}/${system}`); }; const addproject = (values) => { diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesCopyModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesCopyModal.jsx index c4cd26d68a..6f16a97a34 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesCopyModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesCopyModal.jsx @@ -56,11 +56,14 @@ const DataFilesCopyModal = React.memo(() => { }; const excludedSystems = systems - .filter( - (s) => s.hidden || (s.scheme !== 'private' && s.scheme !== 'projects') - ) - .filter((s) => !(s.scheme === 'public' && canMakePublic)) - .map((s) => `${s.system}${s.homeDir || ''}`); + .filter( + (s) => + s.hidden || + (s.scheme !== 'private' && s.scheme !== 'projects') || + (s.scheme === 'projects' && s.readOnly) + ) + .filter((s) => !(s.scheme === 'public' && canMakePublic)) + .map((s) => `${s.system}${s.homeDir || ''}`); const selectedSystem = fetchSelectedSystem(params); diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx index b6e9de7cb6..b982487694 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx @@ -16,8 +16,8 @@ const DataFilesFormModal = () => { const location = useLocation(); const reloadPage = (updatedPath = '') => { - // using regex to get the url up until the project name - let projectUrl = location.pathname.replace(/(\/projects\/[^/]+\/).*/, '$1'); + // Updated regex to capture the URL up until the last project segment + let projectUrl = location.pathname.replace(/(\/projects\/[^/]+\/[^/]+)\/?.*/, '$1'); if (projectUrl.endsWith('/')) { projectUrl = projectUrl.slice(0, -1); diff --git a/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx b/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx index b3f8c65087..4ed1ba0395 100644 --- a/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx +++ b/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx @@ -12,7 +12,7 @@ import { useAddonComponents, useFileListing } from 'hooks/datafiles'; import DataFilesListing from '../DataFilesListing/DataFilesListing'; import styles from './DataFilesProjectFileListing.module.scss'; -const DataFilesProjectFileListing = ({ system, path }) => { +const DataFilesProjectFileListing = ({ rootSystem, system, path }) => { const dispatch = useDispatch(); const { fetchListing } = useFileListing('FilesListing'); @@ -131,7 +131,7 @@ const DataFilesProjectFileListing = ({ system, path }) => { {readOnlyTeam ? 'View' : 'Manage'} Team {DataFilesProjectFileListingAddon && ( - + )} } @@ -163,6 +163,7 @@ const DataFilesProjectFileListing = ({ system, path }) => { scheme="projects" system={system} path={path || '/'} + rootSystem={rootSystem} /> ); diff --git a/client/src/components/DataFiles/DataFilesProjectsList/DataFilesProjectsList.jsx b/client/src/components/DataFiles/DataFilesProjectsList/DataFilesProjectsList.jsx index 02569e1417..9a00f92f7a 100644 --- a/client/src/components/DataFiles/DataFilesProjectsList/DataFilesProjectsList.jsx +++ b/client/src/components/DataFiles/DataFilesProjectsList/DataFilesProjectsList.jsx @@ -12,7 +12,7 @@ import styles from './DataFilesProjectsList.module.scss'; import './DataFilesProjectsList.scss'; import Searchbar from '_common/Searchbar'; -const DataFilesProjectsList = ({ modal }) => { +const DataFilesProjectsList = ({ modal, rootSystem }) => { const { error, loading, projects } = useSelector( (state) => state.projects.listing ); @@ -24,7 +24,15 @@ const DataFilesProjectsList = ({ modal }) => { shallowEqual ); - const sharedWorkspacesDisplayName = systems.find((e) => e.scheme === 'projects')?.name; + let selectedSystem; + + if (rootSystem) { + selectedSystem = systems.find((s) => s.system === rootSystem); + } else { + selectedSystem = systems.find((s) => s.scheme === 'projects'); + } + + const sharedWorkspacesDisplayName = selectedSystem?.name || 'Shared Workspaces'; const infiniteScrollCallback = useCallback(() => {}); const dispatch = useDispatch(); @@ -37,10 +45,11 @@ const DataFilesProjectsList = ({ modal }) => { type: actionType, payload: { queryString: modal ? null : query.query_string, + rootSystem: selectedSystem.system, modal, }, }); - }, [dispatch, query.query_string]); + }, [dispatch, query.query_string, rootSystem]); const listingCallback = (e, el) => { if (!modal) return; @@ -72,7 +81,7 @@ const DataFilesProjectsList = ({ modal }) => { Cell: (el) => ( listingCallback(e, el)} > {el.value} diff --git a/client/src/components/DataFiles/DataFilesSidebar/DataFilesSidebar.jsx b/client/src/components/DataFiles/DataFilesSidebar/DataFilesSidebar.jsx index c177d11a82..a6123b3bd4 100644 --- a/client/src/components/DataFiles/DataFilesSidebar/DataFilesSidebar.jsx +++ b/client/src/components/DataFiles/DataFilesSidebar/DataFilesSidebar.jsx @@ -117,15 +117,25 @@ const DataFilesSidebar = ({ readOnly }) => { var sidebarItems = []; systems.forEach((sys) => { - sidebarItems.push({ - to: `${match.path}/${sys.api}/${sys.scheme}/${ - sys.system ? `${sys.system}${sys.homeDir || ''}/` : '' - }`, - label: sys.name, - iconName: sys.icon || 'my-data', - disabled: false, - hidden: false, - }); + if (sys.scheme === 'projects') { + sidebarItems.push({ + to: `${match.path}/${sys.api}/${sys.scheme}/${sys.system}`, + label: sys.name, + iconName: sys.icon || 'my-data', + disabled: false, + hidden: false, + }); + } else { + sidebarItems.push({ + to: `${match.path}/${sys.api}/${sys.scheme}/${ + sys.system ? `${sys.system}${sys.homeDir || ''}/` : '' + }`, + label: sys.name, + iconName: sys.icon || 'my-data', + disabled: false, + hidden: false, + }); + } }); const addItems = [ diff --git a/client/src/components/Workbench/Workbench.jsx b/client/src/components/Workbench/Workbench.jsx index f71055a1a7..d04da3fd79 100644 --- a/client/src/components/Workbench/Workbench.jsx +++ b/client/src/components/Workbench/Workbench.jsx @@ -72,7 +72,6 @@ function Workbench() { dispatch({ type: 'GET_APPS' }); dispatch({ type: 'GET_APP_START' }); dispatch({ type: 'GET_JOBS', params: { offset: 0 } }); - dispatch({ type: 'PROJECTS_GET_LISTING' }); } }, [setupComplete]); diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx index c5b3019f52..57ddd8c895 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx @@ -7,7 +7,7 @@ import useDrpDatasetModals from '../utils/hooks/useDrpDatasetModals'; import { Link } from 'react-router-dom'; import * as ROUTES from '../../../../constants/routes'; -const DataFilesProjectFileListingAddon = ({ system }) => { +const DataFilesProjectFileListingAddon = ({ rootSystem, system }) => { const portalName = useSelector((state) => state.workbench.portalName); const { projectId } = useSelector((state) => state.projects.metadata); const { metadata } = useSelector((state) => state.projects); @@ -120,7 +120,7 @@ const DataFilesProjectFileListingAddon = ({ system }) => { {canRequestPublication && ( <> | - + Request Publication @@ -128,7 +128,7 @@ const DataFilesProjectFileListingAddon = ({ system }) => { {canReviewPublication && ( <> | - + Review Publication Request diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx index 74d1137f0f..7e82b4ec1c 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx @@ -12,7 +12,7 @@ import { ReviewProjectStructureStep } from './DataFilesProjectPublishWizardSteps import { ReviewAuthorsStep } from './DataFilesProjectPublishWizardSteps/ReviewAuthors'; import { SubmitPublicationRequestStep } from './DataFilesProjectPublishWizardSteps/SubmitPublicationRequest'; -const DataFilesProjectPublish = ({ system }) => { +const DataFilesProjectPublish = ({ rootSystem, system }) => { const dispatch = useDispatch(); const history = useHistory(); const location = useLocation(); @@ -109,7 +109,7 @@ const DataFilesProjectPublish = ({ system }) => { <> Back to Project diff --git a/client/src/components/_custom/drp/DataFilesProjectReview/DataFilesProjectReview.jsx b/client/src/components/_custom/drp/DataFilesProjectReview/DataFilesProjectReview.jsx index 44cd59e399..c767bba98b 100644 --- a/client/src/components/_custom/drp/DataFilesProjectReview/DataFilesProjectReview.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectReview/DataFilesProjectReview.jsx @@ -11,7 +11,7 @@ import { ReviewProjectStructureStep } from '../DataFilesProjectPublish/DataFiles import { ReviewAuthorsStep } from '../DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors'; import { SubmitPublicationReviewStep } from '../DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/SubmitPublicationReview'; -const DataFilesProjectReview = ({ system }) => { +const DataFilesProjectReview = ({ rootSystem, system }) => { const dispatch = useDispatch(); const portalName = useSelector((state) => state.workbench.portalName); const { projectId } = useSelector((state) => state.projects.metadata); @@ -75,7 +75,7 @@ const DataFilesProjectReview = ({ system }) => { <> Back to Project diff --git a/client/src/redux/reducers/projects.reducers.js b/client/src/redux/reducers/projects.reducers.js index b97bffd696..dd8a03d847 100644 --- a/client/src/redux/reducers/projects.reducers.js +++ b/client/src/redux/reducers/projects.reducers.js @@ -37,6 +37,7 @@ export default function projects(state = initialState, action) { ...state, listing: { ...state.listing, + projects: [], error: null, loading: true, }, diff --git a/client/src/redux/sagas/projects.sagas.js b/client/src/redux/sagas/projects.sagas.js index 4173b6cd18..a37c189270 100644 --- a/client/src/redux/sagas/projects.sagas.js +++ b/client/src/redux/sagas/projects.sagas.js @@ -2,10 +2,11 @@ import { put, takeLatest, call } from 'redux-saga/effects'; import queryStringParser from 'query-string'; import { fetchUtil } from 'utils/fetchUtil'; -export async function fetchProjectsListing(queryString) { +export async function fetchProjectsListing(queryString, rootSystem) { const q = queryStringParser.stringify({ query_string: queryString }); + const url = rootSystem ? `api/projects/${rootSystem}` : `/api/projects/`; const result = await fetchUtil({ - url: queryString ? `/api/projects/?${q}` : `/api/projects/`, + url: queryString ? `${url}?${q}` : `${url}`, }); return result.response; } @@ -22,7 +23,8 @@ export function* getProjectsListing(action) { try { const projects = yield call( fetchProjectsListing, - action.payload.queryString + action.payload.queryString, + action.payload.rootSystem ); yield put({ @@ -47,6 +49,7 @@ export function* showSharedWorkspaces(action) { type: 'PROJECTS_GET_LISTING', payload: { queryString: action.payload.queryString, + rootSystem: action.payload.rootSystem, }, }); } diff --git a/server/portal/apps/projects/urls.py b/server/portal/apps/projects/urls.py index a01904be18..1991b11c95 100644 --- a/server/portal/apps/projects/urls.py +++ b/server/portal/apps/projects/urls.py @@ -11,5 +11,6 @@ path('/system-role//', views.get_system_role), path('/', views.ProjectInstanceApiView.as_view(), name='project'), path('/entities/create', views.ProjectEntityView.as_view()), + path('', views.ProjectsApiView.as_view()), path('', views.ProjectsApiView.as_view(), name='projects_api') ] diff --git a/server/portal/apps/projects/views.py b/server/portal/apps/projects/views.py index bb18faea7d..f14969b387 100644 --- a/server/portal/apps/projects/views.py +++ b/server/portal/apps/projects/views.py @@ -53,7 +53,7 @@ class ProjectsApiView(BaseApiView): of the projects. """ - def get(self, request): + def get(self, request, root_system=None): """GET handler. If no 'query_string' is present this view will return a list of every @@ -109,12 +109,12 @@ def get(self, request): # Filter search results to projects specific to user if hits: client = request.user.tapis_oauth.client - listing = list_projects(client) + listing = list_projects(client, root_system) filtered_list = filter(lambda prj: prj['id'] in hits, listing) listing = list(filtered_list) else: client = request.user.tapis_oauth.client - listing = list_projects(client) + listing = list_projects(client, root_system) tapis_project_listing_indexer.delay(listing) diff --git a/server/portal/apps/projects/workspace_operations/shared_workspace_operations.py b/server/portal/apps/projects/workspace_operations/shared_workspace_operations.py index 82d446e0d9..7d8c67c6a8 100644 --- a/server/portal/apps/projects/workspace_operations/shared_workspace_operations.py +++ b/server/portal/apps/projects/workspace_operations/shared_workspace_operations.py @@ -319,13 +319,23 @@ def get_project_user(username): } -def list_projects(client): +def list_projects(client, root_system_id=None): """ List all workspace systems accessible to the user's client. """ fields = "id,host,description,notes,updated,owner,rootDir" - query = f"id.like.{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.*" + query = f"(id.like.{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.*)" + + if root_system_id: + root_system = next( + (system for system in settings.PORTAL_DATAFILES_STORAGE_SYSTEMS if system['system'] == root_system_id), + None + ) + if root_system: + query += f"~(rootDir.like.{root_system['rootDir']}*)" + + # use limit as -1 to allow search to corelate with # all projects available to the api user listing = client.systems.getSystems(listType='ALL', diff --git a/server/portal/settings/settings_default.py b/server/portal/settings/settings_default.py index b9fc824f6e..e1455c97d7 100644 --- a/server/portal/settings/settings_default.py +++ b/server/portal/settings/settings_default.py @@ -108,8 +108,21 @@ 'api': 'tapis', 'icon': 'publications', 'readOnly': False, - 'hideSearchBar': False + 'hideSearchBar': False, + 'defaultProject': True, + 'system': 'cep.project.root', + 'rootDir': '/corral-repl/tacc/aci/CEP/projects', }, + { + 'name': 'Review Projects', + 'scheme': 'projects', + 'api': 'tapis', + 'icon': 'publications', + 'readOnly': True, + 'hideSearchBar': False, + 'system': 'drp.project.review.test', + 'rootDir': '/corral-repl/utexas/pge-nsf/data_pprd/test', + } ] ######################## From 4c175bf69258e772ed2a54aac6cd44a5d889af2c Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Tue, 29 Oct 2024 17:54:47 -0500 Subject: [PATCH 119/328] cleanup commented code --- client/src/redux/sagas/_custom/drp.sagas.js | 205 ++++++++++-------- server/portal/apps/_custom/drp/views.py | 21 +- .../apps/projects/models/project_metadata.py | 68 ------ server/portal/apps/projects/views.py | 6 - 4 files changed, 121 insertions(+), 179 deletions(-) diff --git a/client/src/redux/sagas/_custom/drp.sagas.js b/client/src/redux/sagas/_custom/drp.sagas.js index ffef67d869..dee4f5f241 100644 --- a/client/src/redux/sagas/_custom/drp.sagas.js +++ b/client/src/redux/sagas/_custom/drp.sagas.js @@ -1,17 +1,16 @@ -import { - takeLatest, - put, - call, -} from 'redux-saga/effects'; -import { - mkdirUtil,updateMetadataUtil, -} from '../datafiles.sagas'; +import { takeLatest, put, call } from 'redux-saga/effects'; +import { mkdirUtil, updateMetadataUtil } from '../datafiles.sagas'; import { useHistory, useLocation } from 'react-router-dom'; import { createEntityUtil, patchEntityUtil } from '../projects.sagas'; - -function* executeOperation(isEdit, params, values, reloadCallback, file = null, path = '') { - +function* executeOperation( + isEdit, + params, + values, + reloadCallback, + file = null, + path = '' +) { yield put({ type: 'DATA_FILES_SET_OPERATION_STATUS', payload: { status: 'RUNNING', operation: 'dynamicform' }, @@ -19,8 +18,10 @@ function* executeOperation(isEdit, params, values, reloadCallback, file = null, // filter out empty values from the metadata const filteredValues = Object.fromEntries( - Object.entries(values).filter(([key, value]) => value !== "" && value !== null) - ) + Object.entries(values).filter( + ([key, value]) => value !== '' && value !== null + ) + ); try { if (file && isEdit) { @@ -32,44 +33,28 @@ function* executeOperation(isEdit, params, values, reloadCallback, file = null, '/' + path, filteredValues, file.uuid - ) - // yield call( - // updateMetadataUtil, - // params.api, - // params.scheme, - // params.system, - // '/' + file.path, - // '/' + path, - // file.name, - // values.name || file.name, - // filteredValues - // ); - } else { + ); + } else { yield call( - createEntityUtil, + createEntityUtil, filteredValues.data_type, params.system, - path || "/", + path || '/', filteredValues - ) - - // yield call( - // mkdirUtil, - // params.api, - // params.scheme, - // params.system, - // path, - // values.name, - // filteredValues - // ); + ); } - + if (reloadCallback) { - - const newPath = isEdit && file.path === params.path ? `${path}/${file.path.split('/').pop()}` : path; - - const reloadPath = (isEdit && file.name !== values.name) ? newPath.replace(file.name, values.name) : newPath; - + const newPath = + isEdit && file.path === params.path + ? `${path}/${file.path.split('/').pop()}` + : path; + + const reloadPath = + isEdit && file.name !== values.name + ? newPath.replace(file.name, values.name) + : newPath; + yield call(reloadCallback, reloadPath); } @@ -77,7 +62,7 @@ function* executeOperation(isEdit, params, values, reloadCallback, file = null, type: 'DATA_FILES_SET_OPERATION_STATUS', payload: { status: 'SUCCESS', operation: 'dynamicform' }, }); - + yield put({ type: 'DATA_FILES_TOGGLE_MODAL', payload: { @@ -90,7 +75,6 @@ function* executeOperation(isEdit, params, values, reloadCallback, file = null, type: 'DATA_FILES_SET_OPERATION_STATUS', payload: { status: {}, operation: 'dynamicform' }, }); - } catch (e) { yield put({ type: 'DATA_FILES_SET_OPERATION_STATUS', @@ -100,87 +84,138 @@ function* executeOperation(isEdit, params, values, reloadCallback, file = null, } function* handleSampleData(action, isEdit) { - const { params, values, reloadPage: reloadCallback, selectedFile } = action.payload; + const { + params, + values, + reloadPage: reloadCallback, + selectedFile, + } = action.payload; const metadata = { ...values, data_type: 'sample', - } + }; yield call( - executeOperation, isEdit, params, metadata, reloadCallback, selectedFile - ) + executeOperation, + isEdit, + params, + metadata, + reloadCallback, + selectedFile + ); } function* handleOriginData(action, isEdit) { - const { params, values, reloadPage: reloadCallback, selectedFile, additionalData: { samples } } = action.payload; + const { + params, + values, + reloadPage: reloadCallback, + selectedFile, + additionalData: { samples }, + } = action.payload; - const sample = samples.find( - (sample) => sample.uuid === values.sample - ); + const sample = samples.find((sample) => sample.uuid === values.sample); const metadata = { ...values, data_type: 'digital_dataset', sample: sample.uuid, - } + }; - // get the path without system name - // const path = sample.value.path.split('/').slice(1).join('/'); - const path = sample.value.name; // PROBLEM + const path = sample.value.name; yield call( - executeOperation, isEdit, params, metadata, reloadCallback, selectedFile, path - ) + executeOperation, + isEdit, + params, + metadata, + reloadCallback, + selectedFile, + path + ); } function* handleAnalysisData(action, isEdit) { - const { params, values, reloadPage: reloadCallback, selectedFile, additionalData: { samples, originDatasets } } = action.payload; + const { + params, + values, + reloadPage: reloadCallback, + selectedFile, + additionalData: { samples, originDatasets }, + } = action.payload; - const sample = samples.find( - (sample) => sample.uuid === values.sample - ); + const sample = samples.find((sample) => sample.uuid === values.sample); const originData = originDatasets.find( (originData) => originData.uuid === values.base_origin_data ); - let path = originData ? `${sample.value.name}/${originData.value.name}` : sample.value.name; + let path = originData + ? `${sample.value.name}/${originData.value.name}` + : sample.value.name; const metadata = { - ...values, + ...values, data_type: 'analysis_data', - sample: sample.uuid, - // base_origin_data: originData ? originData.uuid : '' - } + sample: sample.uuid, + }; yield call( - executeOperation, isEdit, params, metadata, reloadCallback, selectedFile, path - ) + executeOperation, + isEdit, + params, + metadata, + reloadCallback, + selectedFile, + path + ); } function* handleFile(action, isEdit) { - - const { params, values, reloadPage: reloadCallback, selectedFile } = action.payload; + const { + params, + values, + reloadPage: reloadCallback, + selectedFile, + } = action.payload; const metadata = { ...values, data_type: 'file', - } + }; - const path = selectedFile.path.split('/').slice(0, -1).join('/') + const path = selectedFile.path.split('/').slice(0, -1).join('/'); yield call( - executeOperation, isEdit, params, metadata, reloadCallback, selectedFile, path - ) + executeOperation, + isEdit, + params, + metadata, + reloadCallback, + selectedFile, + path + ); } export default function* watchDRP() { - yield takeLatest('ADD_SAMPLE_DATA', action => handleSampleData(action, false)); - yield takeLatest('EDIT_SAMPLE_DATA', action => handleSampleData(action, true)); - yield takeLatest('ADD_ANALYSIS_DATASET', action => handleAnalysisData(action, false)); - yield takeLatest('EDIT_ANALYSIS_DATASET', action => handleAnalysisData(action, true)); - yield takeLatest('ADD_ORIGIN_DATASET', action => handleOriginData(action, false)); - yield takeLatest('EDIT_ORIGIN_DATASET', action => handleOriginData(action, true)); - yield takeLatest('EDIT_FILE', action => handleFile(action, true)) + yield takeLatest('ADD_SAMPLE_DATA', (action) => + handleSampleData(action, false) + ); + yield takeLatest('EDIT_SAMPLE_DATA', (action) => + handleSampleData(action, true) + ); + yield takeLatest('ADD_ANALYSIS_DATASET', (action) => + handleAnalysisData(action, false) + ); + yield takeLatest('EDIT_ANALYSIS_DATASET', (action) => + handleAnalysisData(action, true) + ); + yield takeLatest('ADD_ORIGIN_DATASET', (action) => + handleOriginData(action, false) + ); + yield takeLatest('EDIT_ORIGIN_DATASET', (action) => + handleOriginData(action, true) + ); + yield takeLatest('EDIT_FILE', (action) => handleFile(action, true)); } diff --git a/server/portal/apps/_custom/drp/views.py b/server/portal/apps/_custom/drp/views.py index 8b784c9017..600dbbd312 100644 --- a/server/portal/apps/_custom/drp/views.py +++ b/server/portal/apps/_custom/drp/views.py @@ -16,15 +16,11 @@ def get(self, request): full_project_id = f'{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}' - # samples = DataFilesMetadata.objects.filter(project_id=full_project_id, metadata__data_type='sample').values('id', 'name', 'path', 'metadata') - samples = ProjectMetadata.objects.filter(base_project__value__projectId=full_project_id, name=constants.SAMPLE).values('uuid', 'name', 'value') origin_data = [] if get_origin_data == 'true': - # origin_data = DataFilesMetadata.objects.filter(project_id=full_project_id, metadata__data_type='origin_data').values('id', 'name', 'path', 'metadata') - origin_data = ProjectMetadata.objects.filter(base_project__value__projectId=full_project_id, name=constants.DIGITAL_DATASET).values('uuid', 'name', 'value') response_data = { @@ -35,21 +31,6 @@ def get(self, request): return JsonResponse(response_data) class DigitalRocksTreeView(BaseApiView): - - @staticmethod - def construct_tree(records, parent_id=None): - tree = [] - for record in records: - if record.parent_id == parent_id: - node_dict = { - "id": record.id, - "name": record.name, - "path": record.path, - "metadata": record.ordered_metadata, - "children": DigitalRocksTreeView.construct_tree(records, record.id) - } - tree.append(node_dict) - return tree @staticmethod def _get_entity(uuid): @@ -83,7 +64,7 @@ def get(self, request): node = graph.nodes[node_id] - # Get the path from NODE_ROOT to the current node + # Get the path from NODE_ROOT to the current node excluding the root if nx.has_path(graph, 'NODE_ROOT', node_id): path_nodes = shortest_path(graph, 'NODE_ROOT', node_id)[1:] node['path'] = '/'.join(graph.nodes[parent]['label'] for parent in path_nodes if 'label' in graph.nodes[parent]) diff --git a/server/portal/apps/projects/models/project_metadata.py b/server/portal/apps/projects/models/project_metadata.py index ad6c0f3fdb..bf1aad37db 100644 --- a/server/portal/apps/projects/models/project_metadata.py +++ b/server/portal/apps/projects/models/project_metadata.py @@ -77,47 +77,6 @@ def project_graph(self): return self.__class__.objects.get( name=constants.PROJECT_GRAPH, base_project=self.base_project ) - - @property - def ordered_value(self): - """ - Return the metadata in the order defined in the Pydantic model. - Also converts camelCase keys to snake_case. This is a temporary workaround until fields in settings_forms.py can be updated to use camelCase. - """ - schema = SCHEMA_MAPPING.get(self.name) - - if not schema: - return self.value # Return the field value directly if no schema is found - - ordered_metadata = {} - - # Iterate through the model fields to preserve the order - for field in schema.model_fields.keys(): - camel_field = snake_to_camel(field) - field_value = self.value.get(camel_field) - - # Skip the field if there is no value (None or not present) - if field_value is None: - continue - - # if the fiels is a list, then we need to get the model of the list and process it - if isinstance(field_value, list): - field_annotation = schema.model_fields[field].annotation - item_type = get_args(field_annotation)[0] if get_args(field_annotation) else None # returns the model class of the list - - # Check if the item type is a Pydantic model - if item_type and hasattr(item_type, "model_fields"): - # Re-order each item in the list if it's a list of Pydantic models - ordered_metadata[field] = [ - {k: item.get(snake_to_camel(k)) for k in item_type.model_fields.keys()} - for item in field_value - ] - else: - ordered_metadata[field] = field_value - else: - ordered_metadata[field] = field_value - - return ordered_metadata @classmethod def get_project_by_id(cls, project_id: str): @@ -187,30 +146,3 @@ class Meta: name="base_projectId_not_null", ), ] - -# @receiver(pre_save, sender=ProjectMetadata) -# def set_or_update_parent_uuid(sender, instance, **kwargs): -# # Assuming 'folder_path' is stored in instance.value -# current_path = instance.value.get('path') - -# # If new record or the 'folder_path' has changed -# if not instance.pk or current_path != sender.objects.get(pk=instance.pk).value.get('path'): -# parent_path = current_path.rstrip('/').rsplit('/', 1)[0] + '/' # Extract the parent path - -# if len(parent_path.split('/')) > 1: -# # Attempt to find the parent entity -# parent_entity = sender.objects.get(value__folder_path=parent_path) -# parent_uuid = parent_entity.uuid -# else: -# parent_uuid = None - - -# # Update 'parent_uuid' in the instance's 'value' field -# instance.value['parent_uuid'] = parent_uuid - - -# @receiver(models.signals.post_save, sender=ProjectMetadata) -# def handle_save(instance: ProjectMetadata, **_): -# """After saving a project, update the associated users so that listings can -# be performed.""" -# instance.sync_users() diff --git a/server/portal/apps/projects/views.py b/server/portal/apps/projects/views.py index f14969b387..e4185c32e4 100644 --- a/server/portal/apps/projects/views.py +++ b/server/portal/apps/projects/views.py @@ -135,13 +135,7 @@ def post(self, request): # pylint: disable=no-self-use metadata["projectId"] = workspace_id project_meta = create_project_metadata(metadata) initialize_project_graph(project_meta.project_id) - # project_metadata = ProjectsMetadata( - # project_id = workspace_id, - # metadata = validate_project_metadata(metadata) - # ) - # project_metadata.save() - client = request.user.tapis_oauth.client system_id = create_shared_workspace(client, title, request.user.username, description, workspace_number) From 31229317bc72985d5db3812234a7fa55928d3990 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Tue, 29 Oct 2024 17:58:00 -0500 Subject: [PATCH 120/328] client side linting --- client/src/components/DataFiles/DataFiles.jsx | 52 ++-- .../DataFilesDropdown/DataFilesDropdown.jsx | 4 +- .../DataFilesListing/DataFilesListing.jsx | 16 +- .../DataFilesListingCells.jsx | 36 ++- .../DataFilesListingCells.scss | 1 - .../DataFilesAddProjectModal.jsx | 13 +- .../DataFilesModals/DataFilesCopyModal.jsx | 16 +- .../DataFilesModals/DataFilesFormModal.jsx | 18 +- .../DataFilesModals/DataFilesPreviewModal.jsx | 12 +- .../DataFilesProjectEditDescriptionModal.jsx | 16 +- .../DataFilesModals/DataFilesUploadModal.jsx | 2 +- .../DataFilesProjectFileListing.jsx | 49 ++-- .../DataFilesProjectsList.jsx | 3 +- .../DataFilesToolbar/DataFilesToolbar.jsx | 42 ++-- client/src/components/Workbench/AppRouter.jsx | 4 +- .../src/components/_common/Expand/Expand.jsx | 2 +- .../_common/Form/DynamicForm/DynamicForm.jsx | 226 ++++++++++++------ .../components/_common/ShowMore/ShowMore.jsx | 4 +- .../DataFilesAddProjectModalAddon.jsx | 1 - .../DataFilesPreviewModalAddon.jsx | 160 +++++++------ .../DataFilesPreviewModalAddon.module.scss | 16 +- .../DataFilesProjectFileListingAddon.jsx | 208 +++++++++------- ...taFilesProjectFileListingMetadataAddon.jsx | 51 ++-- ...rojectFileListingMetadataAddon.module.scss | 19 +- ...esProjectFileListingMetadataTitleAddon.jsx | 98 ++++---- ...tFileListingMetadataTitleAddon.module.scss | 16 +- .../DataFilesProjectPublish.jsx | 23 +- .../DataFilesProjectPublish.module.scss | 34 ++- .../DataFilesProjectPublishWizard.module.scss | 200 ++++++++-------- .../ProjectDescription.jsx | 87 ++++--- .../ReviewAuthors.jsx | 31 ++- .../ReviewProjectStructure.jsx | 35 +-- .../SubmitPublicationRequest.jsx | 8 +- .../SubmitPublicationReview.jsx | 16 +- .../DataFilesProjectReview.jsx | 134 +++++------ .../DataFilesProjectReview.module.scss | 34 ++- .../DataFilesUploadModalAddon.jsx | 122 +++++----- .../DataFilesUploadModalAddon.module.scss | 10 +- .../drp/utils/DataDisplay/DataDisplay.jsx | 129 +++++----- .../utils/DataDisplay/DataDisplay.module.scss | 18 +- .../DataFilesToolbar/customFilePermissions.js | 45 ++-- .../ProjectMembersList/ProjectMembersList.jsx | 41 ++-- .../ProjectMembersList.module.scss | 34 +-- .../utils/ReorderUserList/ReorderUserList.jsx | 83 ++++--- .../ReorderUserList.module.scss | 2 +- .../drp/utils/hooks/useDrpDatasetModals.js | 16 +- .../hooks/datafiles/mutations/useRename.js | 3 +- .../src/hooks/datafiles/useAddonComponents.js | 6 +- .../src/redux/reducers/datafiles.reducers.js | 10 +- .../src/redux/reducers/projects.reducers.js | 4 +- client/src/redux/sagas/datafiles.sagas.js | 63 ++++- client/src/redux/sagas/projects.sagas.js | 26 +- client/src/utils/dataKeyFormat.js | 11 +- client/src/utils/filePermissions.js | 39 ++- client/src/utils/systems.js | 4 +- 55 files changed, 1341 insertions(+), 1012 deletions(-) diff --git a/client/src/components/DataFiles/DataFiles.jsx b/client/src/components/DataFiles/DataFiles.jsx index d8f930059a..6a57664220 100644 --- a/client/src/components/DataFiles/DataFiles.jsx +++ b/client/src/components/DataFiles/DataFiles.jsx @@ -11,7 +11,11 @@ import { SectionMessage, LoadingSpinner, } from '_common'; -import { useFileListing, useSystems, useAddonComponents } from 'hooks/datafiles'; +import { + useFileListing, + useSystems, + useAddonComponents, +} from 'hooks/datafiles'; import DataFilesToolbar from './DataFilesToolbar/DataFilesToolbar'; import DataFilesListing from './DataFilesListing/DataFilesListing'; import DataFilesSidebar from './DataFilesSidebar/DataFilesSidebar'; @@ -30,7 +34,7 @@ const DefaultSystemRedirect = () => { useEffect(() => { if (systems.length === 0) return; const defaultSystem = systems[0]; - + let path = `/workbench/data/${defaultSystem.api}/${defaultSystem.scheme}`; if (defaultSystem.scheme === 'projects') { @@ -49,44 +53,46 @@ const DataFilesSwitch = React.memo(() => { const portalName = useSelector((state) => state.workbench.portalName); - const { DataFilesProjectPublish, DataFilesProjectReview } = useAddonComponents({ portalName }); + const { DataFilesProjectPublish, DataFilesProjectReview } = + useAddonComponents({ portalName }); return ( - {DataFilesProjectPublish && + {DataFilesProjectPublish && ( { + path={`${path}/tapis/projects/:root_system/:system/publish`} + render={({ match: { params } }) => { return ( - + - ) + ); }} /> - } - { - DataFilesProjectReview && + )} + {DataFilesProjectReview && ( { + path={`${path}/tapis/projects/:root_system/:system/review`} + render={({ match: { params } }) => { return ( - + - ) + ); }} /> - } - { - return ( - - ); + return ; }} /> { ); }} - /> + /> diff --git a/client/src/components/DataFiles/DataFilesDropdown/DataFilesDropdown.jsx b/client/src/components/DataFiles/DataFilesDropdown/DataFilesDropdown.jsx index 7f6f45ce3d..60e90fce26 100644 --- a/client/src/components/DataFiles/DataFilesDropdown/DataFilesDropdown.jsx +++ b/client/src/components/DataFiles/DataFilesDropdown/DataFilesDropdown.jsx @@ -68,7 +68,9 @@ const BreadcrumbsDropdown = ({ shallowEqual ); - const sharedWorkspacesDisplayName = systems.find((e) => e.scheme === 'projects')?.name; + const sharedWorkspacesDisplayName = systems.find( + (e) => e.scheme === 'projects' + )?.name; let currentPath = startingPath; pathComponents.slice(overlapIndex).forEach((component) => { diff --git a/client/src/components/DataFiles/DataFilesListing/DataFilesListing.jsx b/client/src/components/DataFiles/DataFilesListing/DataFilesListing.jsx index 7686252c4b..54a828394f 100644 --- a/client/src/components/DataFiles/DataFilesListing/DataFilesListing.jsx +++ b/client/src/components/DataFiles/DataFilesListing/DataFilesListing.jsx @@ -38,7 +38,14 @@ const fileTypes = [ '3D Visualization', ]; -const DataFilesListing = ({ api, scheme, system, path, isPublic, rootSystem }) => { +const DataFilesListing = ({ + api, + scheme, + system, + path, + isPublic, + rootSystem, +}) => { // Redux hooks const location = useLocation(); const systems = useSelector( @@ -149,10 +156,11 @@ const DataFilesListing = ({ api, scheme, system, path, isPublic, rootSystem }) = width: 0.1, Cell: (el) => , }); - } - + } + if (showDataFileType) { - cells.splice(3, 0, { // Inserting at index 3 after 'Name' + cells.splice(3, 0, { + // Inserting at index 3 after 'Name' Header: 'Data Type', Cell: (el) => , width: 0.2, diff --git a/client/src/components/DataFiles/DataFilesListing/DataFilesListingCells.jsx b/client/src/components/DataFiles/DataFilesListing/DataFilesListingCells.jsx index 9d237e0b96..c3d3593b7d 100644 --- a/client/src/components/DataFiles/DataFilesListing/DataFilesListingCells.jsx +++ b/client/src/components/DataFiles/DataFilesListing/DataFilesListingCells.jsx @@ -50,7 +50,19 @@ CheckboxCell.propTypes = { }; export const FileNavCell = React.memo( - ({ system, path, name, format, api, scheme, href, isPublic, length, metadata, rootSystem }) => { + ({ + system, + path, + name, + format, + api, + scheme, + href, + isPublic, + length, + metadata, + rootSystem, + }) => { const dispatch = useDispatch(); const previewCallback = (e) => { e.stopPropagation(); @@ -75,7 +87,9 @@ export const FileNavCell = React.memo( { - const dataType = file.metadata ? file.metadata.data_type : file.type; - - const formatDatatype = (data_type) => - data_type.split('_') - .map(word => word.charAt(0).toUpperCase() + word.slice(1)) + + const formatDatatype = (data_type) => + data_type + .split('_') + .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) .join(' '); return dataType ? ( - - {formatDatatype(dataType)} - + {formatDatatype(dataType)} ) : null; -} +}; DataTypeCell.propTypes = { file: PropTypes.shape({}).isRequired, -}; \ No newline at end of file +}; diff --git a/client/src/components/DataFiles/DataFilesListing/DataFilesListingCells.scss b/client/src/components/DataFiles/DataFilesListing/DataFilesListingCells.scss index 6868deabc4..cb8b414b0e 100644 --- a/client/src/components/DataFiles/DataFilesListing/DataFilesListingCells.scss +++ b/client/src/components/DataFiles/DataFilesListing/DataFilesListingCells.scss @@ -36,7 +36,6 @@ display: flex; } - .dataTypeBox { display: inline-block; background-color: #808080; /* Light grey background */ diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesAddProjectModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesAddProjectModal.jsx index 3c5a6f1374..cac1d0c7b4 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesAddProjectModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesAddProjectModal.jsx @@ -20,7 +20,7 @@ const DataFilesAddProjectModal = () => { // logic to render addonComponents for DRP const portalName = useSelector((state) => state.workbench.portalName); - const { DataFilesAddProjectModalAddon } = useAddonComponents({portalName}) + const { DataFilesAddProjectModalAddon } = useAddonComponents({ portalName }); useEffect(() => { setMembers([ @@ -45,18 +45,19 @@ const DataFilesAddProjectModal = () => { state.projects.operation.error ); }); - + const systems = useSelector( (state) => state.systems.storage.configuration.filter((s) => !s.hidden), shallowEqual ); - const system = systems.find((s) => s.scheme === 'projects' && s.defaultProject == true); + const system = systems.find( + (s) => s.scheme === 'projects' && s.defaultProject == true + ); const sharedWorkspacesDisplayName = system?.name; const rootSystem = system?.system; - const toggle = () => { dispatch({ type: 'DATA_FILES_TOGGLE_MODAL', @@ -132,7 +133,9 @@ const DataFilesAddProjectModal = () => { } /> - {DataFilesAddProjectModalAddon && } + {DataFilesAddProjectModalAddon && ( + + )} { }; const excludedSystems = systems - .filter( - (s) => - s.hidden || - (s.scheme !== 'private' && s.scheme !== 'projects') || - (s.scheme === 'projects' && s.readOnly) - ) - .filter((s) => !(s.scheme === 'public' && canMakePublic)) - .map((s) => `${s.system}${s.homeDir || ''}`); + .filter( + (s) => + s.hidden || + (s.scheme !== 'private' && s.scheme !== 'projects') || + (s.scheme === 'projects' && s.readOnly) + ) + .filter((s) => !(s.scheme === 'public' && canMakePublic)) + .map((s) => `${s.system}${s.homeDir || ''}`); const selectedSystem = fetchSelectedSystem(params); diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx index b982487694..1cbd2afc2a 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx @@ -17,8 +17,11 @@ const DataFilesFormModal = () => { const reloadPage = (updatedPath = '') => { // Updated regex to capture the URL up until the last project segment - let projectUrl = location.pathname.replace(/(\/projects\/[^/]+\/[^/]+)\/?.*/, '$1'); - + let projectUrl = location.pathname.replace( + /(\/projects\/[^/]+\/[^/]+)\/?.*/, + '$1' + ); + if (projectUrl.endsWith('/')) { projectUrl = projectUrl.slice(0, -1); } @@ -27,9 +30,8 @@ const DataFilesFormModal = () => { history.replace(path); }; - const { form, selectedFile, formName, additionalData, useReloadCallback } = useSelector( - (state) => state.files.modalProps.dynamicform - ); + const { form, selectedFile, formName, additionalData, useReloadCallback } = + useSelector((state) => state.files.modalProps.dynamicform); const isOpen = useSelector((state) => state.files.modals.dynamicform); const { params } = useFileListing('FilesListing'); @@ -55,9 +57,9 @@ const DataFilesFormModal = () => { }, []); const handleSubmit = (values) => { - - Object.keys(values).forEach(key => { - values[key] = typeof(values[key]) === 'string' ? values[key].trim() : values[key]; + Object.keys(values).forEach((key) => { + values[key] = + typeof values[key] === 'string' ? values[key].trim() : values[key]; }); dispatch({ diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesPreviewModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesPreviewModal.jsx index 6233850454..7db89e73bc 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesPreviewModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesPreviewModal.jsx @@ -38,7 +38,7 @@ const DataFilesPreviewModal = () => { const [isFrameLoading, setIsFrameLoading] = useState(true); const portalName = useSelector((state) => state.workbench.portalName); - const { DataFilesPreviewModalAddon } = useAddonComponents({portalName}) + const { DataFilesPreviewModalAddon } = useAddonComponents({ portalName }); useEffect(() => { if (previewUsingBrainmap) setIsFrameLoading(false); @@ -89,7 +89,9 @@ const DataFilesPreviewModal = () => { File Preview: {params.name} - {DataFilesPreviewModalAddon && !isLoading && } + {DataFilesPreviewModalAddon && !isLoading && ( + + )} {(isLoading || (previewUsingHref && isFrameLoading)) && (
@@ -117,7 +119,11 @@ const DataFilesPreviewModal = () => {
)} {hasError && ( -
+
{error} diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesProjectEditDescriptionModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesProjectEditDescriptionModal.jsx index 58543012b4..8172e234da 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesProjectEditDescriptionModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesProjectEditDescriptionModal.jsx @@ -30,7 +30,9 @@ const DataFilesProjectEditDescriptionModal = () => { }); const portalName = useSelector((state) => state.workbench.portalName); - const { DataFilesProjectEditDescriptionModalAddon } = useAddonComponents({ portalName }) + const { DataFilesProjectEditDescriptionModalAddon } = useAddonComponents({ + portalName, + }); const initialValues = useMemo( () => ({ @@ -56,7 +58,7 @@ const DataFilesProjectEditDescriptionModal = () => { data: { title: values.title, description: values.description || '', - metadata: DataFilesProjectEditDescriptionModalAddon ? values : null + metadata: DataFilesProjectEditDescriptionModalAddon ? values : null, }, }, }); @@ -90,7 +92,7 @@ const DataFilesProjectEditDescriptionModal = () => { Edit Project - { type="textarea" className={styles['description-textarea']} /> - {DataFilesProjectEditDescriptionModalAddon && } + {DataFilesProjectEditDescriptionModalAddon && ( + + )}
@@ -139,7 +145,7 @@ const DataFilesProjectEditDescriptionModal = () => { )} - + ); }; diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesUploadModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesUploadModal.jsx index 5b5c1f300a..28671d9094 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesUploadModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesUploadModal.jsx @@ -32,7 +32,7 @@ const DataFilesUploadModal = ({ className, layout }) => { history.push(location.pathname); }; const portalName = useSelector((state) => state.workbench.portalName); - const { DataFilesUploadModalAddon } = useAddonComponents({portalName}) + const { DataFilesUploadModalAddon } = useAddonComponents({ portalName }); const { getStatus: getModalStatus, toggle } = useModal(); const isOpen = getModalStatus('upload'); diff --git a/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx b/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx index 4ed1ba0395..be22cfe3c3 100644 --- a/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx +++ b/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx @@ -32,8 +32,7 @@ const DataFilesProjectFileListing = ({ rootSystem, system, path }) => { dispatch({ type: 'PROJECTS_GET_PUBLICATION_REQUESTS', payload: system, - }) - + }); }, [system]); useEffect(() => { @@ -98,25 +97,24 @@ const DataFilesProjectFileListing = ({ rootSystem, system, path }) => {
); } - + return ( - { - DataFilesProjectFileListingMetadataTitleAddon ? - - - : metadata.title - } + {DataFilesProjectFileListingMetadataTitleAddon ? ( + + ) : ( + metadata.title + )}
} - headerActions={
{canEditSystem ? ( @@ -131,7 +129,10 @@ const DataFilesProjectFileListing = ({ rootSystem, system, path }) => { {readOnlyTeam ? 'View' : 'Manage'} Team {DataFilesProjectFileListingAddon && ( - + )}
} @@ -144,19 +145,19 @@ const DataFilesProjectFileListing = ({ rootSystem, system, path }) => { - (D) __both__ (A) or (B) __and__ (C) */}
- <> - {DataFilesProjectFileListingMetadataAddon ? - - + {DataFilesProjectFileListingMetadataAddon ? ( + + - - - : {metadata.description} - } - + + ) : ( + {metadata.description} + )} +
{ selectedSystem = systems.find((s) => s.scheme === 'projects'); } - const sharedWorkspacesDisplayName = selectedSystem?.name || 'Shared Workspaces'; + const sharedWorkspacesDisplayName = + selectedSystem?.name || 'Shared Workspaces'; const infiniteScrollCallback = useCallback(() => {}); const dispatch = useDispatch(); diff --git a/client/src/components/DataFiles/DataFilesToolbar/DataFilesToolbar.jsx b/client/src/components/DataFiles/DataFilesToolbar/DataFilesToolbar.jsx index d43e1d517f..d4c31b3207 100644 --- a/client/src/components/DataFiles/DataFilesToolbar/DataFilesToolbar.jsx +++ b/client/src/components/DataFiles/DataFilesToolbar/DataFilesToolbar.jsx @@ -53,21 +53,23 @@ const DataFilesToolbar = ({ scheme, api }) => { ); // A project system has different fields than a regular system - const selectedSystem = systemList.find( - (sys) => { - if (params.scheme === 'projects') { - return params.api === sys.api && sys.scheme === params.scheme; - } else { - return sys.system === params.system && sys.scheme === params.scheme - } + const selectedSystem = systemList.find((sys) => { + if (params.scheme === 'projects') { + return params.api === sys.api && sys.scheme === params.scheme; + } else { + return sys.system === params.system && sys.scheme === params.scheme; } - ); + }); const { projectId } = useSelector((state) => state.projects.metadata); // defaults to return true if no custom permission check is provided - const [customPermissionCheck, setCustomPermissionCheck] = useState(() => () => true); - const { hasCustomDataFilesToolbarChecks } = useSelector((state) => state.workbench.config) + const [customPermissionCheck, setCustomPermissionCheck] = useState( + () => () => true + ); + const { hasCustomDataFilesToolbarChecks } = useSelector( + (state) => state.workbench.config + ); const { portalName } = useSelector((state) => state.workbench); useEffect(() => { @@ -82,13 +84,12 @@ const DataFilesToolbar = ({ scheme, api }) => { } catch (error) { console.error('Error loading custom permission check:', error); } - } + }; if (hasCustomDataFilesToolbarChecks && portalName) { loadCustomPermissions(); } - - }, [hasCustomDataFilesToolbarChecks, portalName]) + }, [hasCustomDataFilesToolbarChecks, portalName]); const authenticatedUser = useSelector( (state) => state.authenticatedUser.user.username @@ -110,7 +111,7 @@ const DataFilesToolbar = ({ scheme, api }) => { // remove leading slash from homeDir value const homeDir = selectedSystem?.homeDir?.slice(1); if (!homeDir) return false; - + return state.files.params.FilesListing.path.startsWith( `${homeDir}/${state.workbench.config.trashPath}` ); @@ -130,9 +131,9 @@ const DataFilesToolbar = ({ scheme, api }) => { const showRename = modifiableUserData; - const showTrash = modifiableUserData + const showTrash = modifiableUserData; - const showDownload = api === 'tapis' + const showDownload = api === 'tapis'; const showMakeLink = useSelector( (state) => @@ -140,7 +141,7 @@ const DataFilesToolbar = ({ scheme, api }) => { state.workbench.config.makeLink && api === 'tapis' && (scheme === 'private' || scheme === 'projects') - ) + ); const showCompress = !!useSelector( (state) => state.workbench.config.extractApp && modifiableUserData @@ -223,7 +224,12 @@ const DataFilesToolbar = ({ scheme, api }) => { }); }; - const permissionParams = { files: selectedFiles, scheme, api, customPermissionCheck }; + const permissionParams = { + files: selectedFiles, + scheme, + api, + customPermissionCheck, + }; const canDownload = getFilePermissions('download', permissionParams); const areMultipleFilesOrFolderSelected = getFilePermissions( 'areMultipleFilesOrFolderSelected', diff --git a/client/src/components/Workbench/AppRouter.jsx b/client/src/components/Workbench/AppRouter.jsx index 3ced0725f1..9459696079 100644 --- a/client/src/components/Workbench/AppRouter.jsx +++ b/client/src/components/Workbench/AppRouter.jsx @@ -16,7 +16,9 @@ function AppRouter() { const authenticatedUser = useSelector( (state) => state.authenticatedUser.user ); - const hasCustomSagas = useSelector((state) => state.workbench.config.hasCustomSagas); + const hasCustomSagas = useSelector( + (state) => state.workbench.config.hasCustomSagas + ); useEffect(() => { dispatch({ type: 'FETCH_AUTHENTICATED_USER' }); diff --git a/client/src/components/_common/Expand/Expand.jsx b/client/src/components/_common/Expand/Expand.jsx index 68f4101d64..13293c6bcd 100644 --- a/client/src/components/_common/Expand/Expand.jsx +++ b/client/src/components/_common/Expand/Expand.jsx @@ -5,7 +5,7 @@ import Icon from '../Icon'; import './Expand.global.scss'; import styles from './Expand.module.scss'; -const Expand = ({ className, detail, message, isOpenDefault=false }) => { +const Expand = ({ className, detail, message, isOpenDefault = false }) => { const [isOpen, setIsOpen] = useState(isOpenDefault); const toggleCallback = useCallback(() => { setIsOpen(!isOpen); diff --git a/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx b/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx index 8e51738ecb..0bf7528b84 100644 --- a/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx +++ b/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx @@ -4,21 +4,29 @@ import { Button, Expand, InlineMessage } from '_common'; import { useFormikContext } from 'formik'; import { FormGroup, Input } from 'reactstrap'; import { FieldArray } from 'formik'; -import styles from './DynamicForm.module.scss' +import styles from './DynamicForm.module.scss'; import { useSelector } from 'react-redux'; - const DynamicForm = ({ initialFormFields, onChange }) => { - const [formFields, setFormFields] = useState(initialFormFields); - const { setFieldValue, values, handleChange, handleBlur } = useFormikContext(); + const { setFieldValue, values, handleChange, handleBlur } = + useFormikContext(); - const status = useSelector((state) => state.files.operationStatus.dynamicform); + const status = useSelector( + (state) => state.files.operationStatus.dynamicform + ); - const handleFilterDependency = (field, values, setFieldValue, modifiedField) => { + const handleFilterDependency = ( + field, + values, + setFieldValue, + modifiedField + ) => { const { dependency } = field; - const filteredOptions = field.options.filter(option => option.dependentId == values[dependency.name]); + const filteredOptions = field.options.filter( + (option) => option.dependentId == values[dependency.name] + ); const updatedOptions = [{ value: '', label: '' }, ...filteredOptions]; // Only update the field value if the modified field is the dependency field @@ -26,11 +34,20 @@ const DynamicForm = ({ initialFormFields, onChange }) => { setFieldValue(field.name, updatedOptions[0].value); } - return { ...field, hidden: false, filteredOptions: updatedOptions, value: '' }; - } - - const handleVisibilityDependency = (field, values, setFieldValue, modifiedField) => { + return { + ...field, + hidden: false, + filteredOptions: updatedOptions, + value: '', + }; + }; + const handleVisibilityDependency = ( + field, + values, + setFieldValue, + modifiedField + ) => { const { dependency } = field; // Stores the value of the dependency field that is currently entered in the form @@ -38,12 +55,19 @@ const DynamicForm = ({ initialFormFields, onChange }) => { // If the dependency field is a nested field (e.g. sample.type) if (dependency.name.includes('.')) { - const [dependencyName, nestedDependencyField] = dependency.name.split('.'); - const dependentFormField = formFields.find(field => field.name === dependencyName); + const [dependencyName, nestedDependencyField] = + dependency.name.split('.'); + const dependentFormField = formFields.find( + (field) => field.name === dependencyName + ); // converts both values to string to compare - const dependentField = dependentFormField?.options.find(option => `${option.value}` === `${values[dependencyName]}`); + const dependentField = dependentFormField?.options.find( + (option) => `${option.value}` === `${values[dependencyName]}` + ); - currentDependencyFieldValue = dependentField ? dependentField[nestedDependencyField] : null; + currentDependencyFieldValue = dependentField + ? dependentField[nestedDependencyField] + : null; } else { currentDependencyFieldValue = values[dependency.name]; } @@ -58,18 +82,28 @@ const DynamicForm = ({ initialFormFields, onChange }) => { } return { ...field, hidden: isHidden }; - } + }; const updateFormFieldsBasedOnDependency = useMemo(() => { return (formFields, values, setFieldValue, modifiedField) => { return formFields.map((field) => { const { dependency } = field; - + if (dependency) { if (dependency.type === 'filter') { - return handleFilterDependency(field, values, setFieldValue, modifiedField); + return handleFilterDependency( + field, + values, + setFieldValue, + modifiedField + ); } else if (dependency.type === 'visibility') { - return handleVisibilityDependency(field, values, setFieldValue, modifiedField); + return handleVisibilityDependency( + field, + values, + setFieldValue, + modifiedField + ); } } return field; @@ -78,22 +112,30 @@ const DynamicForm = ({ initialFormFields, onChange }) => { }, []); useEffect(() => { - const updatedFormFields = updateFormFieldsBasedOnDependency(initialFormFields, values, setFieldValue); + const updatedFormFields = updateFormFieldsBasedOnDependency( + initialFormFields, + values, + setFieldValue + ); setFormFields(updatedFormFields); }, [updateFormFieldsBasedOnDependency, values]); useEffect(() => { onChange && onChange(formFields, values); }, [formFields, values]); - + // This function updates and filters any dependant fields. Field dependency is described in the form config file const handleDependentFieldUpdate = (value, modifiedField) => { - const updatedFormFields = updateFormFieldsBasedOnDependency(formFields, { ...values, [modifiedField.name]: value }, setFieldValue, modifiedField); + const updatedFormFields = updateFormFieldsBasedOnDependency( + formFields, + { ...values, [modifiedField.name]: value }, + setFieldValue, + modifiedField + ); setFormFields(updatedFormFields); }; const renderFormField = (field) => { - if (field.hidden) { return null; } @@ -147,68 +189,89 @@ const DynamicForm = ({ initialFormFields, onChange }) => { ); }) - : - // shows only filtered fields - field.filteredOptions ? field.filteredOptions.map((option) => ( - - )) : - // shows all fields - field.options.map((option) => ( + : // shows only filtered fields + field.filteredOptions + ? field.filteredOptions.map((option) => ( + + )) + : // shows all fields + field.options.map((option) => ( ))} ); - // uses FieldArray from formik to handle array fields. arrayHelpers from FieldArray is used to add and remove fields - case 'array': - return ( - <> - ( -
-
-

{field.label}

-
- {values[field.name]?.map((_, index) => ( -
- - {field.fields.map(subField => ( -
- {renderFormField({ - ...subField, - name: `${field.name}[${index}].${subField.name}` - })} -
- ))} - - - } - /> + // uses FieldArray from formik to handle array fields. arrayHelpers from FieldArray is used to add and remove fields + case 'array': + return ( + <> + ( +
+
+

{field.label}

+
+ {values[field.name]?.map((_, index) => ( +
+ + {field.fields.map((subField) => ( +
+ {renderFormField({ + ...subField, + name: `${field.name}[${index}].${subField.name}`, + })}
- ))} - -
- )} - /> - - - - ); + ))} + + + } + /> +
+ ))} + +
+ )} + /> + + ); case 'radio': return ( @@ -250,7 +313,12 @@ const DynamicForm = ({ initialFormFields, onChange }) => { {status === 'ERROR' && ( An error has occurred )} - diff --git a/client/src/components/_common/ShowMore/ShowMore.jsx b/client/src/components/_common/ShowMore/ShowMore.jsx index 59abefecf6..a973b44193 100644 --- a/client/src/components/_common/ShowMore/ShowMore.jsx +++ b/client/src/components/_common/ShowMore/ShowMore.jsx @@ -16,7 +16,9 @@ const ShowMore = ({ className, children }) => { // overflowThreshold to account for minor differences in height for example 84.5 and 85 const overflowThreshold = 1; const hasOverflow = - ref && ref.current ? (ref.current.scrollHeight - height) > overflowThreshold : false; + ref && ref.current + ? ref.current.scrollHeight - height > overflowThreshold + : false; return ( <> diff --git a/client/src/components/_custom/drp/DataFilesAddProjectModalAddon/DataFilesAddProjectModalAddon.jsx b/client/src/components/_custom/drp/DataFilesAddProjectModalAddon/DataFilesAddProjectModalAddon.jsx index e4fc9e3b31..91441a74b6 100644 --- a/client/src/components/_custom/drp/DataFilesAddProjectModalAddon/DataFilesAddProjectModalAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesAddProjectModalAddon/DataFilesAddProjectModalAddon.jsx @@ -4,7 +4,6 @@ import { fetchUtil } from 'utils/fetchUtil'; import { DynamicForm } from '_common/Form/DynamicForm'; const DataFilesAddProjectModalAddon = () => { - const { data: form, isLoading } = useQuery('form_ADD_PROJECT', () => fetchUtil({ url: 'api/forms', diff --git a/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.jsx b/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.jsx index b439c64771..a9f3be5b73 100644 --- a/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.jsx @@ -10,104 +10,106 @@ import { useDispatch, useSelector } from 'react-redux'; import { useHistory, useLocation } from 'react-router-dom'; const DataFilesPreviewModalAddon = ({ metadata }) => { + const dispatch = useDispatch(); + const history = useHistory(); + const location = useLocation(); - const dispatch = useDispatch(); - const history = useHistory(); - const location = useLocation(); + // regex from old digitalrocks portal + const standardImageType = /(\.|\/)(gif|jpe?g|png|tiff?)$/i; - // regex from old digitalrocks portal - const standardImageType = /(\.|\/)(gif|jpe?g|png|tiff?)$/i; + const status = useSelector( + (state) => state.files.operationStatus.dynamicform + ); - const status = useSelector((state) => state.files.operationStatus.dynamicform); + useEffect(() => { + if (status === 'SUCCESS') { + dispatch({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { operation: 'preview', props: {} }, + }); + } + }, [status, dispatch]); - useEffect(() => { - if (status === 'SUCCESS') { - dispatch({ - type: 'DATA_FILES_TOGGLE_MODAL', - payload: { operation: 'preview', props: {} }, - }); - } - }, [status, dispatch]); + const { params } = useFileListing('FilesListing'); - const { params } = useFileListing('FilesListing'); + const { ...file } = useSelector((state) => state.files.modalProps.preview); - const { ...file } = useSelector( - (state) => state.files.modalProps.preview - ); - - const { data: form, isLoading } = useQuery('form_UPLOAD_FILE', () => + const { data: form, isLoading } = useQuery('form_UPLOAD_FILE', () => fetchUtil({ url: 'api/forms', params: { form_name: 'EDIT_FILE', }, - })); + }) + ); + + const initialValues = form?.form_fields.reduce((acc, field) => { + let value = ''; + if (field.optgroups) { + value = field.optgroups[0].options[0]?.value; + } else { + value = + field.options && field.options.length > 0 ? field.options[0].value : ''; + } - const initialValues = form?.form_fields.reduce((acc, field) => { - let value = ''; - if (field.optgroups) { - value = field.optgroups[0].options[0]?.value; - } else { - value = - field.options && field.options.length > 0 ? field.options[0].value : ''; - } - - acc[field.name] = metadata ? metadata[field.name] : value; - return acc; - }, {}); + acc[field.name] = metadata ? metadata[field.name] : value; + return acc; + }, {}); - const reloadPage = () => { - history.replace(location.pathname); - }; + const reloadPage = () => { + history.replace(location.pathname); + }; - const handleSubmit = (values) => { + const handleSubmit = (values) => { + Object.keys(values).forEach((key) => { + values[key] = + typeof values[key] === 'string' ? values[key].trim() : values[key]; + }); - Object.keys(values).forEach(key => { - values[key] = typeof(values[key]) === 'string' ? values[key].trim() : values[key]; - }); - - dispatch({ - type: 'EDIT_FILE', - payload: { - params, - values, - reloadPage, - selectedFile: file - }, - }); - }; + dispatch({ + type: 'EDIT_FILE', + payload: { + params, + values, + reloadPage, + selectedFile: file, + }, + }); + }; - return ( - !isLoading && form && !standardImageType.test(file.name) && - <> + return ( + !isLoading && + form && + !standardImageType.test(file.name) && ( + <> -
-
- -
- } - /> - {form?.footer && ( -
- -
- )} - +
+
+ +
+ } + /> + {form?.footer && ( +
+ +
+ )} + - } - /> - + } + /> + ) - + ); }; -export default DataFilesPreviewModalAddon; \ No newline at end of file +export default DataFilesPreviewModalAddon; diff --git a/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.module.scss b/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.module.scss index c804ae2b43..053f83c75c 100644 --- a/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.module.scss +++ b/client/src/components/_custom/drp/DataFilesPreviewModalAddon/DataFilesPreviewModalAddon.module.scss @@ -1,14 +1,14 @@ .section main { - padding: 0; - margin: 0; + padding: 0; + margin: 0; } .listing-header { - font-size: 20px; - } + font-size: 20px; +} .footer { - display: flex; - justify-content: flex-end; - padding: 10px 0px; -} \ No newline at end of file + display: flex; + justify-content: flex-end; + padding: 10px 0px; +} diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx index 57ddd8c895..9b5735a3c9 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx @@ -15,132 +15,160 @@ const DataFilesProjectFileListingAddon = ({ rootSystem, system }) => { const dispatch = useDispatch(); - const { createSampleModal, createOriginDataModal, createAnalysisDataModal } = useDrpDatasetModals(projectId, portalName); + const { createSampleModal, createOriginDataModal, createAnalysisDataModal } = + useDrpDatasetModals(projectId, portalName); const createPublicationRequestModal = () => { dispatch({ type: 'DATA_FILES_TOGGLE_MODAL', - payload: { operation: 'publicationRequest', props: { publicationRequests: metadata?.publication_requests }}, + payload: { + operation: 'publicationRequest', + props: { publicationRequests: metadata?.publication_requests }, + }, }); }; - const { canEditDataset, canRequestPublication, canReviewPublication } = useSelector((state) => { - const { members } = state.projects.metadata; - const { username } = state.authenticatedUser.user; - const currentUser = members.find((member) => member.user?.username === username); - - if (!currentUser) { + const { canEditDataset, canRequestPublication, canReviewPublication } = + useSelector((state) => { + const { members } = state.projects.metadata; + const { username } = state.authenticatedUser.user; + const currentUser = members.find( + (member) => member.user?.username === username + ); + + if (!currentUser) { + return { + canEditDataset: false, + canRequestPublication: false, + canReviewPublication: false, + }; + } + + const { access } = currentUser; + const { is_review_project, publication_requests } = + state.projects.metadata; + + let canReviewPublication = false; + let canRequestPublication = access === 'owner'; + + if (publication_requests?.length > 0) { + const pendingRequest = publication_requests.find( + (request) => request.status === 'PENDING' + ); + + if (pendingRequest) { + canRequestPublication = false; // Prevent requesting publication if there is a pending request + canReviewPublication = + is_review_project && + pendingRequest.reviewers.some( + (reviewer) => reviewer.username === username + ); + } + } + return { - canEditDataset: false, - canRequestPublication: false, - canReviewPublication: false, + canEditDataset: access === 'owner' || access === 'edit', + canRequestPublication, + canReviewPublication, }; - } - - const { access } = currentUser; - const { is_review_project, publication_requests } = state.projects.metadata; - - let canReviewPublication = false; - let canRequestPublication = access === 'owner'; - - if (publication_requests?.length > 0) { - const pendingRequest = publication_requests.find((request) => request.status === 'PENDING'); - - if (pendingRequest) { - canRequestPublication = false; // Prevent requesting publication if there is a pending request - canReviewPublication = is_review_project && pendingRequest.reviewers.some((reviewer) => reviewer.username === username); - } - } - - return { - canEditDataset: access === 'owner' || access === 'edit', - canRequestPublication, - canReviewPublication, - }; - }); + }); return ( <> {canEditDataset && ( <> | - {selectedFiles.length == 1 && selectedFiles[0]?.metadata && - selectedFiles[0].metadata['data_type'] === 'sample' ? ( - - ) : ( - - )} + {selectedFiles.length == 1 && + selectedFiles[0]?.metadata && + selectedFiles[0].metadata['data_type'] === 'sample' ? ( + + ) : ( + + )} | - {selectedFiles.length == 1 && selectedFiles[0]?.metadata && - selectedFiles[0].metadata['data_type'] === 'digital_dataset' ? ( - - ) : ( - + {selectedFiles.length == 1 && + selectedFiles[0]?.metadata && + selectedFiles[0].metadata['data_type'] === 'digital_dataset' ? ( + + ) : ( + )} | - {selectedFiles.length == 1 && selectedFiles[0]?.metadata && - selectedFiles[0].metadata['data_type'] === 'analysis_data' ? ( - - ) : ( - + {selectedFiles.length == 1 && + selectedFiles[0]?.metadata && + selectedFiles[0].metadata['data_type'] === 'analysis_data' ? ( + + ) : ( + )} )} {canRequestPublication && ( <> | - - Request Publication + + Request Publication )} {canReviewPublication && ( <> | - - Review Publication Request + + Review Publication Request )} {metadata?.publication_requests?.length > 0 && ( <> | - )} diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx index 88ca11d6b3..ffd4141845 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx @@ -5,49 +5,68 @@ import { useFileListing } from 'hooks/datafiles'; import DataDisplay from '../utils/DataDisplay/DataDisplay'; import { formatDate } from 'utils/timeFormat'; -const excludeKeys = ['name', 'description', 'data_type', 'sample', 'digital_dataset', 'file_objs']; - -const DataFilesProjectFileListingMetadataAddon = ({ folderMetadata, metadata, path }) => { +const excludeKeys = [ + 'name', + 'description', + 'data_type', + 'sample', + 'digital_dataset', + 'file_objs', +]; +const DataFilesProjectFileListingMetadataAddon = ({ + folderMetadata, + metadata, + path, +}) => { const { loading } = useFileListing('FilesListing'); const getProjectMetadata = (metadata) => { - - const dateOptions = { month: 'long', day: 'numeric', year: 'numeric' } + const dateOptions = { month: 'long', day: 'numeric', year: 'numeric' }; const formattedMetadata = { - created: new Date(metadata.created).toLocaleDateString('en-US', dateOptions), + created: new Date(metadata.created).toLocaleDateString( + 'en-US', + dateOptions + ), license: metadata.license ?? 'None', - } + }; if (metadata.doi) { formattedMetadata.doi = metadata.doi; } return formattedMetadata; - } + }; return ( <> - {!loading && ( - folderMetadata ? ( + {!loading && + (folderMetadata ? ( <> {folderMetadata.description} - + ) : ( <> {metadata.description} - + - ) - )} + ))} ); -} +}; DataFilesProjectFileListingMetadataAddon.propTypes = { folderMetadata: PropTypes.shape({}).isRequired, }; -export default DataFilesProjectFileListingMetadataAddon; \ No newline at end of file +export default DataFilesProjectFileListingMetadataAddon; diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.module.scss b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.module.scss index 06a134e2cc..47d675087b 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.module.scss +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.module.scss @@ -1,14 +1,13 @@ - .metadata-section main { - padding-left: 0; + padding-left: 0; } .dataset-link { - composes: c-button from '../../../../styles/components/c-button--new.css'; - composes: c-button--size-small from '../../../../styles/components/c-button--new.css'; - composes: c-button__text from '../../../../styles/components/c-button--new.css'; - composes: c-button--as-link from '../../../../styles/components/c-button--new.css'; - font-size: 14px; - padding: 0; - margin-bottom: -3px; - } \ No newline at end of file + composes: c-button from '../../../../styles/components/c-button--new.css'; + composes: c-button--size-small from '../../../../styles/components/c-button--new.css'; + composes: c-button__text from '../../../../styles/components/c-button--new.css'; + composes: c-button--as-link from '../../../../styles/components/c-button--new.css'; + font-size: 14px; + padding: 0; + margin-bottom: -3px; +} diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx index d760641639..6bcb4c6320 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx @@ -7,7 +7,12 @@ import { fetchUtil } from 'utils/fetchUtil'; import { useFileListing } from 'hooks/datafiles'; import useDrpDatasetModals from '../utils/hooks/useDrpDatasetModals'; -const DataFilesProjectFileListingMetadataTitleAddon = ({ folderMetadata, metadata, system, path }) => { +const DataFilesProjectFileListingMetadataTitleAddon = ({ + folderMetadata, + metadata, + system, + path, +}) => { const dispatch = useDispatch(); const portalName = useSelector((state) => state.workbench.portalName); const { projectId } = useSelector((state) => state.projects.metadata); @@ -24,29 +29,31 @@ const DataFilesProjectFileListingMetadataTitleAddon = ({ folderMetadata, metadat ) .map((currentUser) => { return { - canEditDataset: currentUser.access === 'owner' || currentUser.access === 'edit', - } + canEditDataset: + currentUser.access === 'owner' || currentUser.access === 'edit', + }; })[0] ); - const { createSampleModal, createOriginDataModal, createAnalysisDataModal } = useDrpDatasetModals(projectId, portalName) + const { createSampleModal, createOriginDataModal, createAnalysisDataModal } = + useDrpDatasetModals(projectId, portalName); const onEditData = (dataType) => { const name = path.split('/').pop(); // reconstruct editFile to mimic SelectedFile object const editFile = { - "format": "folder", - "id" : system + "/" + path, - "metadata": folderMetadata, - "name": name, - "system": system, - "path": path, - "type": "dir", - "_links": { - "self": { - "href": "tapis://" + system + "/" + path, + format: 'folder', + id: system + '/' + path, + metadata: folderMetadata, + name: name, + system: system, + path: path, + type: 'dir', + _links: { + self: { + href: 'tapis://' + system + '/' + path, }, - } + }, }; switch (dataType) { case 'sample': @@ -61,44 +68,43 @@ const DataFilesProjectFileListingMetadataTitleAddon = ({ folderMetadata, metadat default: break; } - } + }; // Function to format the data_type value from snake_case to Label Case i.e. origin_data -> Origin Data - const formatDatatype = (data_type) => - data_type.split('_') - .map(word => word.charAt(0).toUpperCase() + word.slice(1)) + const formatDatatype = (data_type) => + data_type + .split('_') + .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) .join(' '); - return ( - <> - {loading ? ( - - ) : ( - folderMetadata && folderMetadata.data_type ? ( - <> - {folderMetadata.name} - - {formatDatatype(folderMetadata.data_type)} - - {canEditDataset && ( - - )} - - ) : ( - metadata.title - ) - )} - - ); + return ( + <> + {loading ? ( + + ) : folderMetadata && folderMetadata.data_type ? ( + <> + {folderMetadata.name} + + {formatDatatype(folderMetadata.data_type)} + + {canEditDataset && ( + + )} + + ) : ( + metadata.title + )} + + ); }; DataFilesProjectFileListingMetadataTitleAddon.propTypes = { folderMetadata: PropTypes.shape({}).isRequired, }; -export default DataFilesProjectFileListingMetadataTitleAddon; \ No newline at end of file +export default DataFilesProjectFileListingMetadataTitleAddon; diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.module.scss b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.module.scss index 49443e216a..5d2d21c3f3 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.module.scss +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.module.scss @@ -1,9 +1,9 @@ .dataTypeBox { - display: inline-block; - background-color: #f0f0f0; /* Light grey background */ - color: #333; /* Dark text color */ - padding: 2px 8px; /* Small top/bottom padding, larger left/right padding */ - border-radius: 10px; /* Rounded corners */ - font-size: 0.875rem; /* Smaller font size */ - margin: 8px 8px; /* Space from the title */ - } \ No newline at end of file + display: inline-block; + background-color: #f0f0f0; /* Light grey background */ + color: #333; /* Dark text color */ + padding: 2px 8px; /* Small top/bottom padding, larger left/right padding */ + border-radius: 10px; /* Rounded corners */ + font-size: 0.875rem; /* Smaller font size */ + margin: 8px 8px; /* Space from the title */ +} diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx index 7e82b4ec1c..058cf1fa3d 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx @@ -17,7 +17,9 @@ const DataFilesProjectPublish = ({ rootSystem, system }) => { const history = useHistory(); const location = useLocation(); const portalName = useSelector((state) => state.workbench.portalName); - const { projectId, publication_requests } = useSelector((state) => state.projects.metadata); + const { projectId, publication_requests } = useSelector( + (state) => state.projects.metadata + ); const [authors, setAuthors] = useState([]); const [tree, setTree] = useState([]); @@ -55,7 +57,9 @@ const DataFilesProjectPublish = ({ rootSystem, system }) => { // Check if there is any PENDING publication request if (publication_requests?.some((request) => request.status === 'PENDING')) { // Navigate back to the previous location - history.replace(location.state?.from || `${ROUTES.WORKBENCH}${ROUTES.DATA}`); + history.replace( + location.state?.from || `${ROUTES.WORKBENCH}${ROUTES.DATA}` + ); } }, [publication_requests, history]); @@ -68,26 +72,29 @@ const DataFilesProjectPublish = ({ rootSystem, system }) => { const handleAuthorsUpdate = (authors) => { setAuthors(authors); - } + }; const wizardSteps = [ PublicationInstructionsStep(), ProjectDescriptionStep({ project: metadata }), ReviewProjectStructureStep({ projectTree: tree }), - ReviewAuthorsStep({ project: metadata, onAuthorsUpdate: handleAuthorsUpdate}), + ReviewAuthorsStep({ + project: metadata, + onAuthorsUpdate: handleAuthorsUpdate, + }), SubmitPublicationRequestStep(), ]; const formSubmit = (values) => { const data = { - ...metadata, - authors: authors - } + ...metadata, + authors: authors, + }; if (Object.keys(values).length > 0) { dispatch({ type: 'PROJECTS_CREATE_PUBLICATION_REQUEST', - payload: data + payload: data, }); } }; diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.module.scss b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.module.scss index b736edd364..df63730e95 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.module.scss +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.module.scss @@ -1,23 +1,21 @@ @import '../../../../styles/tools/mixins.scss'; - .root { - /* As a flex child */ - flex-grow: 1; - - /* WARNING: Mimicked on: History, Allocation, DataFiles, DataFilesProjectsList, DataFilesProjectFileListing, PublicData */ - padding-top: 1.75rem; /* ~28px (22.5px * design * 1.2 design-to-app ratio) */ - padding-left: 1.5em; /* ~24px (20px * design * 1.2 design-to-app ratio) */ - } + /* As a flex child */ + flex-grow: 1; + + /* WARNING: Mimicked on: History, Allocation, DataFiles, DataFilesProjectsList, DataFilesProjectFileListing, PublicData */ + padding-top: 1.75rem; /* ~28px (22.5px * design * 1.2 design-to-app ratio) */ + padding-left: 1.5em; /* ~24px (20px * design * 1.2 design-to-app ratio) */ +} - .title { - @include truncate-with-ellipsis; - max-width: 1400px; - font-size: 18px; - } +.title { + @include truncate-with-ellipsis; + max-width: 1400px; + font-size: 18px; +} - .controls { - display: flex; - align-items: flex-start; - } - \ No newline at end of file +.controls { + display: flex; + align-items: flex-start; +} diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss index e029fa7ee8..0e2ebbbd4e 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss @@ -1,103 +1,103 @@ @import '../../../../../styles/tools/mixins.scss'; .root { - /* As a flex child */ - flex-grow: 1; - - /* WARNING: Mimicked on: History, Allocation, DataFiles, DataFilesProjectsList, DataFilesProjectFileListing, PublicData */ - padding-top: 1.75rem; /* ~28px (22.5px * design * 1.2 design-to-app ratio) */ - padding-left: 1.5em; /* ~24px (20px * design * 1.2 design-to-app ratio) */ - } - - .title { - @include truncate-with-ellipsis; - max-width: 1400px; - font-size: 18px; - } - - .controls { - display: flex; - align-items: flex-start; - } - - .project-expand-card { - padding-top: 20px; - } - - .project-expand-card > div:first-child { - padding-bottom: 20px; - } - - .section-project-structure main { - margin-right: 0; - margin-bottom: 10px; - } - - .tree-label { - padding-top: 10px; - padding-bottom: 10px; - border: 1px solid var(--global-color-primary--light); - - // &:hover { - // background-color: white; - // } - } - - .metadata-description-div { - border: 1px solid var(--global-color-primary--light); - margin-left: 2px; - padding: 10px; - } - - .description { - font-size: 0.875rem; - } - - .description-section main { - padding-left: 1px; - } - - .edit-button { - font-size: 0.875rem; - padding-bottom: 5px; - } - - .node-name-div { - display: flex; - justify-content: space-between; - align-items: center; - } - - .data-type-box { - display: inline-block; - background-color: #808080; /* Light grey background */ - color: #fff; /* Dark text color */ - padding: 2px 8px; /* Small top/bottom padding, larger left/right padding */ - border-radius: 12px; /* Rounded corners */ - font-size: 0.875rem; /* Smaller font size */ - margin: 8px 8px; - font-weight: bold; - } - - .citation-box { - border: 1px solid var(--global-color-primary--light); - padding: 10px; - margin-bottom: 20px; - border-radius: 10px; - background-color: white; - } - - .required-text { - color: var(--global-color-danger--normal); - } - - .submit-div { - display: flex; - justify-content: center; - margin-top: 20px; - } - - .submit-button { - min-width: 200px; - margin: 20px; - } + /* As a flex child */ + flex-grow: 1; + + /* WARNING: Mimicked on: History, Allocation, DataFiles, DataFilesProjectsList, DataFilesProjectFileListing, PublicData */ + padding-top: 1.75rem; /* ~28px (22.5px * design * 1.2 design-to-app ratio) */ + padding-left: 1.5em; /* ~24px (20px * design * 1.2 design-to-app ratio) */ +} + +.title { + @include truncate-with-ellipsis; + max-width: 1400px; + font-size: 18px; +} + +.controls { + display: flex; + align-items: flex-start; +} + +.project-expand-card { + padding-top: 20px; +} + +.project-expand-card > div:first-child { + padding-bottom: 20px; +} + +.section-project-structure main { + margin-right: 0; + margin-bottom: 10px; +} + +.tree-label { + padding-top: 10px; + padding-bottom: 10px; + border: 1px solid var(--global-color-primary--light); + + // &:hover { + // background-color: white; + // } +} + +.metadata-description-div { + border: 1px solid var(--global-color-primary--light); + margin-left: 2px; + padding: 10px; +} + +.description { + font-size: 0.875rem; +} + +.description-section main { + padding-left: 1px; +} + +.edit-button { + font-size: 0.875rem; + padding-bottom: 5px; +} + +.node-name-div { + display: flex; + justify-content: space-between; + align-items: center; +} + +.data-type-box { + display: inline-block; + background-color: #808080; /* Light grey background */ + color: #fff; /* Dark text color */ + padding: 2px 8px; /* Small top/bottom padding, larger left/right padding */ + border-radius: 12px; /* Rounded corners */ + font-size: 0.875rem; /* Smaller font size */ + margin: 8px 8px; + font-weight: bold; +} + +.citation-box { + border: 1px solid var(--global-color-primary--light); + padding: 10px; + margin-bottom: 20px; + border-radius: 10px; + background-color: white; +} + +.required-text { + color: var(--global-color-danger--normal); +} + +.submit-div { + display: flex; + justify-content: center; + margin-top: 20px; +} + +.submit-button { + min-width: 200px; + margin: 20px; +} diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx index 671f78e0a7..836e2a48ad 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx @@ -18,12 +18,14 @@ const ProjectDescription = ({ project }) => { const canEdit = useSelector((state) => { const { members } = state.projects.metadata; const { username } = state.authenticatedUser.user; - const currentUser = members.find((member) => member.user?.username === username); - + const currentUser = members.find( + (member) => member.user?.username === username + ); + if (!currentUser) { return false; } - + return currentUser.access === 'owner' || currentUser.access === 'edit'; }); @@ -40,36 +42,37 @@ const ProjectDescription = ({ project }) => { Created: formatDate(new Date(project.created)), Abstract: {project.description} , License: project.license ?? 'None', - } + }; if (project.keywords) { - projectData['Keywords'] = project.keywords + projectData['Keywords'] = project.keywords; } if (project.related_publications?.length > 0) { - const relatedPublicationCards = project.related_publications.map((publication) => { - return ( - { - acc[formatDataKey(key)] = publication[key]; - return acc; - }, {}) - } - direction={'vertical'} - density={'compact'} - /> - } - /> - ) - }) + const relatedPublicationCards = project.related_publications.map( + (publication) => { + return ( + { + acc[formatDataKey(key)] = publication[key]; + return acc; + }, {})} + direction={'vertical'} + density={'compact'} + /> + } + /> + ); + } + ); - projectData['Related Publications'] = relatedPublicationCards + projectData['Related Publications'] = relatedPublicationCards; } else { - projectData['Related Publications'] = 'None' + projectData['Related Publications'] = 'None'; } if (project.related_datasets?.length > 0) { @@ -83,19 +86,18 @@ const ProjectDescription = ({ project }) => { data={Object.keys(dataset).reduce((acc, key) => { acc[formatDataKey(key)] = dataset[key]; return acc; - }, {}) - } + }, {})} direction={'vertical'} density={'compact'} /> } /> - ) - }) + ); + }); - projectData['Related Datasets'] = relatedDatasetCards + projectData['Related Datasets'] = relatedDatasetCards; } else { - projectData['Related Datasets'] = 'None' + projectData['Related Datasets'] = 'None'; } if (project.related_software?.length > 0) { @@ -109,24 +111,22 @@ const ProjectDescription = ({ project }) => { data={Object.keys(software).reduce((acc, key) => { acc[formatDataKey(key)] = software[key]; return acc; - }, {}) - } + }, {})} direction={'vertical'} density={'compact'} /> } /> - ) - }) + ); + }); - projectData['Related Software'] = relatedSoftwareCards + projectData['Related Software'] = relatedSoftwareCards; } else { - projectData['Related Software'] = 'None' + projectData['Related Software'] = 'None'; } - setData(projectData) - - }, [project]) + setData(projectData); + }, [project]); return ( { } > - + ); }; diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx index 0042778999..2ad2edf3b6 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx @@ -42,23 +42,27 @@ const ReviewAuthors = ({ project, onAuthorsUpdate }) => { const [authors, setAuthors] = useState([]); const [members, setMembers] = useState([]); - const canEdit = useSelector((state) => { const { members } = state.projects.metadata; const { username } = state.authenticatedUser.user; - const currentUser = members.find((member) => member.user?.username === username); - + const currentUser = members.find( + (member) => member.user?.username === username + ); + if (!currentUser) { return false; } - + return currentUser.access === 'owner' || currentUser.access === 'edit'; }); useEffect(() => { - const owners = project.authors?.length > 0 ? project.authors : project.members - .filter((user) => user.access === 'owner') - .map((user) => ({ ...user.user, isOwner: true })); + const owners = + project.authors?.length > 0 + ? project.authors + : project.members + .filter((user) => user.access === 'owner') + .map((user) => ({ ...user.user, isOwner: true })); const members = project.members .filter( @@ -102,13 +106,16 @@ const ReviewAuthors = ({ project, onAuthorsUpdate }) => { {canEdit && ( <> - + users={authors} + onReorder={onReorder} + onRemoveAuthor={onRemoveCoAuthor} + /> + )} -
)} diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx index 70dd83f978..66f1c31f69 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx @@ -45,12 +45,14 @@ const ReviewProjectStructure = ({ projectTree }) => { const canEdit = useSelector((state) => { const { members } = state.projects.metadata; const { username } = state.authenticatedUser.user; - const currentUser = members.find((member) => member.user?.username === username); - + const currentUser = members.find( + (member) => member.user?.username === username + ); + if (!currentUser) { return false; } - + return currentUser.access === 'owner' || currentUser.access === 'edit'; }); @@ -108,9 +110,10 @@ const ReviewProjectStructure = ({ projectTree }) => { } }; - const formatDatatype = (data_type) => - data_type.split('_') - .map(word => word.charAt(0).toUpperCase() + word.slice(1)) + const formatDatatype = (data_type) => + data_type + .split('_') + .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) .join(' '); const renderTree = (node) => ( @@ -125,12 +128,12 @@ const ReviewProjectStructure = ({ projectTree }) => { nodeId={node.id} label={
- {node.label ?? node.name} - {node.metadata.data_type && ( - - {formatDatatype(node.metadata.data_type)} - - )} + {node.label ?? node.name} + {node.metadata.data_type && ( + + {formatDatatype(node.metadata.data_type)} + + )}
} classes={{ @@ -146,7 +149,9 @@ const ReviewProjectStructure = ({ projectTree }) => { type="link" onClick={() => onEditData(node)} > - {canEdit && node.metadata.data_type !== 'file' ? 'Edit' : 'View'} + {canEdit && node.metadata.data_type !== 'file' + ? 'Edit' + : 'View'} )}
@@ -164,7 +169,7 @@ const ReviewProjectStructure = ({ projectTree }) => {
)} - {Array.isArray(node.fileObjs) && + {Array.isArray(node.fileObjs) && node.fileObjs.map((fileObj) => renderTree(fileObj))} {Array.isArray(node.children) && node.children.map((child) => renderTree(child))} @@ -173,7 +178,7 @@ const ReviewProjectStructure = ({ projectTree }) => {
); - + return ( { curator Maria Esteva before submitting the data for publication.
-
diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/SubmitPublicationReview.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/SubmitPublicationReview.jsx index f293699842..624aa0741f 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/SubmitPublicationReview.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/SubmitPublicationReview.jsx @@ -45,10 +45,22 @@ const SubmitPublicationReview = () => { curator Maria Esteva before submitting the data for publication.
- -
diff --git a/client/src/components/_custom/drp/DataFilesProjectReview/DataFilesProjectReview.jsx b/client/src/components/_custom/drp/DataFilesProjectReview/DataFilesProjectReview.jsx index c767bba98b..9918fd435b 100644 --- a/client/src/components/_custom/drp/DataFilesProjectReview/DataFilesProjectReview.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectReview/DataFilesProjectReview.jsx @@ -12,82 +12,80 @@ import { ReviewAuthorsStep } from '../DataFilesProjectPublish/DataFilesProjectPu import { SubmitPublicationReviewStep } from '../DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/SubmitPublicationReview'; const DataFilesProjectReview = ({ rootSystem, system }) => { - const dispatch = useDispatch(); - const portalName = useSelector((state) => state.workbench.portalName); - const { projectId } = useSelector((state) => state.projects.metadata); - const [tree, setTree] = useState([]); + const dispatch = useDispatch(); + const portalName = useSelector((state) => state.workbench.portalName); + const { projectId } = useSelector((state) => state.projects.metadata); + const [tree, setTree] = useState([]); - useEffect(() => { - dispatch({ - type: 'PROJECTS_GET_METADATA', - payload: system, - }); - }, [system]); + useEffect(() => { + dispatch({ + type: 'PROJECTS_GET_METADATA', + payload: system, + }); + }, [system]); - const { metadata } = useSelector((state) => state.projects); + const { metadata } = useSelector((state) => state.projects); - const fetchTree = useCallback(async () => { + const fetchTree = useCallback(async () => { if (projectId) { - try { - const response = await fetchUtil({ - url: `api/${portalName.toLowerCase()}/tree`, - params: { - project_id: projectId, - }, - }); - setTree(response); - } catch (error) { - console.error('Error fetching tree data:', error); - setTree([]); - } + try { + const response = await fetchUtil({ + url: `api/${portalName.toLowerCase()}/tree`, + params: { + project_id: projectId, + }, + }); + setTree(response); + } catch (error) { + console.error('Error fetching tree data:', error); + setTree([]); + } } - }, [portalName, projectId]); + }, [portalName, projectId]); - useEffect(() => { - fetchTree(); - }, [portalName, projectId, fetchTree]); + useEffect(() => { + fetchTree(); + }, [portalName, projectId, fetchTree]); - const wizardSteps = [ - ProjectDescriptionStep({ project: metadata }), - ReviewProjectStructureStep({ projectTree: tree }), - ReviewAuthorsStep({ project: metadata, onAuthorsUpdate: () => {}}), - SubmitPublicationReviewStep(), - ] + const wizardSteps = [ + ProjectDescriptionStep({ project: metadata }), + ReviewProjectStructureStep({ projectTree: tree }), + ReviewAuthorsStep({ project: metadata, onAuthorsUpdate: () => {} }), + SubmitPublicationReviewStep(), + ]; - const formSubmit = (values) => { - - } + const formSubmit = (values) => {}; - return ( - <> - {metadata.loading ? ( - - ) : ( - - Review Publication Request | {metadata.title} - - } - headerActions={ -
- <> - - Back to Project - - -
- } + return ( + <> + {metadata.loading ? ( + + ) : ( + + Review Publication Request | {metadata.title} + + } + headerActions={ +
+ <> + - - - )} - - ) -} + Back to Project + + +
+ } + > + + + )} + + ); +}; -export default DataFilesProjectReview; \ No newline at end of file +export default DataFilesProjectReview; diff --git a/client/src/components/_custom/drp/DataFilesProjectReview/DataFilesProjectReview.module.scss b/client/src/components/_custom/drp/DataFilesProjectReview/DataFilesProjectReview.module.scss index b736edd364..df63730e95 100644 --- a/client/src/components/_custom/drp/DataFilesProjectReview/DataFilesProjectReview.module.scss +++ b/client/src/components/_custom/drp/DataFilesProjectReview/DataFilesProjectReview.module.scss @@ -1,23 +1,21 @@ @import '../../../../styles/tools/mixins.scss'; - .root { - /* As a flex child */ - flex-grow: 1; - - /* WARNING: Mimicked on: History, Allocation, DataFiles, DataFilesProjectsList, DataFilesProjectFileListing, PublicData */ - padding-top: 1.75rem; /* ~28px (22.5px * design * 1.2 design-to-app ratio) */ - padding-left: 1.5em; /* ~24px (20px * design * 1.2 design-to-app ratio) */ - } + /* As a flex child */ + flex-grow: 1; + + /* WARNING: Mimicked on: History, Allocation, DataFiles, DataFilesProjectsList, DataFilesProjectFileListing, PublicData */ + padding-top: 1.75rem; /* ~28px (22.5px * design * 1.2 design-to-app ratio) */ + padding-left: 1.5em; /* ~24px (20px * design * 1.2 design-to-app ratio) */ +} - .title { - @include truncate-with-ellipsis; - max-width: 1400px; - font-size: 18px; - } +.title { + @include truncate-with-ellipsis; + max-width: 1400px; + font-size: 18px; +} - .controls { - display: flex; - align-items: flex-start; - } - \ No newline at end of file +.controls { + display: flex; + align-items: flex-start; +} diff --git a/client/src/components/_custom/drp/DataFilesUploadModalAddon/DataFilesUploadModalAddon.jsx b/client/src/components/_custom/drp/DataFilesUploadModalAddon/DataFilesUploadModalAddon.jsx index 6fda57e506..fa95222d2b 100644 --- a/client/src/components/_custom/drp/DataFilesUploadModalAddon/DataFilesUploadModalAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesUploadModalAddon/DataFilesUploadModalAddon.jsx @@ -6,29 +6,30 @@ import { Form, Formik } from 'formik'; import { LoadingSpinner, Section, SectionHeader } from '_common'; import styles from './DataFilesUploadModalAddon.module.scss'; - const DataFilesUploadModalAddon = ({ uploadedFiles, setUploadedFiles }) => { + const [uploadedFilesWithMetadata, setUploadedFilesWithMetadata] = useState( + [] + ); + // regex from old digitalrocks portal + const standardImageType = /(\.|\/)(gif|jpe?g|png|tiff?)$/i; - const [uploadedFilesWithMetadata, setUploadedFilesWithMetadata] = useState([]); - - // regex from old digitalrocks portal - const standardImageType = /(\.|\/)(gif|jpe?g|png|tiff?)$/i; - - useEffect(() => { - - // if the uploaded files don't have basic metadata, add metadata - if (!uploadedFiles.every((file) => file.metadata)) { - setUploadedFiles(uploadedFiles.map(file => { - return {...file, metadata: { data_type: 'file' }} - })) - } - - setUploadedFilesWithMetadata(uploadedFiles.filter((file) => !standardImageType.test(file.data.name))); - }, [uploadedFiles]) + useEffect(() => { + // if the uploaded files don't have basic metadata, add metadata + if (!uploadedFiles.every((file) => file.metadata)) { + setUploadedFiles( + uploadedFiles.map((file) => { + return { ...file, metadata: { data_type: 'file' } }; + }) + ); + } + setUploadedFilesWithMetadata( + uploadedFiles.filter((file) => !standardImageType.test(file.data.name)) + ); + }, [uploadedFiles]); - const { data: form, isLoading } = useQuery('form_UPLOAD_FILE', () => + const { data: form, isLoading } = useQuery('form_UPLOAD_FILE', () => fetchUtil({ url: 'api/forms', params: { @@ -37,47 +38,52 @@ const DataFilesUploadModalAddon = ({ uploadedFiles, setUploadedFiles }) => { }) ); - const handleUploadedFileMetadata = (formFields, values, file) => { - const updatedFiles = uploadedFiles.map((uploadedFile) => { - if (uploadedFile.data.name === file.data.name) { - return { - ...uploadedFile, - metadata: {name: uploadedFile.data.name, ...values} - } - } - return uploadedFile; - }); - - setUploadedFiles(updatedFiles); - } + const handleUploadedFileMetadata = (formFields, values, file) => { + const updatedFiles = uploadedFiles.map((uploadedFile) => { + if (uploadedFile.data.name === file.data.name) { + return { + ...uploadedFile, + metadata: { name: uploadedFile.data.name, ...values }, + }; + } + return uploadedFile; + }); - return uploadedFilesWithMetadata.length > 0 && ( - <> - Metadata -
( -
- {isLoading ? ( -

Loading form...

- ) : ( - -
- {file.data.name} - handleUploadedFileMetadata(formFields, values, file)} - /> - -
- )} -
- ))} - /> - - ); + setUploadedFiles(updatedFiles); + }; + return ( + uploadedFilesWithMetadata.length > 0 && ( + <> + Metadata +
( +
+ {isLoading ? ( +

Loading form...

+ ) : ( + +
+ {file.data.name} + + handleUploadedFileMetadata(formFields, values, file) + } + /> + +
+ )} +
+ ))} + /> + + ) + ); }; -export default DataFilesUploadModalAddon; \ No newline at end of file +export default DataFilesUploadModalAddon; diff --git a/client/src/components/_custom/drp/DataFilesUploadModalAddon/DataFilesUploadModalAddon.module.scss b/client/src/components/_custom/drp/DataFilesUploadModalAddon/DataFilesUploadModalAddon.module.scss index 29de897225..650b7577ce 100644 --- a/client/src/components/_custom/drp/DataFilesUploadModalAddon/DataFilesUploadModalAddon.module.scss +++ b/client/src/components/_custom/drp/DataFilesUploadModalAddon/DataFilesUploadModalAddon.module.scss @@ -1,9 +1,9 @@ .section { - border: 1px solid black; - padding: 20px 0px; + border: 1px solid black; + padding: 20px 0px; } .listing-header { - margin-top: 10px; - font-size: 20px; - } \ No newline at end of file + margin-top: 10px; + font-size: 20px; +} diff --git a/client/src/components/_custom/drp/utils/DataDisplay/DataDisplay.jsx b/client/src/components/_custom/drp/utils/DataDisplay/DataDisplay.jsx index 774ef07cc4..ab3622edb6 100644 --- a/client/src/components/_custom/drp/utils/DataDisplay/DataDisplay.jsx +++ b/client/src/components/_custom/drp/utils/DataDisplay/DataDisplay.jsx @@ -11,80 +11,99 @@ const processSampleAndOriginData = (data, path) => { const origin_data = data.digital_dataset ? path.split('/')[1] : null; // remove trailing / from pathname - const locationPathname = location.pathname.endsWith('/') ? location.pathname.slice(0, -1) : location.pathname; - + const locationPathname = location.pathname.endsWith('/') + ? location.pathname.slice(0, -1) + : location.pathname; + const locationPathnameParts = locationPathname.split('/'); - const sampleAndOriginMetadata = [] + const sampleAndOriginMetadata = []; // construct urls for sample and origin data and add to processed data if (sample) { - const sampleUrl = locationPathnameParts.slice(0, data.digital_dataset ? -2 : -1).join('/') - sampleAndOriginMetadata.push({ label: 'Sample', value: {sample} }); + const sampleUrl = locationPathnameParts + .slice(0, data.digital_dataset ? -2 : -1) + .join('/'); + sampleAndOriginMetadata.push({ + label: 'Sample', + value: ( + + {sample} + + ), + }); } if (origin_data) { const originDataUrl = locationPathnameParts.slice(0, -1).join('/'); - sampleAndOriginMetadata.push({ label: 'Origin Data', value: {origin_data} }); - + sampleAndOriginMetadata.push({ + label: 'Origin Data', + value: ( + + {origin_data} + + ), + }); } return sampleAndOriginMetadata; -} +}; const DataDisplay = ({ data, path, excludeKeys }) => { + const location = useLocation(); + + // Function to format the dict key from snake_case to Label Case i.e. data_type -> Data Type + const formatLabel = (key) => + key + .split('_') + .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) + .join(' '); + + //filter out empty values and unwanted keys + const processedData = Object.entries(data) + .filter(([key, value]) => value !== '' && !excludeKeys.includes(key)) + .map(([key, value]) => ({ + label: formatLabel(key), + value: typeof value === 'string' ? formatLabel(value) : value, + })); - const location = useLocation(); - - // Function to format the dict key from snake_case to Label Case i.e. data_type -> Data Type - const formatLabel = (key) => - key.split('_') - .map(word => word.charAt(0).toUpperCase() + word.slice(1)) - .join(' '); - - //filter out empty values and unwanted keys - const processedData = Object.entries(data) - .filter(([key, value]) => value !== "" && !excludeKeys.includes(key)) - .map(([key, value]) => ({ - label: formatLabel(key), - value: typeof(value) === 'string' ? formatLabel(value) : value - })); - - if (path) { - processedData.unshift(...processSampleAndOriginData(data, path)); - } - - // Divide processed data into chunks for two-column layout display - const chunkSize = Math.ceil(processedData.length / 2); - const chunks = []; - for (let i = 0; i < processedData.length; i += chunkSize) { - chunks.push(processedData.slice(i, i + chunkSize)); - } - - const renderDataEntries = (entries) => ( - entries.map(({ label, value }, index) => ( -
- {label}: {value} -
- )) - ); - - // Render each data entry within its chunk for two-column layout - return ( -
- {chunks.map((chunk, index) => ( - - {renderDataEntries(chunk)} - - ))} -
- ); + if (path) { + processedData.unshift(...processSampleAndOriginData(data, path)); } + // Divide processed data into chunks for two-column layout display + const chunkSize = Math.ceil(processedData.length / 2); + const chunks = []; + for (let i = 0; i < processedData.length; i += chunkSize) { + chunks.push(processedData.slice(i, i + chunkSize)); + } + + const renderDataEntries = (entries) => + entries.map(({ label, value }, index) => ( +
+ {label}: {value} +
+ )); + + // Render each data entry within its chunk for two-column layout + return ( +
+ {chunks.map((chunk, index) => ( + + {renderDataEntries(chunk)} + + ))} +
+ ); +}; + DataDisplay.propTypes = { data: PropTypes.object.isRequired, path: PropTypes.string.isRequired, - excludeKeys: PropTypes.array + excludeKeys: PropTypes.array, }; -export default DataDisplay; \ No newline at end of file +export default DataDisplay; diff --git a/client/src/components/_custom/drp/utils/DataDisplay/DataDisplay.module.scss b/client/src/components/_custom/drp/utils/DataDisplay/DataDisplay.module.scss index fea9e890f0..6c2112c00b 100644 --- a/client/src/components/_custom/drp/utils/DataDisplay/DataDisplay.module.scss +++ b/client/src/components/_custom/drp/utils/DataDisplay/DataDisplay.module.scss @@ -1,13 +1,13 @@ .metadata-section main { - padding-left: 0; + padding-left: 0; } .dataset-link { - composes: c-button from '../../../../../styles/components/c-button--new.css'; - composes: c-button--size-small from '../../../../../styles/components/c-button--new.css'; - composes: c-button__text from '../../../../../styles/components/c-button--new.css'; - composes: c-button--as-link from '../../../../../styles/components/c-button--new.css'; - font-size: 14px; - padding: 0; - margin-bottom: -3px; - } \ No newline at end of file + composes: c-button from '../../../../../styles/components/c-button--new.css'; + composes: c-button--size-small from '../../../../../styles/components/c-button--new.css'; + composes: c-button__text from '../../../../../styles/components/c-button--new.css'; + composes: c-button--as-link from '../../../../../styles/components/c-button--new.css'; + font-size: 14px; + padding: 0; + margin-bottom: -3px; +} diff --git a/client/src/components/_custom/drp/utils/DataFilesToolbar/customFilePermissions.js b/client/src/components/_custom/drp/utils/DataFilesToolbar/customFilePermissions.js index 0b25a0ebd9..398d641bf9 100644 --- a/client/src/components/_custom/drp/utils/DataFilesToolbar/customFilePermissions.js +++ b/client/src/components/_custom/drp/utils/DataFilesToolbar/customFilePermissions.js @@ -1,30 +1,23 @@ - const customFilePermissions = (operation, files) => { + const protectedDataTypes = ['sample', 'digital_dataset', 'analysis_data']; - const protectedDataTypes = [ - 'sample', - 'digital_dataset', - 'analysis_data' - ] - - switch(operation) { - case 'copy': - case 'move': - case 'download': - case 'extract': - case 'compress': - case 'areMultipleFilesOrFolderSelected': - return files.every((file) => { - if (!file.metadata) { - return true; - } - - return !protectedDataTypes.includes(file.metadata['data_type']) - }); - default: - return true; - } + switch (operation) { + case 'copy': + case 'move': + case 'download': + case 'extract': + case 'compress': + case 'areMultipleFilesOrFolderSelected': + return files.every((file) => { + if (!file.metadata) { + return true; + } -} + return !protectedDataTypes.includes(file.metadata['data_type']); + }); + default: + return true; + } +}; -export default customFilePermissions; \ No newline at end of file +export default customFilePermissions; diff --git a/client/src/components/_custom/drp/utils/ProjectMembersList/ProjectMembersList.jsx b/client/src/components/_custom/drp/utils/ProjectMembersList/ProjectMembersList.jsx index 591d05f1c1..c6c5dc048f 100644 --- a/client/src/components/_custom/drp/utils/ProjectMembersList/ProjectMembersList.jsx +++ b/client/src/components/_custom/drp/utils/ProjectMembersList/ProjectMembersList.jsx @@ -1,26 +1,25 @@ import React, { useEffect, useState } from 'react'; import { Button, Icon } from '_common'; -import styles from './ProjectMembersList.module.scss' +import styles from './ProjectMembersList.module.scss'; const ProjectMembersList = ({ members, onAddCoAuthor }) => { + return ( + <> +

Other Members

+ {members.map((member, index) => ( +
+ + {member.last_name}, {member.first_name} + +
+ +
+
+ ))} + + ); +}; - return ( - <> -

- Other Members -

- {members.map((member, index) => ( -
- {member.last_name}, {member.first_name} -
- -
-
- ))} - - ) -} - -export default ProjectMembersList; \ No newline at end of file +export default ProjectMembersList; diff --git a/client/src/components/_custom/drp/utils/ProjectMembersList/ProjectMembersList.module.scss b/client/src/components/_custom/drp/utils/ProjectMembersList/ProjectMembersList.module.scss index 1f0add1a39..d6952f9810 100644 --- a/client/src/components/_custom/drp/utils/ProjectMembersList/ProjectMembersList.module.scss +++ b/client/src/components/_custom/drp/utils/ProjectMembersList/ProjectMembersList.module.scss @@ -1,18 +1,18 @@ .user-div { - display: flex; - align-items: center; - margin-bottom: 10px; - justify-content: flex-start; - width: 35%; - } - - .user-name { - flex-grow: 1; /* Take up available space */ - margin-right: 10px; /* Space between name and buttons */ - } - - .button-group { - display: flex; - gap: 15px; /* Space between buttons */ - min-width: 190px; - } \ No newline at end of file + display: flex; + align-items: center; + margin-bottom: 10px; + justify-content: flex-start; + width: 35%; +} + +.user-name { + flex-grow: 1; /* Take up available space */ + margin-right: 10px; /* Space between name and buttons */ +} + +.button-group { + display: flex; + gap: 15px; /* Space between buttons */ + min-width: 190px; +} diff --git a/client/src/components/_custom/drp/utils/ReorderUserList/ReorderUserList.jsx b/client/src/components/_custom/drp/utils/ReorderUserList/ReorderUserList.jsx index 63f23fbf7e..945df385b5 100644 --- a/client/src/components/_custom/drp/utils/ReorderUserList/ReorderUserList.jsx +++ b/client/src/components/_custom/drp/utils/ReorderUserList/ReorderUserList.jsx @@ -1,44 +1,55 @@ import React, { useEffect, useState } from 'react'; import { Button, Icon } from '_common'; -import styles from './ReorderUserList.module.scss' +import styles from './ReorderUserList.module.scss'; const ReorderUserList = ({ users, onReorder, onRemoveAuthor }) => { + const moveUp = (index) => { + const reordered = [...users]; + const temp = reordered[index - 1]; + reordered[index - 1] = reordered[index]; + reordered[index] = temp; + onReorder(reordered); + }; - const moveUp = (index) => { - const reordered = [...users]; - const temp = reordered[index - 1]; - reordered[index - 1] = reordered[index]; - reordered[index] = temp; - onReorder(reordered); - }; + const moveDown = (index) => { + const reordered = [...users]; + const temp = reordered[index + 1]; + reordered[index + 1] = reordered[index]; + reordered[index] = temp; + onReorder(reordered); + }; - const moveDown = (index) => { - const reordered = [...users]; - const temp = reordered[index + 1]; - reordered[index + 1] = reordered[index]; - reordered[index] = temp; - onReorder(reordered); - } - - return ( - <> -

Authors

- {users.map((user, index) => ( -
- {user.last_name}, {user.first_name} -
- - )} -
-
- ))} - - ); + return ( + <> +

Authors

+ {users.map((user, index) => ( +
+ + {user.last_name}, {user.first_name} + +
+ + )} +
+
+ ))} + + ); }; -export default ReorderUserList; \ No newline at end of file +export default ReorderUserList; diff --git a/client/src/components/_custom/drp/utils/ReorderUserList/ReorderUserList.module.scss b/client/src/components/_custom/drp/utils/ReorderUserList/ReorderUserList.module.scss index a14e1a1284..4a5025174f 100644 --- a/client/src/components/_custom/drp/utils/ReorderUserList/ReorderUserList.module.scss +++ b/client/src/components/_custom/drp/utils/ReorderUserList/ReorderUserList.module.scss @@ -15,4 +15,4 @@ display: flex; gap: 15px; /* Space between buttons */ min-width: 200px; -} \ No newline at end of file +} diff --git a/client/src/components/_custom/drp/utils/hooks/useDrpDatasetModals.js b/client/src/components/_custom/drp/utils/hooks/useDrpDatasetModals.js index 5334f4029c..0b9cb96576 100644 --- a/client/src/components/_custom/drp/utils/hooks/useDrpDatasetModals.js +++ b/client/src/components/_custom/drp/utils/hooks/useDrpDatasetModals.js @@ -2,7 +2,11 @@ import { useCallback } from 'react'; import { useSelector, useDispatch } from 'react-redux'; import { fetchUtil } from 'utils/fetchUtil'; -const useDrpDatasetModals = (projectId, portalName, useReloadCallback = true) => { +const useDrpDatasetModals = ( + projectId, + portalName, + useReloadCallback = true +) => { const getFormFields = async (formName) => { const response = await fetchUtil({ url: 'api/forms', @@ -65,7 +69,13 @@ const useDrpDatasetModals = (projectId, portalName, useReloadCallback = true) => type: 'DATA_FILES_TOGGLE_MODAL', payload: { operation: 'dynamicform', - props: { selectedFile, form, formName, additionalData: { samples }, useReloadCallback }, + props: { + selectedFile, + form, + formName, + additionalData: { samples }, + useReloadCallback, + }, }, }); } @@ -112,7 +122,7 @@ const useDrpDatasetModals = (projectId, portalName, useReloadCallback = true) => form, formName, additionalData: { samples, originDatasets }, - useReloadCallback + useReloadCallback, }, }, }); diff --git a/client/src/hooks/datafiles/mutations/useRename.js b/client/src/hooks/datafiles/mutations/useRename.js index 8c14c2251c..274883ea32 100644 --- a/client/src/hooks/datafiles/mutations/useRename.js +++ b/client/src/hooks/datafiles/mutations/useRename.js @@ -15,9 +15,8 @@ function useRename() { }; const rename = ({ selectedFile, newName, api, scheme, callback }) => { - if (selectedFile.metadata) { - selectedFile.metadata = {...selectedFile.metadata, name: newName} + selectedFile.metadata = { ...selectedFile.metadata, name: newName }; } dispatch({ diff --git a/client/src/hooks/datafiles/useAddonComponents.js b/client/src/hooks/datafiles/useAddonComponents.js index 55c0db2178..cce3871ba9 100644 --- a/client/src/hooks/datafiles/useAddonComponents.js +++ b/client/src/hooks/datafiles/useAddonComponents.js @@ -11,9 +11,9 @@ const useAddonComponents = ({ portalName }) => { const module = await import( `../../components/_custom/${portalName.toLowerCase()}/${addonName}/${addonName}.jsx` ); - setAddonComponents(prevComponents => ({ + setAddonComponents((prevComponents) => ({ ...prevComponents, - [addonName]: module.default + [addonName]: module.default, })); } } catch (error) { @@ -26,6 +26,6 @@ const useAddonComponents = ({ portalName }) => { } }, []); return addonComponents; -} +}; export default useAddonComponents; diff --git a/client/src/redux/reducers/datafiles.reducers.js b/client/src/redux/reducers/datafiles.reducers.js index 6d19a74432..f3a4d135f7 100644 --- a/client/src/redux/reducers/datafiles.reducers.js +++ b/client/src/redux/reducers/datafiles.reducers.js @@ -199,7 +199,10 @@ export function files(state = initialFilesState, action) { ...state, loading: { ...state.loading, [action.payload.section]: true }, error: { ...state.error, [action.payload.section]: false }, - folderMetadata: {...state.folderMetadata,[action.payload.section]: null }, + folderMetadata: { + ...state.folderMetadata, + [action.payload.section]: null, + }, listing: { ...state.listing, [action.payload.section]: [] }, params: { ...state.params, @@ -223,7 +226,10 @@ export function files(state = initialFilesState, action) { ...state, loading: { ...state.loading, [action.payload.section]: false }, error: { ...state.error, [action.payload.section]: false }, - folderMetadata: {...state.folderMetadata,[action.payload.section]: action.payload.folderMetadata }, + folderMetadata: { + ...state.folderMetadata, + [action.payload.section]: action.payload.folderMetadata, + }, listing: { ...state.listing, [action.payload.section]: [...action.payload.files], diff --git a/client/src/redux/reducers/projects.reducers.js b/client/src/redux/reducers/projects.reducers.js index dd8a03d847..5cdb15c08e 100644 --- a/client/src/redux/reducers/projects.reducers.js +++ b/client/src/redux/reducers/projects.reducers.js @@ -263,7 +263,7 @@ export default function projects(state = initialState, action) { }; case 'PROJECTS_GET_PUBLICATION_REQUESTS_SUCCESS': return { - ...state, + ...state, metadata: { ...state.metadata, publication_requests: action.payload, @@ -274,7 +274,7 @@ export default function projects(state = initialState, action) { error: null, result: action.payload, }, - } + }; case 'PROJECTS_GET_PUBLICATION_REQUESTS_FAILED': return { ...state, diff --git a/client/src/redux/sagas/datafiles.sagas.js b/client/src/redux/sagas/datafiles.sagas.js index 19f8b31e58..1134a462d1 100644 --- a/client/src/redux/sagas/datafiles.sagas.js +++ b/client/src/redux/sagas/datafiles.sagas.js @@ -223,7 +223,14 @@ export function* scrollFiles(action) { } } -export async function renameFileUtil(api, scheme, system, path, newName, metadata) { +export async function renameFileUtil( + api, + scheme, + system, + path, + newName, + metadata +) { const url = `/api/datafiles/${api}/rename/${scheme}/${system}/${path}/`; const response = await fetch(url, { method: 'PUT', @@ -287,7 +294,7 @@ export async function moveFileUtil( system, path, destSystem, - destPath, + destPath, index, metadata ) { @@ -296,7 +303,11 @@ export async function moveFileUtil( method: 'PUT', headers: { 'X-CSRFToken': Cookies.get('csrftoken') }, credentials: 'same-origin', - body: JSON.stringify({ dest_system: destSystem, dest_path: destPath, metadata: metadata ?? null }), + body: JSON.stringify({ + dest_system: destSystem, + dest_path: destPath, + metadata: metadata ?? null, + }), }); if (!request.ok) { throw new Error(request.status); @@ -388,7 +399,7 @@ export async function copyFileUtil( file_name: filename, filetype, dest_path_name: destPathName, - metadata: metadata ?? null + metadata: metadata ?? null, }; } else { url = removeDuplicateSlashes(`/api/datafiles/transfer/${filetype}/`); @@ -481,14 +492,24 @@ export function* copyFiles(action) { yield call(action.payload.reloadCallback); } -export async function uploadFileUtil(api, scheme, system, path, file, metadata) { +export async function uploadFileUtil( + api, + scheme, + system, + path, + file, + metadata +) { let apiPath = !path || path[0] === '/' ? path : `/${path}`; if (apiPath === '/') { apiPath = ''; } const formData = new FormData(); formData.append('uploaded_file', file); - formData.append('metadata', metadata ? JSON.stringify({data_type: 'file', ...metadata}) : null); + formData.append( + 'metadata', + metadata ? JSON.stringify({ data_type: 'file', ...metadata }) : null + ); const url = removeDuplicateSlashes( `/api/datafiles/${api}/upload/${scheme}/${system}/${apiPath}/` @@ -769,7 +790,7 @@ export async function trashUtil(api, scheme, system, path, homeDir, metadata) { credentials: 'same-origin', body: JSON.stringify({ homeDir: homeDir, - metadata: metadata ?? null + metadata: metadata ?? null, }), }); @@ -785,7 +806,13 @@ export function* watchTrash() { export function* trashFiles(action) { const trashCalls = action.payload.src.map((file) => { - return call(trashFile, file.system, file.path, action.payload.homeDir, file.metadata); + return call( + trashFile, + file.system, + file.path, + action.payload.homeDir, + file.metadata + ); }); const { result } = yield race({ result: all(trashCalls), @@ -1179,13 +1206,27 @@ export function* watchMakePublic() { yield takeLeading('DATA_FILES_MAKE_PUBLIC', doMakePublic); } -export async function updateMetadataUtil(api, scheme, system, path, newPath, oldName, newName, metadata) { +export async function updateMetadataUtil( + api, + scheme, + system, + path, + newPath, + oldName, + newName, + metadata +) { const url = `/api/datafiles/${api}/update_metadata/${scheme}/${system}/${path}/`; const response = await fetch(url, { method: 'PUT', headers: { 'X-CSRFToken': Cookies.get('csrftoken') }, credentials: 'same-origin', - body: JSON.stringify({ new_path: newPath, old_name: oldName, new_name: newName, metadata: metadata ?? null }), + body: JSON.stringify({ + new_path: newPath, + old_name: oldName, + new_name: newName, + metadata: metadata ?? null, + }), }); if (!response.ok) { throw new Error(response.status); @@ -1193,4 +1234,4 @@ export async function updateMetadataUtil(api, scheme, system, path, newPath, old const responseJson = await response.json(); return responseJson.data; -} \ No newline at end of file +} diff --git a/client/src/redux/sagas/projects.sagas.js b/client/src/redux/sagas/projects.sagas.js index a37c189270..12cd8d1bd7 100644 --- a/client/src/redux/sagas/projects.sagas.js +++ b/client/src/redux/sagas/projects.sagas.js @@ -226,12 +226,14 @@ export async function fetchPublicationRequestsUtil(system) { } export function* getPublicationRequests(action) { - yield put({ type: 'PROJECTS_GET_PUBLICATION_REQUESTS_STARTED', }); try { - const publicationRequests = yield call(fetchPublicationRequestsUtil, action.payload); + const publicationRequests = yield call( + fetchPublicationRequestsUtil, + action.payload + ); yield put({ type: 'PROJECTS_GET_PUBLICATION_REQUESTS_SUCCESS', payload: publicationRequests, @@ -253,15 +255,22 @@ export async function createEntityUtil(entityType, projectId, path, data) { }, body: JSON.stringify({ name: entityType, - value: data, - path: path + value: data, + path: path, }), }); return result.response; } -export async function patchEntityUtil(entityType, projectId, path, updatedPath, data, entityUuid) { +export async function patchEntityUtil( + entityType, + projectId, + path, + updatedPath, + data, + entityUuid +) { const result = await fetchUtil({ url: `/api/projects/${projectId}/entities/create`, method: 'PATCH', @@ -273,7 +282,7 @@ export async function patchEntityUtil(entityType, projectId, path, updatedPath, value: data, uuid: entityUuid ?? null, path: path, - updatedPath: updatedPath + updatedPath: updatedPath, }), }); @@ -287,6 +296,9 @@ export function* watchProjects() { yield takeLatest('PROJECTS_GET_METADATA', getMetadata); yield takeLatest('PROJECTS_SET_MEMBER', setMember); yield takeLatest('PROJECTS_SET_TITLE_DESCRIPTION', setTitleDescription); - yield takeLatest('PROJECTS_CREATE_PUBLICATION_REQUEST', createPublicationRequest); + yield takeLatest( + 'PROJECTS_CREATE_PUBLICATION_REQUEST', + createPublicationRequest + ); yield takeLatest('PROJECTS_GET_PUBLICATION_REQUESTS', getPublicationRequests); } diff --git a/client/src/utils/dataKeyFormat.js b/client/src/utils/dataKeyFormat.js index 4c04837fc3..f6906f5c00 100644 --- a/client/src/utils/dataKeyFormat.js +++ b/client/src/utils/dataKeyFormat.js @@ -2,8 +2,9 @@ * Function to format the data_type value from snake_case to Label Case i.e. project_title -> Project Title * @param {String} key */ -export function formatDataKey(key) { - return key.split('_') - .map(word => word.charAt(0).toUpperCase() + word.slice(1)) - .join(' '); -} \ No newline at end of file +export function formatDataKey(key) { + return key + .split('_') + .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) + .join(' '); +} diff --git a/client/src/utils/filePermissions.js b/client/src/utils/filePermissions.js index 49526b49e7..e6cd54555b 100644 --- a/client/src/utils/filePermissions.js +++ b/client/src/utils/filePermissions.js @@ -6,7 +6,10 @@ * @param {String} api * @returns {Boolean} */ -export default function getFilePermissions(name, { files, scheme, api, customPermissionCheck }) { +export default function getFilePermissions( + name, + { files, scheme, api, customPermissionCheck } +) { const protectedFiles = [ '.bash_profile', '.bashrc', @@ -25,28 +28,50 @@ export default function getFilePermissions(name, { files, scheme, api, customPer switch (name) { case 'rename': return ( - !isProtected && files.length === 1 && isPrivate && api !== 'googledrive' && customPermissionCheck(name, files) + !isProtected && + files.length === 1 && + isPrivate && + api !== 'googledrive' && + customPermissionCheck(name, files) ); case 'download': return ( files.length === 1 && files[0].format !== 'folder' && - api !== 'googledrive' && customPermissionCheck(name, files) + api !== 'googledrive' && + customPermissionCheck(name, files) ); case 'areMultipleFilesOrFolderSelected': return ( (files.length > 1 || files.some((file) => file.format === 'folder')) && - api !== 'googledrive' && customPermissionCheck(name, files) + api !== 'googledrive' && + customPermissionCheck(name, files) ); case 'extract': - return files.length === 1 && isArchive && isPrivate && api === 'tapis' && customPermissionCheck(name, files); + return ( + files.length === 1 && + isArchive && + isPrivate && + api === 'tapis' && + customPermissionCheck(name, files) + ); case 'compress': - return !isArchive && files.length > 0 && isPrivate && api === 'tapis' && customPermissionCheck(name, files); + return ( + !isArchive && + files.length > 0 && + isPrivate && + api === 'tapis' && + customPermissionCheck(name, files) + ); case 'copy': return files.length > 0 && customPermissionCheck(name, files); case 'move': return ( - !isProtected && files.length > 0 && isPrivate && api !== 'googledrive' && customPermissionCheck(name, files) + !isProtected && + files.length > 0 && + isPrivate && + api !== 'googledrive' && + customPermissionCheck(name, files) ); case 'trash': return ( diff --git a/client/src/utils/systems.js b/client/src/utils/systems.js index 334620ef7b..fe105a410a 100644 --- a/client/src/utils/systems.js +++ b/client/src/utils/systems.js @@ -91,7 +91,9 @@ export function findSystemOrProjectDisplayName( case 'projects': let project = findProjectTitle(projectsList, system, projectTitle); if (!project) { - const projectSystem = systemList.find(system => system.scheme === 'projects') + const projectSystem = systemList.find( + (system) => system.scheme === 'projects' + ); return projectSystem ? projectSystem.name : ''; } else { return project; From 79192a95f240453a5c10e4b0dabd4b54484d955f Mon Sep 17 00:00:00 2001 From: Van Go <35277477+van-go@users.noreply.github.com> Date: Fri, 8 Nov 2024 11:37:15 -0600 Subject: [PATCH 121/328] Refactor ReviewAuthors component to use separate citation components (#995) --- .../ReviewAuthors.jsx | 108 +++++++++++++++--- 1 file changed, 92 insertions(+), 16 deletions(-) diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx index 2ad2edf3b6..eaf4ca47dd 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx @@ -15,29 +15,104 @@ import ReorderUserList from '../../utils/ReorderUserList/ReorderUserList'; import ProjectMembersList from '../../utils/ProjectMembersList/ProjectMembersList'; import { useSelector } from 'react-redux'; +const ACMCitation = ({ project, authors }) => { + const authorString = authors.map(a => `${a.first_name} ${a.last_name}`).join(', '); + const projectUrl = `https://www.digitalrocksportal.org/projects/${project.projectId.split('-').pop()}`; + const createdDate = new Date(project.created).toLocaleDateString('en-US', { month: 'long', year: 'numeric' }); + + return ( +
+ {`${authorString}. ${project.title}. `} Digital Rocks Portal {` (${createdDate}). ${projectUrl}`}
+ ); +}; + +const APACitation = ({ project, authors }) => { + const authorString = authors + .map((a) => `${a.last_name}, ${a.first_name.charAt(0)}.`) + .join(', '); + const projectUrl = `https://www.digitalrocksportal.org`; + const createdDateObj = new Date(project.created); + const createdDate = `${createdDateObj.getFullYear()}, ${createdDateObj.toLocaleString('en-US', { month: 'long' })} ${createdDateObj.getDate()}`; + const accessDate = new Date().toLocaleDateString('en-US', { year: 'numeric', month: 'long', day: 'numeric' }); + + return ( +
{`${authorString} (${createdDate}). ${project.title}. Retrieved ${accessDate}, from ${projectUrl}`}
+ ); +}; + +const BibTeXCitation = ({ project, authors }) => { + const authorString = authors.map(a => `${a.last_name}, ${a.first_name}`).join(' and '); + const projectUrl = `https://www.digitalrocksportal.org/projects/${project.projectId.split('-').pop()}`; + const year = new Date(project.created).getFullYear(); + + return ( + 
{`@misc{dataset,
+  author = {${authorString}},
+  title = {${project.title}},
+  year = {${year}},
+  publisher = {Digital Rocks Portal},
+  doi = {},
+  howpublished = {\\url{${projectUrl}}}
+}`}
+ ); +}; + const MLACitation = ({ project, authors }) => { - let authorString; + const authorString = authors.map(a => `${a.last_name}, ${a.first_name}`).join(', '); + const projectUrl = `https://www.digitalrocksportal.org/projects/${project.projectId.split('-').pop()}`; + const createdDate = new Date(project.created).toLocaleDateString('en-GB', { day: 'numeric', month: 'short', year: 'numeric' }); + const accessDate = new Date().toLocaleDateString('en-GB', { day: 'numeric', month: 'short', year: 'numeric' }); + - if (authors.length === 1) { - authorString = `${authors[0].last_name}, ${authors[0].first_name}`; - } else if (authors.length === 2) { - authorString = `${authors[0].last_name}, ${authors[0].first_name}, and ${authors[1].last_name}, ${authors[1].first_name}`; - } else { - authorString = `${authors[0].last_name}, ${authors[0].first_name}, et al`; - } + return ( +
{`${authorString}. "${project.title}."`} Digital Rocks Portal, {` Digital Rocks Portal, ${createdDate}, ${projectUrl} Accessed ${accessDate}.`}
+ ); +}; - const mlaText = `${authorString}. "${project.title}." Digital Rocks Portal `; +const IEEECitation = ({ project, authors }) => { + const authorString = authors.map(a => `${a.first_name[0]}. ${a.last_name}`).join(', '); + const projectUrl = `https://www.digitalrocksportal.org/projects/${project.projectId.split('-').pop()}`; + const date = new Date(project.created); + const year = date.getFullYear(); + const day = date.getDate(); + const month = date.toLocaleString('en-GB', { month: 'short' }); return ( - <> -

MLA

-
-
{mlaText}
-
- +
{`[1] ${authorString}, "${project.title}",`} Digital Rocks Portal, {` ${year}. [Online]. Available: ${projectUrl}. [Accessed: ${day}-${month}-${year}]`}
); }; +const Citations = ({ project, authors }) => ( +
+

ACM ref

+
+ +
+ +

APA

+
+ +
+ +

BibTeX

+
+ +
+ +

MLA

+
+ +
+ +

IEEE

+
+ +
+
+); + + + const ReviewAuthors = ({ project, onAuthorsUpdate }) => { const [authors, setAuthors] = useState([]); const [members, setMembers] = useState([]); @@ -102,7 +177,8 @@ const ReviewAuthors = ({ project, onAuthorsUpdate }) => { > {authors.length > 0 && project && (
- + + {canEdit && ( <> Date: Fri, 8 Nov 2024 12:03:48 -0600 Subject: [PATCH 122/328] Task/WC-93: Add toolbar button and modal to display the tree for DRP projects (#998) * add toolbar button and modal to display the tree for DRP projects * scroll modal when it overflows the screen size --- .../DataFilesModals/DataFilesModals.jsx | 2 + .../DataFilesProjectTreeModal.jsx | 45 ++++ .../DataFilesProjectTreeModal.module.scss | 4 + .../DataFilesProjectFileListingAddon.jsx | 17 +- .../ProjectTreeView.jsx | 199 ++++++++++++++++++ .../ReviewProjectStructure.jsx | 165 +-------------- .../drp/utils/hooks/useDrpDatasetModals.js | 20 +- client/tsconfig.json | 2 + client/vite.config.ts | 1 + 9 files changed, 292 insertions(+), 163 deletions(-) create mode 100644 client/src/components/DataFiles/DataFilesModals/DataFilesProjectTreeModal.jsx create mode 100644 client/src/components/DataFiles/DataFilesModals/DataFilesProjectTreeModal.module.scss create mode 100644 client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectTreeView.jsx diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesModals.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesModals.jsx index 8399ea17fd..d726c9a14a 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesModals.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesModals.jsx @@ -19,6 +19,7 @@ import DataFilesDownloadMessageModal from './DataFilesDownloadMessageModal'; import './DataFilesModals.scss'; import DataFilesFormModal from './DataFilesFormModal'; import DataFilesPublicationRequestModal from './DataFilesPublicationRequestModal'; +import DataFilesProjectTreeModal from './DataFilesProjectTreeModal'; export default function DataFilesModals() { return ( @@ -41,6 +42,7 @@ export default function DataFilesModals() { + ); diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesProjectTreeModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesProjectTreeModal.jsx new file mode 100644 index 0000000000..ad5644b8ee --- /dev/null +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesProjectTreeModal.jsx @@ -0,0 +1,45 @@ +import React, { useCallback } from 'react'; +import { useDispatch, useSelector } from 'react-redux'; +import { Modal, ModalHeader, ModalBody } from 'reactstrap'; +import styles from './DataFilesProjectTreeModal.module.scss'; +import { ProjectTreeView } from '_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectTreeView'; + +const DataFilesProjectTreeModal = () => { + const { projectId } = useSelector((state) => state.projects.metadata); + + const isOpen = useSelector((state) => state.files.modals.projectTree); + const props = useSelector((state) => state.files.modalProps['projectTree']); + + const toggle = useCallback(() => { + dispatch({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { operation: 'projectTree', props: {} }, + }); + }, []); + + const dispatch = useDispatch(); + + return ( + <> + + + Project Tree + + + {' '} + + + + + ); +}; + +export default DataFilesProjectTreeModal; diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesProjectTreeModal.module.scss b/client/src/components/DataFiles/DataFilesModals/DataFilesProjectTreeModal.module.scss new file mode 100644 index 0000000000..d34c1fbe30 --- /dev/null +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesProjectTreeModal.module.scss @@ -0,0 +1,4 @@ +.modal-body { + overflow: auto; + max-height: 80vh; +} \ No newline at end of file diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx index 9b5735a3c9..328f14df83 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx @@ -15,8 +15,12 @@ const DataFilesProjectFileListingAddon = ({ rootSystem, system }) => { const dispatch = useDispatch(); - const { createSampleModal, createOriginDataModal, createAnalysisDataModal } = - useDrpDatasetModals(projectId, portalName); + const { + createSampleModal, + createOriginDataModal, + createAnalysisDataModal, + createTreeModal, + } = useDrpDatasetModals(projectId, portalName); const createPublicationRequestModal = () => { dispatch({ @@ -142,6 +146,15 @@ const DataFilesProjectFileListingAddon = ({ rootSystem, system }) => { )} )} + <> + | + + {canRequestPublication && ( <> | diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectTreeView.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectTreeView.jsx new file mode 100644 index 0000000000..7a559451c6 --- /dev/null +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectTreeView.jsx @@ -0,0 +1,199 @@ +import React, { useEffect, useState, useCallback } from 'react'; +import { + Button, + ShowMore, + Section, + Icon, +} from '_common'; +import { TreeItem, TreeView } from '@material-ui/lab'; +import styles from './DataFilesProjectPublishWizard.module.scss'; +import DataDisplay from '../../utils/DataDisplay/DataDisplay'; +import { useDispatch, useSelector } from 'react-redux'; +import { useFileListing } from 'hooks/datafiles'; +import useDrpDatasetModals from '../../utils/hooks/useDrpDatasetModals'; +import { fetchUtil } from 'utils/fetchUtil'; + +export const ProjectTreeView = ({ projectId, readOnly = false }) => { + const [expandedNodes, setExpandedNodes] = useState([]); + + const dispatch = useDispatch(); + const portalName = useSelector((state) => state.workbench.portalName); + + const [tree, setTree] = useState([]); + + const { dynamicFormModal, previewModal, metadata } = useSelector((state) => ({ + dynamicFormModal: state.files.modals.dynamicform, + previewModal: state.files.modals.preview, + metadata: state.projects.metadata, + })); + + const fetchTree = useCallback(async () => { + if (projectId) { + try { + const response = await fetchUtil({ + url: `api/${portalName.toLowerCase()}/tree`, + params: { + project_id: projectId, + }, + }); + setTree(response); + } catch (error) { + console.error('Error fetching tree data:', error); + setTree([]); + } + } + }, [portalName, projectId]); + + useEffect(() => { + // workaround to get updated data after modal closes + if (!dynamicFormModal || !previewModal) { + fetchTree(); + } + }, [dynamicFormModal, previewModal, fetchTree]); + + const { params } = useFileListing('FilesListing'); + + useEffect(() => { + if (tree && tree.length > 0) { + setExpandedNodes([tree[0].id]); + } + }, []); + + const handleNodeToggle = (event, nodeIds) => { + // Update the list of expanded nodes + setExpandedNodes(nodeIds); + }; + + const { createSampleModal, createOriginDataModal, createAnalysisDataModal } = + useDrpDatasetModals(projectId, portalName, false); + + const onEditData = (node) => { + const dataType = node.metadata.data_type; + // reconstruct editFile to mimic SelectedFile object + const editFile = { + id: node.id, + uuid: node.uuid, + metadata: node.metadata, + name: node.metadata.name, + system: params.system, + type: 'dir', + path: node.path, + }; + switch (dataType) { + case 'sample': + createSampleModal('EDIT_SAMPLE_DATA', editFile); + break; + case 'digital_dataset': + createOriginDataModal('EDIT_ORIGIN_DATASET', editFile); + break; + case 'analysis_data': + createAnalysisDataModal('EDIT_ANALYSIS_DATASET', editFile); + break; + case 'file': + // Dispatch an action to toggle the modal for previewing the file + dispatch({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { + operation: 'preview', + props: { + api: params.api, + scheme: params.scheme, + system: params.system, + path: node.path, + name: node.name, + href: `tapis://${params.system}/${node.path}`, + length: node.length, + metadata: node.metadata, + useReloadCallback: false, + }, + }, + }); + break; + default: + break; + } + }; + + const formatDatatype = (data_type) => + data_type + .split('_') + .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) + .join(' '); + + const renderTree = (node) => ( + <> +
+
+ + {node.label ?? node.name} + {node.metadata.data_type && ( + + {formatDatatype(node.metadata.data_type)} + + )} +
+ } + classes={{ + label: styles['tree-label'], + }} + onLabelClick={() => handleNodeToggle} + > + {expandedNodes.includes(node.id) && node.id !== 'NODE_ROOT' && ( +
+ {(!readOnly || node.metadata.data_type === 'file') && ( + + )} +
+ {node.metadata.description} + +
+
+ )} + {Array.isArray(node.fileObjs) && + node.fileObjs.map((fileObj) => renderTree(fileObj))} + {Array.isArray(node.children) && + node.children.map((child) => renderTree(child))} + + +
+ + ); + + return ( + tree && + tree.length > 0 && ( + } + defaultExpandIcon={} + expanded={expandedNodes} + onNodeToggle={handleNodeToggle} + > + {tree.map((node) => renderTree(node))} + + ) + ); +}; \ No newline at end of file diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx index 66f1c31f69..8d8edef5e1 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx @@ -1,47 +1,17 @@ -import React, { useEffect, useState } from 'react'; -import { - Button, - ShowMore, - LoadingSpinner, - SectionMessage, - SectionTableWrapper, - DescriptionList, - Section, - SectionContent, - Expand, - Icon, -} from '_common'; -import { TreeItem, TreeView } from '@material-ui/lab'; +import React, { useEffect, useState, useCallback } from 'react'; +import { Button, SectionTableWrapper, Section } from '_common'; import styles from './DataFilesProjectPublishWizard.module.scss'; -import DataDisplay from '../../utils/DataDisplay/DataDisplay'; import { useDispatch, useSelector } from 'react-redux'; import { useFileListing } from 'hooks/datafiles'; -import useDrpDatasetModals from '../../utils/hooks/useDrpDatasetModals'; +import { ProjectTreeView } from './ProjectTreeView'; -const ReviewProjectStructure = ({ projectTree }) => { +export const ReviewProjectStructure = ({ projectTree }) => { const dispatch = useDispatch(); - const [expandedNodes, setExpandedNodes] = useState([]); - - const portalName = useSelector((state) => state.workbench.portalName); const { projectId } = useSelector((state) => state.projects.metadata); const { params } = useFileListing('FilesListing'); - useEffect(() => { - if (projectTree && projectTree.length > 0) { - setExpandedNodes([projectTree[0].id]); - } - }, []); - - const handleNodeToggle = (event, nodeIds) => { - // Update the list of expanded nodes - setExpandedNodes(nodeIds); - }; - - const { createSampleModal, createOriginDataModal, createAnalysisDataModal } = - useDrpDatasetModals(projectId, portalName, false); - const canEdit = useSelector((state) => { const { members } = state.projects.metadata; const { username } = state.authenticatedUser.user; @@ -63,122 +33,6 @@ const ReviewProjectStructure = ({ projectTree }) => { }); }; - const onEditData = (node) => { - const dataType = node.metadata.data_type; - // reconstruct editFile to mimic SelectedFile object - const editFile = { - id: node.id, - uuid: node.uuid, - metadata: node.metadata, - name: node.metadata.name, - system: params.system, - type: 'dir', - path: node.path, - }; - switch (dataType) { - case 'sample': - createSampleModal('EDIT_SAMPLE_DATA', editFile); - break; - case 'digital_dataset': - createOriginDataModal('EDIT_ORIGIN_DATASET', editFile); - break; - case 'analysis_data': - createAnalysisDataModal('EDIT_ANALYSIS_DATASET', editFile); - break; - case 'file': - // Dispatch an action to toggle the modal for previewing the file - dispatch({ - type: 'DATA_FILES_TOGGLE_MODAL', - payload: { - operation: 'preview', - props: { - api: params.api, - scheme: params.scheme, - system: params.system, - path: node.path, - name: node.name, - href: `tapis://${params.system}/${node.path}`, - length: node.length, - metadata: node.metadata, - useReloadCallback: false, - }, - }, - }); - break; - default: - break; - } - }; - - const formatDatatype = (data_type) => - data_type - .split('_') - .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) - .join(' '); - - const renderTree = (node) => ( - <> -
-
- - {node.label ?? node.name} - {node.metadata.data_type && ( - - {formatDatatype(node.metadata.data_type)} - - )} -
- } - classes={{ - label: styles['tree-label'], - }} - onLabelClick={() => handleNodeToggle} - > - {expandedNodes.includes(node.id) && node.id !== 'NODE_ROOT' && ( -
- {(canEdit || node.metadata.data_type === 'file') && ( - - )} -
- {node.metadata.description} - -
-
- )} - {Array.isArray(node.fileObjs) && - node.fileObjs.map((fileObj) => renderTree(fileObj))} - {Array.isArray(node.children) && - node.children.map((child) => renderTree(child))} - - -
- - ); - return ( { - {projectTree && projectTree.length > 0 && ( - } - defaultExpandIcon={} - expanded={expandedNodes} - onNodeToggle={handleNodeToggle} - > - {projectTree.map((node) => renderTree(node))} - - )} +
); diff --git a/client/src/components/_custom/drp/utils/hooks/useDrpDatasetModals.js b/client/src/components/_custom/drp/utils/hooks/useDrpDatasetModals.js index 0b9cb96576..6ce6f46aef 100644 --- a/client/src/components/_custom/drp/utils/hooks/useDrpDatasetModals.js +++ b/client/src/components/_custom/drp/utils/hooks/useDrpDatasetModals.js @@ -129,7 +129,25 @@ const useDrpDatasetModals = ( } ); - return { createSampleModal, createOriginDataModal, createAnalysisDataModal }; + const createTreeModal = useCallback( + async ({ readOnly = false }) => { + dispatch({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { + operation: 'projectTree', + props: { readOnly }, + }, + }); + }, + [dispatch] + ); + + return { + createSampleModal, + createOriginDataModal, + createAnalysisDataModal, + createTreeModal, + }; }; export default useDrpDatasetModals; diff --git a/client/tsconfig.json b/client/tsconfig.json index 88baee95c0..09e3d31a37 100644 --- a/client/tsconfig.json +++ b/client/tsconfig.json @@ -5,6 +5,8 @@ "paths": { "_common/*": ["components/_common/*"], "_common": ["components/_common"], + "_custom/*": ["components/_custom/*"], + "_custom": ["components/_custom"], "utils/*": ["utils/*"] }, "useDefineForClassFields": true, diff --git a/client/vite.config.ts b/client/vite.config.ts index db5a4e6caa..540d9fcf2e 100644 --- a/client/vite.config.ts +++ b/client/vite.config.ts @@ -19,6 +19,7 @@ export default defineConfig({ resolve: { alias: { _common: resolve(__dirname, 'src/components/_common'), + _custom: resolve(__dirname, 'src/components/_custom'), hooks: resolve(__dirname, 'src/hooks'), utils: resolve(__dirname, 'src/utils'), styles: resolve(__dirname, 'src/styles'), From c16337df38a5249f870ca41343358a346e2932f2 Mon Sep 17 00:00:00 2001 From: Shayan Khan Date: Fri, 8 Nov 2024 13:39:08 -0600 Subject: [PATCH 123/328] DRP Publications and fixes (#1002) * Publication flow * Refactored code, improved request pub flow * remove incorrect return statement * added versioning support + bug fixes * front end linting * bug fixes with trash, prevent copying of trashed files * settings changes, review system only visible to reviewers * refactor to use project tree to get samples --- client/src/components/DataFiles/DataFiles.jsx | 12 + .../DataFilesPublicationsList.jsx | 122 +++++++++ .../DataFilesPublicationsList.module.scss | 17 ++ .../DataFilesPublicationsList.scss | 22 ++ .../DataFilesSidebar/DataFilesSidebar.jsx | 18 +- ...esProjectFileListingMetadataTitleAddon.jsx | 31 +-- .../DataFilesProjectPublish.jsx | 4 +- .../ProjectDescription.jsx | 4 + .../SubmitPublicationRequest.jsx | 21 +- .../SubmitPublicationReview.jsx | 105 +++++--- .../DataFilesProjectReview.jsx | 27 +- client/src/redux/reducers/index.js | 2 + .../redux/reducers/publications.reducers.js | 143 ++++++++++ client/src/redux/sagas/index.js | 2 + client/src/redux/sagas/publications.sagas.js | 202 ++++++++++++++ server/portal/apps/__init__.py | 5 +- server/portal/apps/_custom/drp/constants.py | 4 +- server/portal/apps/_custom/drp/models.py | 7 +- server/portal/apps/_custom/drp/views.py | 29 ++- server/portal/apps/projects/tasks.py | 2 + server/portal/apps/projects/views.py | 9 +- .../workspace_operations/graph_operations.py | 18 +- .../project_meta_operations.py | 6 +- .../project_publish_operations.py | 246 ++++++++++++++++++ .../shared_workspace_operations.py | 78 ++++-- .../migrations/0003_publication.py | 27 ++ server/portal/apps/publications/models.py | 22 +- server/portal/apps/publications/urls.py | 4 + server/portal/apps/publications/views.py | 201 ++++++++++++-- server/portal/apps/users/views.py | 7 + server/portal/libs/agave/operations.py | 3 +- server/portal/settings/settings.py | 12 + server/portal/settings/settings_default.py | 22 +- 33 files changed, 1299 insertions(+), 135 deletions(-) create mode 100644 client/src/components/DataFiles/DataFilesPublicationsList/DataFilesPublicationsList.jsx create mode 100644 client/src/components/DataFiles/DataFilesPublicationsList/DataFilesPublicationsList.module.scss create mode 100644 client/src/components/DataFiles/DataFilesPublicationsList/DataFilesPublicationsList.scss create mode 100644 client/src/redux/reducers/publications.reducers.js create mode 100644 client/src/redux/sagas/publications.sagas.js create mode 100644 server/portal/apps/publications/migrations/0003_publication.py diff --git a/client/src/components/DataFiles/DataFiles.jsx b/client/src/components/DataFiles/DataFiles.jsx index 6a57664220..e2c4404252 100644 --- a/client/src/components/DataFiles/DataFiles.jsx +++ b/client/src/components/DataFiles/DataFiles.jsx @@ -24,6 +24,7 @@ import DataFilesModals from './DataFilesModals/DataFilesModals'; import DataFilesProjectsList from './DataFilesProjectsList/DataFilesProjectsList'; import DataFilesProjectFileListing from './DataFilesProjectFileListing/DataFilesProjectFileListing'; import { useSystemRole } from './DataFilesProjectMembers/_cells/SystemRoleSelector'; +import DataFilesPublicationsList from './DataFilesPublicationsList/DataFilesPublicationsList'; const DefaultSystemRedirect = () => { const systems = useSelector( @@ -56,6 +57,11 @@ const DataFilesSwitch = React.memo(() => { const { DataFilesProjectPublish, DataFilesProjectReview } = useAddonComponents({ portalName }); + const systems = useSelector( + (state) => state.systems.storage.configuration.filter((s) => !s.hidden), + shallowEqual + ); + return ( {DataFilesProjectPublish && ( @@ -92,6 +98,12 @@ const DataFilesSwitch = React.memo(() => { exact path={`${path}/tapis/projects/:system`} render={({ match: { params } }) => { + const system = systems.find((s) => s.system === params.system); + + if (system.publicationProject) { + return ; + } + return ; }} /> diff --git a/client/src/components/DataFiles/DataFilesPublicationsList/DataFilesPublicationsList.jsx b/client/src/components/DataFiles/DataFilesPublicationsList/DataFilesPublicationsList.jsx new file mode 100644 index 0000000000..a7844bf4ed --- /dev/null +++ b/client/src/components/DataFiles/DataFilesPublicationsList/DataFilesPublicationsList.jsx @@ -0,0 +1,122 @@ +import React, { useEffect, useCallback } from 'react'; +import PropTypes from 'prop-types'; +import { Link, useLocation } from 'react-router-dom'; +import { useSelector, useDispatch, shallowEqual } from 'react-redux'; +import queryStringParser from 'query-string'; +import { + InfiniteScrollTable, + SectionMessage, + SectionTableWrapper, +} from '_common'; +import styles from './DataFilesPublicationsList.module.scss'; +import './DataFilesPublicationsList.scss'; +import Searchbar from '_common/Searchbar'; +import { formatDate, formatDateTimeFromValue } from 'utils/timeFormat'; + +const DataFilesPublicationsList = ({ rootSystem }) => { + const { error, loading, publications } = useSelector( + (state) => state.publications.listing + ); + + const query = queryStringParser.parse(useLocation().search); + + const systems = useSelector( + (state) => state.systems.storage.configuration.filter((s) => !s.hidden), + shallowEqual + ); + + const selectedSystem = systems.find( + (s) => s.scheme === 'projects' && s.publicationProject === true + ); + + const infiniteScrollCallback = useCallback(() => {}); + const dispatch = useDispatch(); + + useEffect(() => { + dispatch({ + type: 'PUBLICATIONS_GET_PUBLICATIONS', + payload: { + queryString: query.query_string, + }, + }); + }, [dispatch, query.query_string]); + + const columns = [ + { + Header: 'Publication Title', + accessor: 'title', + Cell: (el) => ( + + {el.value} + + ), + }, + { + Header: 'Principal Investigator', + accessor: 'authors', + Cell: (el) => ( + + {el.value.length > 0 + ? `${el.value[0].first_name} ${el.value[0].last_name}` + : ''} + + ), + }, + { + Header: 'Keywords', + accessor: 'keywords', + }, + { + Header: 'Publication Date', + accessor: 'publication_date', + Cell: (el) => ( + {el.value ? formatDate(new Date(el.value)) : ''} + ), + }, + ]; + + const noDataText = query.query_string + ? `No Publications match your search term.` + : `No Publications available.`; + + if (error) { + return ( +
+ + There was a problem retrieving Publications. + +
+ ); + } + + return ( + + +
+ +
+ + ); +}; + +export default DataFilesPublicationsList; diff --git a/client/src/components/DataFiles/DataFilesPublicationsList/DataFilesPublicationsList.module.scss b/client/src/components/DataFiles/DataFilesPublicationsList/DataFilesPublicationsList.module.scss new file mode 100644 index 0000000000..28e23efaf6 --- /dev/null +++ b/client/src/components/DataFiles/DataFilesPublicationsList/DataFilesPublicationsList.module.scss @@ -0,0 +1,17 @@ +.root { + /* As a flex child */ + flex-grow: 1; + + /* WARNING: Mimicked on: History, Allocation, DataFiles, DataFilesProjectsList, DataFilesProjectFileListing, PublicData */ + padding-top: 1.75rem; /* ~28px (22.5px * design * 1.2 design-to-app ratio) */ + padding-left: 1.5em; /* ~24px (20px * design * 1.2 design-to-app ratio) */ +} + +/* NOTE: Mimicked on: DataFiles, DataFilesProjectsList, DataFilesProjectFileListing */ +.root-placeholder { + flex-grow: 1; + + display: flex; + align-items: center; + justify-content: center; +} diff --git a/client/src/components/DataFiles/DataFilesPublicationsList/DataFilesPublicationsList.scss b/client/src/components/DataFiles/DataFilesPublicationsList/DataFilesPublicationsList.scss new file mode 100644 index 0000000000..a87507046a --- /dev/null +++ b/client/src/components/DataFiles/DataFilesPublicationsList/DataFilesPublicationsList.scss @@ -0,0 +1,22 @@ +.publications-listing { + /* title */ + th:nth-child(1), + td:nth-child(1) { + width: 40%; + } + /* authors */ + th:nth-child(2), + td:nth-child(2) { + width: 20%; + } + /* keywords */ + th:nth-child(3), + td:nth-child(3) { + width: 25%; + } + /* date */ + th:nth-child(4), + td:nth-child(4) { + width: 15%; + } +} diff --git a/client/src/components/DataFiles/DataFilesSidebar/DataFilesSidebar.jsx b/client/src/components/DataFiles/DataFilesSidebar/DataFilesSidebar.jsx index a6123b3bd4..fe9903d995 100644 --- a/client/src/components/DataFiles/DataFilesSidebar/DataFilesSidebar.jsx +++ b/client/src/components/DataFiles/DataFilesSidebar/DataFilesSidebar.jsx @@ -112,19 +112,23 @@ const DataFilesSidebar = ({ readOnly }) => { shallowEqual ); + const user = useSelector((state) => state.authenticatedUser.user); + const match = useRouteMatch(); var sidebarItems = []; systems.forEach((sys) => { if (sys.scheme === 'projects') { - sidebarItems.push({ - to: `${match.path}/${sys.api}/${sys.scheme}/${sys.system}`, - label: sys.name, - iconName: sys.icon || 'my-data', - disabled: false, - hidden: false, - }); + if (!sys.reviewProject || user.groups?.includes('PROJECT_REVIEWER')) { + sidebarItems.push({ + to: `${match.path}/${sys.api}/${sys.scheme}/${sys.system}`, + label: sys.name, + iconName: sys.icon || 'my-data', + disabled: false, + hidden: false, + }); + } } else { sidebarItems.push({ to: `${match.path}/${sys.api}/${sys.scheme}/${ diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx index 6bcb4c6320..a1acd41c65 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx @@ -19,21 +19,22 @@ const DataFilesProjectFileListingMetadataTitleAddon = ({ const { loading } = useFileListing('FilesListing'); - const { canEditDataset } = useSelector( - (state) => - state.projects.metadata.members - .filter((member) => - member.user - ? member.user.username === state.authenticatedUser.user.username - : { access: null } - ) - .map((currentUser) => { - return { - canEditDataset: - currentUser.access === 'owner' || currentUser.access === 'edit', - }; - })[0] - ); + const { canEditDataset } = useSelector((state) => { + const userAccess = state.projects.metadata.members + .filter((member) => + member.user + ? member.user.username === state.authenticatedUser.user.username + : { access: null } + ) + .map((currentUser) => { + return { + canEditDataset: + currentUser.access === 'owner' || currentUser.access === 'edit', + }; + })[0]; + + return userAccess || { canEditDataset: false }; + }); const { createSampleModal, createOriginDataModal, createAnalysisDataModal } = useDrpDatasetModals(projectId, portalName); diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx index 058cf1fa3d..25dc6d63ef 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx @@ -82,7 +82,9 @@ const DataFilesProjectPublish = ({ rootSystem, system }) => { project: metadata, onAuthorsUpdate: handleAuthorsUpdate, }), - SubmitPublicationRequestStep(), + SubmitPublicationRequestStep({ + callbackUrl: `${ROUTES.WORKBENCH}${ROUTES.DATA}/tapis/projects/${rootSystem}/${system}`, + }), ]; const formSubmit = (values) => { diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx index 836e2a48ad..e314d0a3b9 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx @@ -48,6 +48,10 @@ const ProjectDescription = ({ project }) => { projectData['Keywords'] = project.keywords; } + if (project.doi) { + projectData['DOI'] = project.doi; + } + if (project.related_publications?.length > 0) { const relatedPublicationCards = project.related_publications.map( (publication) => { diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/SubmitPublicationRequest.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/SubmitPublicationRequest.jsx index 57cba02d71..3ccff7617a 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/SubmitPublicationRequest.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/SubmitPublicationRequest.jsx @@ -5,18 +5,21 @@ import { SectionTableWrapper, Section, Button } from '_common'; import * as Yup from 'yup'; import styles from './DataFilesProjectPublishWizard.module.scss'; import { useSelector } from 'react-redux'; +import { useHistory } from 'react-router-dom'; const validationSchema = Yup.object({ reviewInfo: Yup.boolean().oneOf([true], 'Must be checked'), reviewRelatedPublications: Yup.boolean().oneOf([true], 'Must be checked'), }); -const SubmitPublicationRequest = () => { - const { handleChange, handleBlur, values, submitForm } = useFormikContext(); +const SubmitPublicationRequest = ({ callbackUrl }) => { + const { handleChange, handleBlur, values, submitForm, resetForm } = + useFormikContext(); + const history = useHistory(); const [submitDisabled, setSubmitDisabled] = useState(true); - const { loading, error } = useSelector((state) => { + const { loading, error, result } = useSelector((state) => { if ( state.projects.operation && state.projects.operation.name === 'publicationRequest' @@ -29,6 +32,14 @@ const SubmitPublicationRequest = () => { }; }); + useEffect(() => { + if (result && !error && !loading) { + setSubmitDisabled(false); + resetForm(); + history.replace(callbackUrl); + } + }, [result, error, loading]); + useEffect(() => { validationSchema.isValid(values).then((valid) => { setSubmitDisabled(!valid); @@ -92,10 +103,10 @@ const SubmitPublicationRequest = () => { ); }; -export const SubmitPublicationRequestStep = () => ({ +export const SubmitPublicationRequestStep = ({ callbackUrl }) => ({ id: 'submit_publication_request', name: 'Submit Publication Request', - render: , + render: , initialValues: { reviewInfo: false, reviewRelatedPublications: false, diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/SubmitPublicationReview.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/SubmitPublicationReview.jsx index 624aa0741f..f17b286aca 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/SubmitPublicationReview.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/SubmitPublicationReview.jsx @@ -1,39 +1,62 @@ import React, { useEffect, useState } from 'react'; -import { FormGroup, Input } from 'reactstrap'; import { useFormikContext } from 'formik'; import { SectionTableWrapper, Section, Button } from '_common'; import * as Yup from 'yup'; import styles from './DataFilesProjectPublishWizard.module.scss'; -import { useSelector } from 'react-redux'; +import { useDispatch, useSelector } from 'react-redux'; +import { useHistory } from 'react-router-dom'; -const validationSchema = Yup.object({ - reviewInfo: Yup.boolean().oneOf([true], 'Must be checked'), - reviewRelatedPublications: Yup.boolean().oneOf([true], 'Must be checked'), -}); +const validationSchema = Yup.object({}); -const SubmitPublicationReview = () => { - const { handleChange, handleBlur, values, submitForm } = useFormikContext(); +const SubmitPublicationReview = ({ callbackUrl }) => { + const { submitForm, setFieldValue, resetForm } = useFormikContext(); - const [submitDisabled, setSubmitDisabled] = useState(true); + const { doi } = useSelector((state) => state.projects.metadata); - const { loading, error } = useSelector((state) => { - if ( - state.projects.operation && - state.projects.operation.name === 'publicationRequest' - ) { - return state.projects.operation; - } + const history = useHistory(); + + const [submitDisabled, setSubmitDisabled] = useState(false); + + const { + isApproveLoading, + isRejectLoading, + isApproveSuccess, + isRejectSuccess, + } = useSelector((state) => { + const { name, loading, error, result } = state.publications.operation; return { - loading: false, - error: false, + isApproveLoading: name === 'approve' && loading, + isRejectLoading: name === 'reject' && loading, + isApproveSuccess: name === 'approve' && !loading && !error && result, + isRejectSuccess: name === 'reject' && !loading && !error && result, }; }); useEffect(() => { - validationSchema.isValid(values).then((valid) => { - setSubmitDisabled(!valid); - }); - }, [values]); + if (isApproveSuccess || isRejectSuccess) { + setSubmitDisabled(false); + resetForm(); + history.replace(callbackUrl); + } + }, [isApproveSuccess, isRejectSuccess]); + + const handleApproveAndPublish = () => { + setFieldValue('publicationApproved', true); + setSubmitDisabled(true); + submitForm(); + }; + + const handleReject = () => { + setFieldValue('publicationRejected', true); + setSubmitDisabled(true); + submitForm(); + }; + + const handleVersioning = () => { + setFieldValue('versionApproved', true); + setSubmitDisabled(true); + submitForm(); + }; return ( { curator Maria Esteva before submitting the data for publication.
- + {doi ? ( + + ) : ( + + )} @@ -69,10 +104,10 @@ const SubmitPublicationReview = () => { ); }; -export const SubmitPublicationReviewStep = () => ({ +export const SubmitPublicationReviewStep = ({ callbackUrl }) => ({ id: 'submit_publication_review', name: 'Submit Publication Review', - render: , + render: , initialValues: {}, validationSchema, }); diff --git a/client/src/components/_custom/drp/DataFilesProjectReview/DataFilesProjectReview.jsx b/client/src/components/_custom/drp/DataFilesProjectReview/DataFilesProjectReview.jsx index 9918fd435b..6c660720a2 100644 --- a/client/src/components/_custom/drp/DataFilesProjectReview/DataFilesProjectReview.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectReview/DataFilesProjectReview.jsx @@ -51,10 +51,33 @@ const DataFilesProjectReview = ({ rootSystem, system }) => { ProjectDescriptionStep({ project: metadata }), ReviewProjectStructureStep({ projectTree: tree }), ReviewAuthorsStep({ project: metadata, onAuthorsUpdate: () => {} }), - SubmitPublicationReviewStep(), + SubmitPublicationReviewStep({ + callbackUrl: `${ROUTES.WORKBENCH}${ROUTES.DATA}/tapis/projects/${rootSystem}`, + }), ]; - const formSubmit = (values) => {}; + const formSubmit = (values) => { + const data = { + ...metadata, + }; + + if (values && values.publicationApproved) { + dispatch({ + type: 'PUBLICATIONS_APPROVE_PUBLICATION', + payload: data, + }); + } else if (values && values.publicationRejected) { + dispatch({ + type: 'PUBLICATIONS_REJECT_PUBLICATION', + payload: data, + }); + } else if (values && values.versionApproved) { + dispatch({ + type: 'PUBLICATIONS_APPROVE_VERSION', + payload: data, + }); + } + }; return ( <> diff --git a/client/src/redux/reducers/index.js b/client/src/redux/reducers/index.js index 62bb5ae483..82514359a1 100644 --- a/client/src/redux/reducers/index.js +++ b/client/src/redux/reducers/index.js @@ -25,6 +25,7 @@ import { onboarding } from './onboarding.reducers'; import projects from './projects.reducers'; import { users } from './users.reducers'; import siteSearch from './siteSearch.reducers'; +import publications from './publications.reducers'; export default combineReducers({ jobs, @@ -53,4 +54,5 @@ export default combineReducers({ projects, users, siteSearch, + publications, }); diff --git a/client/src/redux/reducers/publications.reducers.js b/client/src/redux/reducers/publications.reducers.js new file mode 100644 index 0000000000..6588f6e090 --- /dev/null +++ b/client/src/redux/reducers/publications.reducers.js @@ -0,0 +1,143 @@ +export const initialState = { + listing: { + publications: [], + error: null, + loading: false, + }, + operation: { + name: '', + loading: false, + error: null, + result: null, + }, +}; + +export default function publications(state = initialState, action) { + switch (action.type) { + case 'PUBLICATIONS_GET_PUBLICATIONS_STARTED': + return { + ...state, + listing: { + ...state.listing, + publications: [], + error: null, + loading: true, + }, + }; + case 'PUBLICATIONS_GET_PUBLICATIONS_SUCCESS': + return { + ...state, + listing: { + publications: action.payload, + error: null, + loading: false, + }, + }; + case 'PUBLICATIONS_GET_PUBLICATIONS_FAILED': + return { + ...state, + listing: { + ...state.listing, + error: action.payload, + loading: false, + }, + }; + case 'PUBLICATIONS_APPROVE_PUBLICATION_STARTED': + return { + ...state, + operation: { + name: 'approve', + loading: true, + error: null, + result: null, + }, + }; + case 'PUBLICATIONS_APPROVE_PUBLICATION_SUCCESS': + return { + ...state, + operation: { + name: 'approve', + loading: false, + error: null, + result: action.payload, + }, + }; + case 'PUBLICATIONS_APPROVE_PUBLICATION_FAILED': + return { + ...state, + operation: { + name: 'approve', + loading: false, + error: action.payload, + result: null, + }, + }; + case 'PUBLICATIONS_REJECT_PUBLICATION_STARTED': + return { + ...state, + operation: { + name: 'reject', + loading: true, + error: null, + result: null, + }, + }; + case 'PUBLICATIONS_REJECT_PUBLICATION_SUCCESS': + return { + ...state, + operation: { + name: 'reject', + loading: false, + error: null, + result: action.payload, + }, + }; + case 'PUBLICATIONS_REJECT_PUBLICATION_FAILED': + return { + ...state, + operation: { + name: 'reject', + loading: false, + error: action.payload, + result: null, + }, + }; + case 'PUBLICATIONS_APPROVE_VERSION_STARTED': + return { + ...state, + operation: { + name: 'approve', + loading: true, + error: null, + result: null, + }, + }; + case 'PUBLICATIONS_APPROVE_VERSION_SUCCESS': + return { + ...state, + operation: { + name: 'approve', + loading: false, + error: null, + result: action.payload, + }, + }; + case 'PUBLICATIONS_APPROVE_VERSION_FAILED': + return { + ...state, + operation: { + name: 'approve', + loading: false, + error: action.payload, + result: null, + }, + }; + case 'PUBLICATIONS_OPERATION_RESET': + return { + ...state, + operation: initialState.operation, + }; + default: + return state; + } +} diff --git a/client/src/redux/sagas/index.js b/client/src/redux/sagas/index.js index 928da69c95..288de701d5 100644 --- a/client/src/redux/sagas/index.js +++ b/client/src/redux/sagas/index.js @@ -52,6 +52,7 @@ import { import { watchProjects } from './projects.sagas'; import { watchUsers } from './users.sagas'; import { watchSiteSearch } from './siteSearch.sagas'; +import { watchPublications } from './publications.sagas'; function* watchStartCustomSaga() { yield takeEvery('START_CUSTOM_SAGA', startCustomSaga); @@ -121,5 +122,6 @@ export default function* rootSaga() { watchUsers(), watchSiteSearch(), watchStartCustomSaga(), + watchPublications(), ]); } diff --git a/client/src/redux/sagas/publications.sagas.js b/client/src/redux/sagas/publications.sagas.js new file mode 100644 index 0000000000..e4f32db23e --- /dev/null +++ b/client/src/redux/sagas/publications.sagas.js @@ -0,0 +1,202 @@ +import { fetchUtil } from 'utils/fetchUtil'; +import { put, takeLatest, call } from 'redux-saga/effects'; +import queryStringParser from 'query-string'; + +export async function createPublicationRequestUtil(data) { + const result = await fetchUtil({ + url: `/api/publications/publication-request/`, + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(data), + }); + return result.response; +} + +export function* createPublicationRequest(action) { + yield put({ + type: 'PUBLICATIONS_CREATE_PUBLICATION_REQUEST_STARTED', + }); + try { + const result = yield call(createPublicationRequestUtil, action.payload); + yield put({ + type: 'PUBLICATIONS_CREATE_PUBLICATION_REQUEST_SUCCESS', + payload: result, + }); + } catch (error) { + yield put({ + type: 'PUBLICATIONS_CREATE_PUBLICATION_REQUEST_FAILED', + payload: error, + }); + } +} + +export async function fetchPublicationRequestsUtil(system) { + const result = await fetchUtil({ + url: `/api/publications/publication-request/${system}`, + }); + return result.response; +} + +export function* getPublicationRequests(action) { + yield put({ + type: 'PUBLICATIONS_GET_PUBLICATION_REQUESTS_STARTED', + }); + try { + const publicationRequests = yield call( + fetchPublicationRequestsUtil, + action.payload + ); + yield put({ + type: 'PUBLICATIONS_GET_PUBLICATION_REQUESTS_SUCCESS', + payload: publicationRequests, + }); + } catch (error) { + yield put({ + type: 'PUBLICATIONS_GET_PUBLICATION_REQUESTS_FAILED', + payload: error, + }); + } +} + +export async function approvePublicationUtil(data) { + const result = await fetchUtil({ + url: `/api/publications/publish/`, + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(data), + }); + return result.response; +} + +export function* approvePublication(action) { + yield put({ + type: 'PUBLICATIONS_APPROVE_PUBLICATION_STARTED', + }); + try { + const result = yield call(approvePublicationUtil, action.payload); + yield put({ + type: 'PUBLICATIONS_APPROVE_PUBLICATION_SUCCESS', + payload: result, + }); + } catch (error) { + yield put({ + type: 'PUBLICATIONS_APPROVE_PUBLICATION_FAILED', + payload: error, + }); + } finally { + yield put({ type: 'PUBLICATIONS_OPERATION_RESET' }); + } +} + +export async function rejectPublicationUtil(data) { + const result = await fetchUtil({ + url: `/api/publications/reject/`, + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(data), + }); + return result.response; +} + +export function* rejectPublication(action) { + yield put({ + type: 'PUBLICATIONS_REJECT_PUBLICATION_STARTED', + }); + try { + const result = yield call(rejectPublicationUtil, action.payload); + yield put({ + type: 'PUBLICATIONS_REJECT_PUBLICATION_SUCCESS', + payload: result, + }); + } catch (error) { + yield put({ + type: 'PUBLICATIONS_REJECT_PUBLICATION_FAILED', + payload: error, + }); + } finally { + yield put({ type: 'PUBLICATIONS_OPERATION_RESET' }); + } +} + +export async function fetchPublicationsUtil(queryString) { + const q = queryStringParser.stringify({ query_string: queryString }); + + const result = await fetchUtil({ + url: queryString ? `/api/publications?${q}` : '/api/publications', + }); + return result.response; +} + +export function* getPublications(action) { + yield put({ + type: 'PUBLICATIONS_GET_PUBLICATIONS_STARTED', + }); + try { + const result = yield call( + fetchPublicationsUtil, + action.payload.queryString + ); + yield put({ + type: 'PUBLICATIONS_GET_PUBLICATIONS_SUCCESS', + payload: result, + }); + } catch (error) { + yield put({ + type: 'PUBLICATIONS_GET_PUBLICATIONS_FAILED', + payload: error, + }); + } +} + +export async function versionPublicationUtil(data) { + const result = await fetchUtil({ + url: `/api/publications/version/`, + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(data), + }); + return result.response; +} + +export function* versionPublication(action) { + yield put({ + type: 'PUBLICATIONS_APPROVE_VERSION_STARTED', + }); + try { + const result = yield call(versionPublicationUtil, action.payload); + yield put({ + type: 'PUBLICATIONS_APPROVE_VERSION_SUCCESS', + payload: result, + }); + } catch (error) { + yield put({ + type: 'PUBLICATIONS_APPROVE_VERSION_FAILED', + payload: error, + }); + } finally { + yield put({ type: 'PUBLICATIONS_OPERATION_RESET' }); + } +} + +export function* watchPublications() { + yield takeLatest('PUBLICATIONS_GET_PUBLICATIONS', getPublications); + yield takeLatest('PUBLICATIONS_APPROVE_PUBLICATION', approvePublication); + yield takeLatest('PUBLICATIONS_REJECT_PUBLICATION', rejectPublication); + yield takeLatest('PUBLICATIONS_APPROVE_VERSION', versionPublication); + yield takeLatest( + 'PUBLICATIONS_CREATE_PUBLICATION_REQUEST', + createPublicationRequest + ); + yield takeLatest( + 'PUBLICATIONS_GET_PUBLICATION_REQUESTS', + getPublicationRequests + ); +} diff --git a/server/portal/apps/__init__.py b/server/portal/apps/__init__.py index 36e0a49af2..862ffb0044 100644 --- a/server/portal/apps/__init__.py +++ b/server/portal/apps/__init__.py @@ -1,5 +1,5 @@ -from portal.apps._custom.drp.models import DrpProjectMetadata, DrpSampleMetadata, DrpOriginDatasetMetadata, DrpAnalysisDatasetMetadata, DrpFileMetadata +from portal.apps._custom.drp.models import DrpProjectMetadata, DrpSampleMetadata, DrpOriginDatasetMetadata, DrpAnalysisDatasetMetadata, DrpFileMetadata, PartialTrashEntity from portal.apps._custom.drp import constants SCHEMA_MAPPING = { @@ -8,5 +8,6 @@ constants.ORIGIN_DATA: DrpOriginDatasetMetadata, constants.DIGITAL_DATASET: DrpOriginDatasetMetadata, constants.ANALYSIS_DATA: DrpAnalysisDatasetMetadata, - constants.FILE: DrpFileMetadata + constants.FILE: DrpFileMetadata, + constants.TRASH: PartialTrashEntity, } \ No newline at end of file diff --git a/server/portal/apps/_custom/drp/constants.py b/server/portal/apps/_custom/drp/constants.py index 481aae583d..8a96600d40 100644 --- a/server/portal/apps/_custom/drp/constants.py +++ b/server/portal/apps/_custom/drp/constants.py @@ -9,4 +9,6 @@ ANALYSIS_DATA = "drp.project.analysis_dataset" ORIGIN_DATA = "drp.project.origin_data" -FILE = "drp.project.file" \ No newline at end of file +FILE = "drp.project.file" + +TRASH = "drp.project.trash" \ No newline at end of file diff --git a/server/portal/apps/_custom/drp/models.py b/server/portal/apps/_custom/drp/models.py index 8f6b599bef..99662cc6e3 100644 --- a/server/portal/apps/_custom/drp/models.py +++ b/server/portal/apps/_custom/drp/models.py @@ -54,6 +54,10 @@ class FileObj(DrpMetadataModel): uuid: Optional[str] = None value: Optional[DrpFileMetadata] = None +class PartialTrashEntity(DrpMetadataModel): + """Model for representing a trash entity.""" + + model_config = ConfigDict(extra="ignore") class PartialEntityWithFiles(DrpMetadataModel): """Model for representing an entity with associated files.""" @@ -116,7 +120,8 @@ class DrpProjectMetadata(DrpMetadataModel): publication_date: Optional[str] = None authors: list[dict] = [] file_objs: list[FileObj] = [] - is_review_project : Optional[bool] = None + is_review_project: Optional[bool] = None + is_published_project: Optional[bool] = None class DrpDatasetMetadata(DrpMetadataModel): """Model for Base DRP Dataset Metadata""" diff --git a/server/portal/apps/_custom/drp/views.py b/server/portal/apps/_custom/drp/views.py index 600dbbd312..0da3c9e86b 100644 --- a/server/portal/apps/_custom/drp/views.py +++ b/server/portal/apps/_custom/drp/views.py @@ -8,6 +8,7 @@ import networkx as nx from networkx import shortest_path from portal.apps.projects.workspace_operations.project_meta_operations import get_ordered_value +from portal.apps.projects.workspace_operations.graph_operations import remove_trash_nodes class DigitalRocksSampleView(BaseApiView): def get(self, request): @@ -16,7 +17,20 @@ def get(self, request): full_project_id = f'{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}' - samples = ProjectMetadata.objects.filter(base_project__value__projectId=full_project_id, name=constants.SAMPLE).values('uuid', 'name', 'value') + graph_model = ProjectMetadata.objects.get( + name=constants.PROJECT_GRAPH, base_project__value__projectId=full_project_id + ) + + project_graph = nx.node_link_graph(graph_model.value) + + sample_uuids = [] + + for node_id in list(project_graph.successors('NODE_ROOT')): + node = project_graph.nodes[node_id] + if (node.get('name') == constants.SAMPLE): + sample_uuids.append(node.get('uuid')) + + samples = ProjectMetadata.objects.filter(uuid__in=sample_uuids).values('uuid', 'name', 'value') origin_data = [] @@ -47,18 +61,7 @@ def get(self, request): graph = nx.node_link_graph(graph_model.value) - trash_node_id = None - - for node_id in graph.nodes: - trash_node = graph.nodes[node_id].get('name') == settings.TAPIS_DEFAULT_TRASH_NAME - if trash_node: - trash_node_id = node_id - break - - if trash_node_id: - trash_descendants = nx.descendants(graph, trash_node_id) - nodes_to_remove = {trash_node_id} | trash_descendants - graph.remove_nodes_from(nodes_to_remove) + graph = remove_trash_nodes(graph) for node_id in graph.nodes: diff --git a/server/portal/apps/projects/tasks.py b/server/portal/apps/projects/tasks.py index 2f5d0af129..379b4bc128 100644 --- a/server/portal/apps/projects/tasks.py +++ b/server/portal/apps/projects/tasks.py @@ -16,6 +16,8 @@ import networkx as nx import uuid +# TODO: Cleanup this file + logger = logging.getLogger(__name__) def _transfer_files(user_access_token, source_system_id, review_system_id): diff --git a/server/portal/apps/projects/views.py b/server/portal/apps/projects/views.py index e4185c32e4..83d6473d18 100644 --- a/server/portal/apps/projects/views.py +++ b/server/portal/apps/projects/views.py @@ -224,13 +224,12 @@ def patch( project_id_full = f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}" client = request.user.tapis_oauth.client - if metadata is not None: - patch_project_entity(project_id_full, metadata) - workspace_def = update_project(client, project_id, data['title'], data['description']) - if metadata is not None: - workspace_def.update(metadata) + if metadata is not None: + entity = patch_project_entity(project_id_full, metadata) + workspace_def.update(get_ordered_value(entity.name, entity.value)) + workspace_def["projectId"] = project_id return JsonResponse( { diff --git a/server/portal/apps/projects/workspace_operations/graph_operations.py b/server/portal/apps/projects/workspace_operations/graph_operations.py index e1df18c823..8c3ae4d96d 100644 --- a/server/portal/apps/projects/workspace_operations/graph_operations.py +++ b/server/portal/apps/projects/workspace_operations/graph_operations.py @@ -3,7 +3,7 @@ from django.db import transaction import uuid import copy - +from django.conf import settings from portal.apps._custom.drp import constants from portal.apps.projects.models.project_metadata import ProjectMetadata @@ -168,4 +168,18 @@ def get_node_from_uuid(project_id: str, uuid: str): for node_id in project_graph.nodes: if project_graph.nodes[node_id]["uuid"] == uuid: return {"id": node_id, **project_graph.nodes[node_id]} - return None \ No newline at end of file + return None + +def remove_trash_nodes(graph: nx.DiGraph): + trash_node_id = None + for node_id in graph.nodes: + trash_node = graph.nodes[node_id].get("name") == constants.TRASH + if trash_node: + trash_node_id = node_id + break + + if trash_node_id: + trash_descendants = nx.descendants(graph, trash_node_id) + nodes_to_remove = {trash_node_id} | trash_descendants + graph.remove_nodes_from(nodes_to_remove) + return graph \ No newline at end of file diff --git a/server/portal/apps/projects/workspace_operations/project_meta_operations.py b/server/portal/apps/projects/workspace_operations/project_meta_operations.py index a6cea1d2f0..0429b67899 100644 --- a/server/portal/apps/projects/workspace_operations/project_meta_operations.py +++ b/server/portal/apps/projects/workspace_operations/project_meta_operations.py @@ -155,7 +155,11 @@ def patch_project_entity(project_id, value): entity = ProjectMetadata.get_project_by_id(project_id) schema_model = SCHEMA_MAPPING[entity.name] - patched_metadata = {**value, 'projectId': project_id, 'fileObjs': entity.value.get('fileObjs', [])} + patched_metadata = {**value, + 'projectId': project_id, + 'fileObjs': entity.value.get('fileObjs', []), + 'doi': entity.value.get('doi', None) + } update_node_in_project(project_id, 'NODE_ROOT', None, value.get('title')) diff --git a/server/portal/apps/projects/workspace_operations/project_publish_operations.py b/server/portal/apps/projects/workspace_operations/project_publish_operations.py index e69de29bb2..481ea9f16e 100644 --- a/server/portal/apps/projects/workspace_operations/project_publish_operations.py +++ b/server/portal/apps/projects/workspace_operations/project_publish_operations.py @@ -0,0 +1,246 @@ +from typing import Optional +from django.conf import settings +import logging +from portal.apps.projects.workspace_operations.shared_workspace_operations import remove_user +from portal.apps.projects.models.project_metadata import ProjectMetadata +import networkx as nx +from celery import shared_task +from portal.apps._custom.drp import constants +from portal.libs.agave.utils import user_account, service_account +from portal.apps.publications.models import Publication, PublicationRequest +from django.db import transaction +from portal.apps.projects.workspace_operations.graph_operations import remove_trash_nodes +from tapipy.errors import NotFoundError, BaseTapyException +from django.contrib.auth import get_user_model +from django.core.exceptions import ObjectDoesNotExist + +logger = logging.getLogger(__name__) + +def _transfer_files(client, source_system_id, dest_system_id): + + service_client = service_account() + + source_system_files = client.files.listFiles(systemId=source_system_id, path='/') + + # Filter out the trash folder + filtered_files = [file for file in source_system_files if file.name != settings.TAPIS_DEFAULT_TRASH_NAME] + + transfer_elements = [ + { + 'sourceURI': file.url, + 'destinationURI': f'tapis://{dest_system_id}/{file.path}' + } + for file in filtered_files + ] + + transfer = service_client.files.createTransferTask(elements=transfer_elements) + return transfer + +def _check_transfer_status(service_client, transfer_task_id): + transfer_details = service_client.files.getTransferTask(transferTaskId=transfer_task_id) + return transfer_details.status + +def _add_values_to_tree(project_id): + project_meta = ProjectMetadata.get_project_by_id(project_id) + prj_entities = ProjectMetadata.get_entities_by_project_id(project_id) + + entity_map = {entity.uuid: entity for entity in prj_entities} + + publication_tree: nx.DiGraph = nx.node_link_graph(project_meta.project_graph.value) + + publication_tree = remove_trash_nodes(publication_tree) + + for node_id in publication_tree: + uuid = publication_tree.nodes[node_id]["uuid"] + if uuid is not None: + publication_tree.nodes[node_id]["value"] = entity_map[uuid].value + publication_tree.nodes[node_id]["uuid"] = None # Clear the uuid field + + return publication_tree + +def publish_project_callback(review_project_id, published_project_id): + service_client = service_account() + update_and_cleanup_review_project(review_project_id, PublicationRequest.Status.APPROVED) + + # Make system public for listing + service_client.systems.shareSystemPublic(systemId=published_project_id) + +def publication_request_callback(user_access_token, source_workspace_id, review_workspace_id, source_system_id, review_system_id): + service_client = service_account() + user_client = user_account(user_access_token) + portal_admin_username = settings.PORTAL_ADMIN_USERNAME + + publication_reviewers = get_user_model().objects.filter(groups__name=settings.PORTAL_PUBLICATION_REVIEWERS_GROUP_NAME).values_list('username', flat=True) + + with transaction.atomic(): + # Remove admin from source workspace + user_client.systems.unShareSystem(systemId=source_system_id, users=[portal_admin_username]) + user_client.systems.revokeUserPerms(systemId=source_system_id, userName=portal_admin_username, permissions=["READ", "MODIFY", "EXECUTE"]) + user_client.files.deletePermissions(systemId=source_system_id, username=portal_admin_username, path="/") + logger.info(f'Removed service account from workspace {source_workspace_id}') + + # Add reviewers to review workspace + from portal.apps.projects.workspace_operations.shared_workspace_operations import add_user_to_workspace + + for reviewer in publication_reviewers: + add_user_to_workspace( + service_client, + review_workspace_id, + reviewer, + "reader", + f"{settings.PORTAL_PROJECTS_REVIEW_SYSTEM_PREFIX}.{review_workspace_id}", + settings.PORTAL_PROJECTS_ROOT_REVIEW_SYSTEM_NAME, + ) + logger.info(f'Added reviewer {reviewer} to review system {review_system_id}') + +@shared_task(bind=True, max_retries=3, queue='default') +def publish_project(self, project_id: str, version: Optional[int] = 1): + + review_system_prefix = settings.PORTAL_PROJECTS_REVIEW_SYSTEM_PREFIX + published_system_prefix = settings.PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX + + published_workspace_id = f"{project_id}{f'v{version}' if version and version > 1 else ''}" + published_system_id = f'{published_system_prefix}.{published_workspace_id}' + review_system_id = f'{review_system_prefix}.{project_id}' + + with transaction.atomic(): + + project_meta = ProjectMetadata.get_project_by_id(review_system_id) + publication_tree: nx.DiGraph = nx.node_link_graph(project_meta.project_graph.value) + + published_project = ProjectMetadata.get_project_by_id(published_system_id) + + ProjectMetadata.objects.create( + name=constants.PROJECT_GRAPH, + base_project=published_project, + value=nx.node_link_data(publication_tree), + ) + + doi = 'test_doi' # Replace with actual DOI retrieval logic + + # Update project metadata with datacite doi + source_project_id = f'{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}' + source_project = ProjectMetadata.get_project_by_id(source_project_id) + source_project.value['doi'] = doi + source_project.save() + + pub_tree = nx.node_link_graph(published_project.project_graph.value) + pub_tree.nodes["NODE_ROOT"]["version"] = version + published_project.project_graph.value = nx.node_link_data(pub_tree) + published_project.value['doi'] = doi + published_project.save() + + + pub_metadata = Publication.objects.update_or_create( + project_id=project_id, + defaults={"value": published_project.value, "tree": nx.node_link_data(pub_tree), "version": version}, + ) + + # transfer files + client = service_account() + transfer = _transfer_files(client, review_system_id, published_system_id) + + poll_tapis_file_transfer.apply_async( + args=(transfer.uuid, False), + kwargs={ + 'review_project_id': review_system_id, + 'published_project_id': published_system_id, + }, countdown=30) + +@shared_task(bind=True, max_retries=3, queue='default') +def copy_graph_and_files_for_review_system(self, user_access_token, source_workspace_id, review_workspace_id, source_system_id, review_system_id): + logger.info(f'Starting copy task for system {source_system_id} to system {review_system_id}') + + with transaction.atomic(): + pub_tree = _add_values_to_tree(source_system_id) + + graph_model_value = nx.node_link_data(pub_tree) + review_project = ProjectMetadata.get_project_by_id(review_system_id) + ProjectMetadata.objects.update_or_create( + name=constants.PROJECT_GRAPH, + base_project=review_project, + defaults={"value": graph_model_value}, + ) + + client = user_account(user_access_token) + transfer = _transfer_files(client, source_system_id, review_system_id) + + logger.info(f'Transfer task submmited with id {transfer.uuid}') + + poll_tapis_file_transfer.apply_async( + args=(transfer.uuid, True), + kwargs={ + 'user_access_token': user_access_token, + 'source_workspace_id': source_workspace_id, + 'review_workspace_id': review_workspace_id, + 'source_system_id': source_system_id, + 'review_system_id': review_system_id, + }, countdown=30) + +@shared_task(bind=True, queue='default') +def poll_tapis_file_transfer(self, transfer_task_id, is_review, **kwargs): + logger.info(f'Starting post transfer task for transfer id {transfer_task_id} with arguments: {kwargs}') + + try: + service_client = service_account() + + # Check the transfer status + transfer_status = _check_transfer_status(service_client, transfer_task_id) + + # Handle pending or in-progress transfer + if transfer_status in ['PENDING', 'IN_PROGRESS']: + logger.info(f'Transfer {transfer_task_id} is still pending with status {transfer_status}, retrying in 30 seconds.') + self.apply_async(args=(transfer_task_id, is_review), kwargs=kwargs, countdown=30) + return + + # Handle completed transfer + elif transfer_status == 'COMPLETED': + logger.info(f'Transfer {transfer_task_id} completed successfully with arguments: {kwargs}') + + # Call the callback function with any passed arguments + if is_review: + publication_request_callback(**kwargs) + else: + publish_project_callback(**kwargs) + + else: + logger.error(f'Error processing transfer {transfer_task_id}: Transfer status is {transfer_status}') + raise Exception(f'Transfer {transfer_task_id} failed with status {transfer_status}') + + except Exception as e: + logger.error(f'Error processing transfer {transfer_task_id} with arguments {kwargs}: {e}') + self.retry(exc=e, countdown=30) + +@transaction.atomic +def update_and_cleanup_review_project(review_project_id: str, status: PublicationRequest.Status): + + client = service_account() + + workspace_id = review_project_id.split(f"{settings.PORTAL_PROJECTS_REVIEW_SYSTEM_PREFIX}.")[1] + + # update the publication request + review_project = ProjectMetadata.get_project_by_id(review_project_id) + pub_request = PublicationRequest.objects.get(review_project=review_project, status=PublicationRequest.Status.PENDING) + pub_request.status = status + pub_request.save() + + logger.info(f'Updated publication request for review project {review_project_id} to {status}.') + + # delete the review project and data inside it + reviewers = pub_request.reviewers.all() + + for reviewer in reviewers: + try: + remove_user(client, workspace_id, reviewer.username, review_project_id, settings.PORTAL_PROJECTS_ROOT_REVIEW_SYSTEM_NAME) + logger.info(f'Removed reviewer {reviewer.username} from review system {review_project_id}') + except: + logger.error(f'Error removing reviewer {reviewer.username} from review system {review_project_id}') + continue + + client.files.delete(systemId=review_project_id, path='/') + client.systems.deleteSystem(systemId=review_project_id) + review_project_graph = ProjectMetadata.objects.get(name=constants.PROJECT_GRAPH, base_project=review_project) + review_project_graph.delete() + review_project.delete() + + logger.info(f'Deleted review project {review_project_id} and its associated data.') \ No newline at end of file diff --git a/server/portal/apps/projects/workspace_operations/shared_workspace_operations.py b/server/portal/apps/projects/workspace_operations/shared_workspace_operations.py index 7d8c67c6a8..347e976eb8 100644 --- a/server/portal/apps/projects/workspace_operations/shared_workspace_operations.py +++ b/server/portal/apps/projects/workspace_operations/shared_workspace_operations.py @@ -1,6 +1,7 @@ # from portal.utils.encryption import createKeyPair from portal.libs.agave.utils import service_account from portal.apps.projects.models.project_metadata import ProjectMetadata +from portal.apps._custom.drp import constants from tapipy.tapis import Tapis from typing import Literal from django.db import transaction @@ -8,7 +9,8 @@ from django.contrib.auth import get_user_model from portal.apps.projects.models.metadata import ProjectsMetadata from django.db import models -from portal.apps.projects.workspace_operations.project_meta_operations import create_project_metadata +from portal.apps.projects.workspace_operations.project_meta_operations import create_project_metadata, get_ordered_value +from portal.apps.onboarding.steps.system_access_v3 import create_system_credentials import logging logger = logging.getLogger(__name__) @@ -248,15 +250,17 @@ def change_user_role(client, workspace_id: str, username: str, new_role): set_workspace_permissions(client, username, system_id, new_role) -def remove_user(client, workspace_id: str, username: str): +def remove_user(client, workspace_id: str, username: str, system_id=None, system_name=None): """ Unshare the system and remove all permissions and credentials. """ service_client = service_account() - system_id = f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{workspace_id}" + system_id = system_id or f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{workspace_id}" + system_name = system_name or f"{settings.PORTAL_PROJECTS_ROOT_SYSTEM_NAME}" + set_workspace_acls(service_client, - settings.PORTAL_PROJECTS_ROOT_SYSTEM_NAME, + system_name, workspace_id, username, "remove", @@ -270,7 +274,7 @@ def remove_user(client, workspace_id: str, username: str): username=username, path="/") - return get_project(client, workspace_id) + return get_project(client, workspace_id, system_id) def transfer_ownership(client, workspace_id: str, new_owner: str, old_owner: str): @@ -335,7 +339,6 @@ def list_projects(client, root_system_id=None): if root_system: query += f"~(rootDir.like.{root_system['rootDir']}*)" - # use limit as -1 to allow search to corelate with # all projects available to the api user listing = client.systems.getSystems(listType='ALL', @@ -401,32 +404,55 @@ def get_workspace_role(client, workspace_id, username): return None @transaction.atomic -def create_publication_review_shared_workspace(client, source_workspace_id: str, source_system_id: str, review_workspace_id: str, - review_system_id: str, title: str, description=""): +def create_publication_workspace(client, source_workspace_id: str, source_system_id: str, target_workspace_id: str, + target_system_id: str, title: str, description="", is_review=False): portal_admin_username = settings.PORTAL_ADMIN_USERNAME service_client = service_account() - # add admin to the source workspace to allow for file copying - resp = add_user_to_workspace(client, source_workspace_id, portal_admin_username) + # Determine workspace and system-specific settings based on the project type + system_prefix = settings.PORTAL_PROJECTS_REVIEW_SYSTEM_PREFIX if is_review else settings.PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX + root_system_name = settings.PORTAL_PROJECTS_ROOT_REVIEW_SYSTEM_NAME if is_review else settings.PORTAL_PROJECTS_PUBLISHED_ROOT_SYSTEM_NAME + root_dir = settings.PORTAL_PROJECTS_REVIEW_ROOT_DIR if is_review else settings.PORTAL_PROJECTS_PUBLISHED_ROOT_DIR + if is_review: + # Add admin to the source workspace to allow for file copying + add_user_to_workspace(client, source_workspace_id, portal_admin_username) + + # Retrieve the source project and adjust project data based on review/published status source_project = ProjectMetadata.get_project_by_id(source_system_id) + project_value = get_ordered_value(constants.PROJECT, source_project.value) - review_project = create_project_metadata({**source_project.value, "projectId": review_system_id, "is_review_project": True}) + project_data = { + **project_value, + "project_id": target_system_id, + "is_review_project": is_review, + "is_published_project": not is_review + } - review_project.save() - - create_workspace_dir(review_workspace_id, settings.PORTAL_PROJECTS_ROOT_REVIEW_SYSTEM_NAME) - - set_workspace_acls(service_client, - settings.PORTAL_PROJECTS_ROOT_REVIEW_SYSTEM_NAME, - review_workspace_id, - portal_admin_username, - "add", - "writer") - - system_id = create_workspace_system(service_client, review_workspace_id, title, description, None, - f"{settings.PORTAL_PROJECTS_REVIEW_SYSTEM_PREFIX}.{review_workspace_id}", - f"{settings.PORTAL_PROJECTS_REVIEW_ROOT_DIR}/{review_workspace_id}") + # Create and save the new project metadata + new_project = create_project_metadata(project_data) + new_project.save() + + # Set up the target workspace directory + create_workspace_dir(target_workspace_id, root_system_name) + + # Configure workspace ACLs + set_workspace_acls(service_client, root_system_name, target_workspace_id, portal_admin_username, "add", "writer") + + query = f"(id.eq.{target_system_id})" + listing = service_client.systems.getSystems(listType='ALL', search=query, select="id,deleted", + showDeleted=True, limit=-1) - return system_id + if listing and listing[0].deleted: + service_client.systems.undeleteSystem(systemId=target_system_id) + # Add back system credentials since the system was previously deleted + create_system_credentials(service_client, portal_admin_username, settings.PORTAL_PROJECTS_PUBLIC_KEY, + settings.PORTAL_PROJECTS_PRIVATE_KEY, target_system_id) + else: + # Create the target workspace system + create_workspace_system( + service_client, target_workspace_id, title, description, None, + f"{system_prefix}.{target_workspace_id}", + f"{root_dir}/{target_workspace_id}" + ) diff --git a/server/portal/apps/publications/migrations/0003_publication.py b/server/portal/apps/publications/migrations/0003_publication.py new file mode 100644 index 0000000000..9c5363b3ef --- /dev/null +++ b/server/portal/apps/publications/migrations/0003_publication.py @@ -0,0 +1,27 @@ +# Generated by Django 4.2.16 on 2024-11-01 19:52 + +import django.core.serializers.json +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('publications', '0002_alter_publicationrequest_review_project_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='Publication', + fields=[ + ('project_id', models.CharField(editable=False, max_length=100, primary_key=True, serialize=False)), + ('created', models.DateTimeField(default=django.utils.timezone.now)), + ('is_published', models.BooleanField(default=True)), + ('last_updated', models.DateTimeField(auto_now=True)), + ('version', models.IntegerField(default=1)), + ('value', models.JSONField(encoder=django.core.serializers.json.DjangoJSONEncoder, help_text="Value for the project's base metadata, including title/description/users")), + ('tree', models.JSONField(encoder=django.core.serializers.json.DjangoJSONEncoder, help_text='JSON document containing the serialized publication tree')), + ], + ), + ] diff --git a/server/portal/apps/publications/models.py b/server/portal/apps/publications/models.py index 99d360cb66..a9bba80765 100644 --- a/server/portal/apps/publications/models.py +++ b/server/portal/apps/publications/models.py @@ -8,6 +8,7 @@ from django.db import models from django.utils import timezone from portal.apps.projects.models.project_metadata import ProjectMetadata +from django.core.serializers.json import DjangoJSONEncoder # pylint: disable=invalid-name logger = logging.getLogger(__name__) @@ -30,4 +31,23 @@ class Status(models.TextChoices): last_updated = models.DateTimeField(auto_now=True) def __str__(self): - return f'Review for {self.review_project.project_id}' \ No newline at end of file + return f'Review for {self.review_project.project_id}' + +class Publication(models.Model): + + project_id = models.CharField(max_length=100, primary_key=True, editable=False) + created = models.DateTimeField(default=timezone.now) + is_published = models.BooleanField(default=True) + last_updated = models.DateTimeField(auto_now=True) + version = models.IntegerField(default=1) + value = models.JSONField( + encoder=DjangoJSONEncoder, + help_text=( + "Value for the project's base metadata, including title/description/users" + ), + ) + + tree = models.JSONField( + encoder=DjangoJSONEncoder, + help_text=("JSON document containing the serialized publication tree"), + ) \ No newline at end of file diff --git a/server/portal/apps/publications/urls.py b/server/portal/apps/publications/urls.py index ab73238c5c..41c4aee40a 100644 --- a/server/portal/apps/publications/urls.py +++ b/server/portal/apps/publications/urls.py @@ -7,4 +7,8 @@ urlpatterns = [ path('publication-request/', views.PublicationRequestView.as_view(), name='publication_request'), path('publication-request//', views.PublicationRequestView.as_view(), name='publication_request_detail'), + path('publish/', views.PublicationPublishView.as_view(), name='publication_publish'), + path('reject/', views.PublicationRejectView.as_view(), name='publication_reject'), + path('version/', views.PublicationVersionView.as_view(), name='publication_version'), + path('', views.PublicationListingView.as_view(), name='publication_listing'), ] \ No newline at end of file diff --git a/server/portal/apps/publications/views.py b/server/portal/apps/publications/views.py index 8cccae9f1e..ade977e389 100644 --- a/server/portal/apps/publications/views.py +++ b/server/portal/apps/publications/views.py @@ -11,19 +11,20 @@ from django.utils.decorators import method_decorator from portal.exceptions.api import ApiException from portal.views.base import BaseApiView -from portal.apps.projects.workspace_operations.shared_workspace_operations import create_publication_review_shared_workspace +from portal.apps.projects.workspace_operations.shared_workspace_operations import create_publication_workspace +from portal.apps.projects.workspace_operations.project_publish_operations import copy_graph_and_files_for_review_system, publish_project, update_and_cleanup_review_project from portal.apps.projects.models.metadata import ProjectsMetadata from django.db import transaction -from portal.apps.projects.tasks import copy_graph_and_files from portal.apps.notifications.models import Notification from django.http import HttpResponse -from portal.apps.publications.models import PublicationRequest +from portal.apps.publications.models import Publication, PublicationRequest from portal.apps.projects.models.project_metadata import ProjectMetadata from django.db import models +from django.core.exceptions import ObjectDoesNotExist +from django.contrib.auth import get_user_model +from portal.libs.agave.utils import service_account - - -LOGGER = logging.getLogger(__name__) +logger = logging.getLogger(__name__) class PublicationRequestView(BaseApiView): @@ -67,27 +68,54 @@ def get(self, request, project_id: str): @method_decorator(login_required, name='dispatch') def post(self, request): - data = json.loads(request.body) + request_body = json.loads(request.body) client = request.user.tapis_oauth.client + service_client = service_account() - source_workspace_id = data['projectId'] + full_project_id = request_body.get('project_id') + + if not full_project_id: + raise ApiException("Missing project ID", status=400) + + source_workspace_id = full_project_id.split(f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.")[1] review_workspace_id = f"{source_workspace_id}" source_system_id = f'{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{source_workspace_id}' review_system_id = f"{settings.PORTAL_PROJECTS_REVIEW_SYSTEM_PREFIX}.{review_workspace_id}" with transaction.atomic(): - # Update authors for the source project - source_project = ProjectMetadata.get_project_by_id(source_system_id) + # Update authors for the source project # TODO: use pydantic to validate data - source_project.value['authors'] = data['authors'] + source_project = ProjectMetadata.get_project_by_id(source_system_id) + source_project.value['authors'] = request_body.get('authors') source_project.save() - system_id = create_publication_review_shared_workspace(client, source_workspace_id, source_system_id, review_workspace_id, - review_system_id, data['title'], data['description']) + create_publication_workspace(client, source_workspace_id, source_system_id, review_workspace_id, + review_system_id, request_body.get('title'), request_body.get('description'), True) + + # Create publication request + review_project = ProjectMetadata.get_project_by_id(review_system_id) + source_project = ProjectMetadata.get_project_by_id(source_system_id) + publication_reviewers = get_user_model().objects.filter(groups__name=settings.PORTAL_PUBLICATION_REVIEWERS_GROUP_NAME) + + publication_request = PublicationRequest( + review_project=review_project, + source_project=source_project, + ) + + publication_request.save() + + for reviewer in publication_reviewers: + try: + publication_request.reviewers.add(reviewer) + except ObjectDoesNotExist: + continue + + publication_request.save() + logger.info(f'Created publication review for system {review_system_id}') # Start task to copy files and metadata - copy_graph_and_files.apply_async(kwargs={ + copy_graph_and_files_for_review_system.apply_async(kwargs={ 'user_access_token': client.access_token.access_token, 'source_workspace_id': source_workspace_id, 'review_workspace_id': review_workspace_id, @@ -106,4 +134,147 @@ def post(self, request): with transaction.atomic(): Notification.objects.create(**event_data) - return HttpResponse('OK') \ No newline at end of file + return JsonResponse({'response': 'OK'}) + +class PublicationListingView(BaseApiView): + + def get(self, request): + + publications = Publication.objects.all() + + publications_data = [ + { + 'id': publication.value.get('projectId'), + 'title': publication.value.get('title'), + 'description': publication.value.get('description'), + 'keywords': publication.value.get('keywords'), + 'authors': publication.value.get('authors'), + 'publication_date': publication.last_updated, + } + for publication in publications + ] + + return JsonResponse({'response': publications_data}, safe=False) + +class PublicationPublishView(BaseApiView): + + def post(self, request): + """view for publishing a project""" + + client = request.user.tapis_oauth.client + request_body = json.loads(request.body) + + full_project_id = request_body.get('project_id') + is_review = request_body.get('is_review_project', False) + + if not full_project_id: + raise ApiException("Missing project ID", status=400) + + if is_review: + project_id = full_project_id.split(f"{settings.PORTAL_PROJECTS_REVIEW_SYSTEM_PREFIX}.")[1] + else: + project_id = full_project_id.split(f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.")[1] + + source_system_id = f'{settings.PORTAL_PROJECTS_REVIEW_SYSTEM_PREFIX}.{project_id}' + published_workspace_id = f"{project_id}" + published_system_id = f"{settings.PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX}.{published_workspace_id}" + + create_publication_workspace(client, project_id, source_system_id, published_workspace_id, published_system_id, + request_body.get('title'), request_body.get('description'), False) + + publish_project.apply_async(kwargs={ + 'project_id': project_id, + 'version': 1 + }) + + # Create notification + event_data = { + Notification.EVENT_TYPE: 'default', + Notification.STATUS: Notification.INFO, + Notification.USER: request.user.username, + Notification.MESSAGE: f'{project_id} submitted for publication', + } + + with transaction.atomic(): + Notification.objects.create(**event_data) + + return JsonResponse({'response': 'OK'}) + + +class PublicationVersionView(BaseApiView): + + def post(self, request): + """view for publishing a project""" + + client = request.user.tapis_oauth.client + request_body = json.loads(request.body) + + full_project_id = request_body.get('project_id') + is_review = request_body.get('is_review_project', False) + + if not full_project_id: + raise ApiException("Missing project ID", status=400) + + if is_review: + project_id = full_project_id.split(f"{settings.PORTAL_PROJECTS_REVIEW_SYSTEM_PREFIX}.")[1] + else: + project_id = full_project_id.split(f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.")[1] + + print('project_id:', project_id) + + publication = Publication.objects.get(project_id=project_id) + version = publication.version + 1 + + print(f"Version: {version}") + + source_system_id = f'{settings.PORTAL_PROJECTS_REVIEW_SYSTEM_PREFIX}.{project_id}' + published_workspace_id = f"{project_id}{f'v{version}' if version and version > 1 else ''}" + published_system_id = f"{settings.PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX}.{published_workspace_id}" + + print(f"Published Workspace ID: {published_workspace_id}") + + create_publication_workspace(client, project_id, source_system_id, published_workspace_id, published_system_id, + request_body.get('title'), request_body.get('description'), False) + + publish_project.apply_async(kwargs={ + 'project_id': project_id, + 'version': version + }) + + # Create notification + event_data = { + Notification.EVENT_TYPE: 'default', + Notification.STATUS: Notification.INFO, + Notification.USER: request.user.username, + Notification.MESSAGE: f'{project_id} submitted for publication', + } + + with transaction.atomic(): + Notification.objects.create(**event_data) + + return JsonResponse({'response': 'OK'}) + +class PublicationRejectView(BaseApiView): + + def post(self, request): + + request_body = json.loads(request.body) + full_project_id = request_body.get('project_id') + + if not full_project_id: + raise ApiException("Missing project ID", status=400) + + update_and_cleanup_review_project(full_project_id, PublicationRequest.Status.REJECTED) + + # Create notification + event_data = { + Notification.EVENT_TYPE: 'default', + Notification.STATUS: Notification.INFO, + Notification.USER: request.user.username, + Notification.MESSAGE: f'{full_project_id} was rejected', + } + + with transaction.atomic(): + Notification.objects.create(**event_data) + + return JsonResponse({'response': 'OK'}) \ No newline at end of file diff --git a/server/portal/apps/users/views.py b/server/portal/apps/users/views.py index 3946233fe9..84dac63387 100644 --- a/server/portal/apps/users/views.py +++ b/server/portal/apps/users/views.py @@ -30,6 +30,12 @@ def get(self, request): if request.user.is_authenticated: u = request.user + try: + user = get_user_model().objects.get(username=u.username) + groups = [group.name for group in user.groups.all()] + except ObjectDoesNotExist: + groups = [] + out = { "first_name": u.first_name, "username": u.username, @@ -39,6 +45,7 @@ def get(self, request): "expires_in": u.tapis_oauth.expires_in, }, "isStaff": u.is_staff, + "groups": groups } return JsonResponse(out) diff --git a/server/portal/libs/agave/operations.py b/server/portal/libs/agave/operations.py index c8ff60308b..c957b5bd61 100644 --- a/server/portal/libs/agave/operations.py +++ b/server/portal/libs/agave/operations.py @@ -507,7 +507,8 @@ def trash(client, system, path, homeDir, metadata=None): if err.response.status_code != 404: logger.error(f'Unexpected exception listing .trash path in {system}') raise - add_node_to_project(system, 'NODE_ROOT', None, settings.TAPIS_DEFAULT_TRASH_NAME, settings.TAPIS_DEFAULT_TRASH_NAME) + trash_entity = create_entity_metadata(system, constants.TRASH, {}) + add_node_to_project(system, 'NODE_ROOT', trash_entity.uuid, trash_entity.name, settings.TAPIS_DEFAULT_TRASH_NAME) mkdir(client, system, homeDir, settings.TAPIS_DEFAULT_TRASH_NAME) resp = move(client, system, path, system, diff --git a/server/portal/settings/settings.py b/server/portal/settings/settings.py index b2b2705995..bdd8f86339 100644 --- a/server/portal/settings/settings.py +++ b/server/portal/settings/settings.py @@ -580,6 +580,18 @@ PORTAL_PROJECTS_ROOT_REVIEW_SYSTEM_NAME = settings_custom.\ _PORTAL_PROJECTS_ROOT_REVIEW_SYSTEM_NAME +PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX = settings_custom.\ + _PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX + +PORTAL_PROJECTS_PUBLISHED_ROOT_DIR = settings_custom.\ + _PORTAL_PROJECTS_PUBLISHED_ROOT_DIR + +PORTAL_PROJECTS_PUBLISHED_ROOT_SYSTEM_NAME = settings_custom.\ + _PORTAL_PROJECTS_PUBLISHED_ROOT_SYSTEM_NAME + +PORTAL_PUBLICATION_REVIEWERS_GROUP_NAME = settings_custom.\ + _PORTAL_PUBLICATION_REVIEWERS_GROUP_NAME + PORTAL_PROJECTS_PRIVATE_KEY = settings_secret.\ _PORTAL_PROJECTS_PRIVATE_KEY diff --git a/server/portal/settings/settings_default.py b/server/portal/settings/settings_default.py index e1455c97d7..eb12d24d9c 100644 --- a/server/portal/settings/settings_default.py +++ b/server/portal/settings/settings_default.py @@ -103,7 +103,7 @@ 'siteSearchPriority': 0 }, { - 'name': 'Project', + 'name': 'Projects', 'scheme': 'projects', 'api': 'tapis', 'icon': 'publications', @@ -112,9 +112,20 @@ 'defaultProject': True, 'system': 'cep.project.root', 'rootDir': '/corral-repl/tacc/aci/CEP/projects', + }, + { + 'name': 'Published', + 'scheme': 'projects', + 'api': 'tapis', + 'icon': 'publications', + 'readOnly': True, + 'hideSearchBar': False, + 'system': 'drp.project.published.test', + 'rootDir': '/corral-repl/utexas/pge-nsf/data_pprd/published', + 'publicationProject': True, }, { - 'name': 'Review Projects', + 'name': 'Review', 'scheme': 'projects', 'api': 'tapis', 'icon': 'publications', @@ -122,6 +133,7 @@ 'hideSearchBar': False, 'system': 'drp.project.review.test', 'rootDir': '/corral-repl/utexas/pge-nsf/data_pprd/test', + 'reviewProject': True, } ] @@ -205,6 +217,12 @@ _PORTAL_PROJECTS_REVIEW_ROOT_DIR = '/corral-repl/utexas/pge-nsf/data_pprd/test' _PORTAL_PROJECTS_ROOT_REVIEW_SYSTEM_NAME = 'drp.project.review.test' +_PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX = 'cep.project.published' +_PORTAL_PROJECTS_PUBLISHED_ROOT_DIR = '/corral-repl/utexas/pge-nsf/data_pprd/published' +_PORTAL_PROJECTS_PUBLISHED_ROOT_SYSTEM_NAME = 'drp.project.published.test' + +_PORTAL_PUBLICATION_REVIEWERS_GROUP_NAME = 'PROJECT_REVIEWER' + ######################## # Custom Portal Template Assets # Asset path root is static files output dir. From dfa5161b7ed0ef1491484dc253ba6a059a7efb98 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Fri, 8 Nov 2024 13:50:38 -0600 Subject: [PATCH 124/328] quick css fix --- .../DataFilesProjectPublishWizard.module.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss index 0e2ebbbd4e..4dc5e34c10 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/DataFilesProjectPublishWizard.module.scss @@ -98,6 +98,6 @@ } .submit-button { - min-width: 200px; + min-width: 200px !important; margin: 20px; } From 4dbe146ef4570ba11e0a7e033ac6ba8942b142b1 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Mon, 11 Nov 2024 14:56:18 -0600 Subject: [PATCH 125/328] new columns/info for review and published projects --- client/src/components/DataFiles/DataFiles.jsx | 3 + .../DataFilesModals/DataFilesModals.jsx | 4 +- .../DataFilesProjectDescriptionModal.jsx | 40 +++++ ...taFilesProjectDescriptionModal.module.scss | 0 .../DataFilesPublicationsList.jsx | 41 ++++- .../DataFilesPublicationsList.scss | 17 ++- .../DataFilesReviewProjectList.jsx | 141 ++++++++++++++++++ .../DataFilesReviewProjectList.module.scss | 17 +++ .../DataFilesReviewProjectList.scss | 27 ++++ server/portal/apps/projects/views.py | 8 + 10 files changed, 284 insertions(+), 14 deletions(-) create mode 100644 client/src/components/DataFiles/DataFilesModals/DataFilesProjectDescriptionModal.jsx create mode 100644 client/src/components/DataFiles/DataFilesModals/DataFilesProjectDescriptionModal.module.scss create mode 100644 client/src/components/DataFiles/DataFilesReviewProjectsList/DataFilesReviewProjectList.jsx create mode 100644 client/src/components/DataFiles/DataFilesReviewProjectsList/DataFilesReviewProjectList.module.scss create mode 100644 client/src/components/DataFiles/DataFilesReviewProjectsList/DataFilesReviewProjectList.scss diff --git a/client/src/components/DataFiles/DataFiles.jsx b/client/src/components/DataFiles/DataFiles.jsx index e2c4404252..68dfc97c68 100644 --- a/client/src/components/DataFiles/DataFiles.jsx +++ b/client/src/components/DataFiles/DataFiles.jsx @@ -25,6 +25,7 @@ import DataFilesProjectsList from './DataFilesProjectsList/DataFilesProjectsList import DataFilesProjectFileListing from './DataFilesProjectFileListing/DataFilesProjectFileListing'; import { useSystemRole } from './DataFilesProjectMembers/_cells/SystemRoleSelector'; import DataFilesPublicationsList from './DataFilesPublicationsList/DataFilesPublicationsList'; +import DataFilesReviewProjectList from './DataFilesReviewProjectsList/DataFilesReviewProjectList'; const DefaultSystemRedirect = () => { const systems = useSelector( @@ -102,6 +103,8 @@ const DataFilesSwitch = React.memo(() => { if (system.publicationProject) { return ; + } else if (system.reviewProject) { + return ; } return ; diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesModals.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesModals.jsx index d726c9a14a..7f4746499d 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesModals.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesModals.jsx @@ -20,6 +20,7 @@ import './DataFilesModals.scss'; import DataFilesFormModal from './DataFilesFormModal'; import DataFilesPublicationRequestModal from './DataFilesPublicationRequestModal'; import DataFilesProjectTreeModal from './DataFilesProjectTreeModal'; +import DataFilesProjectDescriptionModal from './DataFilesProjectDescriptionModal'; export default function DataFilesModals() { return ( @@ -42,8 +43,9 @@ export default function DataFilesModals() { - + + ); } diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesProjectDescriptionModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesProjectDescriptionModal.jsx new file mode 100644 index 0000000000..621add8014 --- /dev/null +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesProjectDescriptionModal.jsx @@ -0,0 +1,40 @@ +import React, { useCallback } from 'react'; +import { useDispatch, useSelector } from 'react-redux'; +import { Modal, ModalHeader, ModalBody } from 'reactstrap'; +import styles from './DataFilesProjectDescriptionModal.module.scss'; + +const DataFilesProjectDescriptionModal = () => { + const dispatch = useDispatch(); + + const isOpen = useSelector((state) => state.files.modals.projectDescription); + const props = useSelector( + (state) => state.files.modalProps.projectDescription + ); + + const toggle = useCallback(() => { + dispatch({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { operation: 'projectDescription', props: {} }, + }); + }, []); + + return ( + <> + + + {props?.title} + + +

{props?.description}

+ + + + ); +}; + +export default DataFilesProjectDescriptionModal; diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesProjectDescriptionModal.module.scss b/client/src/components/DataFiles/DataFilesModals/DataFilesProjectDescriptionModal.module.scss new file mode 100644 index 0000000000..e69de29bb2 diff --git a/client/src/components/DataFiles/DataFilesPublicationsList/DataFilesPublicationsList.jsx b/client/src/components/DataFiles/DataFilesPublicationsList/DataFilesPublicationsList.jsx index a7844bf4ed..4bd69ecf4a 100644 --- a/client/src/components/DataFiles/DataFilesPublicationsList/DataFilesPublicationsList.jsx +++ b/client/src/components/DataFiles/DataFilesPublicationsList/DataFilesPublicationsList.jsx @@ -4,6 +4,7 @@ import { Link, useLocation } from 'react-router-dom'; import { useSelector, useDispatch, shallowEqual } from 'react-redux'; import queryStringParser from 'query-string'; import { + Button, InfiniteScrollTable, SectionMessage, SectionTableWrapper, @@ -41,6 +42,16 @@ const DataFilesPublicationsList = ({ rootSystem }) => { }); }, [dispatch, query.query_string]); + const createProjectDescriptionModal = (title, description) => { + dispatch({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { + operation: 'projectDescription', + props: { title, description }, + }, + }); + }; + const columns = [ { Header: 'Publication Title', @@ -54,6 +65,13 @@ const DataFilesPublicationsList = ({ rootSystem }) => { ), }, + { + Header: 'Publication Date', + accessor: 'publication_date', + Cell: (el) => ( + {el.value ? formatDate(new Date(el.value)) : ''} + ), + }, { Header: 'Principal Investigator', accessor: 'authors', @@ -66,15 +84,24 @@ const DataFilesPublicationsList = ({ rootSystem }) => { ), }, { - Header: 'Keywords', - accessor: 'keywords', + Header: 'Description', + accessor: 'description', + Cell: (el) => { + return ( + + ); + }, }, { - Header: 'Publication Date', - accessor: 'publication_date', - Cell: (el) => ( - {el.value ? formatDate(new Date(el.value)) : ''} - ), + Header: 'Keywords', + accessor: 'keywords', }, ]; diff --git a/client/src/components/DataFiles/DataFilesPublicationsList/DataFilesPublicationsList.scss b/client/src/components/DataFiles/DataFilesPublicationsList/DataFilesPublicationsList.scss index a87507046a..c25dafd70a 100644 --- a/client/src/components/DataFiles/DataFilesPublicationsList/DataFilesPublicationsList.scss +++ b/client/src/components/DataFiles/DataFilesPublicationsList/DataFilesPublicationsList.scss @@ -4,19 +4,24 @@ td:nth-child(1) { width: 40%; } - /* authors */ + /* date */ th:nth-child(2), td:nth-child(2) { - width: 20%; + width: 10%; } - /* keywords */ + /* author */ th:nth-child(3), td:nth-child(3) { - width: 25%; + width: 15%; } - /* date */ + /* description */ th:nth-child(4), td:nth-child(4) { - width: 15%; + width: 10%; + } + /* keywords */ + th:nth-child(5), + td:nth-child(5) { + width: 25%; } } diff --git a/client/src/components/DataFiles/DataFilesReviewProjectsList/DataFilesReviewProjectList.jsx b/client/src/components/DataFiles/DataFilesReviewProjectsList/DataFilesReviewProjectList.jsx new file mode 100644 index 0000000000..ff13e444e1 --- /dev/null +++ b/client/src/components/DataFiles/DataFilesReviewProjectsList/DataFilesReviewProjectList.jsx @@ -0,0 +1,141 @@ +import React, { useEffect, useCallback } from 'react'; +import { + Button, + InfiniteScrollTable, + SectionMessage, + SectionTableWrapper, +} from '_common'; +import { useDispatch, useSelector } from 'react-redux'; +import { Link, useLocation } from 'react-router-dom'; +import Searchbar from '_common/Searchbar'; +import './DataFilesReviewProjectList.scss'; +import styles from './DataFilesReviewProjectList.module.scss'; +import queryStringParser from 'query-string'; +import { formatDate } from 'utils/timeFormat'; + +const DataFilesReviewProjectList = ({ rootSystem }) => { + const { error, loading, projects } = useSelector( + (state) => state.projects.listing + ); + + const query = queryStringParser.parse(useLocation().search); + + const infiniteScrollCallback = useCallback(() => {}); + const dispatch = useDispatch(); + + useEffect(() => { + const actionType = 'PROJECTS_SHOW_SHARED_WORKSPACES'; + dispatch({ + type: actionType, + payload: { + queryString: query.query_string, + rootSystem: rootSystem, + }, + }); + }, [dispatch, query.query_string, rootSystem]); + + const createProjectDescriptionModal = (title, description) => { + dispatch({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { + operation: 'projectDescription', + props: { title, description }, + }, + }); + }; + + const columns = [ + { + Header: `Request Title`, + accessor: 'title', + Cell: (el) => ( + + {el.value} + + ), + }, + { + Header: 'Requested Date', + accessor: 'updated', + Cell: (el) => ( + {el.value ? formatDate(new Date(el.value)) : ''} + ), + }, + { + Header: 'Principal Investigator', + accessor: 'authors', + Cell: (el) => ( + + {el.value?.length > 0 + ? `${el.value[0].first_name} ${el.value[0].last_name}` + : ''} + + ), + }, + { + Header: 'Description', + accessor: 'description', + Cell: (el) => { + return ( + + ); + }, + }, + { + Header: 'Keywords', + accessor: 'keywords', + }, + ]; + + const noDataText = query.query_string + ? `No Projects match your search term.` + : `You don't have any requests to review`; + + if (error) { + return ( +
+ + There was a problem retrieving your {sharedWorkspacesDisplayName}. + +
+ ); + } + + return ( + + +
+ +
+ + ); +}; + +export default DataFilesReviewProjectList; diff --git a/client/src/components/DataFiles/DataFilesReviewProjectsList/DataFilesReviewProjectList.module.scss b/client/src/components/DataFiles/DataFilesReviewProjectsList/DataFilesReviewProjectList.module.scss new file mode 100644 index 0000000000..28e23efaf6 --- /dev/null +++ b/client/src/components/DataFiles/DataFilesReviewProjectsList/DataFilesReviewProjectList.module.scss @@ -0,0 +1,17 @@ +.root { + /* As a flex child */ + flex-grow: 1; + + /* WARNING: Mimicked on: History, Allocation, DataFiles, DataFilesProjectsList, DataFilesProjectFileListing, PublicData */ + padding-top: 1.75rem; /* ~28px (22.5px * design * 1.2 design-to-app ratio) */ + padding-left: 1.5em; /* ~24px (20px * design * 1.2 design-to-app ratio) */ +} + +/* NOTE: Mimicked on: DataFiles, DataFilesProjectsList, DataFilesProjectFileListing */ +.root-placeholder { + flex-grow: 1; + + display: flex; + align-items: center; + justify-content: center; +} diff --git a/client/src/components/DataFiles/DataFilesReviewProjectsList/DataFilesReviewProjectList.scss b/client/src/components/DataFiles/DataFilesReviewProjectsList/DataFilesReviewProjectList.scss new file mode 100644 index 0000000000..21188ad57f --- /dev/null +++ b/client/src/components/DataFiles/DataFilesReviewProjectsList/DataFilesReviewProjectList.scss @@ -0,0 +1,27 @@ +.review-projects-listing { + /* title */ + th:nth-child(1), + td:nth-child(1) { + width: 40%; + } + /* date */ + th:nth-child(2), + td:nth-child(2) { + width: 10%; + } + /* author */ + th:nth-child(3), + td:nth-child(3) { + width: 15%; + } + /* description */ + th:nth-child(4), + td:nth-child(4) { + width: 10%; + } + /* keywords */ + th:nth-child(5), + td:nth-child(5) { + width: 25%; + } +} diff --git a/server/portal/apps/projects/views.py b/server/portal/apps/projects/views.py index 83d6473d18..f2ff4b05d2 100644 --- a/server/portal/apps/projects/views.py +++ b/server/portal/apps/projects/views.py @@ -116,6 +116,14 @@ def get(self, request, root_system=None): client = request.user.tapis_oauth.client listing = list_projects(client, root_system) + for project in listing: + try: + project_meta = ProjectMetadata.objects.get(models.Q(value__projectId=project['id'])) + project.update(get_ordered_value(project_meta.name, project_meta.value)) + project["projectId"] = project['id'] + except ProjectMetadata.DoesNotExist: + pass + tapis_project_listing_indexer.delay(listing) return JsonResponse({"status": 200, "response": listing}) From 11117afb98ef5c5c9bb1f9ce5a5ca7d36ee3d789 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Mon, 11 Nov 2024 15:08:17 -0600 Subject: [PATCH 126/328] fix keywords not appearing --- .../DataFilesProjectFileListingMetadataAddon.jsx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx index ffd4141845..246af5fa99 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx @@ -36,6 +36,10 @@ const DataFilesProjectFileListingMetadataAddon = ({ formattedMetadata.doi = metadata.doi; } + if (metadata.keywords) { + formattedMetadata.keywords = metadata.keywords; + } + return formattedMetadata; }; From f5becf8e62ff8c198b63712e41a0c5df61ead0cb Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Tue, 12 Nov 2024 12:09:57 -0600 Subject: [PATCH 127/328] Publication UI improvements - Added citation box and citation modal for publications - Removed Last Modified column on publication listing - Changed Created to Published Date - Added data view modal to view related project information --- .../DataFilesListing/DataFilesListing.jsx | 18 ++++ .../DataFilesModals/DataFilesModals.jsx | 4 + .../DataFilesProjectCitationModal.jsx | 44 ++++++++++ .../DataFilesProjectCitationModal.module.scss | 0 .../DataFilesViewDataModal.jsx | 64 ++++++++++++++ .../DataFilesViewDataModal.module.scss | 49 +++++++++++ .../DataFilesProjectFileListing.jsx | 13 ++- ...taFilesProjectFileListingMetadataAddon.jsx | 69 ++++++++++++--- ...rojectFileListingMetadataAddon.module.scss | 10 +++ .../ReviewAuthors.jsx | 84 ++++++++++++++----- .../drp/utils/DataDisplay/DataDisplay.jsx | 50 ++++++++--- .../project_publish_operations.py | 2 + 12 files changed, 362 insertions(+), 45 deletions(-) create mode 100644 client/src/components/DataFiles/DataFilesModals/DataFilesProjectCitationModal.jsx create mode 100644 client/src/components/DataFiles/DataFilesModals/DataFilesProjectCitationModal.module.scss create mode 100644 client/src/components/DataFiles/DataFilesModals/DataFilesViewDataModal.jsx create mode 100644 client/src/components/DataFiles/DataFilesModals/DataFilesViewDataModal.module.scss diff --git a/client/src/components/DataFiles/DataFilesListing/DataFilesListing.jsx b/client/src/components/DataFiles/DataFilesListing/DataFilesListing.jsx index 54a828394f..0552a10a30 100644 --- a/client/src/components/DataFiles/DataFilesListing/DataFilesListing.jsx +++ b/client/src/components/DataFiles/DataFilesListing/DataFilesListing.jsx @@ -166,6 +166,24 @@ const DataFilesListing = ({ width: 0.2, }); } + + if (scheme === 'projects' && rootSystem) { + const projectSystem = systems.find( + (s) => s.scheme === 'projects' && s.system === rootSystem + ); + + if (projectSystem && projectSystem.publicationProject) { + const index = cells.findIndex( + (cell) => cell.Header === 'Last Modified' + ); + cells.splice(index, 1); + ['Name', 'Size'].forEach((header) => { + const column = cells.find((col) => col.Header === header); + column.width += 0.1; + }); + } + } + return cells; }, [api, showViewPath, fileNavCellCallback]); diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesModals.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesModals.jsx index 7f4746499d..e250ac0c8f 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesModals.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesModals.jsx @@ -21,6 +21,8 @@ import DataFilesFormModal from './DataFilesFormModal'; import DataFilesPublicationRequestModal from './DataFilesPublicationRequestModal'; import DataFilesProjectTreeModal from './DataFilesProjectTreeModal'; import DataFilesProjectDescriptionModal from './DataFilesProjectDescriptionModal'; +import DataFilesViewDataModal from './DataFilesViewDataModal'; +import DataFilesProjectCitationModal from './DataFilesProjectCitationModal'; export default function DataFilesModals() { return ( @@ -46,6 +48,8 @@ export default function DataFilesModals() { + + ); } diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesProjectCitationModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesProjectCitationModal.jsx new file mode 100644 index 0000000000..602ca07c6f --- /dev/null +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesProjectCitationModal.jsx @@ -0,0 +1,44 @@ +import React, { useCallback } from 'react'; +import { useDispatch, useSelector } from 'react-redux'; +import { Modal, ModalHeader, ModalBody } from 'reactstrap'; +import styles from './DataFilesProjectCitationModal.module.scss'; +import { Citations } from '_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors'; + +const DataFilesProjectCitationModal = () => { + const dispatch = useDispatch(); + + const isOpen = useSelector((state) => state.files.modals.projectCitation); + const props = useSelector((state) => state.files.modalProps.projectCitation); + + const toggle = useCallback(() => { + dispatch({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { operation: 'projectCitation', props: {} }, + }); + }, []); + + return ( + <> + {props?.project && ( + + + Citations + + + + + + )} + + ); +}; + +export default DataFilesProjectCitationModal; diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesProjectCitationModal.module.scss b/client/src/components/DataFiles/DataFilesModals/DataFilesProjectCitationModal.module.scss new file mode 100644 index 0000000000..e69de29bb2 diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesViewDataModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesViewDataModal.jsx new file mode 100644 index 0000000000..090d59a22c --- /dev/null +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesViewDataModal.jsx @@ -0,0 +1,64 @@ +import React, { useCallback, useEffect, useState } from 'react'; +import { useDispatch, useSelector } from 'react-redux'; +import { Modal, ModalHeader, ModalBody } from 'reactstrap'; +import styles from './DataFilesViewDataModal.module.scss'; +import DescriptionList from '_common/DescriptionList'; + +const formatLabel = (key) => + key + .split('_') + .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) + .join(' '); + +const DataFilesViewDataModal = () => { + const dispatch = useDispatch(); + const isOpen = useSelector((state) => state.files.modals.viewData); + const props = useSelector((state) => state.files.modalProps.viewData); + + const [descriptionListData, setDescriptionListData] = useState({}); + + useEffect(() => { + if (!props?.value) return; + + const values = Array.isArray(props.value) ? props.value : [props.value]; + + const descriptionListFormattedData = values.map((val) => { + const formattedData = Object.entries(val).reduce((acc, [key, value]) => { + acc[formatLabel(key)] = value; + return acc; + }, {}); + return ; + }); + + // Prevents description list from having a header + setDescriptionListData({ '': descriptionListFormattedData }); + }, [props]); + + const toggle = useCallback(() => { + dispatch({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { operation: 'viewData', props: {} }, + }); + }, [dispatch]); + + return ( + + + {props?.key ? formatLabel(props.key) : ''} + + + + + + ); +}; + +export default DataFilesViewDataModal; diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesViewDataModal.module.scss b/client/src/components/DataFiles/DataFilesModals/DataFilesViewDataModal.module.scss new file mode 100644 index 0000000000..27a26df3b9 --- /dev/null +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesViewDataModal.module.scss @@ -0,0 +1,49 @@ +.panel-content { + width: 100%; + overflow-y: scroll; + + /* Cross-browser solution to padding ignored by overflow (in spec-compliant Firefox) */ + /* SEE: https://stackoverflow.com/a/38997047/11817077 */ + padding-bottom: 0; + &::after { + content: ''; + display: block; + height: var(--padding); + } +} + +dl.panel-content { + --buffer-horz: 12px; /* ~10px design * 1.2 design-to-app ratio */ + --buffer-vert: 10px; /* gut feel based loosely on random space from design */ + --border: var(--global-border-width--normal) solid + var(--global-color-primary--light); +} + +dl.panel-content > dt, +dl.panel-content > dd { + padding-left: var(--buffer-horz); + padding-right: var(--buffer-horz); + padding-top: var(--buffer-vert); +} + +dl.panel-content > dt { + border-top: var(--border); +} +dl.panel-content > dt:first-of-type { + border-top: none; +} + +dl.panel-content > dt:nth-of-type(even), +dl.panel-content > dd:nth-of-type(even) { + background-color: var(--global-color-primary--x-light); +} + +/* Remove the colon from top-level labels */ +dl.panel-content > dt::after { + display: none; +} + +.modal-body { + overflow: auto; + max-height: 80vh; +} diff --git a/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx b/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx index be22cfe3c3..25467417d0 100644 --- a/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx +++ b/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx @@ -1,5 +1,5 @@ import React, { useEffect, useState } from 'react'; -import { useDispatch, useSelector } from 'react-redux'; +import { shallowEqual, useDispatch, useSelector } from 'react-redux'; import PropTypes from 'prop-types'; import { Button, @@ -15,6 +15,11 @@ import styles from './DataFilesProjectFileListing.module.scss'; const DataFilesProjectFileListing = ({ rootSystem, system, path }) => { const dispatch = useDispatch(); const { fetchListing } = useFileListing('FilesListing'); + const systems = useSelector( + (state) => state.systems.storage.configuration.filter((s) => !s.hidden), + shallowEqual + ); + const [isPublicationSystem, setIsPublicationSystem] = useState(false); // logic to render addonComponents for DRP const portalName = useSelector((state) => state.workbench.portalName); @@ -39,6 +44,11 @@ const DataFilesProjectFileListing = ({ rootSystem, system, path }) => { fetchListing({ api: 'tapis', scheme: 'projects', system, path }); }, [system, path, fetchListing]); + useEffect(() => { + const system = systems.find((s) => s.system === rootSystem); + setIsPublicationSystem(system?.publicationProject); + }, [systems, rootSystem]); + const metadata = useSelector((state) => state.projects.metadata); const folderMetadata = useSelector( (state) => state.files.folderMetadata?.FilesListing @@ -152,6 +162,7 @@ const DataFilesProjectFileListing = ({ rootSystem, system, path }) => { folderMetadata={folderMetadata} metadata={metadata} path={path} + showCitation={isPublicationSystem} /> ) : ( diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx index ffd4141845..32950382aa 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx @@ -4,6 +4,9 @@ import styles from './DataFilesProjectFileListingMetadataAddon.module.scss'; import { useFileListing } from 'hooks/datafiles'; import DataDisplay from '../utils/DataDisplay/DataDisplay'; import { formatDate } from 'utils/timeFormat'; +import { MLACitation } from '../DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors'; +import { Button } from '_common'; +import { useDispatch } from 'react-redux'; const excludeKeys = [ 'name', @@ -18,25 +21,53 @@ const DataFilesProjectFileListingMetadataAddon = ({ folderMetadata, metadata, path, + showCitation, }) => { + const dispatch = useDispatch(); + const { loading } = useFileListing('FilesListing'); - const getProjectMetadata = (metadata) => { + const getProjectMetadata = ({ + publication_date, + created, + license, + doi, + keywords, + }) => { const dateOptions = { month: 'long', day: 'numeric', year: 'numeric' }; - const formattedMetadata = { - created: new Date(metadata.created).toLocaleDateString( - 'en-US', - dateOptions - ), - license: metadata.license ?? 'None', + return { + publication_date: new Date( + publication_date || created + ).toLocaleDateString('en-US', dateOptions), + license: license ?? 'None', + ...(doi && { doi }), + ...(keywords && { keywords }), }; + }; - if (metadata.doi) { - formattedMetadata.doi = metadata.doi; - } + const getProjectModalMetadata = (metadata) => { + const fields = [ + 'related_publications', + 'related_software', + 'related_datasets', + ]; + return fields.reduce((formattedMetadata, field) => { + if (metadata[field] && metadata[field].length > 0) { + formattedMetadata[field] = metadata[field]; + } + return formattedMetadata; + }, {}); + }; - return formattedMetadata; + const createProjectCitationModal = (project) => { + dispatch({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { + operation: 'projectCitation', + props: { project }, + }, + }); }; return ( @@ -53,11 +84,27 @@ const DataFilesProjectFileListingMetadataAddon = ({ ) : ( <> + {showCitation && ( +
+

Cite This Data:

+ +
+ +
+
+ )} {metadata.description} ))} diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.module.scss b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.module.scss index 47d675087b..e4923b4915 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.module.scss +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.module.scss @@ -11,3 +11,13 @@ padding: 0; margin-bottom: -3px; } + +.citation-box { + padding: 10px; + margin-bottom: 20px; + background-color: var(--global-color-primary--x-light); +} + +.citation-button { + margin-top: 5px; +} diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx index eaf4ca47dd..49572392cb 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx @@ -16,13 +16,22 @@ import ProjectMembersList from '../../utils/ProjectMembersList/ProjectMembersLis import { useSelector } from 'react-redux'; const ACMCitation = ({ project, authors }) => { - const authorString = authors.map(a => `${a.first_name} ${a.last_name}`).join(', '); - const projectUrl = `https://www.digitalrocksportal.org/projects/${project.projectId.split('-').pop()}`; - const createdDate = new Date(project.created).toLocaleDateString('en-US', { month: 'long', year: 'numeric' }); + const authorString = authors + .map((a) => `${a.first_name} ${a.last_name}`) + .join(', '); + const projectUrl = `https://www.digitalrocksportal.org/projects/${project.projectId + .split('-') + .pop()}`; + const createdDate = new Date(project.created).toLocaleDateString('en-US', { + month: 'long', + year: 'numeric', + }); return (
- {`${authorString}. ${project.title}. `} Digital Rocks Portal {` (${createdDate}). ${projectUrl}`}
+ {`${authorString}. ${project.title}. `} Digital Rocks Portal{' '} + {` (${createdDate}). ${projectUrl}`}{' '} +
); }; @@ -32,8 +41,15 @@ const APACitation = ({ project, authors }) => { .join(', '); const projectUrl = `https://www.digitalrocksportal.org`; const createdDateObj = new Date(project.created); - const createdDate = `${createdDateObj.getFullYear()}, ${createdDateObj.toLocaleString('en-US', { month: 'long' })} ${createdDateObj.getDate()}`; - const accessDate = new Date().toLocaleDateString('en-US', { year: 'numeric', month: 'long', day: 'numeric' }); + const createdDate = `${createdDateObj.getFullYear()}, ${createdDateObj.toLocaleString( + 'en-US', + { month: 'long' } + )} ${createdDateObj.getDate()}`; + const accessDate = new Date().toLocaleDateString('en-US', { + year: 'numeric', + month: 'long', + day: 'numeric', + }); return (
{`${authorString} (${createdDate}). ${project.title}. Retrieved ${accessDate}, from ${projectUrl}`}
@@ -41,8 +57,12 @@ const APACitation = ({ project, authors }) => { }; const BibTeXCitation = ({ project, authors }) => { - const authorString = authors.map(a => `${a.last_name}, ${a.first_name}`).join(' and '); - const projectUrl = `https://www.digitalrocksportal.org/projects/${project.projectId.split('-').pop()}`; + const authorString = authors + .map((a) => `${a.last_name}, ${a.first_name}`) + .join(' and '); + const projectUrl = `https://www.digitalrocksportal.org/projects/${project.projectId + .split('-') + .pop()}`; const year = new Date(project.created).getFullYear(); return ( @@ -57,38 +77,60 @@ const BibTeXCitation = ({ project, authors }) => { ); }; -const MLACitation = ({ project, authors }) => { - const authorString = authors.map(a => `${a.last_name}, ${a.first_name}`).join(', '); - const projectUrl = `https://www.digitalrocksportal.org/projects/${project.projectId.split('-').pop()}`; - const createdDate = new Date(project.created).toLocaleDateString('en-GB', { day: 'numeric', month: 'short', year: 'numeric' }); - const accessDate = new Date().toLocaleDateString('en-GB', { day: 'numeric', month: 'short', year: 'numeric' }); - +export const MLACitation = ({ project, authors }) => { + const authorString = authors + .map((a) => `${a.last_name}, ${a.first_name}`) + .join(', '); + const projectUrl = `https://www.digitalrocksportal.org/projects/${project.projectId + .split('-') + .pop()}`; + const createdDate = new Date(project.created).toLocaleDateString('en-GB', { + day: 'numeric', + month: 'short', + year: 'numeric', + }); + const accessDate = new Date().toLocaleDateString('en-GB', { + day: 'numeric', + month: 'short', + year: 'numeric', + }); return ( -
{`${authorString}. "${project.title}."`} Digital Rocks Portal, {` Digital Rocks Portal, ${createdDate}, ${projectUrl} Accessed ${accessDate}.`}
+
+ {`${authorString}. "${project.title}."`} Digital Rocks Portal,{' '} + {` Digital Rocks Portal, ${createdDate}, ${projectUrl} Accessed ${accessDate}.`} +
); }; const IEEECitation = ({ project, authors }) => { - const authorString = authors.map(a => `${a.first_name[0]}. ${a.last_name}`).join(', '); - const projectUrl = `https://www.digitalrocksportal.org/projects/${project.projectId.split('-').pop()}`; + const authorString = authors + .map((a) => `${a.first_name[0]}. ${a.last_name}`) + .join(', '); + const projectUrl = `https://www.digitalrocksportal.org/projects/${project.projectId + .split('-') + .pop()}`; const date = new Date(project.created); const year = date.getFullYear(); const day = date.getDate(); const month = date.toLocaleString('en-GB', { month: 'short' }); return ( -
{`[1] ${authorString}, "${project.title}",`} Digital Rocks Portal, {` ${year}. [Online]. Available: ${projectUrl}. [Accessed: ${day}-${month}-${year}]`}
+
+ {`[1] ${authorString}, "${project.title}",`}{' '} + Digital Rocks Portal,{' '} + {` ${year}. [Online]. Available: ${projectUrl}. [Accessed: ${day}-${month}-${year}]`} +
); }; -const Citations = ({ project, authors }) => ( +export const Citations = ({ project, authors }) => (

ACM ref

- +

APA

@@ -111,8 +153,6 @@ const Citations = ({ project, authors }) => (
); - - const ReviewAuthors = ({ project, onAuthorsUpdate }) => { const [authors, setAuthors] = useState([]); const [members, setMembers] = useState([]); diff --git a/client/src/components/_custom/drp/utils/DataDisplay/DataDisplay.jsx b/client/src/components/_custom/drp/utils/DataDisplay/DataDisplay.jsx index ab3622edb6..f807ccd128 100644 --- a/client/src/components/_custom/drp/utils/DataDisplay/DataDisplay.jsx +++ b/client/src/components/_custom/drp/utils/DataDisplay/DataDisplay.jsx @@ -1,9 +1,17 @@ import React from 'react'; import PropTypes from 'prop-types'; -import { Section, SectionContent, LoadingSpinner } from '_common'; +import { Section, SectionContent, LoadingSpinner, Button } from '_common'; import { useLocation, Link } from 'react-router-dom'; import styles from './DataDisplay.module.scss'; import { useFileListing } from 'hooks/datafiles'; +import { useDispatch } from 'react-redux'; + +// Function to format the dict key from snake_case to Label Case i.e. data_type -> Data Type +const formatLabel = (key) => + key + .split('_') + .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) + .join(' '); const processSampleAndOriginData = (data, path) => { // use the path to get sample and origin data names @@ -49,28 +57,48 @@ const processSampleAndOriginData = (data, path) => { return sampleAndOriginMetadata; }; -const DataDisplay = ({ data, path, excludeKeys }) => { - const location = useLocation(); +const processModalViewableData = (data) => { + const createViewDataModal = (key, value) => { + dispatch({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { + operation: 'viewData', + props: { key, value }, + }, + }); + }; + + const dispatch = useDispatch(); + + return Object.entries(data).map(([key, value]) => ({ + label: formatLabel(key), + value: ( + + ), + })); +}; - // Function to format the dict key from snake_case to Label Case i.e. data_type -> Data Type - const formatLabel = (key) => - key - .split('_') - .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) - .join(' '); +const DataDisplay = ({ data, path, excludeKeys, modalData }) => { + const location = useLocation(); //filter out empty values and unwanted keys - const processedData = Object.entries(data) + let processedData = Object.entries(data) .filter(([key, value]) => value !== '' && !excludeKeys.includes(key)) .map(([key, value]) => ({ label: formatLabel(key), - value: typeof value === 'string' ? formatLabel(value) : value, + value: typeof value === 'string' ? value : value, })); if (path) { processedData.unshift(...processSampleAndOriginData(data, path)); } + if (modalData) { + processedData.push(...processModalViewableData(modalData)); + } + // Divide processed data into chunks for two-column layout display const chunkSize = Math.ceil(processedData.length / 2); const chunks = []; diff --git a/server/portal/apps/projects/workspace_operations/project_publish_operations.py b/server/portal/apps/projects/workspace_operations/project_publish_operations.py index 481ea9f16e..fc5579acb9 100644 --- a/server/portal/apps/projects/workspace_operations/project_publish_operations.py +++ b/server/portal/apps/projects/workspace_operations/project_publish_operations.py @@ -122,12 +122,14 @@ def publish_project(self, project_id: str, version: Optional[int] = 1): source_project_id = f'{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}' source_project = ProjectMetadata.get_project_by_id(source_project_id) source_project.value['doi'] = doi + source_project.value['publication_date'] = published_project.created source_project.save() pub_tree = nx.node_link_graph(published_project.project_graph.value) pub_tree.nodes["NODE_ROOT"]["version"] = version published_project.project_graph.value = nx.node_link_data(pub_tree) published_project.value['doi'] = doi + published_project.value['publication_date'] = published_project.created published_project.save() From d1b16a5c5314c641f3d74c5a1630e8ef1f260f0a Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Wed, 13 Nov 2024 10:31:31 -0600 Subject: [PATCH 128/328] Added publication search functionality --- .../project_publish_operations.py | 3 ++ server/portal/apps/publications/views.py | 52 ++++++++++++++++++- server/portal/apps/search/tasks.py | 16 +++++- server/portal/libs/elasticsearch/docs/base.py | 9 ++++ server/portal/libs/elasticsearch/indexes.py | 8 ++- 5 files changed, 85 insertions(+), 3 deletions(-) diff --git a/server/portal/apps/projects/workspace_operations/project_publish_operations.py b/server/portal/apps/projects/workspace_operations/project_publish_operations.py index fc5579acb9..ffbc23c41f 100644 --- a/server/portal/apps/projects/workspace_operations/project_publish_operations.py +++ b/server/portal/apps/projects/workspace_operations/project_publish_operations.py @@ -10,6 +10,7 @@ from portal.apps.publications.models import Publication, PublicationRequest from django.db import transaction from portal.apps.projects.workspace_operations.graph_operations import remove_trash_nodes +from portal.apps.search.tasks import index_publication from tapipy.errors import NotFoundError, BaseTapyException from django.contrib.auth import get_user_model from django.core.exceptions import ObjectDoesNotExist @@ -138,6 +139,8 @@ def publish_project(self, project_id: str, version: Optional[int] = 1): defaults={"value": published_project.value, "tree": nx.node_link_data(pub_tree), "version": version}, ) + index_publication(project_id) + # transfer files client = service_account() transfer = _transfer_files(client, review_system_id, published_system_id) diff --git a/server/portal/apps/publications/views.py b/server/portal/apps/publications/views.py index ade977e389..55aeef777a 100644 --- a/server/portal/apps/publications/views.py +++ b/server/portal/apps/publications/views.py @@ -23,6 +23,8 @@ from django.core.exceptions import ObjectDoesNotExist from django.contrib.auth import get_user_model from portal.libs.agave.utils import service_account +from portal.libs.elasticsearch.docs.base import IndexedPublication +from elasticsearch_dsl import Q logger = logging.getLogger(__name__) @@ -140,7 +142,55 @@ class PublicationListingView(BaseApiView): def get(self, request): - publications = Publication.objects.all() + query_string = request.GET.get('query_string') + offset = int(request.GET.get('offset', 0)) + limit = int(request.GET.get('limit', 100)) + + if query_string: + query = IndexedPublication.search() + + qs_query = Q( + "query_string", + # Elasticsearch can't parse query strings with unescaped slashes + query=query_string.replace("/", "\\/"), + default_operator="AND", + type="cross_fields", + fields=[ + "nodes.value.doi", + "nodes.value.description", + "nodes.value.keywords", + "nodes.value.title", + "nodes.value.projectId", + "nodes.value.authors", + "nodes.value.authors.first_name", + "nodes.value.authors.last_name", + "nodes.value.authors.username", + ], + ) + term_query = Q( + { + "term": { + "nodes.value.projectId.keyword": query_string.replace("/", "\\/") + } + } + ) + + query = query.filter(qs_query | term_query) + query = query.extra(from_=int(offset), size=int(limit)) + + res = query.execute() + hits = [hit.meta.id for hit in res if hasattr(hit.meta, 'id') and hit.meta.id is not None] + + if hits: + publications = ( + Publication.objects.filter(project_id__in=hits, is_published=True) + .defer("tree") + .order_by("-created") + ) + else: + publications = Publication.objects.none() + else: + publications = Publication.objects.filter(is_published=True).order_by("-created") publications_data = [ { diff --git a/server/portal/apps/search/tasks.py b/server/portal/apps/search/tasks.py index f72f145258..2fdd5c17fa 100644 --- a/server/portal/apps/search/tasks.py +++ b/server/portal/apps/search/tasks.py @@ -7,8 +7,10 @@ from portal.apps.users.utils import get_tas_allocations from portal.apps.projects.models.metadata import LegacyProjectMetadata from portal.libs.elasticsearch.docs.base import (IndexedAllocation, - IndexedProject) + IndexedProject, IndexedPublication) from portal.libs.elasticsearch.utils import get_sha256_hash +from portal.apps.publications.models import Publication +from elasticsearch.exceptions import NotFoundError logger = logging.getLogger(__name__) @@ -83,3 +85,15 @@ def index_project(self, project_id): @shared_task(bind=True, max_retries=3, queue='default') def tapis_project_listing_indexer(self, projects): index_project_listing(projects) + +def index_publication(project_id): + """Util to index a publication by its project ID""" + pub = Publication.objects.get(project_id=project_id) + try: + pub_es = IndexedPublication.get(project_id) + pub_es.update(**pub.tree, created=pub.created) + + except NotFoundError: + pub_es = IndexedPublication(**pub.tree, created=pub.created) + pub_es.meta["id"] = project_id + pub_es.save() \ No newline at end of file diff --git a/server/portal/libs/elasticsearch/docs/base.py b/server/portal/libs/elasticsearch/docs/base.py index c52b36be70..fb70ef6515 100644 --- a/server/portal/libs/elasticsearch/docs/base.py +++ b/server/portal/libs/elasticsearch/docs/base.py @@ -185,3 +185,12 @@ def from_username(cls, username): class Index: name = settings.ES_INDEX_PREFIX.format('allocations') + +class IndexedPublication(Document): + """Elasticsearch model for published works""" + + # pylint: disable=too-few-public-methods + class Index: + """Index meta settings""" + + name = settings.ES_INDEX_PREFIX.format('publications') diff --git a/server/portal/libs/elasticsearch/indexes.py b/server/portal/libs/elasticsearch/indexes.py index 56492dd23b..dfe5fc5f7f 100644 --- a/server/portal/libs/elasticsearch/indexes.py +++ b/server/portal/libs/elasticsearch/indexes.py @@ -8,7 +8,7 @@ from elasticsearch_dsl import Index from portal.libs.elasticsearch.docs.base import (IndexedFile, IndexedAllocation, - IndexedProject) + IndexedProject, IndexedPublication) from portal.libs.elasticsearch.analyzers import file_query_analyzer @@ -75,3 +75,9 @@ def setup_projects_index(reindex=False, force=False): if not index.exists(): index.document(IndexedProject) index.create() + +def setup_publications_index(reindex=False, force=False): + index = setup_indexes('publications', reindex, force) + if not index.exists(): + index.document(IndexedPublication) + index.create() \ No newline at end of file From 8eb1c0a17a9403ffa63725a2be43c17e817d9484 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Wed, 13 Nov 2024 15:15:15 -0600 Subject: [PATCH 129/328] Added metadata sync for externally added files --- server/portal/apps/projects/tasks.py | 57 +++++++++++++++++-- server/portal/apps/projects/views.py | 8 ++- .../workspace_operations/graph_operations.py | 15 ++++- 3 files changed, 74 insertions(+), 6 deletions(-) diff --git a/server/portal/apps/projects/tasks.py b/server/portal/apps/projects/tasks.py index 379b4bc128..f36703dc2b 100644 --- a/server/portal/apps/projects/tasks.py +++ b/server/portal/apps/projects/tasks.py @@ -1,18 +1,17 @@ import logging +from pathlib import Path from django.conf import settings from celery import shared_task from django.db import transaction -from portal.apps.projects.models.metadata import ProjectsMetadata from portal.apps.publications.models import PublicationRequest -from portal.apps.datafiles.models import DataFilesMetadata from portal.apps import SCHEMA_MAPPING from portal.libs.agave.utils import user_account, service_account from django.contrib.auth import get_user_model from django.core.exceptions import ObjectDoesNotExist from portal.apps.projects.models.project_metadata import ProjectMetadata from portal.apps._custom.drp import constants -from portal.apps.projects.workspace_operations.project_meta_operations import create_entity_metadata -from portal.apps.projects.workspace_operations.graph_operations import initialize_project_graph +from portal.apps.projects.workspace_operations.project_meta_operations import add_file_associations, create_file_obj, get_ordered_value +from portal.apps.projects.workspace_operations.graph_operations import get_path_uuid_mapping import networkx as nx import uuid @@ -183,3 +182,53 @@ def post_file_transfer(self, user_access_token, source_workspace_id, review_work except Exception as e: logger.error(f'Error processing transfer {transfer_task_id} for system {source_system_id} to system {review_system_id}: {e}') self.retry(exc=e, countdown=30) + + +@shared_task(bind=True, max_retries=3, queue='default') +def sync_files_without_metadata(self, user_access_token, project_id: str): + client = user_account(user_access_token) + + path_uuid_map = get_path_uuid_mapping(project_id) + tapis_files_listing = client.files.listFiles(systemId=project_id, path='/', recurse=True) + files = [file for file in tapis_files_listing if file.type != 'dir'] + + required_uuids = set(path_uuid_map.values()) + + # Cache to avoid repeated database queries for the same parent path + entity_cache = { + entity.uuid: entity for entity in ProjectMetadata.objects.filter(uuid__in=required_uuids) + } + + files_to_add_dict = {} + + for file in files: + file_path = file.path + parent_path = str(Path(file_path).parent) + + if parent_path == '.': + parent_path = '' + + # Check if the parent path exists in path_uuid_map + if parent_path not in path_uuid_map: + continue + + # Fetch the cached entity + parent_uuid = path_uuid_map[parent_path] + entity = entity_cache.get(parent_uuid) + + if not entity: + logger.warning(f"Parent entity {parent_uuid} for file {file_path} does not exist") + continue + + entity_value = get_ordered_value(entity.name, entity.value) + file_objs = entity_value.get('file_objs', []) + file_paths_set = {file_obj.get('path') for file_obj in file_objs} + + if file_path not in file_paths_set: + new_file_obj = create_file_obj(project_id, file.name, file.size, file_path, {'data_type': 'file'}) + files_to_add_dict[entity.uuid] = files_to_add_dict.get(entity.uuid, []) + [new_file_obj] + + for entity_uuid, file_objs in files_to_add_dict.items(): + logger.info(f'Adding {len(file_objs)} files to entity {entity_uuid} in project {project_id}') + add_file_associations(entity_uuid, file_objs) + \ No newline at end of file diff --git a/server/portal/apps/projects/views.py b/server/portal/apps/projects/views.py index f2ff4b05d2..e1ac3d179d 100644 --- a/server/portal/apps/projects/views.py +++ b/server/portal/apps/projects/views.py @@ -33,6 +33,7 @@ from pathlib import Path from portal.apps._custom.drp import constants from portal.apps.projects.workspace_operations.graph_operations import add_node_to_project, initialize_project_graph, get_node_from_path +from portal.apps.projects.tasks import sync_files_without_metadata LOGGER = logging.getLogger(__name__) @@ -178,12 +179,17 @@ def get(self, request, project_id=None, system_id=None): if system_id is not None: project_id = system_id.split(f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.")[1] - prj = get_project(request.user.tapis_oauth.client, project_id) + client = request.user.tapis_oauth.client + + prj = get_project(client, project_id) try: project = ProjectMetadata.objects.get(models.Q(value__projectId=f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}")) prj.update(get_ordered_value(project.name, project.value)) prj["projectId"] = project_id + + if not getattr(prj, 'is_review_project', False) and not getattr(prj, 'is_published_project', False): + sync_files_without_metadata.delay(client.access_token.access_token, f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}") except: pass diff --git a/server/portal/apps/projects/workspace_operations/graph_operations.py b/server/portal/apps/projects/workspace_operations/graph_operations.py index 8c3ae4d96d..3c98e807c5 100644 --- a/server/portal/apps/projects/workspace_operations/graph_operations.py +++ b/server/portal/apps/projects/workspace_operations/graph_operations.py @@ -182,4 +182,17 @@ def remove_trash_nodes(graph: nx.DiGraph): trash_descendants = nx.descendants(graph, trash_node_id) nodes_to_remove = {trash_node_id} | trash_descendants graph.remove_nodes_from(nodes_to_remove) - return graph \ No newline at end of file + return graph + +def get_path_uuid_mapping(project_id: str): + """Return a mapping of node paths to UUIDs for a project graph.""" + graph_model = ProjectMetadata.objects.get( + name=constants.PROJECT_GRAPH, base_project__value__projectId=project_id + ) + project_graph = nx.node_link_graph(graph_model.value) + path_uuid_mapping = {} + for node_id in project_graph.nodes: + path_nodes = nx.shortest_path(project_graph, 'NODE_ROOT', node_id)[1:] + path = '/'.join(project_graph.nodes[parent]['label'] for parent in path_nodes if 'label' in project_graph.nodes[parent]) + path_uuid_mapping[path] = project_graph.nodes[node_id]["uuid"] + return path_uuid_mapping \ No newline at end of file From c504a911ebdbd44312adcc90f3c1122900cad7b1 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Wed, 13 Nov 2024 17:15:16 -0600 Subject: [PATCH 130/328] Added validation for links --- .../DataFilesModals/DataFilesFormModal.jsx | 9 +++- .../_common/Form/DynamicForm/DynamicForm.jsx | 1 + ...aFilesProjectEditDescriptionModalAddon.jsx | 54 ++++++++++++------- 3 files changed, 43 insertions(+), 21 deletions(-) diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx index 1cbd2afc2a..29b5be8e6a 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx @@ -77,10 +77,17 @@ const DataFilesFormModal = () => { const validationSchema = Yup.object().shape({ ...(form?.form_fields ?? []).reduce((schema, field) => { if (field.validation?.required) { - schema[field.name] = Yup.string().required( + schema[field.name] = (schema[field.name] || Yup.string()).required( `${field.label} is required` ); } + + if (field.type === 'link') { + schema[field.name] = (schema[field.name] || Yup.string()) + .url(`${field.label} must be a valid URL`) + .matches(/^https:\/\//, `${field.label} must start with https://`); + } + return schema; }, {}), }); diff --git a/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx b/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx index 0bf7528b84..f7f74f177e 100644 --- a/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx +++ b/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx @@ -143,6 +143,7 @@ const DynamicForm = ({ initialFormFields, onChange }) => { switch (field.type) { case 'text': case 'number': + case 'link': return ( { }, [form]); const onFormChange = (formFields, values) => { - let schema = {}; - - Object.keys(values).forEach((key) => { - const field = formFields.find((f) => f.name === key); - - if (field) { - if (field.type === 'array') { - schema[key] = Yup.array().of( - Yup.object().shape( - field.fields.reduce((acc, subField) => { - if (subField.validation?.required) { - acc[subField.name] = Yup.string().required( - `${subField.label} is required` + const schema = formFields.reduce((acc, field) => { + if (field.type === 'array') { + acc[field.name] = Yup.array().of( + Yup.object().shape( + field.fields.reduce((subAcc, subField) => { + if (subField.type === 'link') { + subAcc[subField.name] = (subAcc[subField.name] || Yup.string()) + .url(`${subField.label} must be a valid URL`) + .matches( + /^https:\/\//, + `${subField.label} must start with https://` ); - } - return acc; - }, {}) - ) + } + if (subField.validation?.required) { + subAcc[subField.name] = ( + subAcc[subField.name] || Yup.string() + ).required(`${subField.label} is required`); + } + + return subAcc; + }, {}) + ) + ); + } else { + if (field.validation?.required) { + acc[field.name] = (acc[field.name] || Yup.string()).required( + `${field.label} is required` ); - } else { - schema[key] = Yup.string().required(`${field.label} is required`); + } + if (field.type === 'link') { + acc[field.name] = (acc[field.name] || Yup.string()) + .required(`${field.label} is required`) + .url(`${field.label} must be a valid URL`) + .matches(/^https:\/\//, `${field.label} must start with https://`); } } - }); + return acc; + }, {}); setValidationSchema((prevSchema) => { return Yup.object().shape({ From 5c0c98cd230839d8ebb6244a8649701d349f7f53 Mon Sep 17 00:00:00 2001 From: Jake Rosenberg Date: Wed, 13 Nov 2024 17:20:54 -0600 Subject: [PATCH 131/328] Close upload modal after successful upload (#1007) --- client/src/redux/sagas/datafiles.sagas.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/client/src/redux/sagas/datafiles.sagas.js b/client/src/redux/sagas/datafiles.sagas.js index 1134a462d1..0a4f504d99 100644 --- a/client/src/redux/sagas/datafiles.sagas.js +++ b/client/src/redux/sagas/datafiles.sagas.js @@ -561,6 +561,10 @@ export function* uploadFiles(action) { }); yield call(action.payload.reloadCallback); + yield put({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { operation: 'upload', props: {} }, + }); } export function* uploadFile(api, scheme, system, path, file, index, metadata) { From 23cd979ade168cfb41f672f4f568aac24ece2160 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Wed, 13 Nov 2024 17:26:34 -0600 Subject: [PATCH 132/328] minor bug fixes --- .../DataFilesProjectFileListingMetadataAddon.jsx | 3 ++- .../components/_custom/drp/utils/DataDisplay/DataDisplay.jsx | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx index 32950382aa..687965457d 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx @@ -35,9 +35,10 @@ const DataFilesProjectFileListingMetadataAddon = ({ keywords, }) => { const dateOptions = { month: 'long', day: 'numeric', year: 'numeric' }; + const dateLabel = publication_date ? 'Publication Date' : 'Created'; return { - publication_date: new Date( + [dateLabel]: new Date( publication_date || created ).toLocaleDateString('en-US', dateOptions), license: license ?? 'None', diff --git a/client/src/components/_custom/drp/utils/DataDisplay/DataDisplay.jsx b/client/src/components/_custom/drp/utils/DataDisplay/DataDisplay.jsx index f807ccd128..8059132262 100644 --- a/client/src/components/_custom/drp/utils/DataDisplay/DataDisplay.jsx +++ b/client/src/components/_custom/drp/utils/DataDisplay/DataDisplay.jsx @@ -88,7 +88,7 @@ const DataDisplay = ({ data, path, excludeKeys, modalData }) => { .filter(([key, value]) => value !== '' && !excludeKeys.includes(key)) .map(([key, value]) => ({ label: formatLabel(key), - value: typeof value === 'string' ? value : value, + value: typeof value === 'string' ? formatLabel(value) : value, })); if (path) { From 1611b6fd4889a19108d05a0c0d961951a3ef8bcd Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Wed, 13 Nov 2024 21:44:33 -0600 Subject: [PATCH 133/328] Fix for publication date --- .../workspace_operations/project_publish_operations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/portal/apps/projects/workspace_operations/project_publish_operations.py b/server/portal/apps/projects/workspace_operations/project_publish_operations.py index ffbc23c41f..bad0c781ff 100644 --- a/server/portal/apps/projects/workspace_operations/project_publish_operations.py +++ b/server/portal/apps/projects/workspace_operations/project_publish_operations.py @@ -123,14 +123,14 @@ def publish_project(self, project_id: str, version: Optional[int] = 1): source_project_id = f'{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}' source_project = ProjectMetadata.get_project_by_id(source_project_id) source_project.value['doi'] = doi - source_project.value['publication_date'] = published_project.created + source_project.value['publicationDate'] = published_project.created source_project.save() pub_tree = nx.node_link_graph(published_project.project_graph.value) pub_tree.nodes["NODE_ROOT"]["version"] = version published_project.project_graph.value = nx.node_link_data(pub_tree) published_project.value['doi'] = doi - published_project.value['publication_date'] = published_project.created + published_project.value['publicationDate'] = published_project.created published_project.save() From 1dbb30449d3ce33e9e60501f5007d969fd23dd4f Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Fri, 15 Nov 2024 16:46:25 -0600 Subject: [PATCH 134/328] Fixed project system naming, improved project/pub state handling --- .../DataFilesBreadcrumbs/DataFilesBreadcrumbs.jsx | 6 +++--- .../DataFilesModals/DataFilesShowPathModal.jsx | 6 +++++- .../DataFilesPublicationsList.jsx | 1 + client/src/hooks/datafiles/useSystems.js | 10 +++++++++- client/src/redux/reducers/datafiles.reducers.js | 2 +- client/src/redux/reducers/projects.reducers.js | 5 +++++ client/src/redux/sagas/projects.sagas.js | 11 +++++++++++ client/src/redux/sagas/publications.sagas.js | 9 +++++++++ client/src/utils/systems.js | 4 ++-- 9 files changed, 46 insertions(+), 8 deletions(-) diff --git a/client/src/components/DataFiles/DataFilesBreadcrumbs/DataFilesBreadcrumbs.jsx b/client/src/components/DataFiles/DataFilesBreadcrumbs/DataFilesBreadcrumbs.jsx index 636a4e5f70..829339069e 100644 --- a/client/src/components/DataFiles/DataFilesBreadcrumbs/DataFilesBreadcrumbs.jsx +++ b/client/src/components/DataFiles/DataFilesBreadcrumbs/DataFilesBreadcrumbs.jsx @@ -141,7 +141,7 @@ const DataFilesBreadcrumbs = ({ }); }; - const { fetchSelectedSystem } = useSystems(); + const { fetchSelectedSystem, isRootProjectSystem } = useSystems(); const selectedSystem = fetchSelectedSystem({ scheme, system, path }); @@ -184,7 +184,7 @@ const DataFilesBreadcrumbs = ({
{currentDirectory.length === 0 ? ( - {truncateMiddle(systemName || 'Shared Workspaces', 30)} + {truncateMiddle(systemName, 30)} ) : ( currentDirectory.map((pathComp, i) => { @@ -194,7 +194,7 @@ const DataFilesBreadcrumbs = ({ }) )}
- {systemName && api === 'tapis' && ( + {systemName && api === 'tapis' && !isRootProjectSystem(selectedSystem ?? '') && ( diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesShowPathModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesShowPathModal.jsx index 0b49d5840a..79cf2447eb 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesShowPathModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesShowPathModal.jsx @@ -3,6 +3,7 @@ import { useSelector, useDispatch, shallowEqual } from 'react-redux'; import { Modal, ModalHeader, ModalBody } from 'reactstrap'; import { TextCopyField } from '_common'; import styles from './DataFilesShowPathModal.module.scss'; +import { useSystems } from 'hooks/datafiles'; const DataFilesShowPathModal = React.memo(() => { const dispatch = useDispatch(); @@ -27,8 +28,11 @@ const DataFilesShowPathModal = React.memo(() => { } }, [modalParams, dispatch]); + const { isRootProjectSystem } = useSystems(); + useEffect(() => { - if (params.api === 'tapis' && params.system) { + + if (params.api === 'tapis' && params.system && !isRootProjectSystem({ system: params.system })) { dispatch({ type: 'FETCH_SYSTEM_DEFINITION', payload: params.system, diff --git a/client/src/components/DataFiles/DataFilesPublicationsList/DataFilesPublicationsList.jsx b/client/src/components/DataFiles/DataFilesPublicationsList/DataFilesPublicationsList.jsx index 4bd69ecf4a..e17c81016a 100644 --- a/client/src/components/DataFiles/DataFilesPublicationsList/DataFilesPublicationsList.jsx +++ b/client/src/components/DataFiles/DataFilesPublicationsList/DataFilesPublicationsList.jsx @@ -38,6 +38,7 @@ const DataFilesPublicationsList = ({ rootSystem }) => { type: 'PUBLICATIONS_GET_PUBLICATIONS', payload: { queryString: query.query_string, + system: selectedSystem.system, }, }); }, [dispatch, query.query_string]); diff --git a/client/src/hooks/datafiles/useSystems.js b/client/src/hooks/datafiles/useSystems.js index c5fa5a65f3..884c116746 100644 --- a/client/src/hooks/datafiles/useSystems.js +++ b/client/src/hooks/datafiles/useSystems.js @@ -29,7 +29,15 @@ function useSystems() { [data] ); - return { data, loading, error, fetchSystems, fetchSelectedSystem }; + const isRootProjectSystem = useCallback( + ({ system = '' }) => { + return data.some((s) => + s.scheme === 'projects' && s.system === system + ) + } + ) + + return { data, loading, error, fetchSystems, fetchSelectedSystem, isRootProjectSystem }; } export default useSystems; diff --git a/client/src/redux/reducers/datafiles.reducers.js b/client/src/redux/reducers/datafiles.reducers.js index f3a4d135f7..1482d82d40 100644 --- a/client/src/redux/reducers/datafiles.reducers.js +++ b/client/src/redux/reducers/datafiles.reducers.js @@ -418,7 +418,7 @@ export function files(state = initialFilesState, action) { FilesListing: { api: 'tapis', scheme: 'projects', - system: '', + system: action.payload.system ?? '', path: '', }, }, diff --git a/client/src/redux/reducers/projects.reducers.js b/client/src/redux/reducers/projects.reducers.js index 5cdb15c08e..5dd4acf2fc 100644 --- a/client/src/redux/reducers/projects.reducers.js +++ b/client/src/redux/reducers/projects.reducers.js @@ -221,6 +221,11 @@ export default function projects(state = initialState, action) { ...state, operation: initialState.operation, }; + case 'PROJECTS_CLEAR_METADATA': + return { + ...state, + metadata: initialState.metadata, + }; case 'PROJECTS_CREATE_PUBLICATION_REQUEST_STARTED': return { ...state, diff --git a/client/src/redux/sagas/projects.sagas.js b/client/src/redux/sagas/projects.sagas.js index 12cd8d1bd7..e5fc236b55 100644 --- a/client/src/redux/sagas/projects.sagas.js +++ b/client/src/redux/sagas/projects.sagas.js @@ -43,7 +43,15 @@ export function* showSharedWorkspaces(action) { // Clear FileListing params to reset breadcrumbs yield put({ type: 'DATA_FILES_CLEAR_PROJECT_SELECTION', + payload: { + system: action.payload.rootSystem, + } }); + + yield put({ + type: 'PROJECTS_CLEAR_METADATA', + }) + // Load projects list yield put({ type: 'PROJECTS_GET_LISTING', @@ -93,6 +101,9 @@ export async function fetchMetadata(system) { } export function* getMetadata(action) { + yield put({ + type: 'PROJECTS_CLEAR_METADATA', + }) yield put({ type: 'PROJECTS_GET_METADATA_STARTED', }); diff --git a/client/src/redux/sagas/publications.sagas.js b/client/src/redux/sagas/publications.sagas.js index e4f32db23e..6d1bffe052 100644 --- a/client/src/redux/sagas/publications.sagas.js +++ b/client/src/redux/sagas/publications.sagas.js @@ -134,6 +134,15 @@ export async function fetchPublicationsUtil(queryString) { } export function* getPublications(action) { + yield put({ + type: 'DATA_FILES_CLEAR_PROJECT_SELECTION', + payload: { + system: action.payload.system, + } + }); + yield put({ + type: 'PROJECTS_CLEAR_METADATA', + }) yield put({ type: 'PUBLICATIONS_GET_PUBLICATIONS_STARTED', }); diff --git a/client/src/utils/systems.js b/client/src/utils/systems.js index fe105a410a..cef773e735 100644 --- a/client/src/utils/systems.js +++ b/client/src/utils/systems.js @@ -90,9 +90,9 @@ export function findSystemOrProjectDisplayName( switch (scheme) { case 'projects': let project = findProjectTitle(projectsList, system, projectTitle); - if (!project) { + if (!project) { const projectSystem = systemList.find( - (system) => system.scheme === 'projects' + (sys) => sys.scheme === 'projects' && sys.system === system ); return projectSystem ? projectSystem.name : ''; } else { From 0e5319a3761faa4a47d3000f3e895cbc5deafbf4 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Tue, 19 Nov 2024 13:24:35 -0600 Subject: [PATCH 135/328] fix for projects breadcrumbs Go To button --- .../DataFilesDropdown/DataFilesDropdown.jsx | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/client/src/components/DataFiles/DataFilesDropdown/DataFilesDropdown.jsx b/client/src/components/DataFiles/DataFilesDropdown/DataFilesDropdown.jsx index 60e90fce26..981aa944cf 100644 --- a/client/src/components/DataFiles/DataFilesDropdown/DataFilesDropdown.jsx +++ b/client/src/components/DataFiles/DataFilesDropdown/DataFilesDropdown.jsx @@ -26,7 +26,12 @@ const BreadcrumbsDropdown = ({ const location = useLocation(); const pathParts = location.pathname.split('/'); + const projectId = pathParts.includes('projects') + ? pathParts[pathParts.indexOf('projects') + 2] + : null; + + const rootProjectSystem = pathParts.includes('projects') ? pathParts[pathParts.indexOf('projects') + 1] : null; @@ -35,13 +40,15 @@ const BreadcrumbsDropdown = ({ let url; if (scheme === 'projects' && targetPath === systemName) { - url = `${basePath}/${api}/projects/${projectId}/`; + url = `${basePath}/${api}/projects/${rootProjectSystem}/${projectId}/`; } else if (scheme === 'projects' && !targetPath) { - url = `${basePath}/${api}/projects/`; + url = `${basePath}/${api}/projects/${rootProjectSystem}`; } else if (api === 'googledrive' && !targetPath) { url = `${basePath}/${api}/${scheme}/${system}/`; } else if (api === 'tapis' && scheme !== 'projects' && !targetPath) { url = `${basePath}/${api}/${scheme}/${system}/`; + } else if (scheme === 'projects') { + url = `${basePath}/${api}/projects/${rootProjectSystem}/${system}${targetPath}`; } else { url = `${basePath}/${api}/${scheme}/${system}${targetPath}/`; } @@ -69,7 +76,7 @@ const BreadcrumbsDropdown = ({ ); const sharedWorkspacesDisplayName = systems.find( - (e) => e.scheme === 'projects' + (e) => e.scheme === 'projects' && e.system === rootProjectSystem )?.name; let currentPath = startingPath; @@ -81,7 +88,7 @@ const BreadcrumbsDropdown = ({ const fullPath = paths.reverse(); const displayPaths = scheme === 'projects' ? [...fullPath, systemName] : fullPath; - const sliceStart = scheme === 'projects' && systemName ? 0 : 1; + const sliceStart = 1 return (
Date: Tue, 19 Nov 2024 17:26:09 -0600 Subject: [PATCH 136/328] Added View Authors modal, added extra logic for View Team button --- .../DataFilesModals/DataFilesModals.jsx | 2 + .../DataFilesPublicationAuthorsModal.jsx | 61 +++++++++++++++++++ ...taFilesPublicationAuthorsModal.module.scss | 13 ++++ .../DataFilesProjectFileListing.jsx | 22 +++---- .../DataFilesProjectFileListingAddon.jsx | 14 ++++- .../drp/utils/hooks/useDrpDatasetModals.js | 14 +++++ client/src/hooks/datafiles/useSystems.js | 16 ++++- 7 files changed, 126 insertions(+), 16 deletions(-) create mode 100644 client/src/components/DataFiles/DataFilesModals/DataFilesPublicationAuthorsModal.jsx create mode 100644 client/src/components/DataFiles/DataFilesModals/DataFilesPublicationAuthorsModal.module.scss diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesModals.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesModals.jsx index e250ac0c8f..1e8a081d7b 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesModals.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesModals.jsx @@ -23,6 +23,7 @@ import DataFilesProjectTreeModal from './DataFilesProjectTreeModal'; import DataFilesProjectDescriptionModal from './DataFilesProjectDescriptionModal'; import DataFilesViewDataModal from './DataFilesViewDataModal'; import DataFilesProjectCitationModal from './DataFilesProjectCitationModal'; +import DataFilesPublicationAuthorsModal from './DataFilesPublicationAuthorsModal'; export default function DataFilesModals() { return ( @@ -50,6 +51,7 @@ export default function DataFilesModals() { + ); } diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesPublicationAuthorsModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesPublicationAuthorsModal.jsx new file mode 100644 index 0000000000..96e8969b3d --- /dev/null +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesPublicationAuthorsModal.jsx @@ -0,0 +1,61 @@ +import InfiniteScrollTable from '_common/InfiniteScrollTable'; +import React, { useCallback, useEffect, useState } from 'react'; +import { useDispatch, useSelector } from 'react-redux'; +import { Modal, ModalHeader, ModalBody } from 'reactstrap'; +import styles from './DataFilesPublicationAuthorsModal.module.scss' + + +const DataFilesPublicationAuthorsModal = () => { + + const dispatch = useDispatch(); + const isOpen = useSelector((state) => state.files.modals.publicationAuthors); + const props = useSelector((state) => state.files.modalProps.publicationAuthors); + + const columns = [ + { + Header: 'Name', + accessor: (el) => el, + Cell: (el) => { + const { first_name, last_name } = el.value; + return ( + + {first_name} {last_name} + + ); + }, + }, + { + Header: 'Email', + accessor: 'email', + }, + ]; + + const toggle = useCallback(() => { + dispatch({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { operation: 'publicationAuthors', props: {} }, + }); + }, [dispatch]); + + return ( + + + Authors + + + + + + ); +}; + +export default DataFilesPublicationAuthorsModal; \ No newline at end of file diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesPublicationAuthorsModal.module.scss b/client/src/components/DataFiles/DataFilesModals/DataFilesPublicationAuthorsModal.module.scss new file mode 100644 index 0000000000..2a1979deba --- /dev/null +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesPublicationAuthorsModal.module.scss @@ -0,0 +1,13 @@ +.author-listing { + /* title */ + th:nth-child(1), + td:nth-child(1) { + width: 50%; + } + /* date */ + th:nth-child(2), + td:nth-child(2) { + width: 50%; + } + } + \ No newline at end of file diff --git a/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx b/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx index 25467417d0..cebf71cd79 100644 --- a/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx +++ b/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx @@ -8,18 +8,14 @@ import { SectionMessage, SectionTableWrapper, } from '_common'; -import { useAddonComponents, useFileListing } from 'hooks/datafiles'; +import { useAddonComponents, useFileListing, useSystems } from 'hooks/datafiles'; import DataFilesListing from '../DataFilesListing/DataFilesListing'; import styles from './DataFilesProjectFileListing.module.scss'; const DataFilesProjectFileListing = ({ rootSystem, system, path }) => { const dispatch = useDispatch(); const { fetchListing } = useFileListing('FilesListing'); - const systems = useSelector( - (state) => state.systems.storage.configuration.filter((s) => !s.hidden), - shallowEqual - ); - const [isPublicationSystem, setIsPublicationSystem] = useState(false); + const { isPublicationSystem, isReviewSystem } = useSystems(); // logic to render addonComponents for DRP const portalName = useSelector((state) => state.workbench.portalName); @@ -44,10 +40,6 @@ const DataFilesProjectFileListing = ({ rootSystem, system, path }) => { fetchListing({ api: 'tapis', scheme: 'projects', system, path }); }, [system, path, fetchListing]); - useEffect(() => { - const system = systems.find((s) => s.system === rootSystem); - setIsPublicationSystem(system?.publicationProject); - }, [systems, rootSystem]); const metadata = useSelector((state) => state.projects.metadata); const folderMetadata = useSelector( @@ -135,9 +127,11 @@ const DataFilesProjectFileListing = ({ rootSystem, system, path }) => { | ) : null} - + {!isPublicationSystem(rootSystem) && !isReviewSystem(rootSystem) && ( + + )} {DataFilesProjectFileListingAddon && ( { folderMetadata={folderMetadata} metadata={metadata} path={path} - showCitation={isPublicationSystem} + showCitation={isPublicationSystem(rootSystem)} /> ) : ( diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx index 328f14df83..a44232dfbb 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx @@ -2,7 +2,7 @@ import React, { useEffect, useState } from 'react'; import { Button } from '_common'; import { useDispatch, useSelector } from 'react-redux'; import styles from './DataFilesProjectFileListingAddon.module.scss'; -import { useSelectedFiles } from 'hooks/datafiles'; +import { useSelectedFiles, useSystems } from 'hooks/datafiles'; import useDrpDatasetModals from '../utils/hooks/useDrpDatasetModals'; import { Link } from 'react-router-dom'; import * as ROUTES from '../../../../constants/routes'; @@ -12,6 +12,7 @@ const DataFilesProjectFileListingAddon = ({ rootSystem, system }) => { const { projectId } = useSelector((state) => state.projects.metadata); const { metadata } = useSelector((state) => state.projects); const { selectedFiles } = useSelectedFiles(); + const { isPublicationSystem, isReviewSystem } = useSystems(); const dispatch = useDispatch(); @@ -20,6 +21,7 @@ const DataFilesProjectFileListingAddon = ({ rootSystem, system }) => { createOriginDataModal, createAnalysisDataModal, createTreeModal, + createPublicationAuthorsModal, } = useDrpDatasetModals(projectId, portalName); const createPublicationRequestModal = () => { @@ -79,6 +81,16 @@ const DataFilesProjectFileListingAddon = ({ rootSystem, system }) => { return ( <> + {(isPublicationSystem(rootSystem) || isReviewSystem(rootSystem)) && ( + <> + + + )} {canEditDataset && ( <> | diff --git a/client/src/components/_custom/drp/utils/hooks/useDrpDatasetModals.js b/client/src/components/_custom/drp/utils/hooks/useDrpDatasetModals.js index 6ce6f46aef..010e80ce13 100644 --- a/client/src/components/_custom/drp/utils/hooks/useDrpDatasetModals.js +++ b/client/src/components/_custom/drp/utils/hooks/useDrpDatasetModals.js @@ -142,11 +142,25 @@ const useDrpDatasetModals = ( [dispatch] ); + const createPublicationAuthorsModal = useCallback( + async ({ authors }) => { + dispatch({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { + operation: 'publicationAuthors', + props: { authors }, + }, + }); + }, + [dispatch] + ) + return { createSampleModal, createOriginDataModal, createAnalysisDataModal, createTreeModal, + createPublicationAuthorsModal }; }; diff --git a/client/src/hooks/datafiles/useSystems.js b/client/src/hooks/datafiles/useSystems.js index c5fa5a65f3..2bad304fd3 100644 --- a/client/src/hooks/datafiles/useSystems.js +++ b/client/src/hooks/datafiles/useSystems.js @@ -29,7 +29,21 @@ function useSystems() { [data] ); - return { data, loading, error, fetchSystems, fetchSelectedSystem }; + const isPublicationSystem = useCallback( + (system) => { + return data.some((s) => s.system === system && s.publicationProject); + }, + [data] + ); + + const isReviewSystem = useCallback( + (system) => { + return data.some((s) => s.system === system && s.reviewProject); + }, + [data] + ); + + return { data, loading, error, fetchSystems, fetchSelectedSystem, isPublicationSystem, isReviewSystem }; } export default useSystems; From 9c5543cb7f1665ade725cf428722c7159dfb5e93 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Wed, 20 Nov 2024 10:54:06 -0600 Subject: [PATCH 137/328] front end linting --- .../DataFilesPublicationAuthorsModal.jsx | 100 +++++++++--------- ...taFilesPublicationAuthorsModal.module.scss | 21 ++-- .../DataFilesProjectFileListing.jsx | 9 +- .../DataFilesProjectFileListingAddon.jsx | 4 +- ...taFilesProjectFileListingMetadataAddon.jsx | 7 +- .../drp/utils/hooks/useDrpDatasetModals.js | 4 +- client/src/hooks/datafiles/useSystems.js | 10 +- 7 files changed, 84 insertions(+), 71 deletions(-) diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesPublicationAuthorsModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesPublicationAuthorsModal.jsx index 96e8969b3d..25666ec2d4 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesPublicationAuthorsModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesPublicationAuthorsModal.jsx @@ -2,60 +2,60 @@ import InfiniteScrollTable from '_common/InfiniteScrollTable'; import React, { useCallback, useEffect, useState } from 'react'; import { useDispatch, useSelector } from 'react-redux'; import { Modal, ModalHeader, ModalBody } from 'reactstrap'; -import styles from './DataFilesPublicationAuthorsModal.module.scss' - +import styles from './DataFilesPublicationAuthorsModal.module.scss'; const DataFilesPublicationAuthorsModal = () => { + const dispatch = useDispatch(); + const isOpen = useSelector((state) => state.files.modals.publicationAuthors); + const props = useSelector( + (state) => state.files.modalProps.publicationAuthors + ); - const dispatch = useDispatch(); - const isOpen = useSelector((state) => state.files.modals.publicationAuthors); - const props = useSelector((state) => state.files.modalProps.publicationAuthors); - - const columns = [ - { - Header: 'Name', - accessor: (el) => el, - Cell: (el) => { - const { first_name, last_name } = el.value; - return ( - - {first_name} {last_name} - - ); - }, - }, - { - Header: 'Email', - accessor: 'email', - }, - ]; + const columns = [ + { + Header: 'Name', + accessor: (el) => el, + Cell: (el) => { + const { first_name, last_name } = el.value; + return ( + + {first_name} {last_name} + + ); + }, + }, + { + Header: 'Email', + accessor: 'email', + }, + ]; - const toggle = useCallback(() => { - dispatch({ - type: 'DATA_FILES_TOGGLE_MODAL', - payload: { operation: 'publicationAuthors', props: {} }, - }); - }, [dispatch]); + const toggle = useCallback(() => { + dispatch({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { operation: 'publicationAuthors', props: {} }, + }); + }, [dispatch]); - return ( - - - Authors - - - - - - ); + return ( + + + Authors + + + + + + ); }; -export default DataFilesPublicationAuthorsModal; \ No newline at end of file +export default DataFilesPublicationAuthorsModal; diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesPublicationAuthorsModal.module.scss b/client/src/components/DataFiles/DataFilesModals/DataFilesPublicationAuthorsModal.module.scss index 2a1979deba..dbd530a026 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesPublicationAuthorsModal.module.scss +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesPublicationAuthorsModal.module.scss @@ -1,13 +1,12 @@ .author-listing { - /* title */ - th:nth-child(1), - td:nth-child(1) { - width: 50%; - } - /* date */ - th:nth-child(2), - td:nth-child(2) { - width: 50%; - } + /* title */ + th:nth-child(1), + td:nth-child(1) { + width: 50%; } - \ No newline at end of file + /* date */ + th:nth-child(2), + td:nth-child(2) { + width: 50%; + } +} diff --git a/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx b/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx index cebf71cd79..2828e91516 100644 --- a/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx +++ b/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx @@ -8,7 +8,11 @@ import { SectionMessage, SectionTableWrapper, } from '_common'; -import { useAddonComponents, useFileListing, useSystems } from 'hooks/datafiles'; +import { + useAddonComponents, + useFileListing, + useSystems, +} from 'hooks/datafiles'; import DataFilesListing from '../DataFilesListing/DataFilesListing'; import styles from './DataFilesProjectFileListing.module.scss'; @@ -40,7 +44,6 @@ const DataFilesProjectFileListing = ({ rootSystem, system, path }) => { fetchListing({ api: 'tapis', scheme: 'projects', system, path }); }, [system, path, fetchListing]); - const metadata = useSelector((state) => state.projects.metadata); const folderMetadata = useSelector( (state) => state.files.folderMetadata?.FilesListing @@ -129,7 +132,7 @@ const DataFilesProjectFileListing = ({ rootSystem, system, path }) => { ) : null} {!isPublicationSystem(rootSystem) && !isReviewSystem(rootSystem) && ( )} {DataFilesProjectFileListingAddon && ( diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx index a44232dfbb..a2099a34c5 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingAddon/DataFilesProjectFileListingAddon.jsx @@ -85,7 +85,9 @@ const DataFilesProjectFileListingAddon = ({ rootSystem, system }) => { <> diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx index 687965457d..e64b4318db 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx @@ -38,9 +38,10 @@ const DataFilesProjectFileListingMetadataAddon = ({ const dateLabel = publication_date ? 'Publication Date' : 'Created'; return { - [dateLabel]: new Date( - publication_date || created - ).toLocaleDateString('en-US', dateOptions), + [dateLabel]: new Date(publication_date || created).toLocaleDateString( + 'en-US', + dateOptions + ), license: license ?? 'None', ...(doi && { doi }), ...(keywords && { keywords }), diff --git a/client/src/components/_custom/drp/utils/hooks/useDrpDatasetModals.js b/client/src/components/_custom/drp/utils/hooks/useDrpDatasetModals.js index 010e80ce13..2ba2ecab0d 100644 --- a/client/src/components/_custom/drp/utils/hooks/useDrpDatasetModals.js +++ b/client/src/components/_custom/drp/utils/hooks/useDrpDatasetModals.js @@ -153,14 +153,14 @@ const useDrpDatasetModals = ( }); }, [dispatch] - ) + ); return { createSampleModal, createOriginDataModal, createAnalysisDataModal, createTreeModal, - createPublicationAuthorsModal + createPublicationAuthorsModal, }; }; diff --git a/client/src/hooks/datafiles/useSystems.js b/client/src/hooks/datafiles/useSystems.js index 2bad304fd3..ded5f08b0a 100644 --- a/client/src/hooks/datafiles/useSystems.js +++ b/client/src/hooks/datafiles/useSystems.js @@ -43,7 +43,15 @@ function useSystems() { [data] ); - return { data, loading, error, fetchSystems, fetchSelectedSystem, isPublicationSystem, isReviewSystem }; + return { + data, + loading, + error, + fetchSystems, + fetchSelectedSystem, + isPublicationSystem, + isReviewSystem, + }; } export default useSystems; From 5f7f8b9f9cf389f41770621931415957f21b9d6e Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Wed, 20 Nov 2024 10:55:16 -0600 Subject: [PATCH 138/328] front end linting --- .../DataFilesBreadcrumbs.jsx | 16 +++++++------- .../DataFilesDropdown/DataFilesDropdown.jsx | 4 ++-- .../DataFilesShowPathModal.jsx | 7 +++++-- ...taFilesProjectFileListingMetadataAddon.jsx | 7 ++++--- client/src/hooks/datafiles/useSystems.js | 21 +++++++++++-------- client/src/redux/sagas/projects.sagas.js | 6 +++--- client/src/redux/sagas/publications.sagas.js | 4 ++-- client/src/utils/systems.js | 2 +- 8 files changed, 37 insertions(+), 30 deletions(-) diff --git a/client/src/components/DataFiles/DataFilesBreadcrumbs/DataFilesBreadcrumbs.jsx b/client/src/components/DataFiles/DataFilesBreadcrumbs/DataFilesBreadcrumbs.jsx index 829339069e..010d880650 100644 --- a/client/src/components/DataFiles/DataFilesBreadcrumbs/DataFilesBreadcrumbs.jsx +++ b/client/src/components/DataFiles/DataFilesBreadcrumbs/DataFilesBreadcrumbs.jsx @@ -183,9 +183,7 @@ const DataFilesBreadcrumbs = ({
{currentDirectory.length === 0 ? ( - - {truncateMiddle(systemName, 30)} - + {truncateMiddle(systemName, 30)} ) : ( currentDirectory.map((pathComp, i) => { if (i === fullPath.length - 1) { @@ -194,11 +192,13 @@ const DataFilesBreadcrumbs = ({ }) )}
- {systemName && api === 'tapis' && !isRootProjectSystem(selectedSystem ?? '') && ( - - )} + {systemName && + api === 'tapis' && + !isRootProjectSystem(selectedSystem ?? '') && ( + + )}
); }; diff --git a/client/src/components/DataFiles/DataFilesDropdown/DataFilesDropdown.jsx b/client/src/components/DataFiles/DataFilesDropdown/DataFilesDropdown.jsx index 981aa944cf..33affeef4f 100644 --- a/client/src/components/DataFiles/DataFilesDropdown/DataFilesDropdown.jsx +++ b/client/src/components/DataFiles/DataFilesDropdown/DataFilesDropdown.jsx @@ -30,7 +30,7 @@ const BreadcrumbsDropdown = ({ const projectId = pathParts.includes('projects') ? pathParts[pathParts.indexOf('projects') + 2] : null; - + const rootProjectSystem = pathParts.includes('projects') ? pathParts[pathParts.indexOf('projects') + 1] : null; @@ -88,7 +88,7 @@ const BreadcrumbsDropdown = ({ const fullPath = paths.reverse(); const displayPaths = scheme === 'projects' ? [...fullPath, systemName] : fullPath; - const sliceStart = 1 + const sliceStart = 1; return (
{ const { isRootProjectSystem } = useSystems(); useEffect(() => { - - if (params.api === 'tapis' && params.system && !isRootProjectSystem({ system: params.system })) { + if ( + params.api === 'tapis' && + params.system && + !isRootProjectSystem({ system: params.system }) + ) { dispatch({ type: 'FETCH_SYSTEM_DEFINITION', payload: params.system, diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx index 687965457d..e64b4318db 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx @@ -38,9 +38,10 @@ const DataFilesProjectFileListingMetadataAddon = ({ const dateLabel = publication_date ? 'Publication Date' : 'Created'; return { - [dateLabel]: new Date( - publication_date || created - ).toLocaleDateString('en-US', dateOptions), + [dateLabel]: new Date(publication_date || created).toLocaleDateString( + 'en-US', + dateOptions + ), license: license ?? 'None', ...(doi && { doi }), ...(keywords && { keywords }), diff --git a/client/src/hooks/datafiles/useSystems.js b/client/src/hooks/datafiles/useSystems.js index 884c116746..f540ee8353 100644 --- a/client/src/hooks/datafiles/useSystems.js +++ b/client/src/hooks/datafiles/useSystems.js @@ -29,15 +29,18 @@ function useSystems() { [data] ); - const isRootProjectSystem = useCallback( - ({ system = '' }) => { - return data.some((s) => - s.scheme === 'projects' && s.system === system - ) - } - ) - - return { data, loading, error, fetchSystems, fetchSelectedSystem, isRootProjectSystem }; + const isRootProjectSystem = useCallback(({ system = '' }) => { + return data.some((s) => s.scheme === 'projects' && s.system === system); + }); + + return { + data, + loading, + error, + fetchSystems, + fetchSelectedSystem, + isRootProjectSystem, + }; } export default useSystems; diff --git a/client/src/redux/sagas/projects.sagas.js b/client/src/redux/sagas/projects.sagas.js index e5fc236b55..220dea631c 100644 --- a/client/src/redux/sagas/projects.sagas.js +++ b/client/src/redux/sagas/projects.sagas.js @@ -45,12 +45,12 @@ export function* showSharedWorkspaces(action) { type: 'DATA_FILES_CLEAR_PROJECT_SELECTION', payload: { system: action.payload.rootSystem, - } + }, }); yield put({ type: 'PROJECTS_CLEAR_METADATA', - }) + }); // Load projects list yield put({ @@ -103,7 +103,7 @@ export async function fetchMetadata(system) { export function* getMetadata(action) { yield put({ type: 'PROJECTS_CLEAR_METADATA', - }) + }); yield put({ type: 'PROJECTS_GET_METADATA_STARTED', }); diff --git a/client/src/redux/sagas/publications.sagas.js b/client/src/redux/sagas/publications.sagas.js index 6d1bffe052..355a4c4505 100644 --- a/client/src/redux/sagas/publications.sagas.js +++ b/client/src/redux/sagas/publications.sagas.js @@ -138,11 +138,11 @@ export function* getPublications(action) { type: 'DATA_FILES_CLEAR_PROJECT_SELECTION', payload: { system: action.payload.system, - } + }, }); yield put({ type: 'PROJECTS_CLEAR_METADATA', - }) + }); yield put({ type: 'PUBLICATIONS_GET_PUBLICATIONS_STARTED', }); diff --git a/client/src/utils/systems.js b/client/src/utils/systems.js index cef773e735..594fe8dcf1 100644 --- a/client/src/utils/systems.js +++ b/client/src/utils/systems.js @@ -90,7 +90,7 @@ export function findSystemOrProjectDisplayName( switch (scheme) { case 'projects': let project = findProjectTitle(projectsList, system, projectTitle); - if (!project) { + if (!project) { const projectSystem = systemList.find( (sys) => sys.scheme === 'projects' && sys.system === system ); From 6e19162a9070efba89e1b7b524609e144b5decca Mon Sep 17 00:00:00 2001 From: Jake Rosenberg Date: Wed, 20 Nov 2024 11:28:30 -0600 Subject: [PATCH 139/328] Email project authors when their publication request is accepted/rejected (#1008) --- .../project_publish_operations.py | 71 ++++++++++++++++++- server/portal/apps/publications/views.py | 6 +- server/portal/settings/settings.py | 10 +++ .../settings/settings_custom.example.py | 4 ++ 4 files changed, 89 insertions(+), 2 deletions(-) diff --git a/server/portal/apps/projects/workspace_operations/project_publish_operations.py b/server/portal/apps/projects/workspace_operations/project_publish_operations.py index bad0c781ff..bbcc860317 100644 --- a/server/portal/apps/projects/workspace_operations/project_publish_operations.py +++ b/server/portal/apps/projects/workspace_operations/project_publish_operations.py @@ -14,6 +14,7 @@ from tapipy.errors import NotFoundError, BaseTapyException from django.contrib.auth import get_user_model from django.core.exceptions import ObjectDoesNotExist +from django.core.mail import send_mail logger = logging.getLogger(__name__) @@ -248,4 +249,72 @@ def update_and_cleanup_review_project(review_project_id: str, status: Publicatio review_project_graph.delete() review_project.delete() - logger.info(f'Deleted review project {review_project_id} and its associated data.') \ No newline at end of file + logger.info(f'Deleted review project {review_project_id} and its associated data.') + + +def get_project_user_emails(project_id): + """Return a list of emails for users in a project.""" + prj = ProjectMetadata.get_project_by_id(project_id) + return [user["email"] for user in prj.value["authors"] if user.get("email")] + + +@shared_task(bind=True, queue='default') +def send_publication_accept_email(project_id): + """ + Alert project authors that their request has been accepted. + """ + user_emails = get_project_user_emails() + for user_email in user_emails: + email_body = f""" +

Hello,

+

+ Congratulations! The following project has been accepted for publication: +
+ {project_id} +
+

+

+ Your publication should appear in the portal within 1 business day. +

+ + This is a programmatically generated message. Do NOT reply to this message. + """ + + send_mail( + "DigitalRocks Alert: Your Publication Request has been Accepted", + email_body, + settings.DEFAULT_FROM_EMAIL, + [user_email], + html_message=email_body, + ) + + +@shared_task(bind=True, queue='default') +def send_publication_reject_email(project_id: str, version: Optional[int], error: str): + """ + Alert project authors that their request has been rejected. + """ + user_emails = get_project_user_emails() + for user_email in user_emails: + email_body = f""" +

Hello,

+

+ The following project has been rejected by a reviewer and cannot be published at this time: +
+ {project_id} +
+

+

+ You are welcome to revise this project and re-submit for publication. +

+ + This is a programmatically generated message. Do NOT reply to this message. + """ + + send_mail( + "DigitalRocks Alert: Your Publication Request has been Rejected", + email_body, + settings.DEFAULT_FROM_EMAIL, + [user_email], + html_message=email_body, + ) \ No newline at end of file diff --git a/server/portal/apps/publications/views.py b/server/portal/apps/publications/views.py index 55aeef777a..2eab42e1fd 100644 --- a/server/portal/apps/publications/views.py +++ b/server/portal/apps/publications/views.py @@ -12,7 +12,7 @@ from portal.exceptions.api import ApiException from portal.views.base import BaseApiView from portal.apps.projects.workspace_operations.shared_workspace_operations import create_publication_workspace -from portal.apps.projects.workspace_operations.project_publish_operations import copy_graph_and_files_for_review_system, publish_project, update_and_cleanup_review_project +from portal.apps.projects.workspace_operations.project_publish_operations import copy_graph_and_files_for_review_system, publish_project, update_and_cleanup_review_project, send_publication_accept_email, send_publication_reject_email from portal.apps.projects.models.metadata import ProjectsMetadata from django.db import transaction from portal.apps.notifications.models import Notification @@ -220,6 +220,8 @@ def post(self, request): if not full_project_id: raise ApiException("Missing project ID", status=400) + send_publication_accept_email.apply_async(args=[full_project_id]) + if is_review: project_id = full_project_id.split(f"{settings.PORTAL_PROJECTS_REVIEW_SYSTEM_PREFIX}.")[1] else: @@ -314,6 +316,8 @@ def post(self, request): if not full_project_id: raise ApiException("Missing project ID", status=400) + send_publication_reject_email.apply_async(args=[full_project_id]) + update_and_cleanup_review_project(full_project_id, PublicationRequest.Status.REJECTED) # Create notification diff --git a/server/portal/settings/settings.py b/server/portal/settings/settings.py index bdd8f86339..8daa52252c 100644 --- a/server/portal/settings/settings.py +++ b/server/portal/settings/settings.py @@ -755,6 +755,16 @@ PORTAL_ELEVATED_ROLES = getattr(settings_custom, '_PORTAL_ELEVATED_ROLES', {}) +""" +SETTINGS: EMAIL +""" +EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' +EMAIL_HOST = getattr(settings_custom, '_SMTP_HOST', 'localhost') +EMAIL_PORT = getattr(settings_custom, '_SMTP_PORT', 25) +EMAIL_HOST_USER = getattr(settings_custom, '_SMTP_USER', '') +EMAIL_HOST_PASSWORD = getattr(settings_custom, '_SMTP_PASSWORD', '') +DEFAULT_FROM_EMAIL = getattr(settings_custom, '_DEFAULT_FROM_EMAIL', '') + """ SETTINGS: INTERNAL DOCS """ diff --git a/server/portal/settings/settings_custom.example.py b/server/portal/settings/settings_custom.example.py index 021148513c..dee98e741f 100644 --- a/server/portal/settings/settings_custom.example.py +++ b/server/portal/settings/settings_custom.example.py @@ -273,3 +273,7 @@ "usernames": [] } } + + +_SMTP_HOST = "relay.tacc.utexas.edu" +_DEFAULT_FROM_EMAIL="no-reply@digitalrocksportal.org" \ No newline at end of file From 941a2d24e77ecec02c59d1e4ace143429ac6cffa Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Wed, 20 Nov 2024 11:42:41 -0600 Subject: [PATCH 140/328] Changed curator name to email link when requesting publications --- .../SubmitPublicationRequest.jsx | 2 +- .../SubmitPublicationReview.jsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/SubmitPublicationRequest.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/SubmitPublicationRequest.jsx index 3ccff7617a..c110696abf 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/SubmitPublicationRequest.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/SubmitPublicationRequest.jsx @@ -85,7 +85,7 @@ const SubmitPublicationRequest = ({ callbackUrl }) => {
If you have any doubts about the process please contact the data - curator Maria Esteva before submitting the data for publication. + curator Maria Esteva before submitting the data for publication.
) : null}
+ {DataFilesManageProjectModalAddon && ( + + )}
diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesProjectEditDescriptionModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesProjectEditDescriptionModal.jsx index 8172e234da..0a07f9f1f0 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesProjectEditDescriptionModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesProjectEditDescriptionModal.jsx @@ -60,6 +60,7 @@ const DataFilesProjectEditDescriptionModal = () => { description: values.description || '', metadata: DataFilesProjectEditDescriptionModalAddon ? values : null, }, + modal: 'editproject', }, }); }, diff --git a/client/src/components/_custom/drp/DataFilesManageProjectModalAddon/DataFilesManageProjectModalAddon.jsx b/client/src/components/_custom/drp/DataFilesManageProjectModalAddon/DataFilesManageProjectModalAddon.jsx new file mode 100644 index 0000000000..fed1aaaa5b --- /dev/null +++ b/client/src/components/_custom/drp/DataFilesManageProjectModalAddon/DataFilesManageProjectModalAddon.jsx @@ -0,0 +1,251 @@ +import React, { useState, useCallback, useEffect, useMemo } from 'react'; +import * as Yup from 'yup'; +import styles from './DataFilesManageProjectModalAddon.module.scss'; +import { useDispatch, useSelector } from 'react-redux'; +import { useSystemRole } from '../../../DataFiles/DataFilesProjectMembers/_cells/SystemRoleSelector'; +import { Input, Label } from 'reactstrap'; +import { FieldArray, Form, Formik, useFormikContext } from 'formik'; +import { + Button, + FormField, + InfiniteScrollTable, + LoadingSpinner, + Section, +} from '_common'; + +const DataFilesManageProjectModalAddon = ({ projectId }) => { + const dispatch = useDispatch(); + + const { metadata } = useSelector((state) => state.projects); + + const { loading, error } = useSelector((state) => { + if ( + state.projects.operation && + state.projects.operation.name === 'titleDescription' + ) { + return state.projects.operation; + } + return { + loading: false, + error: false, + }; + }); + + const authenticatedUser = useSelector( + (state) => state.authenticatedUser.user.username + ); + + const { query: authenticatedUserQuery } = useSystemRole( + projectId ?? null, + authenticatedUser ?? null + ); + + const canEditSystem = ['OWNER', 'ADMIN'].includes( + authenticatedUserQuery?.data?.role + ); + + const readOnlyTeam = useSelector((state) => { + const projectSystem = state.systems.storage.configuration.find( + (s) => s.scheme === 'projects' + ); + + return projectSystem?.readOnly || !canEditSystem; + }); + + const handleRemoveGuestUser = useCallback( + (email) => { + const updatedGuestUsers = metadata.guest_users.filter( + (user) => user.email !== email + ); + + dispatch({ + type: 'PROJECTS_SET_TITLE_DESCRIPTION', + payload: { + projectId, + data: { + title: metadata.title, + description: metadata.description || '', + metadata: { + guest_users: updatedGuestUsers, + }, + }, + modal: '', + }, + }); + }, + [metadata, dispatch, projectId] // Dependency array + ); + + const columns = [ + { + Header: 'Guest Members', + accessor: (el) => el, + Cell: (el) => { + const { first_name, last_name, email } = el.value; + return ( + + {`${first_name} ${last_name}`} + {` (${email})`} + + ); + }, + }, + { + Header: loading ? ( + + ) : ( + '' + ), + accessor: 'email', + Cell: (el) => { + return !readOnlyTeam ? ( + + ) : null; + }, + }, + ]; + + const onSubmit = useCallback( + (values) => { + dispatch({ + type: 'PROJECTS_SET_TITLE_DESCRIPTION', + payload: { + projectId, + data: { + title: metadata.title, + description: metadata.description || '', + metadata: { + guest_users: [...metadata.guest_users, ...values.guestUsers], + }, + }, + modal: '', + }, + }); + }, + [projectId, dispatch, metadata] + ); + + const validationSchema = Yup.object().shape({ + guestUsers: Yup.array().of( + Yup.object().shape({ + first_name: Yup.string().required('First Name is required'), + last_name: Yup.string().required('Last Name is required'), + email: Yup.string() + .email('Invalid email address') + .required('Email is required'), + }) + ), + }); + + return ( +
+ {metadata?.guest_users?.length > 0 && ( + + )} + + {!readOnlyTeam && ( + <> + + + {({ values, isValid, resetForm }) => { + useEffect(() => { + resetForm(); + }, [metadata, resetForm]); + + return ( +
+ + {({ remove, push }) => ( + <> + {/* Render only if there are guest users */} + {values.guestUsers.length > 0 && + values.guestUsers.map((_, index) => ( +
+ + + + + + + } + /> + ))} + + {/* Button to add a new guest user */} + + + )} + + + ); + }} + + + )} +
+ ); +}; + +export default DataFilesManageProjectModalAddon; diff --git a/client/src/components/_custom/drp/DataFilesManageProjectModalAddon/DataFilesManageProjectModalAddon.module.scss b/client/src/components/_custom/drp/DataFilesManageProjectModalAddon/DataFilesManageProjectModalAddon.module.scss new file mode 100644 index 0000000000..17b8c2b5ac --- /dev/null +++ b/client/src/components/_custom/drp/DataFilesManageProjectModalAddon/DataFilesManageProjectModalAddon.module.scss @@ -0,0 +1,45 @@ +.form-div { + display: flex; + justify-content: space-between; + width: 100%; + // align-items: center; +} + +.form-div > *:not(:nth-last-child(-n + 2)) { + width: 25%; + margin-bottom: 50px; +} + +.remove-button { + align-self: center; + margin-bottom: 15px; +} + +.button-full { + min-width: 100% !important; + margin-bottom: 30px; +} + +.printed-name { + font-weight: bold; +} + +.guest-user-listing { + /* title */ + th:nth-child(1), + td:nth-child(1) { + width: 80%; + } + /* date */ + th:nth-child(2), + td:nth-child(2) { + width: 20%; + } + + padding-bottom: 10px; +} + +.guest-members__loading { + font-size: 1em; + justify-content: flex-end; +} diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx index 49572392cb..aef8fe648b 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewAuthors.jsx @@ -1,19 +1,13 @@ import React, { useEffect, useState } from 'react'; import { Button, - ShowMore, - LoadingSpinner, - SectionMessage, SectionTableWrapper, - DescriptionList, Section, - SectionContent, - Expand, } from '_common'; import styles from './DataFilesProjectPublishWizard.module.scss'; import ReorderUserList from '../../utils/ReorderUserList/ReorderUserList'; import ProjectMembersList from '../../utils/ProjectMembersList/ProjectMembersList'; -import { useSelector } from 'react-redux'; +import { useDispatch, useSelector } from 'react-redux'; const ACMCitation = ({ project, authors }) => { const authorString = authors @@ -157,6 +151,8 @@ const ReviewAuthors = ({ project, onAuthorsUpdate }) => { const [authors, setAuthors] = useState([]); const [members, setMembers] = useState([]); + const dispatch = useDispatch(); + const canEdit = useSelector((state) => { const { members } = state.projects.metadata; const { username } = state.authenticatedUser.user; @@ -187,8 +183,10 @@ const ReviewAuthors = ({ project, onAuthorsUpdate }) => { ) .map((user) => user.user); + const guestUsers = project.guest_users || []; + setAuthors(owners); - setMembers(members); + setMembers([...members, ...guestUsers]); onAuthorsUpdate(owners); }, [project]); @@ -211,9 +209,29 @@ const ReviewAuthors = ({ project, onAuthorsUpdate }) => { onAuthorsUpdate(newAuthors); }; + const onManageTeam = () => { + dispatch({ + type: 'DATA_FILES_TOGGLE_MODAL', + payload: { operation: 'manageproject', props: {} }, + }); + }; + return ( Review Authors and Citation
} + header={
Review Authors and Citations
} + headerActions={ + <> + {canEdit && ( +
+ <> + + +
+ )} + + } > {authors.length > 0 && project && (
diff --git a/client/src/redux/reducers/projects.reducers.js b/client/src/redux/reducers/projects.reducers.js index 5dd4acf2fc..86e0079121 100644 --- a/client/src/redux/reducers/projects.reducers.js +++ b/client/src/redux/reducers/projects.reducers.js @@ -150,7 +150,8 @@ export default function projects(state = initialState, action) { return { ...state, metadata: { - ...action.payload, + ...state.metadata, + members: action.payload.members, loading: false, error: null, }, diff --git a/client/src/redux/sagas/projects.sagas.js b/client/src/redux/sagas/projects.sagas.js index 220dea631c..4528262532 100644 --- a/client/src/redux/sagas/projects.sagas.js +++ b/client/src/redux/sagas/projects.sagas.js @@ -175,7 +175,7 @@ export function* setTitleDescription(action) { type: 'PROJECTS_SET_TITLE_DESCRIPTION_STARTED', }); try { - const { projectId, data } = action.payload; + const { projectId, data, modal } = action.payload; const metadata = yield call(setTitleDescriptionUtil, projectId, data); yield put({ type: 'PROJECTS_SET_TITLE_DESCRIPTION_SUCCESS', @@ -183,7 +183,7 @@ export function* setTitleDescription(action) { }); yield put({ type: 'DATA_FILES_TOGGLE_MODAL', - payload: { operation: 'editproject', props: {} }, + payload: { operation: modal, props: {} }, }); yield put({ type: 'PROJECTS_GET_LISTING', diff --git a/server/portal/apps/_custom/drp/models.py b/server/portal/apps/_custom/drp/models.py index 99662cc6e3..999aa14349 100644 --- a/server/portal/apps/_custom/drp/models.py +++ b/server/portal/apps/_custom/drp/models.py @@ -101,6 +101,17 @@ class DrpProjectRelatedPublications(DrpMetadataModel): publication_description: Optional[str] = None publication_link: Optional[str] = None +class DrpGuestUser(DrpMetadataModel): + """Model for DRP Guest User""" + + model_config = ConfigDict( + extra="forbid", + ) + + first_name: str + last_name: str + email: str + class DrpProjectMetadata(DrpMetadataModel): """Model for DRP Project Metadata""" @@ -122,6 +133,7 @@ class DrpProjectMetadata(DrpMetadataModel): file_objs: list[FileObj] = [] is_review_project: Optional[bool] = None is_published_project: Optional[bool] = None + guest_users: list[DrpGuestUser] = [] class DrpDatasetMetadata(DrpMetadataModel): """Model for Base DRP Dataset Metadata""" diff --git a/server/portal/apps/projects/workspace_operations/project_meta_operations.py b/server/portal/apps/projects/workspace_operations/project_meta_operations.py index 0429b67899..8cc480dbef 100644 --- a/server/portal/apps/projects/workspace_operations/project_meta_operations.py +++ b/server/portal/apps/projects/workspace_operations/project_meta_operations.py @@ -155,11 +155,8 @@ def patch_project_entity(project_id, value): entity = ProjectMetadata.get_project_by_id(project_id) schema_model = SCHEMA_MAPPING[entity.name] - patched_metadata = {**value, - 'projectId': project_id, - 'fileObjs': entity.value.get('fileObjs', []), - 'doi': entity.value.get('doi', None) - } + entity_value = get_ordered_value(entity.name, entity.value) + patched_metadata = {**entity_value, **value} update_node_in_project(project_id, 'NODE_ROOT', None, value.get('title')) From c414ddec822a607b11b0befbb66a135fc94f637f Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Sun, 24 Nov 2024 19:19:29 -0600 Subject: [PATCH 146/328] small bug fix --- .../workspace_operations/shared_workspace_operations.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/portal/apps/projects/workspace_operations/shared_workspace_operations.py b/server/portal/apps/projects/workspace_operations/shared_workspace_operations.py index 49ce1c05a8..062d480696 100644 --- a/server/portal/apps/projects/workspace_operations/shared_workspace_operations.py +++ b/server/portal/apps/projects/workspace_operations/shared_workspace_operations.py @@ -341,6 +341,9 @@ def list_projects(client, root_system_id=None): is_review_system = root_system.get('reviewProject', False) is_publication_system = root_system.get('publicationProject', False) + else: + is_review_system = False + is_publication_system = False community_system = next(system for system in settings.PORTAL_DATAFILES_STORAGE_SYSTEMS if system['scheme'] == 'community') From e536d7ba0e9865bb2f60962b44616c8a846e4811 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Sun, 24 Nov 2024 20:10:11 -0600 Subject: [PATCH 147/328] update default settings --- server/portal/settings/settings_default.py | 47 +++++++++++----------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/server/portal/settings/settings_default.py b/server/portal/settings/settings_default.py index eb12d24d9c..1c15ed03e0 100644 --- a/server/portal/settings/settings_default.py +++ b/server/portal/settings/settings_default.py @@ -84,15 +84,6 @@ 'homeDir': '/home1/{tasdir}', 'icon': None, }, - { - 'name': 'Community Data', - 'system': 'cloud.data', - 'scheme': 'community', - 'api': 'tapis', - 'homeDir': '/corral/tacc/aci/CEP/community', - 'icon': None, - 'siteSearchPriority': 1 - }, { 'name': 'Public Data', 'system': 'cloud.data', @@ -102,6 +93,15 @@ 'icon': 'publications', 'siteSearchPriority': 0 }, + { + "name": "Community Data", + "system": "cloud.data", + "scheme": "community", + "api": "tapis", + "homeDir": "/corral-repl/utexas/OTH21076/data_pprd/community", + "icon": None, + "siteSearchPriority": 0, + }, { 'name': 'Projects', 'scheme': 'projects', @@ -110,18 +110,18 @@ 'readOnly': False, 'hideSearchBar': False, 'defaultProject': True, - 'system': 'cep.project.root', - 'rootDir': '/corral-repl/tacc/aci/CEP/projects', + 'system': 'drp.pprd.project.root', + 'rootDir': '/corral-repl/utexas/OTH21076/data_pprd/projects', }, - { + { 'name': 'Published', 'scheme': 'projects', 'api': 'tapis', 'icon': 'publications', 'readOnly': True, 'hideSearchBar': False, - 'system': 'drp.project.published.test', - 'rootDir': '/corral-repl/utexas/pge-nsf/data_pprd/published', + 'system': 'drp.pprd.project.published', + 'rootDir': '/corral-repl/utexas/OTH21076/data_pprd/published', 'publicationProject': True, }, { @@ -131,8 +131,8 @@ 'icon': 'publications', 'readOnly': True, 'hideSearchBar': False, - 'system': 'drp.project.review.test', - 'rootDir': '/corral-repl/utexas/pge-nsf/data_pprd/test', + 'system': 'drp.pprd.project.review', + 'rootDir': '/corral-repl/utexas/OTH21076/data_pprd/review', 'reviewProject': True, } ] @@ -206,20 +206,20 @@ _PORTAL_PROJECTS_SYSTEM_PREFIX = 'cep.project' _PORTAL_PROJECTS_ID_PREFIX = 'CEPV3-DEV' -_PORTAL_PROJECTS_ROOT_DIR = '/corral-repl/tacc/aci/CEP/projects' -_PORTAL_PROJECTS_ROOT_SYSTEM_NAME = 'cep.project.root' +_PORTAL_PROJECTS_ROOT_DIR = '/corral-repl/utexas/OTH21076/data_pprd/projects' +_PORTAL_PROJECTS_ROOT_SYSTEM_NAME = 'drp.pprd.project.root' _PORTAL_PROJECTS_ROOT_HOST = 'cloud.data.tacc.utexas.edu' _PORTAL_PROJECTS_SYSTEM_PORT = "22" _PORTAL_PROJECTS_PEMS_APP_ID = "" # Defunct in v3 _PORTAL_PROJECTS_USE_SET_FACL_JOB = False _PORTAL_PROJECTS_REVIEW_SYSTEM_PREFIX = 'cep.project.review' -_PORTAL_PROJECTS_REVIEW_ROOT_DIR = '/corral-repl/utexas/pge-nsf/data_pprd/test' -_PORTAL_PROJECTS_ROOT_REVIEW_SYSTEM_NAME = 'drp.project.review.test' +_PORTAL_PROJECTS_REVIEW_ROOT_DIR = '/corral-repl/utexas/OTH21076/data_pprd/review' +_PORTAL_PROJECTS_ROOT_REVIEW_SYSTEM_NAME = 'drp.pprd.project.review' _PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX = 'cep.project.published' -_PORTAL_PROJECTS_PUBLISHED_ROOT_DIR = '/corral-repl/utexas/pge-nsf/data_pprd/published' -_PORTAL_PROJECTS_PUBLISHED_ROOT_SYSTEM_NAME = 'drp.project.published.test' +_PORTAL_PROJECTS_PUBLISHED_ROOT_DIR = '/corral-repl/utexas/OTH21076/data_pprd/published' +_PORTAL_PROJECTS_PUBLISHED_ROOT_SYSTEM_NAME = 'drp.pprd.project.published' _PORTAL_PUBLICATION_REVIEWERS_GROUP_NAME = 'PROJECT_REVIEWER' @@ -272,7 +272,8 @@ "hasCustomDataFilesToolbarChecks": True, "addons": ['DataFilesProjectFileListingAddon', 'DataFilesAddProjectModalAddon', 'DataFilesProjectEditDescriptionModalAddon', 'DataFilesProjectFileListingMetadataAddon', 'DataFilesProjectFileListingMetadataTitleAddon', - 'DataFilesUploadModalAddon', 'DataFilesPreviewModalAddon', 'DataFilesProjectPublish', 'DataFilesProjectReview'], + 'DataFilesUploadModalAddon', 'DataFilesPreviewModalAddon', 'DataFilesProjectPublish', 'DataFilesProjectReview', + 'DataFilesManageProjectModalAddon'], "showDataFileType": True, "onboardingCompleteRedirect": '/workbench/', "noPHISystem": "", From de6d2e1c585ee0295232f593291ec5195be7a633 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Mon, 25 Nov 2024 10:20:39 -0600 Subject: [PATCH 148/328] project publish bug fix --- .../workspace_operations/project_publish_operations.py | 4 ++-- server/portal/apps/publications/views.py | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/server/portal/apps/projects/workspace_operations/project_publish_operations.py b/server/portal/apps/projects/workspace_operations/project_publish_operations.py index bbcc860317..1d8a83fffb 100644 --- a/server/portal/apps/projects/workspace_operations/project_publish_operations.py +++ b/server/portal/apps/projects/workspace_operations/project_publish_operations.py @@ -259,7 +259,7 @@ def get_project_user_emails(project_id): @shared_task(bind=True, queue='default') -def send_publication_accept_email(project_id): +def send_publication_accept_email(self, project_id): """ Alert project authors that their request has been accepted. """ @@ -290,7 +290,7 @@ def send_publication_accept_email(project_id): @shared_task(bind=True, queue='default') -def send_publication_reject_email(project_id: str, version: Optional[int], error: str): +def send_publication_reject_email(self, project_id: str, version: Optional[int], error: str): """ Alert project authors that their request has been rejected. """ diff --git a/server/portal/apps/publications/views.py b/server/portal/apps/publications/views.py index 2eab42e1fd..f4f17397c8 100644 --- a/server/portal/apps/publications/views.py +++ b/server/portal/apps/publications/views.py @@ -220,7 +220,8 @@ def post(self, request): if not full_project_id: raise ApiException("Missing project ID", status=400) - send_publication_accept_email.apply_async(args=[full_project_id]) + if not settings.DEBUG: + send_publication_accept_email.apply_async(args=[full_project_id]) if is_review: project_id = full_project_id.split(f"{settings.PORTAL_PROJECTS_REVIEW_SYSTEM_PREFIX}.")[1] @@ -316,7 +317,8 @@ def post(self, request): if not full_project_id: raise ApiException("Missing project ID", status=400) - send_publication_reject_email.apply_async(args=[full_project_id]) + if not settings.DEBUG: + send_publication_reject_email.apply_async(args=[full_project_id]) update_and_cleanup_review_project(full_project_id, PublicationRequest.Status.REJECTED) From 3a92605f720a2687148d6bafc6dc6f639dc7594b Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Mon, 25 Nov 2024 10:53:47 -0600 Subject: [PATCH 149/328] publish send email bug fix, disable publish button --- .../SubmitPublicationReview.jsx | 4 +++- .../workspace_operations/project_publish_operations.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/SubmitPublicationReview.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/SubmitPublicationReview.jsx index ec1cb8aa2f..7bde41e0a9 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/SubmitPublicationReview.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/SubmitPublicationReview.jsx @@ -17,6 +17,8 @@ const SubmitPublicationReview = ({ callbackUrl }) => { const [submitDisabled, setSubmitDisabled] = useState(false); + const { debug } = useSelector((state) => state.workbench.config); + const { isApproveLoading, isRejectLoading, @@ -83,7 +85,7 @@ const SubmitPublicationReview = ({ callbackUrl }) => { | diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx index 25dc6d63ef..bee5d4bd29 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublish.jsx @@ -110,7 +110,7 @@ const DataFilesProjectPublish = ({ rootSystem, system }) => { className={styles.root} header={
- Request Project Publication | {metadata.title} + Request Dataset Publication | {metadata.title}
} headerActions={ @@ -120,7 +120,7 @@ const DataFilesProjectPublish = ({ rootSystem, system }) => { className="wb-link" to={`${ROUTES.WORKBENCH}${ROUTES.DATA}/tapis/projects/${rootSystem}/${system}`} > - Back to Project + Back to Dataset
diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx index e314d0a3b9..99c7a593bb 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx @@ -134,14 +134,14 @@ const ProjectDescription = ({ project }) => { return ( Proofread Project} + header={
Proofread Dataset
} headerActions={ <> {canEdit && (
<>
diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx index 8d8edef5e1..9c8c787da1 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ReviewProjectStructure.jsx @@ -46,7 +46,7 @@ export const ReviewProjectStructure = ({ projectTree }) => {
<>
diff --git a/client/src/components/_custom/drp/DataFilesProjectReview/DataFilesProjectReview.jsx b/client/src/components/_custom/drp/DataFilesProjectReview/DataFilesProjectReview.jsx index 6c660720a2..f11acc2249 100644 --- a/client/src/components/_custom/drp/DataFilesProjectReview/DataFilesProjectReview.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectReview/DataFilesProjectReview.jsx @@ -98,7 +98,7 @@ const DataFilesProjectReview = ({ rootSystem, system }) => { className="wb-link" to={`${ROUTES.WORKBENCH}${ROUTES.DATA}/tapis/projects/${rootSystem}/${system}`} > - Back to Project + Back to Dataset From 271ce8c2b88fdc9d92613a954af42f6489f52e09 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Wed, 4 Dec 2024 16:58:42 -0600 Subject: [PATCH 153/328] publication rejection email bug fix --- .../workspace_operations/project_publish_operations.py | 2 +- server/portal/settings/settings_default.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/portal/apps/projects/workspace_operations/project_publish_operations.py b/server/portal/apps/projects/workspace_operations/project_publish_operations.py index 0157b41b84..1cd37ac710 100644 --- a/server/portal/apps/projects/workspace_operations/project_publish_operations.py +++ b/server/portal/apps/projects/workspace_operations/project_publish_operations.py @@ -290,7 +290,7 @@ def send_publication_accept_email(self, project_id): @shared_task(bind=True, queue='default') -def send_publication_reject_email(self, project_id: str, version: Optional[int], error: str): +def send_publication_reject_email(self, project_id: str): """ Alert project authors that their request has been rejected. """ diff --git a/server/portal/settings/settings_default.py b/server/portal/settings/settings_default.py index 813ec04056..6ba54927fc 100644 --- a/server/portal/settings/settings_default.py +++ b/server/portal/settings/settings_default.py @@ -103,7 +103,7 @@ "siteSearchPriority": 0, }, { - 'name': 'Projects', + 'name': 'Dataset', 'scheme': 'projects', 'api': 'tapis', 'icon': 'publications', @@ -114,7 +114,7 @@ 'rootDir': '/corral-repl/utexas/OTH21076/data_pprd/projects', }, { - 'name': 'Published', + 'name': 'Published Datasets', 'scheme': 'projects', 'api': 'tapis', 'icon': 'publications', From c3dfa0029975d73ba8f6105c911ea2f2ef3a563d Mon Sep 17 00:00:00 2001 From: Jake Rosenberg Date: Tue, 10 Dec 2024 11:44:37 -0600 Subject: [PATCH 154/328] Fix case bug in project metadata search (#1001) Co-authored-by: Shayan Khan --- server/portal/apps/projects/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/portal/apps/projects/views.py b/server/portal/apps/projects/views.py index e1ac3d179d..e5b6246c18 100644 --- a/server/portal/apps/projects/views.py +++ b/server/portal/apps/projects/views.py @@ -94,12 +94,12 @@ def get(self, request, root_system=None): if query_string: search = IndexedProject.search() - ngram_query = Q("query_string", query=query_string, + ngram_query = Q("query_string", query=query_string.lower(), fields=["title", "id"], minimum_should_match='100%', default_operator='or') - wildcard_query = Q("wildcard", title=f'*{query_string}*') | Q("wildcard", id=f'*{query_string}*') + wildcard_query = Q("wildcard", title=f'*{query_string.lower()}*') | Q("wildcard", id=f'*{query_string.lower()}*') search = search.query(ngram_query | wildcard_query) search = search.extra(from_=int(offset), size=int(limit)) From 7787b4a344226b2688e0397202e132778f60e606 Mon Sep 17 00:00:00 2001 From: Jake Rosenberg Date: Tue, 10 Dec 2024 12:06:24 -0600 Subject: [PATCH 155/328] show toast notification when changing project owner (#1016) Co-authored-by: Shayan Khan --- client/src/redux/sagas/projects.sagas.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/client/src/redux/sagas/projects.sagas.js b/client/src/redux/sagas/projects.sagas.js index 4528262532..ae4edc6daf 100644 --- a/client/src/redux/sagas/projects.sagas.js +++ b/client/src/redux/sagas/projects.sagas.js @@ -144,6 +144,13 @@ export function* setMember(action) { type: 'PROJECTS_SET_MEMBER_SUCCESS', payload: metadata, }); + if (data.action === 'transfer_ownership') + yield put({ + type: 'ADD_TOAST', + payload: { + message: `Project ownership transferred to ${data.newOwner}.`, + }, + }); yield put({ type: 'PROJECTS_GET_LISTING', payload: { From 761c23d224b501f7d33752cccf30e831005b217d Mon Sep 17 00:00:00 2001 From: Jake Rosenberg Date: Tue, 10 Dec 2024 12:59:39 -0600 Subject: [PATCH 156/328] sort pulication requests in descending date order (#1014) Co-authored-by: Shayan Khan --- .../DataFilesPublicationRequestModal.jsx | 16 +++++- .../DescriptionList/DescriptionList.jsx | 53 +++++++++++++------ 2 files changed, 51 insertions(+), 18 deletions(-) diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesPublicationRequestModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesPublicationRequestModal.jsx index 979b5f65b9..87e8e97f41 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesPublicationRequestModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesPublicationRequestModal.jsx @@ -13,10 +13,23 @@ const DataFilesPublicationRequestModal = () => { const { publicationRequests } = useSelector((state) => state.files.modalProps.publicationRequest) || []; + const compareFn = (req1, req2) => { + // sort more recent requests first + const date1 = new Date(req1.created_at); + const date2 = new Date(req2.created_at); + if (date1 < date2) { + return 1; + } + if (date1 > date2) { + return -1; + } + return 0; + }; + useEffect(() => { const data = {}; - publicationRequests?.forEach((request, index) => { + publicationRequests?.sort(compareFn).forEach((request, index) => { const publicationRequestDataObj = { Status: request.status, Reviewers: request.reviewers.reduce((acc, reviewer, index) => { @@ -27,6 +40,7 @@ const DataFilesPublicationRequestModal = () => { ); }, ''), Submitted: formatDateTime(new Date(request.created_at)), + _order: index, }; const heading = `Publication Request ${index + 1}`; diff --git a/client/src/components/_common/DescriptionList/DescriptionList.jsx b/client/src/components/_common/DescriptionList/DescriptionList.jsx index 86dc7a0397..0cf30f56d1 100644 --- a/client/src/components/_common/DescriptionList/DescriptionList.jsx +++ b/client/src/components/_common/DescriptionList/DescriptionList.jsx @@ -33,26 +33,45 @@ const DescriptionList = ({ className, data, density, direction }) => { shouldTruncateValues ? 'value-truncated' : '' }`; + const compareFn = (entry1, entry2) => { + const [, val1] = entry1; + const [, val2] = entry2; + if ((val1._order ?? 0) < (val2._order ?? 0)) { + return -1; + } + if ((val1._order ?? 0) > (val2._order ?? 0)) { + return 1; + } + return 0; + }; + return (
- {Object.entries(data).map(([key, value]) => ( - -
- {key} -
- {Array.isArray(value) ? ( - value.map((val) => ( -
- {val} + {Object.entries(data) + .sort(compareFn) + .filter(([key, _]) => !key.startsWith('_')) + .map(([key, value]) => ( + +
+ {key} +
+ {Array.isArray(value) ? ( + value.map((val) => ( +
+ {val} +
+ )) + ) : ( +
+ {value}
- )) - ) : ( -
- {value} -
- )} - - ))} + )} + + ))}
); }; From ae6dffa960ab482aadcadb7bf03ddb1c8863785e Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Tue, 10 Dec 2024 13:14:15 -0600 Subject: [PATCH 157/328] updated pub request modal heading --- .../DataFilesModals/DataFilesPublicationRequestModal.jsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesPublicationRequestModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesPublicationRequestModal.jsx index 87e8e97f41..d81433537c 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesPublicationRequestModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesPublicationRequestModal.jsx @@ -3,7 +3,7 @@ import { useDispatch, useSelector, shallowEqual } from 'react-redux'; import { DescriptionList } from '_common'; import { Modal, ModalHeader, ModalBody } from 'reactstrap'; import styles from './DataFilesPublicationRequestModal.module.scss'; -import { formatDateTime } from 'utils/timeFormat'; +import { formatDate, formatDateTime } from 'utils/timeFormat'; const DataFilesPublicationRequestModal = () => { const dispatch = useDispatch(); @@ -43,7 +43,7 @@ const DataFilesPublicationRequestModal = () => { _order: index, }; - const heading = `Publication Request ${index + 1}`; + const heading = `Publication Request | ${formatDate(new Date(request.created_at))}`; data[heading] = ; }); From c746fb249d1e9be951aec1e552ab6609a432bbaa Mon Sep 17 00:00:00 2001 From: Shayan Khan Date: Fri, 14 Feb 2025 15:21:22 -0600 Subject: [PATCH 158/328] task/WC-156 - DRP Image Processing (#1043) * Added image processing for files with metadata * Image processing runs on upload and whenever the metadata is updated * added notifications for image generation status * cleanup imports --- server/poetry.lock | 483 +++++++++++++++++++- server/portal/apps/_custom/drp/models.py | 1 + server/portal/apps/projects/tasks.py | 110 ++++- server/portal/apps/projects/views.py | 3 +- server/portal/libs/agave/operations.py | 7 + server/portal/libs/files/file_processing.py | 183 ++++++++ server/pyproject.toml | 3 + 7 files changed, 784 insertions(+), 6 deletions(-) create mode 100644 server/portal/libs/files/file_processing.py diff --git a/server/poetry.lock b/server/poetry.lock index 3cfa8a3d0c..fdfd41b117 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -596,6 +596,79 @@ files = [ {file = "constantly-15.1.0.tar.gz", hash = "sha256:586372eb92059873e29eba4f9dec8381541b4d3834660707faf8ba59146dfc35"}, ] +[[package]] +name = "contourpy" +version = "1.3.1" +description = "Python library for calculating contours of 2D quadrilateral grids" +optional = false +python-versions = ">=3.10" +files = [ + {file = "contourpy-1.3.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a045f341a77b77e1c5de31e74e966537bba9f3c4099b35bf4c2e3939dd54cdab"}, + {file = "contourpy-1.3.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:500360b77259914f7805af7462e41f9cb7ca92ad38e9f94d6c8641b089338124"}, + {file = "contourpy-1.3.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b2f926efda994cdf3c8d3fdb40b9962f86edbc4457e739277b961eced3d0b4c1"}, + {file = "contourpy-1.3.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:adce39d67c0edf383647a3a007de0a45fd1b08dedaa5318404f1a73059c2512b"}, + {file = "contourpy-1.3.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:abbb49fb7dac584e5abc6636b7b2a7227111c4f771005853e7d25176daaf8453"}, + {file = "contourpy-1.3.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0cffcbede75c059f535725c1680dfb17b6ba8753f0c74b14e6a9c68c29d7ea3"}, + {file = "contourpy-1.3.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ab29962927945d89d9b293eabd0d59aea28d887d4f3be6c22deaefbb938a7277"}, + {file = "contourpy-1.3.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:974d8145f8ca354498005b5b981165b74a195abfae9a8129df3e56771961d595"}, + {file = "contourpy-1.3.1-cp310-cp310-win32.whl", hash = "sha256:ac4578ac281983f63b400f7fe6c101bedc10651650eef012be1ccffcbacf3697"}, + {file = "contourpy-1.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:174e758c66bbc1c8576992cec9599ce8b6672b741b5d336b5c74e35ac382b18e"}, + {file = "contourpy-1.3.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3e8b974d8db2c5610fb4e76307e265de0edb655ae8169e8b21f41807ccbeec4b"}, + {file = "contourpy-1.3.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:20914c8c973f41456337652a6eeca26d2148aa96dd7ac323b74516988bea89fc"}, + {file = "contourpy-1.3.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19d40d37c1c3a4961b4619dd9d77b12124a453cc3d02bb31a07d58ef684d3d86"}, + {file = "contourpy-1.3.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:113231fe3825ebf6f15eaa8bc1f5b0ddc19d42b733345eae0934cb291beb88b6"}, + {file = "contourpy-1.3.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4dbbc03a40f916a8420e420d63e96a1258d3d1b58cbdfd8d1f07b49fcbd38e85"}, + {file = "contourpy-1.3.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a04ecd68acbd77fa2d39723ceca4c3197cb2969633836ced1bea14e219d077c"}, + {file = "contourpy-1.3.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c414fc1ed8ee1dbd5da626cf3710c6013d3d27456651d156711fa24f24bd1291"}, + {file = "contourpy-1.3.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:31c1b55c1f34f80557d3830d3dd93ba722ce7e33a0b472cba0ec3b6535684d8f"}, + {file = "contourpy-1.3.1-cp311-cp311-win32.whl", hash = "sha256:f611e628ef06670df83fce17805c344710ca5cde01edfdc72751311da8585375"}, + {file = "contourpy-1.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:b2bdca22a27e35f16794cf585832e542123296b4687f9fd96822db6bae17bfc9"}, + {file = "contourpy-1.3.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0ffa84be8e0bd33410b17189f7164c3589c229ce5db85798076a3fa136d0e509"}, + {file = "contourpy-1.3.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:805617228ba7e2cbbfb6c503858e626ab528ac2a32a04a2fe88ffaf6b02c32bc"}, + {file = "contourpy-1.3.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ade08d343436a94e633db932e7e8407fe7de8083967962b46bdfc1b0ced39454"}, + {file = "contourpy-1.3.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:47734d7073fb4590b4a40122b35917cd77be5722d80683b249dac1de266aac80"}, + {file = "contourpy-1.3.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2ba94a401342fc0f8b948e57d977557fbf4d515f03c67682dd5c6191cb2d16ec"}, + {file = "contourpy-1.3.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efa874e87e4a647fd2e4f514d5e91c7d493697127beb95e77d2f7561f6905bd9"}, + {file = "contourpy-1.3.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1bf98051f1045b15c87868dbaea84f92408337d4f81d0e449ee41920ea121d3b"}, + {file = "contourpy-1.3.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:61332c87493b00091423e747ea78200659dc09bdf7fd69edd5e98cef5d3e9a8d"}, + {file = "contourpy-1.3.1-cp312-cp312-win32.whl", hash = "sha256:e914a8cb05ce5c809dd0fe350cfbb4e881bde5e2a38dc04e3afe1b3e58bd158e"}, + {file = "contourpy-1.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:08d9d449a61cf53033612cb368f3a1b26cd7835d9b8cd326647efe43bca7568d"}, + {file = "contourpy-1.3.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a761d9ccfc5e2ecd1bf05534eda382aa14c3e4f9205ba5b1684ecfe400716ef2"}, + {file = "contourpy-1.3.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:523a8ee12edfa36f6d2a49407f705a6ef4c5098de4f498619787e272de93f2d5"}, + {file = "contourpy-1.3.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ece6df05e2c41bd46776fbc712e0996f7c94e0d0543af1656956d150c4ca7c81"}, + {file = "contourpy-1.3.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:573abb30e0e05bf31ed067d2f82500ecfdaec15627a59d63ea2d95714790f5c2"}, + {file = "contourpy-1.3.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a9fa36448e6a3a1a9a2ba23c02012c43ed88905ec80163f2ffe2421c7192a5d7"}, + {file = "contourpy-1.3.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ea9924d28fc5586bf0b42d15f590b10c224117e74409dd7a0be3b62b74a501c"}, + {file = "contourpy-1.3.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5b75aa69cb4d6f137b36f7eb2ace9280cfb60c55dc5f61c731fdf6f037f958a3"}, + {file = "contourpy-1.3.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:041b640d4ec01922083645a94bb3b2e777e6b626788f4095cf21abbe266413c1"}, + {file = "contourpy-1.3.1-cp313-cp313-win32.whl", hash = "sha256:36987a15e8ace5f58d4d5da9dca82d498c2bbb28dff6e5d04fbfcc35a9cb3a82"}, + {file = "contourpy-1.3.1-cp313-cp313-win_amd64.whl", hash = "sha256:a7895f46d47671fa7ceec40f31fae721da51ad34bdca0bee83e38870b1f47ffd"}, + {file = "contourpy-1.3.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:9ddeb796389dadcd884c7eb07bd14ef12408aaae358f0e2ae24114d797eede30"}, + {file = "contourpy-1.3.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:19c1555a6801c2f084c7ddc1c6e11f02eb6a6016ca1318dd5452ba3f613a1751"}, + {file = "contourpy-1.3.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:841ad858cff65c2c04bf93875e384ccb82b654574a6d7f30453a04f04af71342"}, + {file = "contourpy-1.3.1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4318af1c925fb9a4fb190559ef3eec206845f63e80fb603d47f2d6d67683901c"}, + {file = "contourpy-1.3.1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:14c102b0eab282427b662cb590f2e9340a9d91a1c297f48729431f2dcd16e14f"}, + {file = "contourpy-1.3.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05e806338bfeaa006acbdeba0ad681a10be63b26e1b17317bfac3c5d98f36cda"}, + {file = "contourpy-1.3.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4d76d5993a34ef3df5181ba3c92fabb93f1eaa5729504fb03423fcd9f3177242"}, + {file = "contourpy-1.3.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:89785bb2a1980c1bd87f0cb1517a71cde374776a5f150936b82580ae6ead44a1"}, + {file = "contourpy-1.3.1-cp313-cp313t-win32.whl", hash = "sha256:8eb96e79b9f3dcadbad2a3891672f81cdcab7f95b27f28f1c67d75f045b6b4f1"}, + {file = "contourpy-1.3.1-cp313-cp313t-win_amd64.whl", hash = "sha256:287ccc248c9e0d0566934e7d606201abd74761b5703d804ff3df8935f523d546"}, + {file = "contourpy-1.3.1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:b457d6430833cee8e4b8e9b6f07aa1c161e5e0d52e118dc102c8f9bd7dd060d6"}, + {file = "contourpy-1.3.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cb76c1a154b83991a3cbbf0dfeb26ec2833ad56f95540b442c73950af2013750"}, + {file = "contourpy-1.3.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:44a29502ca9c7b5ba389e620d44f2fbe792b1fb5734e8b931ad307071ec58c53"}, + {file = "contourpy-1.3.1.tar.gz", hash = "sha256:dfd97abd83335045a913e3bcc4a09c0ceadbe66580cf573fe961f4a825efa699"}, +] + +[package.dependencies] +numpy = ">=1.23" + +[package.extras] +bokeh = ["bokeh", "selenium"] +docs = ["furo", "sphinx (>=7.2)", "sphinx-copybutton"] +mypy = ["contourpy[bokeh,docs]", "docutils-stubs", "mypy (==1.11.1)", "types-Pillow"] +test = ["Pillow", "contourpy[test-no-images]", "matplotlib"] +test-no-images = ["pytest", "pytest-cov", "pytest-rerunfailures", "pytest-xdist", "wurlitzer"] + [[package]] name = "coverage" version = "7.3.2" @@ -709,6 +782,21 @@ ssh = ["bcrypt (>=3.1.5)"] test = ["certifi", "cryptography-vectors (==43.0.1)", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] test-randomorder = ["pytest-randomly"] +[[package]] +name = "cycler" +version = "0.12.1" +description = "Composable style cycles" +optional = false +python-versions = ">=3.8" +files = [ + {file = "cycler-0.12.1-py3-none-any.whl", hash = "sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30"}, + {file = "cycler-0.12.1.tar.gz", hash = "sha256:88bb128f02ba341da8ef447245a9e138fae777f6a23943da4540077d3601eb1c"}, +] + +[package.extras] +docs = ["ipython", "matplotlib", "numpydoc", "sphinx"] +tests = ["pytest", "pytest-cov", "pytest-xdist"] + [[package]] name = "daphne" version = "4.0.0" @@ -866,6 +954,79 @@ mccabe = ">=0.7.0,<0.8.0" pycodestyle = ">=2.11.0,<2.12.0" pyflakes = ">=3.1.0,<3.2.0" +[[package]] +name = "fonttools" +version = "4.55.3" +description = "Tools to manipulate font files" +optional = false +python-versions = ">=3.8" +files = [ + {file = "fonttools-4.55.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:1dcc07934a2165ccdc3a5a608db56fb3c24b609658a5b340aee4ecf3ba679dc0"}, + {file = "fonttools-4.55.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f7d66c15ba875432a2d2fb419523f5d3d347f91f48f57b8b08a2dfc3c39b8a3f"}, + {file = "fonttools-4.55.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27e4ae3592e62eba83cd2c4ccd9462dcfa603ff78e09110680a5444c6925d841"}, + {file = "fonttools-4.55.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:62d65a3022c35e404d19ca14f291c89cc5890032ff04f6c17af0bd1927299674"}, + {file = "fonttools-4.55.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d342e88764fb201286d185093781bf6628bbe380a913c24adf772d901baa8276"}, + {file = "fonttools-4.55.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:dd68c87a2bfe37c5b33bcda0fba39b65a353876d3b9006fde3adae31f97b3ef5"}, + {file = "fonttools-4.55.3-cp310-cp310-win32.whl", hash = "sha256:1bc7ad24ff98846282eef1cbeac05d013c2154f977a79886bb943015d2b1b261"}, + {file = "fonttools-4.55.3-cp310-cp310-win_amd64.whl", hash = "sha256:b54baf65c52952db65df39fcd4820668d0ef4766c0ccdf32879b77f7c804d5c5"}, + {file = "fonttools-4.55.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:8c4491699bad88efe95772543cd49870cf756b019ad56294f6498982408ab03e"}, + {file = "fonttools-4.55.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5323a22eabddf4b24f66d26894f1229261021dacd9d29e89f7872dd8c63f0b8b"}, + {file = "fonttools-4.55.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5480673f599ad410695ca2ddef2dfefe9df779a9a5cda89503881e503c9c7d90"}, + {file = "fonttools-4.55.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da9da6d65cd7aa6b0f806556f4985bcbf603bf0c5c590e61b43aa3e5a0f822d0"}, + {file = "fonttools-4.55.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e894b5bd60d9f473bed7a8f506515549cc194de08064d829464088d23097331b"}, + {file = "fonttools-4.55.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:aee3b57643827e237ff6ec6d28d9ff9766bd8b21e08cd13bff479e13d4b14765"}, + {file = "fonttools-4.55.3-cp311-cp311-win32.whl", hash = "sha256:eb6ca911c4c17eb51853143624d8dc87cdcdf12a711fc38bf5bd21521e79715f"}, + {file = "fonttools-4.55.3-cp311-cp311-win_amd64.whl", hash = "sha256:6314bf82c54c53c71805318fcf6786d986461622dd926d92a465199ff54b1b72"}, + {file = "fonttools-4.55.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:f9e736f60f4911061235603a6119e72053073a12c6d7904011df2d8fad2c0e35"}, + {file = "fonttools-4.55.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7a8aa2c5e5b8b3bcb2e4538d929f6589a5c6bdb84fd16e2ed92649fb5454f11c"}, + {file = "fonttools-4.55.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:07f8288aacf0a38d174445fc78377a97fb0b83cfe352a90c9d9c1400571963c7"}, + {file = "fonttools-4.55.3-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8d5e8916c0970fbc0f6f1bece0063363bb5857a7f170121a4493e31c3db3314"}, + {file = "fonttools-4.55.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ae3b6600565b2d80b7c05acb8e24d2b26ac407b27a3f2e078229721ba5698427"}, + {file = "fonttools-4.55.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:54153c49913f45065c8d9e6d0c101396725c5621c8aee744719300f79771d75a"}, + {file = "fonttools-4.55.3-cp312-cp312-win32.whl", hash = "sha256:827e95fdbbd3e51f8b459af5ea10ecb4e30af50221ca103bea68218e9615de07"}, + {file = "fonttools-4.55.3-cp312-cp312-win_amd64.whl", hash = "sha256:e6e8766eeeb2de759e862004aa11a9ea3d6f6d5ec710551a88b476192b64fd54"}, + {file = "fonttools-4.55.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a430178ad3e650e695167cb53242dae3477b35c95bef6525b074d87493c4bf29"}, + {file = "fonttools-4.55.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:529cef2ce91dc44f8e407cc567fae6e49a1786f2fefefa73a294704c415322a4"}, + {file = "fonttools-4.55.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e75f12c82127486fac2d8bfbf5bf058202f54bf4f158d367e41647b972342ca"}, + {file = "fonttools-4.55.3-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:859c358ebf41db18fb72342d3080bce67c02b39e86b9fbcf1610cca14984841b"}, + {file = "fonttools-4.55.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:546565028e244a701f73df6d8dd6be489d01617863ec0c6a42fa25bf45d43048"}, + {file = "fonttools-4.55.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:aca318b77f23523309eec4475d1fbbb00a6b133eb766a8bdc401faba91261abe"}, + {file = "fonttools-4.55.3-cp313-cp313-win32.whl", hash = "sha256:8c5ec45428edaa7022f1c949a632a6f298edc7b481312fc7dc258921e9399628"}, + {file = "fonttools-4.55.3-cp313-cp313-win_amd64.whl", hash = "sha256:11e5de1ee0d95af4ae23c1a138b184b7f06e0b6abacabf1d0db41c90b03d834b"}, + {file = "fonttools-4.55.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:caf8230f3e10f8f5d7593eb6d252a37caf58c480b19a17e250a63dad63834cf3"}, + {file = "fonttools-4.55.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b586ab5b15b6097f2fb71cafa3c98edfd0dba1ad8027229e7b1e204a58b0e09d"}, + {file = "fonttools-4.55.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a8c2794ded89399cc2169c4d0bf7941247b8d5932b2659e09834adfbb01589aa"}, + {file = "fonttools-4.55.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cf4fe7c124aa3f4e4c1940880156e13f2f4d98170d35c749e6b4f119a872551e"}, + {file = "fonttools-4.55.3-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:86721fbc389ef5cc1e2f477019e5069e8e4421e8d9576e9c26f840dbb04678de"}, + {file = "fonttools-4.55.3-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:89bdc5d88bdeec1b15af790810e267e8332d92561dce4f0748c2b95c9bdf3926"}, + {file = "fonttools-4.55.3-cp38-cp38-win32.whl", hash = "sha256:bc5dbb4685e51235ef487e4bd501ddfc49be5aede5e40f4cefcccabc6e60fb4b"}, + {file = "fonttools-4.55.3-cp38-cp38-win_amd64.whl", hash = "sha256:cd70de1a52a8ee2d1877b6293af8a2484ac82514f10b1c67c1c5762d38073e56"}, + {file = "fonttools-4.55.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:bdcc9f04b36c6c20978d3f060e5323a43f6222accc4e7fcbef3f428e216d96af"}, + {file = "fonttools-4.55.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c3ca99e0d460eff46e033cd3992a969658c3169ffcd533e0a39c63a38beb6831"}, + {file = "fonttools-4.55.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22f38464daa6cdb7b6aebd14ab06609328fe1e9705bb0fcc7d1e69de7109ee02"}, + {file = "fonttools-4.55.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed63959d00b61959b035c7d47f9313c2c1ece090ff63afea702fe86de00dbed4"}, + {file = "fonttools-4.55.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:5e8d657cd7326eeaba27de2740e847c6b39dde2f8d7cd7cc56f6aad404ddf0bd"}, + {file = "fonttools-4.55.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:fb594b5a99943042c702c550d5494bdd7577f6ef19b0bc73877c948a63184a32"}, + {file = "fonttools-4.55.3-cp39-cp39-win32.whl", hash = "sha256:dc5294a3d5c84226e3dbba1b6f61d7ad813a8c0238fceea4e09aa04848c3d851"}, + {file = "fonttools-4.55.3-cp39-cp39-win_amd64.whl", hash = "sha256:aedbeb1db64496d098e6be92b2e63b5fac4e53b1b92032dfc6988e1ea9134a4d"}, + {file = "fonttools-4.55.3-py3-none-any.whl", hash = "sha256:f412604ccbeee81b091b420272841e5ec5ef68967a9790e80bffd0e30b8e2977"}, + {file = "fonttools-4.55.3.tar.gz", hash = "sha256:3983313c2a04d6cc1fe9251f8fc647754cf49a61dac6cb1e7249ae67afaafc45"}, +] + +[package.extras] +all = ["brotli (>=1.0.1)", "brotlicffi (>=0.8.0)", "fs (>=2.2.0,<3)", "lxml (>=4.0)", "lz4 (>=1.7.4.2)", "matplotlib", "munkres", "pycairo", "scipy", "skia-pathops (>=0.5.0)", "sympy", "uharfbuzz (>=0.23.0)", "unicodedata2 (>=15.1.0)", "xattr", "zopfli (>=0.1.4)"] +graphite = ["lz4 (>=1.7.4.2)"] +interpolatable = ["munkres", "pycairo", "scipy"] +lxml = ["lxml (>=4.0)"] +pathops = ["skia-pathops (>=0.5.0)"] +plot = ["matplotlib"] +repacker = ["uharfbuzz (>=0.23.0)"] +symfont = ["sympy"] +type1 = ["xattr"] +ufo = ["fs (>=2.2.0,<3)"] +unicode = ["unicodedata2 (>=15.1.0)"] +woff = ["brotli (>=1.0.1)", "brotlicffi (>=0.8.0)", "zopfli (>=0.1.4)"] + [[package]] name = "gevent" version = "23.9.1" @@ -1318,6 +1479,95 @@ files = [ [package.dependencies] referencing = ">=0.28.0" +[[package]] +name = "kiwisolver" +version = "1.4.8" +description = "A fast implementation of the Cassowary constraint solver" +optional = false +python-versions = ">=3.10" +files = [ + {file = "kiwisolver-1.4.8-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88c6f252f6816a73b1f8c904f7bbe02fd67c09a69f7cb8a0eecdbf5ce78e63db"}, + {file = "kiwisolver-1.4.8-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c72941acb7b67138f35b879bbe85be0f6c6a70cab78fe3ef6db9c024d9223e5b"}, + {file = "kiwisolver-1.4.8-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ce2cf1e5688edcb727fdf7cd1bbd0b6416758996826a8be1d958f91880d0809d"}, + {file = "kiwisolver-1.4.8-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c8bf637892dc6e6aad2bc6d4d69d08764166e5e3f69d469e55427b6ac001b19d"}, + {file = "kiwisolver-1.4.8-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:034d2c891f76bd3edbdb3ea11140d8510dca675443da7304205a2eaa45d8334c"}, + {file = "kiwisolver-1.4.8-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d47b28d1dfe0793d5e96bce90835e17edf9a499b53969b03c6c47ea5985844c3"}, + {file = "kiwisolver-1.4.8-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eb158fe28ca0c29f2260cca8c43005329ad58452c36f0edf298204de32a9a3ed"}, + {file = "kiwisolver-1.4.8-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5536185fce131780ebd809f8e623bf4030ce1b161353166c49a3c74c287897f"}, + {file = "kiwisolver-1.4.8-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:369b75d40abedc1da2c1f4de13f3482cb99e3237b38726710f4a793432b1c5ff"}, + {file = "kiwisolver-1.4.8-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:641f2ddf9358c80faa22e22eb4c9f54bd3f0e442e038728f500e3b978d00aa7d"}, + {file = "kiwisolver-1.4.8-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:d561d2d8883e0819445cfe58d7ddd673e4015c3c57261d7bdcd3710d0d14005c"}, + {file = "kiwisolver-1.4.8-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:1732e065704b47c9afca7ffa272f845300a4eb959276bf6970dc07265e73b605"}, + {file = "kiwisolver-1.4.8-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:bcb1ebc3547619c3b58a39e2448af089ea2ef44b37988caf432447374941574e"}, + {file = "kiwisolver-1.4.8-cp310-cp310-win_amd64.whl", hash = "sha256:89c107041f7b27844179ea9c85d6da275aa55ecf28413e87624d033cf1f6b751"}, + {file = "kiwisolver-1.4.8-cp310-cp310-win_arm64.whl", hash = "sha256:b5773efa2be9eb9fcf5415ea3ab70fc785d598729fd6057bea38d539ead28271"}, + {file = "kiwisolver-1.4.8-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a4d3601908c560bdf880f07d94f31d734afd1bb71e96585cace0e38ef44c6d84"}, + {file = "kiwisolver-1.4.8-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:856b269c4d28a5c0d5e6c1955ec36ebfd1651ac00e1ce0afa3e28da95293b561"}, + {file = "kiwisolver-1.4.8-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c2b9a96e0f326205af81a15718a9073328df1173a2619a68553decb7097fd5d7"}, + {file = "kiwisolver-1.4.8-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c5020c83e8553f770cb3b5fc13faac40f17e0b205bd237aebd21d53d733adb03"}, + {file = "kiwisolver-1.4.8-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dace81d28c787956bfbfbbfd72fdcef014f37d9b48830829e488fdb32b49d954"}, + {file = "kiwisolver-1.4.8-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:11e1022b524bd48ae56c9b4f9296bce77e15a2e42a502cceba602f804b32bb79"}, + {file = "kiwisolver-1.4.8-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3b9b4d2892fefc886f30301cdd80debd8bb01ecdf165a449eb6e78f79f0fabd6"}, + {file = "kiwisolver-1.4.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a96c0e790ee875d65e340ab383700e2b4891677b7fcd30a699146f9384a2bb0"}, + {file = "kiwisolver-1.4.8-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:23454ff084b07ac54ca8be535f4174170c1094a4cff78fbae4f73a4bcc0d4dab"}, + {file = "kiwisolver-1.4.8-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:87b287251ad6488e95b4f0b4a79a6d04d3ea35fde6340eb38fbd1ca9cd35bbbc"}, + {file = "kiwisolver-1.4.8-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:b21dbe165081142b1232a240fc6383fd32cdd877ca6cc89eab93e5f5883e1c25"}, + {file = "kiwisolver-1.4.8-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:768cade2c2df13db52475bd28d3a3fac8c9eff04b0e9e2fda0f3760f20b3f7fc"}, + {file = "kiwisolver-1.4.8-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d47cfb2650f0e103d4bf68b0b5804c68da97272c84bb12850d877a95c056bd67"}, + {file = "kiwisolver-1.4.8-cp311-cp311-win_amd64.whl", hash = "sha256:ed33ca2002a779a2e20eeb06aea7721b6e47f2d4b8a8ece979d8ba9e2a167e34"}, + {file = "kiwisolver-1.4.8-cp311-cp311-win_arm64.whl", hash = "sha256:16523b40aab60426ffdebe33ac374457cf62863e330a90a0383639ce14bf44b2"}, + {file = "kiwisolver-1.4.8-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d6af5e8815fd02997cb6ad9bbed0ee1e60014438ee1a5c2444c96f87b8843502"}, + {file = "kiwisolver-1.4.8-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bade438f86e21d91e0cf5dd7c0ed00cda0f77c8c1616bd83f9fc157fa6760d31"}, + {file = "kiwisolver-1.4.8-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b83dc6769ddbc57613280118fb4ce3cd08899cc3369f7d0e0fab518a7cf37fdb"}, + {file = "kiwisolver-1.4.8-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:111793b232842991be367ed828076b03d96202c19221b5ebab421ce8bcad016f"}, + {file = "kiwisolver-1.4.8-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:257af1622860e51b1a9d0ce387bf5c2c4f36a90594cb9514f55b074bcc787cfc"}, + {file = "kiwisolver-1.4.8-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:69b5637c3f316cab1ec1c9a12b8c5f4750a4c4b71af9157645bf32830e39c03a"}, + {file = "kiwisolver-1.4.8-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:782bb86f245ec18009890e7cb8d13a5ef54dcf2ebe18ed65f795e635a96a1c6a"}, + {file = "kiwisolver-1.4.8-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc978a80a0db3a66d25767b03688f1147a69e6237175c0f4ffffaaedf744055a"}, + {file = "kiwisolver-1.4.8-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:36dbbfd34838500a31f52c9786990d00150860e46cd5041386f217101350f0d3"}, + {file = "kiwisolver-1.4.8-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:eaa973f1e05131de5ff3569bbba7f5fd07ea0595d3870ed4a526d486fe57fa1b"}, + {file = "kiwisolver-1.4.8-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a66f60f8d0c87ab7f59b6fb80e642ebb29fec354a4dfad687ca4092ae69d04f4"}, + {file = "kiwisolver-1.4.8-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:858416b7fb777a53f0c59ca08190ce24e9abbd3cffa18886a5781b8e3e26f65d"}, + {file = "kiwisolver-1.4.8-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:085940635c62697391baafaaeabdf3dd7a6c3643577dde337f4d66eba021b2b8"}, + {file = "kiwisolver-1.4.8-cp312-cp312-win_amd64.whl", hash = "sha256:01c3d31902c7db5fb6182832713d3b4122ad9317c2c5877d0539227d96bb2e50"}, + {file = "kiwisolver-1.4.8-cp312-cp312-win_arm64.whl", hash = "sha256:a3c44cb68861de93f0c4a8175fbaa691f0aa22550c331fefef02b618a9dcb476"}, + {file = "kiwisolver-1.4.8-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:1c8ceb754339793c24aee1c9fb2485b5b1f5bb1c2c214ff13368431e51fc9a09"}, + {file = "kiwisolver-1.4.8-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:54a62808ac74b5e55a04a408cda6156f986cefbcf0ada13572696b507cc92fa1"}, + {file = "kiwisolver-1.4.8-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:68269e60ee4929893aad82666821aaacbd455284124817af45c11e50a4b42e3c"}, + {file = "kiwisolver-1.4.8-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:34d142fba9c464bc3bbfeff15c96eab0e7310343d6aefb62a79d51421fcc5f1b"}, + {file = "kiwisolver-1.4.8-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ddc373e0eef45b59197de815b1b28ef89ae3955e7722cc9710fb91cd77b7f47"}, + {file = "kiwisolver-1.4.8-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:77e6f57a20b9bd4e1e2cedda4d0b986ebd0216236f0106e55c28aea3d3d69b16"}, + {file = "kiwisolver-1.4.8-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:08e77738ed7538f036cd1170cbed942ef749137b1311fa2bbe2a7fda2f6bf3cc"}, + {file = "kiwisolver-1.4.8-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5ce1e481a74b44dd5e92ff03ea0cb371ae7a0268318e202be06c8f04f4f1246"}, + {file = "kiwisolver-1.4.8-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:fc2ace710ba7c1dfd1a3b42530b62b9ceed115f19a1656adefce7b1782a37794"}, + {file = "kiwisolver-1.4.8-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:3452046c37c7692bd52b0e752b87954ef86ee2224e624ef7ce6cb21e8c41cc1b"}, + {file = "kiwisolver-1.4.8-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:7e9a60b50fe8b2ec6f448fe8d81b07e40141bfced7f896309df271a0b92f80f3"}, + {file = "kiwisolver-1.4.8-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:918139571133f366e8362fa4a297aeba86c7816b7ecf0bc79168080e2bd79957"}, + {file = "kiwisolver-1.4.8-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e063ef9f89885a1d68dd8b2e18f5ead48653176d10a0e324e3b0030e3a69adeb"}, + {file = "kiwisolver-1.4.8-cp313-cp313-win_amd64.whl", hash = "sha256:a17b7c4f5b2c51bb68ed379defd608a03954a1845dfed7cc0117f1cc8a9b7fd2"}, + {file = "kiwisolver-1.4.8-cp313-cp313-win_arm64.whl", hash = "sha256:3cd3bc628b25f74aedc6d374d5babf0166a92ff1317f46267f12d2ed54bc1d30"}, + {file = "kiwisolver-1.4.8-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:370fd2df41660ed4e26b8c9d6bbcad668fbe2560462cba151a721d49e5b6628c"}, + {file = "kiwisolver-1.4.8-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:84a2f830d42707de1d191b9490ac186bf7997a9495d4e9072210a1296345f7dc"}, + {file = "kiwisolver-1.4.8-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:7a3ad337add5148cf51ce0b55642dc551c0b9d6248458a757f98796ca7348712"}, + {file = "kiwisolver-1.4.8-cp313-cp313t-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7506488470f41169b86d8c9aeff587293f530a23a23a49d6bc64dab66bedc71e"}, + {file = "kiwisolver-1.4.8-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2f0121b07b356a22fb0414cec4666bbe36fd6d0d759db3d37228f496ed67c880"}, + {file = "kiwisolver-1.4.8-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d6d6bd87df62c27d4185de7c511c6248040afae67028a8a22012b010bc7ad062"}, + {file = "kiwisolver-1.4.8-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:291331973c64bb9cce50bbe871fb2e675c4331dab4f31abe89f175ad7679a4d7"}, + {file = "kiwisolver-1.4.8-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:893f5525bb92d3d735878ec00f781b2de998333659507d29ea4466208df37bed"}, + {file = "kiwisolver-1.4.8-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:b47a465040146981dc9db8647981b8cb96366fbc8d452b031e4f8fdffec3f26d"}, + {file = "kiwisolver-1.4.8-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:99cea8b9dd34ff80c521aef46a1dddb0dcc0283cf18bde6d756f1e6f31772165"}, + {file = "kiwisolver-1.4.8-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:151dffc4865e5fe6dafce5480fab84f950d14566c480c08a53c663a0020504b6"}, + {file = "kiwisolver-1.4.8-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:577facaa411c10421314598b50413aa1ebcf5126f704f1e5d72d7e4e9f020d90"}, + {file = "kiwisolver-1.4.8-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:be4816dc51c8a471749d664161b434912eee82f2ea66bd7628bd14583a833e85"}, + {file = "kiwisolver-1.4.8-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:e7a019419b7b510f0f7c9dceff8c5eae2392037eae483a7f9162625233802b0a"}, + {file = "kiwisolver-1.4.8-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:286b18e86682fd2217a48fc6be6b0f20c1d0ed10958d8dc53453ad58d7be0bf8"}, + {file = "kiwisolver-1.4.8-pp310-pypy310_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4191ee8dfd0be1c3666ccbac178c5a05d5f8d689bbe3fc92f3c4abec817f8fe0"}, + {file = "kiwisolver-1.4.8-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7cd2785b9391f2873ad46088ed7599a6a71e762e1ea33e87514b1a441ed1da1c"}, + {file = "kiwisolver-1.4.8-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c07b29089b7ba090b6f1a669f1411f27221c3662b3a1b7010e67b59bb5a6f10b"}, + {file = "kiwisolver-1.4.8-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:65ea09a5a3faadd59c2ce96dc7bf0f364986a315949dc6374f04396b0d60e09b"}, + {file = "kiwisolver-1.4.8.tar.gz", hash = "sha256:23d5f023bdc8c7e54eb65f03ca5d5bb25b601eac4d7f1a042888a1f45237987e"}, +] + [[package]] name = "kombu" version = "5.3.2" @@ -1571,6 +1821,63 @@ files = [ {file = "MarkupSafe-2.1.3.tar.gz", hash = "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad"}, ] +[[package]] +name = "matplotlib" +version = "3.10.0" +description = "Python plotting package" +optional = false +python-versions = ">=3.10" +files = [ + {file = "matplotlib-3.10.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2c5829a5a1dd5a71f0e31e6e8bb449bc0ee9dbfb05ad28fc0c6b55101b3a4be6"}, + {file = "matplotlib-3.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a2a43cbefe22d653ab34bb55d42384ed30f611bcbdea1f8d7f431011a2e1c62e"}, + {file = "matplotlib-3.10.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:607b16c8a73943df110f99ee2e940b8a1cbf9714b65307c040d422558397dac5"}, + {file = "matplotlib-3.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:01d2b19f13aeec2e759414d3bfe19ddfb16b13a1250add08d46d5ff6f9be83c6"}, + {file = "matplotlib-3.10.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e6c6461e1fc63df30bf6f80f0b93f5b6784299f721bc28530477acd51bfc3d1"}, + {file = "matplotlib-3.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:994c07b9d9fe8d25951e3202a68c17900679274dadfc1248738dcfa1bd40d7f3"}, + {file = "matplotlib-3.10.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:fd44fc75522f58612ec4a33958a7e5552562b7705b42ef1b4f8c0818e304a363"}, + {file = "matplotlib-3.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c58a9622d5dbeb668f407f35f4e6bfac34bb9ecdcc81680c04d0258169747997"}, + {file = "matplotlib-3.10.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:845d96568ec873be63f25fa80e9e7fae4be854a66a7e2f0c8ccc99e94a8bd4ef"}, + {file = "matplotlib-3.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5439f4c5a3e2e8eab18e2f8c3ef929772fd5641876db71f08127eed95ab64683"}, + {file = "matplotlib-3.10.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4673ff67a36152c48ddeaf1135e74ce0d4bce1bbf836ae40ed39c29edf7e2765"}, + {file = "matplotlib-3.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:7e8632baebb058555ac0cde75db885c61f1212e47723d63921879806b40bec6a"}, + {file = "matplotlib-3.10.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4659665bc7c9b58f8c00317c3c2a299f7f258eeae5a5d56b4c64226fca2f7c59"}, + {file = "matplotlib-3.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d44cb942af1693cced2604c33a9abcef6205601c445f6d0dc531d813af8a2f5a"}, + {file = "matplotlib-3.10.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a994f29e968ca002b50982b27168addfd65f0105610b6be7fa515ca4b5307c95"}, + {file = "matplotlib-3.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9b0558bae37f154fffda54d779a592bc97ca8b4701f1c710055b609a3bac44c8"}, + {file = "matplotlib-3.10.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:503feb23bd8c8acc75541548a1d709c059b7184cde26314896e10a9f14df5f12"}, + {file = "matplotlib-3.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:c40ba2eb08b3f5de88152c2333c58cee7edcead0a2a0d60fcafa116b17117adc"}, + {file = "matplotlib-3.10.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:96f2886f5c1e466f21cc41b70c5a0cd47bfa0015eb2d5793c88ebce658600e25"}, + {file = "matplotlib-3.10.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:12eaf48463b472c3c0f8dbacdbf906e573013df81a0ab82f0616ea4b11281908"}, + {file = "matplotlib-3.10.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2fbbabc82fde51391c4da5006f965e36d86d95f6ee83fb594b279564a4c5d0d2"}, + {file = "matplotlib-3.10.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad2e15300530c1a94c63cfa546e3b7864bd18ea2901317bae8bbf06a5ade6dcf"}, + {file = "matplotlib-3.10.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3547d153d70233a8496859097ef0312212e2689cdf8d7ed764441c77604095ae"}, + {file = "matplotlib-3.10.0-cp313-cp313-win_amd64.whl", hash = "sha256:c55b20591ced744aa04e8c3e4b7543ea4d650b6c3c4b208c08a05b4010e8b442"}, + {file = "matplotlib-3.10.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:9ade1003376731a971e398cc4ef38bb83ee8caf0aee46ac6daa4b0506db1fd06"}, + {file = "matplotlib-3.10.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:95b710fea129c76d30be72c3b38f330269363fbc6e570a5dd43580487380b5ff"}, + {file = "matplotlib-3.10.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdbaf909887373c3e094b0318d7ff230b2ad9dcb64da7ade654182872ab2593"}, + {file = "matplotlib-3.10.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d907fddb39f923d011875452ff1eca29a9e7f21722b873e90db32e5d8ddff12e"}, + {file = "matplotlib-3.10.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3b427392354d10975c1d0f4ee18aa5844640b512d5311ef32efd4dd7db106ede"}, + {file = "matplotlib-3.10.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5fd41b0ec7ee45cd960a8e71aea7c946a28a0b8a4dcee47d2856b2af051f334c"}, + {file = "matplotlib-3.10.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:81713dd0d103b379de4516b861d964b1d789a144103277769238c732229d7f03"}, + {file = "matplotlib-3.10.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:359f87baedb1f836ce307f0e850d12bb5f1936f70d035561f90d41d305fdacea"}, + {file = "matplotlib-3.10.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ae80dc3a4add4665cf2faa90138384a7ffe2a4e37c58d83e115b54287c4f06ef"}, + {file = "matplotlib-3.10.0.tar.gz", hash = "sha256:b886d02a581b96704c9d1ffe55709e49b4d2d52709ccebc4be42db856e511278"}, +] + +[package.dependencies] +contourpy = ">=1.0.1" +cycler = ">=0.10" +fonttools = ">=4.22.0" +kiwisolver = ">=1.3.1" +numpy = ">=1.23" +packaging = ">=20.0" +pillow = ">=8" +pyparsing = ">=2.3.1" +python-dateutil = ">=2.7" + +[package.extras] +dev = ["meson-python (>=0.13.1,<0.17.0)", "pybind11 (>=2.13.2,!=2.13.3)", "setuptools (>=64)", "setuptools_scm (>=7)"] + [[package]] name = "matplotlib-inline" version = "0.1.6" @@ -1706,6 +2013,70 @@ doc = ["myst-nb (>=1.0)", "numpydoc (>=1.7)", "pillow (>=9.4)", "pydata-sphinx-t extra = ["lxml (>=4.6)", "pydot (>=2.0)", "pygraphviz (>=1.12)", "sympy (>=1.10)"] test = ["pytest (>=7.2)", "pytest-cov (>=4.0)"] +[[package]] +name = "numpy" +version = "2.2.1" +description = "Fundamental package for array computing in Python" +optional = false +python-versions = ">=3.10" +files = [ + {file = "numpy-2.2.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5edb4e4caf751c1518e6a26a83501fda79bff41cc59dac48d70e6d65d4ec4440"}, + {file = "numpy-2.2.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:aa3017c40d513ccac9621a2364f939d39e550c542eb2a894b4c8da92b38896ab"}, + {file = "numpy-2.2.1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:61048b4a49b1c93fe13426e04e04fdf5a03f456616f6e98c7576144677598675"}, + {file = "numpy-2.2.1-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:7671dc19c7019103ca44e8d94917eba8534c76133523ca8406822efdd19c9308"}, + {file = "numpy-2.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4250888bcb96617e00bfa28ac24850a83c9f3a16db471eca2ee1f1714df0f957"}, + {file = "numpy-2.2.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a7746f235c47abc72b102d3bce9977714c2444bdfaea7888d241b4c4bb6a78bf"}, + {file = "numpy-2.2.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:059e6a747ae84fce488c3ee397cee7e5f905fd1bda5fb18c66bc41807ff119b2"}, + {file = "numpy-2.2.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f62aa6ee4eb43b024b0e5a01cf65a0bb078ef8c395e8713c6e8a12a697144528"}, + {file = "numpy-2.2.1-cp310-cp310-win32.whl", hash = "sha256:48fd472630715e1c1c89bf1feab55c29098cb403cc184b4859f9c86d4fcb6a95"}, + {file = "numpy-2.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:b541032178a718c165a49638d28272b771053f628382d5e9d1c93df23ff58dbf"}, + {file = "numpy-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:40f9e544c1c56ba8f1cf7686a8c9b5bb249e665d40d626a23899ba6d5d9e1484"}, + {file = "numpy-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f9b57eaa3b0cd8db52049ed0330747b0364e899e8a606a624813452b8203d5f7"}, + {file = "numpy-2.2.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:bc8a37ad5b22c08e2dbd27df2b3ef7e5c0864235805b1e718a235bcb200cf1cb"}, + {file = "numpy-2.2.1-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:9036d6365d13b6cbe8f27a0eaf73ddcc070cae584e5ff94bb45e3e9d729feab5"}, + {file = "numpy-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:51faf345324db860b515d3f364eaa93d0e0551a88d6218a7d61286554d190d73"}, + {file = "numpy-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:38efc1e56b73cc9b182fe55e56e63b044dd26a72128fd2fbd502f75555d92591"}, + {file = "numpy-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:31b89fa67a8042e96715c68e071a1200c4e172f93b0fbe01a14c0ff3ff820fc8"}, + {file = "numpy-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4c86e2a209199ead7ee0af65e1d9992d1dce7e1f63c4b9a616500f93820658d0"}, + {file = "numpy-2.2.1-cp311-cp311-win32.whl", hash = "sha256:b34d87e8a3090ea626003f87f9392b3929a7bbf4104a05b6667348b6bd4bf1cd"}, + {file = "numpy-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:360137f8fb1b753c5cde3ac388597ad680eccbbbb3865ab65efea062c4a1fd16"}, + {file = "numpy-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:694f9e921a0c8f252980e85bce61ebbd07ed2b7d4fa72d0e4246f2f8aa6642ab"}, + {file = "numpy-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3683a8d166f2692664262fd4900f207791d005fb088d7fdb973cc8d663626faa"}, + {file = "numpy-2.2.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:780077d95eafc2ccc3ced969db22377b3864e5b9a0ea5eb347cc93b3ea900315"}, + {file = "numpy-2.2.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:55ba24ebe208344aa7a00e4482f65742969a039c2acfcb910bc6fcd776eb4355"}, + {file = "numpy-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9b1d07b53b78bf84a96898c1bc139ad7f10fda7423f5fd158fd0f47ec5e01ac7"}, + {file = "numpy-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5062dc1a4e32a10dc2b8b13cedd58988261416e811c1dc4dbdea4f57eea61b0d"}, + {file = "numpy-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:fce4f615f8ca31b2e61aa0eb5865a21e14f5629515c9151850aa936c02a1ee51"}, + {file = "numpy-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:67d4cda6fa6ffa073b08c8372aa5fa767ceb10c9a0587c707505a6d426f4e046"}, + {file = "numpy-2.2.1-cp312-cp312-win32.whl", hash = "sha256:32cb94448be47c500d2c7a95f93e2f21a01f1fd05dd2beea1ccd049bb6001cd2"}, + {file = "numpy-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:ba5511d8f31c033a5fcbda22dd5c813630af98c70b2661f2d2c654ae3cdfcfc8"}, + {file = "numpy-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f1d09e520217618e76396377c81fba6f290d5f926f50c35f3a5f72b01a0da780"}, + {file = "numpy-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3ecc47cd7f6ea0336042be87d9e7da378e5c7e9b3c8ad0f7c966f714fc10d821"}, + {file = "numpy-2.2.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:f419290bc8968a46c4933158c91a0012b7a99bb2e465d5ef5293879742f8797e"}, + {file = "numpy-2.2.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:5b6c390bfaef8c45a260554888966618328d30e72173697e5cabe6b285fb2348"}, + {file = "numpy-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:526fc406ab991a340744aad7e25251dd47a6720a685fa3331e5c59fef5282a59"}, + {file = "numpy-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f74e6fdeb9a265624ec3a3918430205dff1df7e95a230779746a6af78bc615af"}, + {file = "numpy-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:53c09385ff0b72ba79d8715683c1168c12e0b6e84fb0372e97553d1ea91efe51"}, + {file = "numpy-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f3eac17d9ec51be534685ba877b6ab5edc3ab7ec95c8f163e5d7b39859524716"}, + {file = "numpy-2.2.1-cp313-cp313-win32.whl", hash = "sha256:9ad014faa93dbb52c80d8f4d3dcf855865c876c9660cb9bd7553843dd03a4b1e"}, + {file = "numpy-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:164a829b6aacf79ca47ba4814b130c4020b202522a93d7bff2202bfb33b61c60"}, + {file = "numpy-2.2.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:4dfda918a13cc4f81e9118dea249e192ab167a0bb1966272d5503e39234d694e"}, + {file = "numpy-2.2.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:733585f9f4b62e9b3528dd1070ec4f52b8acf64215b60a845fa13ebd73cd0712"}, + {file = "numpy-2.2.1-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:89b16a18e7bba224ce5114db863e7029803c179979e1af6ad6a6b11f70545008"}, + {file = "numpy-2.2.1-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:676f4eebf6b2d430300f1f4f4c2461685f8269f94c89698d832cdf9277f30b84"}, + {file = "numpy-2.2.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27f5cdf9f493b35f7e41e8368e7d7b4bbafaf9660cba53fb21d2cd174ec09631"}, + {file = "numpy-2.2.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1ad395cf254c4fbb5b2132fee391f361a6e8c1adbd28f2cd8e79308a615fe9d"}, + {file = "numpy-2.2.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:08ef779aed40dbc52729d6ffe7dd51df85796a702afbf68a4f4e41fafdc8bda5"}, + {file = "numpy-2.2.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:26c9c4382b19fcfbbed3238a14abf7ff223890ea1936b8890f058e7ba35e8d71"}, + {file = "numpy-2.2.1-cp313-cp313t-win32.whl", hash = "sha256:93cf4e045bae74c90ca833cba583c14b62cb4ba2cba0abd2b141ab52548247e2"}, + {file = "numpy-2.2.1-cp313-cp313t-win_amd64.whl", hash = "sha256:bff7d8ec20f5f42607599f9994770fa65d76edca264a87b5e4ea5629bce12268"}, + {file = "numpy-2.2.1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7ba9cc93a91d86365a5d270dee221fdc04fb68d7478e6bf6af650de78a8339e3"}, + {file = "numpy-2.2.1-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:3d03883435a19794e41f147612a77a8f56d4e52822337844fff3d4040a142964"}, + {file = "numpy-2.2.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4511d9e6071452b944207c8ce46ad2f897307910b402ea5fa975da32e0102800"}, + {file = "numpy-2.2.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5c5cc0cbabe9452038ed984d05ac87910f89370b9242371bd9079cb4af61811e"}, + {file = "numpy-2.2.1.tar.gz", hash = "sha256:45681fd7128c8ad1c379f0ca0776a8b0c6583d2f69889ddac01559dfe4390918"}, +] + [[package]] name = "oauthlib" version = "3.2.2" @@ -1884,6 +2255,94 @@ files = [ {file = "pickleshare-0.7.5.tar.gz", hash = "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca"}, ] +[[package]] +name = "pillow" +version = "11.1.0" +description = "Python Imaging Library (Fork)" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pillow-11.1.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:e1abe69aca89514737465752b4bcaf8016de61b3be1397a8fc260ba33321b3a8"}, + {file = "pillow-11.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c640e5a06869c75994624551f45e5506e4256562ead981cce820d5ab39ae2192"}, + {file = "pillow-11.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a07dba04c5e22824816b2615ad7a7484432d7f540e6fa86af60d2de57b0fcee2"}, + {file = "pillow-11.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e267b0ed063341f3e60acd25c05200df4193e15a4a5807075cd71225a2386e26"}, + {file = "pillow-11.1.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:bd165131fd51697e22421d0e467997ad31621b74bfc0b75956608cb2906dda07"}, + {file = "pillow-11.1.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:abc56501c3fd148d60659aae0af6ddc149660469082859fa7b066a298bde9482"}, + {file = "pillow-11.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:54ce1c9a16a9561b6d6d8cb30089ab1e5eb66918cb47d457bd996ef34182922e"}, + {file = "pillow-11.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:73ddde795ee9b06257dac5ad42fcb07f3b9b813f8c1f7f870f402f4dc54b5269"}, + {file = "pillow-11.1.0-cp310-cp310-win32.whl", hash = "sha256:3a5fe20a7b66e8135d7fd617b13272626a28278d0e578c98720d9ba4b2439d49"}, + {file = "pillow-11.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:b6123aa4a59d75f06e9dd3dac5bf8bc9aa383121bb3dd9a7a612e05eabc9961a"}, + {file = "pillow-11.1.0-cp310-cp310-win_arm64.whl", hash = "sha256:a76da0a31da6fcae4210aa94fd779c65c75786bc9af06289cd1c184451ef7a65"}, + {file = "pillow-11.1.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:e06695e0326d05b06833b40b7ef477e475d0b1ba3a6d27da1bb48c23209bf457"}, + {file = "pillow-11.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:96f82000e12f23e4f29346e42702b6ed9a2f2fea34a740dd5ffffcc8c539eb35"}, + {file = "pillow-11.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3cd561ded2cf2bbae44d4605837221b987c216cff94f49dfeed63488bb228d2"}, + {file = "pillow-11.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f189805c8be5ca5add39e6f899e6ce2ed824e65fb45f3c28cb2841911da19070"}, + {file = "pillow-11.1.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:dd0052e9db3474df30433f83a71b9b23bd9e4ef1de13d92df21a52c0303b8ab6"}, + {file = "pillow-11.1.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:837060a8599b8f5d402e97197d4924f05a2e0d68756998345c829c33186217b1"}, + {file = "pillow-11.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:aa8dd43daa836b9a8128dbe7d923423e5ad86f50a7a14dc688194b7be5c0dea2"}, + {file = "pillow-11.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0a2f91f8a8b367e7a57c6e91cd25af510168091fb89ec5146003e424e1558a96"}, + {file = "pillow-11.1.0-cp311-cp311-win32.whl", hash = "sha256:c12fc111ef090845de2bb15009372175d76ac99969bdf31e2ce9b42e4b8cd88f"}, + {file = "pillow-11.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:fbd43429d0d7ed6533b25fc993861b8fd512c42d04514a0dd6337fb3ccf22761"}, + {file = "pillow-11.1.0-cp311-cp311-win_arm64.whl", hash = "sha256:f7955ecf5609dee9442cbface754f2c6e541d9e6eda87fad7f7a989b0bdb9d71"}, + {file = "pillow-11.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2062ffb1d36544d42fcaa277b069c88b01bb7298f4efa06731a7fd6cc290b81a"}, + {file = "pillow-11.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a85b653980faad27e88b141348707ceeef8a1186f75ecc600c395dcac19f385b"}, + {file = "pillow-11.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9409c080586d1f683df3f184f20e36fb647f2e0bc3988094d4fd8c9f4eb1b3b3"}, + {file = "pillow-11.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7fdadc077553621911f27ce206ffcbec7d3f8d7b50e0da39f10997e8e2bb7f6a"}, + {file = "pillow-11.1.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:93a18841d09bcdd774dcdc308e4537e1f867b3dec059c131fde0327899734aa1"}, + {file = "pillow-11.1.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:9aa9aeddeed452b2f616ff5507459e7bab436916ccb10961c4a382cd3e03f47f"}, + {file = "pillow-11.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3cdcdb0b896e981678eee140d882b70092dac83ac1cdf6b3a60e2216a73f2b91"}, + {file = "pillow-11.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:36ba10b9cb413e7c7dfa3e189aba252deee0602c86c309799da5a74009ac7a1c"}, + {file = "pillow-11.1.0-cp312-cp312-win32.whl", hash = "sha256:cfd5cd998c2e36a862d0e27b2df63237e67273f2fc78f47445b14e73a810e7e6"}, + {file = "pillow-11.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:a697cd8ba0383bba3d2d3ada02b34ed268cb548b369943cd349007730c92bddf"}, + {file = "pillow-11.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:4dd43a78897793f60766563969442020e90eb7847463eca901e41ba186a7d4a5"}, + {file = "pillow-11.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ae98e14432d458fc3de11a77ccb3ae65ddce70f730e7c76140653048c71bfcbc"}, + {file = "pillow-11.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cc1331b6d5a6e144aeb5e626f4375f5b7ae9934ba620c0ac6b3e43d5e683a0f0"}, + {file = "pillow-11.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:758e9d4ef15d3560214cddbc97b8ef3ef86ce04d62ddac17ad39ba87e89bd3b1"}, + {file = "pillow-11.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b523466b1a31d0dcef7c5be1f20b942919b62fd6e9a9be199d035509cbefc0ec"}, + {file = "pillow-11.1.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:9044b5e4f7083f209c4e35aa5dd54b1dd5b112b108648f5c902ad586d4f945c5"}, + {file = "pillow-11.1.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:3764d53e09cdedd91bee65c2527815d315c6b90d7b8b79759cc48d7bf5d4f114"}, + {file = "pillow-11.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:31eba6bbdd27dde97b0174ddf0297d7a9c3a507a8a1480e1e60ef914fe23d352"}, + {file = "pillow-11.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b5d658fbd9f0d6eea113aea286b21d3cd4d3fd978157cbf2447a6035916506d3"}, + {file = "pillow-11.1.0-cp313-cp313-win32.whl", hash = "sha256:f86d3a7a9af5d826744fabf4afd15b9dfef44fe69a98541f666f66fbb8d3fef9"}, + {file = "pillow-11.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:593c5fd6be85da83656b93ffcccc2312d2d149d251e98588b14fbc288fd8909c"}, + {file = "pillow-11.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:11633d58b6ee5733bde153a8dafd25e505ea3d32e261accd388827ee987baf65"}, + {file = "pillow-11.1.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:70ca5ef3b3b1c4a0812b5c63c57c23b63e53bc38e758b37a951e5bc466449861"}, + {file = "pillow-11.1.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:8000376f139d4d38d6851eb149b321a52bb8893a88dae8ee7d95840431977081"}, + {file = "pillow-11.1.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ee85f0696a17dd28fbcfceb59f9510aa71934b483d1f5601d1030c3c8304f3c"}, + {file = "pillow-11.1.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:dd0e081319328928531df7a0e63621caf67652c8464303fd102141b785ef9547"}, + {file = "pillow-11.1.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:e63e4e5081de46517099dc30abe418122f54531a6ae2ebc8680bcd7096860eab"}, + {file = "pillow-11.1.0-cp313-cp313t-win32.whl", hash = "sha256:dda60aa465b861324e65a78c9f5cf0f4bc713e4309f83bc387be158b077963d9"}, + {file = "pillow-11.1.0-cp313-cp313t-win_amd64.whl", hash = "sha256:ad5db5781c774ab9a9b2c4302bbf0c1014960a0a7be63278d13ae6fdf88126fe"}, + {file = "pillow-11.1.0-cp313-cp313t-win_arm64.whl", hash = "sha256:67cd427c68926108778a9005f2a04adbd5e67c442ed21d95389fe1d595458756"}, + {file = "pillow-11.1.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:bf902d7413c82a1bfa08b06a070876132a5ae6b2388e2712aab3a7cbc02205c6"}, + {file = "pillow-11.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c1eec9d950b6fe688edee07138993e54ee4ae634c51443cfb7c1e7613322718e"}, + {file = "pillow-11.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e275ee4cb11c262bd108ab2081f750db2a1c0b8c12c1897f27b160c8bd57bbc"}, + {file = "pillow-11.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4db853948ce4e718f2fc775b75c37ba2efb6aaea41a1a5fc57f0af59eee774b2"}, + {file = "pillow-11.1.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:ab8a209b8485d3db694fa97a896d96dd6533d63c22829043fd9de627060beade"}, + {file = "pillow-11.1.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:54251ef02a2309b5eec99d151ebf5c9904b77976c8abdcbce7891ed22df53884"}, + {file = "pillow-11.1.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:5bb94705aea800051a743aa4874bb1397d4695fb0583ba5e425ee0328757f196"}, + {file = "pillow-11.1.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:89dbdb3e6e9594d512780a5a1c42801879628b38e3efc7038094430844e271d8"}, + {file = "pillow-11.1.0-cp39-cp39-win32.whl", hash = "sha256:e5449ca63da169a2e6068dd0e2fcc8d91f9558aba89ff6d02121ca8ab11e79e5"}, + {file = "pillow-11.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:3362c6ca227e65c54bf71a5f88b3d4565ff1bcbc63ae72c34b07bbb1cc59a43f"}, + {file = "pillow-11.1.0-cp39-cp39-win_arm64.whl", hash = "sha256:b20be51b37a75cc54c2c55def3fa2c65bb94ba859dde241cd0a4fd302de5ae0a"}, + {file = "pillow-11.1.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:8c730dc3a83e5ac137fbc92dfcfe1511ce3b2b5d7578315b63dbbb76f7f51d90"}, + {file = "pillow-11.1.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:7d33d2fae0e8b170b6a6c57400e077412240f6f5bb2a342cf1ee512a787942bb"}, + {file = "pillow-11.1.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a8d65b38173085f24bc07f8b6c505cbb7418009fa1a1fcb111b1f4961814a442"}, + {file = "pillow-11.1.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:015c6e863faa4779251436db398ae75051469f7c903b043a48f078e437656f83"}, + {file = "pillow-11.1.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d44ff19eea13ae4acdaaab0179fa68c0c6f2f45d66a4d8ec1eda7d6cecbcc15f"}, + {file = "pillow-11.1.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d3d8da4a631471dfaf94c10c85f5277b1f8e42ac42bade1ac67da4b4a7359b73"}, + {file = "pillow-11.1.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:4637b88343166249fe8aa94e7c4a62a180c4b3898283bb5d3d2fd5fe10d8e4e0"}, + {file = "pillow-11.1.0.tar.gz", hash = "sha256:368da70808b36d73b4b390a8ffac11069f8a5c85f29eff1f1b01bcf3ef5b2a20"}, +] + +[package.extras] +docs = ["furo", "olefile", "sphinx (>=8.1)", "sphinx-copybutton", "sphinx-inline-tabs", "sphinxext-opengraph"] +fpx = ["olefile"] +mic = ["olefile"] +tests = ["check-manifest", "coverage (>=7.4.2)", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout", "trove-classifiers (>=2024.10.12)"] +typing = ["typing-extensions"] +xmp = ["defusedxml"] + [[package]] name = "platformdirs" version = "3.11.0" @@ -2880,6 +3339,28 @@ setuptools = ">=21.0.0" six = ">=1.10,<2.0" urllib3 = ">=1.26.5,<2.0.0" +[[package]] +name = "tifffile" +version = "2025.1.10" +description = "Read and write TIFF files" +optional = false +python-versions = ">=3.10" +files = [ + {file = "tifffile-2025.1.10-py3-none-any.whl", hash = "sha256:ed24cf4c99fb13b4f5fb29f8a0d5605e60558c950bccbdca2a6470732a27cfb3"}, + {file = "tifffile-2025.1.10.tar.gz", hash = "sha256:baaf0a3b87bf7ec375fa1537503353f70497eabe1bdde590f2e41cc0346e612f"}, +] + +[package.dependencies] +numpy = "*" + +[package.extras] +all = ["defusedxml", "fsspec", "imagecodecs (>=2024.12.30)", "lxml", "matplotlib", "zarr (<3)"] +codecs = ["imagecodecs (>=2024.12.30)"] +plot = ["matplotlib"] +test = ["cmapfile", "czifile", "dask", "defusedxml", "fsspec", "imagecodecs", "lfdfiles", "lxml", "ndtiff", "oiffile", "psdtags", "pytest", "roifile", "xarray", "zarr (<3)"] +xml = ["defusedxml", "lxml"] +zarr = ["fsspec", "zarr (<3)"] + [[package]] name = "traitlets" version = "5.12.0" @@ -3148,4 +3629,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "3fad1a8073278577998125ec05cb516de5b4ed9973b4c267fe40d53854c1613b" +content-hash = "031b706da68ac6b9eacdb2c7febde1918019a91c79477dcff2fdd78d219d1fd5" diff --git a/server/portal/apps/_custom/drp/models.py b/server/portal/apps/_custom/drp/models.py index 999aa14349..095fe853c3 100644 --- a/server/portal/apps/_custom/drp/models.py +++ b/server/portal/apps/_custom/drp/models.py @@ -41,6 +41,7 @@ class DrpFileMetadata(DrpMetadataModel): offset_to_first_image: Optional[int] = None gap_between_images: Optional[int] = None byte_order: Optional[Literal['big_endian', 'little_endian']] = None + use_binary_correction: Optional[bool] = None class FileObj(DrpMetadataModel): """Model for associated files""" diff --git a/server/portal/apps/projects/tasks.py b/server/portal/apps/projects/tasks.py index f36703dc2b..79a7d6b3b0 100644 --- a/server/portal/apps/projects/tasks.py +++ b/server/portal/apps/projects/tasks.py @@ -1,19 +1,34 @@ +import os +import base64 import logging from pathlib import Path from django.conf import settings from celery import shared_task from django.db import transaction from portal.apps.publications.models import PublicationRequest -from portal.apps import SCHEMA_MAPPING from portal.libs.agave.utils import user_account, service_account from django.contrib.auth import get_user_model from django.core.exceptions import ObjectDoesNotExist from portal.apps.projects.models.project_metadata import ProjectMetadata from portal.apps._custom.drp import constants -from portal.apps.projects.workspace_operations.project_meta_operations import add_file_associations, create_file_obj, get_ordered_value +from portal.apps.projects.workspace_operations.project_meta_operations import ( + add_file_associations, + create_file_obj, + get_file_obj, + get_ordered_value + ) from portal.apps.projects.workspace_operations.graph_operations import get_path_uuid_mapping import networkx as nx -import uuid +from portal.apps._custom.drp.models import FileObj +from portal.libs.files.file_processing import ( + binary_correction, + conf_raw, + conf_tiff, + create_animation, + create_histogram, + create_thumbnail + ) +from portal.apps.notifications.models import Notification # TODO: Cleanup this file @@ -231,4 +246,91 @@ def sync_files_without_metadata(self, user_access_token, project_id: str): for entity_uuid, file_objs in files_to_add_dict.items(): logger.info(f'Adding {len(file_objs)} files to entity {entity_uuid} in project {project_id}') add_file_associations(entity_uuid, file_objs) - \ No newline at end of file + +@shared_task(bind=True, queue='default') +def process_file(self, project_id: str, path: str, user_access_token: str, encoded_file=None): + + client = user_account(user_access_token) + username = client.access_token.claims['tapis/username'] + + Notification.objects.create(**{ + Notification.EVENT_TYPE: 'default', + Notification.STATUS: Notification.INFO, + Notification.USER: username, + Notification.MESSAGE: f'Generating Images', + }) + + logger.info(f'Processing file {path} in project {project_id}') + + if encoded_file: + logger.info('Decoding file') + file = base64.b64decode(encoded_file) + else: + logger.info('Retrieving file using Tapis') + file = client.files.getContents(systemId=project_id, path=path) + + logger.info('File retrieved') + + parent_path = str(Path(path).parent) + + file_obj: FileObj = get_file_obj(project_id, path) + + if file and file_obj: + value = get_ordered_value(constants.FILE, file_obj.get('value')) + + file_name = file_obj.get('name') + + _, file_ext = os.path.splitext(file_obj.get('name')) + + if file_ext in ['.tif', '.tiff']: + adv_image = conf_tiff(file) + else: + adv_image = conf_raw(value, file) + + try: + if value.get('use_binary_correction'): + adv_image = binary_correction(adv_image) + except Exception as e: + logger.error(f'Error applying binary correction: {e}') + + try: + thumbnail = create_thumbnail(adv_image) + + thumbnail_path = f'{parent_path}/{file_name}.thumb.jpg' + + logger.info('Uploading generated thumbnail') + client.files.insert(systemId=project_id, path=thumbnail_path, file=thumbnail) + except Exception as e: + logger.error(f'Error generating thumbnail: {e}') + + try: + histogram_img, histogram_csv = create_histogram(adv_image) + + histogram_img_path = f'{parent_path}/{file_name}.histogram.jpg' + histogram_csv_path = f'{parent_path}/{file_name}.histogram.csv' + + logger.info('Uploading generated histogram') + client.files.insert(systemId=project_id, path=histogram_img_path, file=histogram_img) + client.files.insert(systemId=project_id, path=histogram_csv_path, file=histogram_csv) + except Exception as e: + logger.error(f'Error generating histogram: {e}') + + try: + animation = create_animation(adv_image) + + animation_path = f'{parent_path}/{file_name}.gif' + + logger.info('Uploading generated animation') + client.files.insert(systemId=project_id, path=animation_path, file=animation) + except Exception as e: + logger.error(f'Error generating animation: {e}') + + with transaction.atomic(): + Notification.objects.create(**{ + Notification.EVENT_TYPE: 'default', + Notification.STATUS: Notification.INFO, + Notification.USER: username, + Notification.MESSAGE: f'Image generation complete. Please refresh the page.', + }) + else: + print(f"File {path} does not exist in project {project_id}") diff --git a/server/portal/apps/projects/views.py b/server/portal/apps/projects/views.py index e5b6246c18..8350cdcd3a 100644 --- a/server/portal/apps/projects/views.py +++ b/server/portal/apps/projects/views.py @@ -33,7 +33,7 @@ from pathlib import Path from portal.apps._custom.drp import constants from portal.apps.projects.workspace_operations.graph_operations import add_node_to_project, initialize_project_graph, get_node_from_path -from portal.apps.projects.tasks import sync_files_without_metadata +from portal.apps.projects.tasks import process_file, sync_files_without_metadata LOGGER = logging.getLogger(__name__) @@ -398,6 +398,7 @@ def patch(self, request: HttpRequest, project_id: str): if value['data_type'] == 'file': try: patch_file_obj_entity(client, project_id, value, path) + process_file.delay(project_id, path.lstrip("/"), client.access_token.access_token) except Exception as exc: raise ApiException("Error updating file metadata", status=500) from exc else: diff --git a/server/portal/libs/agave/operations.py b/server/portal/libs/agave/operations.py index 761f63dbec..66129947fa 100644 --- a/server/portal/libs/agave/operations.py +++ b/server/portal/libs/agave/operations.py @@ -1,3 +1,4 @@ +import base64 import os import io from django.conf import settings @@ -11,6 +12,7 @@ from portal.libs.agave.filter_mapping import filter_mapping from pathlib import Path from portal.apps._custom.drp.models import FileObj +from portal.apps.projects.tasks import process_file from tapipy.errors import BaseTapyException from portal.apps.projects.models.metadata import ProjectsMetadata from portal.apps.datafiles.models import DataFilesMetadata @@ -558,6 +560,11 @@ def upload(client, system, path, uploaded_file, metadata=None): root_node = get_root_node(system) add_file_associations(root_node['uuid'], [file_obj]) + # additional processing for files + encoded_file = base64.b64encode(uploaded_file.read()).decode('utf-8') + uploaded_file.seek(0) + transaction.on_commit(lambda: process_file.delay(file_obj.system, file_obj.path, client.access_token.access_token, encoded_file)) + response_json = client.files.insert(systemId=system, path=dest_path, file=uploaded_file) tapis_indexer.apply_async(kwargs={'access_token': client.access_token.access_token, 'systemId': system, diff --git a/server/portal/libs/files/file_processing.py b/server/portal/libs/files/file_processing.py new file mode 100644 index 0000000000..c86e128ece --- /dev/null +++ b/server/portal/libs/files/file_processing.py @@ -0,0 +1,183 @@ +import numpy as np +import io +import logging +from matplotlib import pyplot as plt +import csv +import matplotlib.animation as anim +import tempfile +import tifffile as tiff + +logger = logging.getLogger(__name__) + +def conf_raw(img, file): + # NOTE: As mentioned above, this datatype should be an auto generated field + # in the adv image upload form for RAW files. The image parameters for + # slices, width, height should be converted to ints + # NOTE: If an 8-bit raw comes through, we need to set the datatype for that to unsigned. + + logger.info(f'img: {img}') + + prefix_map = {'little_endian': '<', 'big_endian': '>'} + suffix_map = { + '8_bit': 'u1', '16_bit_unsigned': 'u2', '32_bit_unsigned': 'u4', '64_bit_unsigned': 'u8', + '8_bit_signed': 'i1', '16_bit_signed': 'i2', '32_bit_signed': 'i4', '64_bit_signed': 'i8', + '32_bit_real': 'f4', '64_bit_real': 'f8' + } + + prefix = prefix_map.get(img['byte_order'], '|') # Default to native byte order if unknown + suffix = suffix_map.get(img['image_type']) + datatype = prefix + suffix + + file_data = np.frombuffer(file, dtype=datatype) + return file_data.reshape([ + int(img['number_of_images']), + int(img['height']), + int(img['width']) + ]) + +def conf_tiff(file): + with io.BytesIO(file) as buffer: + image = tiff.imread(buffer) + return image + +def binary_correction(img): + logger.debug('Correcting for Binary values...') + min_value = np.min(img) + max_value = np.max(img) + k=255/(max_value-min_value) + l=-k*min_value + + image1=np.floor(img * k + l) + del img + return image1.astype('uint8') + +def create_thumbnail(img): + dpi = 100 + dim_max = 5 + width = img.shape[1] + height = img.shape[0] + depth_slice = None + if len(img.shape) == 3 and 3 not in img.shape: + width = img.shape[2] + height = img.shape[1] + depth_slice = int(np.ceil(img.shape[0]/2)) + elif len(img.shape) == 3 and 3 in img.shape: + # TODO: Handle RGB files shape ==> (h, w, 3) + logger.debug('handle RGB') + + # preserve aspect ratio and resize to fit. + modifier = dim_max/width if width>height else dim_max/height + resized_width = width*modifier + resized_height = height*modifier + + fig = plt.figure() + fig.set_size_inches(resized_width, resized_height, dpi) + ax = plt.Axes(fig, [0., 0., 1., 1.]) + ax.set_axis_off() + fig.add_axes(ax) + # TODO: Swap color mapping if image is bitmap/8bit + # plt.set_cmap('gray') if img.invert_colors else plt.set_cmap('Greys') + plt.set_cmap('Greys') + if depth_slice: + logger.debug('Creating Thumbnail from 3D tif') + ax.imshow(img[depth_slice,:,:], aspect='equal', vmin=0, vmax=255) + else: + logger.debug('Creating Thumbnail from FLAT tif') + ax.imshow(img, aspect='equal') + + buffer = io.BytesIO() + plt.savefig(buffer, format='jpeg', dpi=dpi) + buffer.seek(0) + + plt.close(fig) + + return buffer.getvalue() + +def create_histogram(img): + logger.debug('Creating Histogram') + nbins=256 + fig_hist = plt.figure(figsize=(4,2.4)) + freq, bins, patches = plt.hist(img.reshape([np.size(img),]), nbins, density=True) + plt.xlabel('Gray value') + plt.ylabel('Probability') + plt.tight_layout() + + image_buffer = io.BytesIO() + fig_hist.savefig(image_buffer, format='jpeg', dpi=200) + image_buffer.seek(0) + plt.close(fig_hist) + + csv_buffer = io.StringIO() + histwriter = csv.writer(csv_buffer,delimiter=',') + histwriter.writerow(('Value','Probability')) + for i in range(np.size(freq)): + histwriter.writerow((bins[i],freq[i])) + csv_buffer.seek(0) + + logger.debug('Histogram Created') + + return image_buffer.getvalue(), csv_buffer.getvalue() + +def create_animation(img): + """ + Creates an animated GIF in memory using matplotlib and returns its binary data. + + Args: + img (ndarray): The input 3D image as a NumPy array. + + Returns: + bytes: Binary data of the animated GIF. + """ + if len(img.shape) < 3 or (len(img.shape) == 3 and 3 in img.shape): + logger.debug('Image is not a 3D array') + return # Exit if the image is not a 3D array + + logger.debug('Creating Animated Gif') + + class AnimatedGif: + def __init__(self): + self.fig = plt.figure() + self.images = [] + + def add(self, image, h, w, dpi=100): + self.fig.set_size_inches(w, h, dpi) + ax1 = plt.Axes(self.fig, [0., 0., 1., 1.]) + ax1.set_axis_off() + self.fig.add_axes(ax1) + plt.set_cmap('Greys') + plt_im = ax1.imshow(image, aspect='equal', vmin=0, vmax=255) + self.images.append([plt_im]) + + def save_to_tempfile(self): + """Saves the animation to a temporary file and returns its binary content.""" + with tempfile.NamedTemporaryFile(suffix=".gif", delete=True) as temp_file: + animation = anim.ArtistAnimation(self.fig, self.images) + animation.save(temp_file.name, writer='imagemagick', fps=6) + temp_file.seek(0) # Reset pointer to the beginning + return temp_file.read() # Read binary content + + # Resize the image while preserving aspect ratio + dim_max = 5 + width, height = img.shape[2], img.shape[1] + modifier = dim_max / width if width > height else dim_max / height + resized_width, resized_height = width * modifier, height * modifier + + # Create the animation + sl1 = img[0, :, :] + animated_gif = AnimatedGif() + animated_gif.add(sl1, h=resized_height, w=resized_width) + + if img.shape[0] < 100: + slicesave = 1 + else: + slicesave = round((img.shape[0] / 100) * 1) + + for i in range(1, img.shape[0], slicesave): + sl = img[i, :, :] + animated_gif.add(sl, h=resized_height, w=resized_width) + + # Save the animation to a temporary file and return its binary data + gif_binary_data = animated_gif.save_to_tempfile() + + logger.debug('Animated Gif Created') + return gif_binary_data \ No newline at end of file diff --git a/server/pyproject.toml b/server/pyproject.toml index b7163c5312..3944c5e4d3 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -41,6 +41,9 @@ gevent = "^23.9.1" pymemcache = "^4.0.0" pydantic = "^2.5.0" networkx = "^3.2.1" +numpy = "^2.2.1" +matplotlib = "^3.10.0" +tifffile = "^2025.1.10" [tool.poetry.group.dev.dependencies] pytest = "^7.3.1" From cd9f8449b9924ef469cae8af87b61a3bdb19e3a2 Mon Sep 17 00:00:00 2001 From: Jake Rosenberg Date: Tue, 18 Feb 2025 12:41:57 -0600 Subject: [PATCH 159/328] Serve published projects at a world-readable /publications URL (#1046) Co-authored-by: Shayan Khan --- .../CombinedBreadcrumbs.jsx | 1 + .../DataFilesDropdown/DataFilesDropdown.jsx | 3 +- .../DataFilesListing/DataFilesListing.jsx | 4 +- .../DataFilesListingCells.jsx | 3 +- .../DataFilesProjectFileListing.jsx | 3 +- .../DataFilesPublicationsList.jsx | 9 ++-- .../PublicationDetailPublicView.jsx | 43 +++++++++++++++++++ .../Publications/PublicationsPublicView.jsx | 14 ++++++ client/src/components/Workbench/AppRouter.jsx | 15 ++++++- .../DataFilesProjectFileListingAddon.jsx | 2 +- ...esProjectFileListingMetadataTitleAddon.jsx | 2 +- server/conf/nginx/nginx.conf | 2 +- server/portal/apps/datafiles/views.py | 6 ++- server/portal/apps/projects/views.py | 9 ++-- server/portal/urls.py | 1 + 15 files changed, 102 insertions(+), 15 deletions(-) create mode 100644 client/src/components/Publications/PublicationDetailPublicView.jsx create mode 100644 client/src/components/Publications/PublicationsPublicView.jsx diff --git a/client/src/components/DataFiles/CombinedBreadcrumbs/CombinedBreadcrumbs.jsx b/client/src/components/DataFiles/CombinedBreadcrumbs/CombinedBreadcrumbs.jsx index 7f5ce80c59..914773b185 100644 --- a/client/src/components/DataFiles/CombinedBreadcrumbs/CombinedBreadcrumbs.jsx +++ b/client/src/components/DataFiles/CombinedBreadcrumbs/CombinedBreadcrumbs.jsx @@ -20,6 +20,7 @@ CombinedBreadcrumbs.propTypes = { path: PropTypes.string.isRequired, section: PropTypes.string.isRequired, isPublic: PropTypes.bool, + basePath: PropTypes.string, className: PropTypes.string, }; diff --git a/client/src/components/DataFiles/DataFilesDropdown/DataFilesDropdown.jsx b/client/src/components/DataFiles/DataFilesDropdown/DataFilesDropdown.jsx index 33affeef4f..e91ebd3173 100644 --- a/client/src/components/DataFiles/DataFilesDropdown/DataFilesDropdown.jsx +++ b/client/src/components/DataFiles/DataFilesDropdown/DataFilesDropdown.jsx @@ -17,6 +17,7 @@ const BreadcrumbsDropdown = ({ scheme, system, path, + basePath, section, isPublic, }) => { @@ -36,7 +37,7 @@ const BreadcrumbsDropdown = ({ : null; const handleNavigation = (targetPath) => { - const basePath = isPublic ? '/public-data' : '/workbench/data'; + if (!basePath) basePath = isPublic ? '/public-data' : '/workbench/data'; let url; if (scheme === 'projects' && targetPath === systemName) { diff --git a/client/src/components/DataFiles/DataFilesListing/DataFilesListing.jsx b/client/src/components/DataFiles/DataFilesListing/DataFilesListing.jsx index 0552a10a30..98991cda11 100644 --- a/client/src/components/DataFiles/DataFilesListing/DataFilesListing.jsx +++ b/client/src/components/DataFiles/DataFilesListing/DataFilesListing.jsx @@ -45,6 +45,7 @@ const DataFilesListing = ({ path, isPublic, rootSystem, + basePath, }) => { // Redux hooks const location = useLocation(); @@ -54,7 +55,7 @@ const DataFilesListing = ({ ); const sharedWorkspaces = systems.find((e) => e.scheme === 'projects'); const isPortalProject = scheme === 'projects'; - const hideSearchBar = isPortalProject && sharedWorkspaces.hideSearchBar; + const hideSearchBar = isPortalProject && sharedWorkspaces?.hideSearchBar; const showViewPath = useSelector( (state) => @@ -101,6 +102,7 @@ const DataFilesListing = ({ scheme={scheme} href={row.original._links.self.href} isPublic={isPublic} + basePath={basePath} length={row.original.length} metadata={row.original.metadata} /> diff --git a/client/src/components/DataFiles/DataFilesListing/DataFilesListingCells.jsx b/client/src/components/DataFiles/DataFilesListing/DataFilesListingCells.jsx index c3d3593b7d..7899e3f8ff 100644 --- a/client/src/components/DataFiles/DataFilesListing/DataFilesListingCells.jsx +++ b/client/src/components/DataFiles/DataFilesListing/DataFilesListingCells.jsx @@ -59,6 +59,7 @@ export const FileNavCell = React.memo( scheme, href, isPublic, + basePath, length, metadata, rootSystem, @@ -80,7 +81,7 @@ export const FileNavCell = React.memo( }); }; - const basePath = isPublic ? '/public-data' : '/workbench/data'; + if (!basePath) basePath = isPublic ? '/public-data' : '/workbench/data'; return ( <> diff --git a/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx b/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx index cde79cdeee..95698a10a4 100644 --- a/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx +++ b/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx @@ -54,7 +54,7 @@ const DataFilesProjectFileListing = ({ rootSystem, system, path }) => { metadata.members .filter((member) => member.user - ? member.user.username === state.authenticatedUser.user.username + ? member.user.username === state.authenticatedUser?.user?.username : { access: null } ) .map((currentUser) => currentUser.access === 'owner')[0] @@ -172,6 +172,7 @@ const DataFilesProjectFileListing = ({ rootSystem, system, path }) => { scheme="projects" system={system} path={path || '/'} + basePath="/publications" rootSystem={rootSystem} /> diff --git a/client/src/components/DataFiles/DataFilesPublicationsList/DataFilesPublicationsList.jsx b/client/src/components/DataFiles/DataFilesPublicationsList/DataFilesPublicationsList.jsx index e17c81016a..85558fcd3c 100644 --- a/client/src/components/DataFiles/DataFilesPublicationsList/DataFilesPublicationsList.jsx +++ b/client/src/components/DataFiles/DataFilesPublicationsList/DataFilesPublicationsList.jsx @@ -14,11 +14,13 @@ import './DataFilesPublicationsList.scss'; import Searchbar from '_common/Searchbar'; import { formatDate, formatDateTimeFromValue } from 'utils/timeFormat'; -const DataFilesPublicationsList = ({ rootSystem }) => { +const DataFilesPublicationsList = ({ rootSystem, basePath }) => { const { error, loading, publications } = useSelector( (state) => state.publications.listing ); + const _basePath = basePath ?? '/workbench/data'; + const query = queryStringParser.parse(useLocation().search); const systems = useSelector( @@ -38,7 +40,7 @@ const DataFilesPublicationsList = ({ rootSystem }) => { type: 'PUBLICATIONS_GET_PUBLICATIONS', payload: { queryString: query.query_string, - system: selectedSystem.system, + system: selectedSystem?.system, }, }); }, [dispatch, query.query_string]); @@ -60,7 +62,7 @@ const DataFilesPublicationsList = ({ rootSystem }) => { Cell: (el) => ( {el.value} @@ -141,6 +143,7 @@ const DataFilesPublicationsList = ({ rootSystem }) => { isLoading={loading} noDataText={noDataText} className="publications-listing" + columnMemoProps={[selectedSystem]} /> diff --git a/client/src/components/Publications/PublicationDetailPublicView.jsx b/client/src/components/Publications/PublicationDetailPublicView.jsx new file mode 100644 index 0000000000..8db841f096 --- /dev/null +++ b/client/src/components/Publications/PublicationDetailPublicView.jsx @@ -0,0 +1,43 @@ +import React from 'react'; +import CombinedBreadcrumbs from '../DataFiles/CombinedBreadcrumbs/CombinedBreadcrumbs'; +import DataFilesPreviewModal from '../DataFiles/DataFilesModals/DataFilesPreviewModal'; +import DataFilesProjectCitationModal from '../DataFiles/DataFilesModals/DataFilesProjectCitationModal'; +import DataFilesProjectTreeModal from '../DataFiles/DataFilesModals/DataFilesProjectTreeModal'; +import DataFilesPublicationAuthorsModal from '../DataFiles/DataFilesModals/DataFilesPublicationAuthorsModal'; +import DataFilesShowPathModal from '../DataFiles/DataFilesModals/DataFilesShowPathModal'; +import DataFilesViewDataModal from '../DataFiles/DataFilesModals/DataFilesViewDataModal'; +import DataFilesProjectFileListing from '../DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing'; + +function PublicationDetailPublicView({params}) { + return ( +
+ + + + + + + + +
+ ); +} + +export default PublicationDetailPublicView; diff --git a/client/src/components/Publications/PublicationsPublicView.jsx b/client/src/components/Publications/PublicationsPublicView.jsx new file mode 100644 index 0000000000..609ea73cf5 --- /dev/null +++ b/client/src/components/Publications/PublicationsPublicView.jsx @@ -0,0 +1,14 @@ +import React from 'react'; +import DataFilesPublicationsList from '../DataFiles/DataFilesPublicationsList/DataFilesPublicationsList'; +import DataFilesProjectDescriptionModal from '../DataFiles/DataFilesModals/DataFilesProjectDescriptionModal'; + +function PublicationsPublicView() { + return ( +
+ + +
+ ); +} + +export default PublicationsPublicView \ No newline at end of file diff --git a/client/src/components/Workbench/AppRouter.jsx b/client/src/components/Workbench/AppRouter.jsx index 9459696079..66a8741bf5 100644 --- a/client/src/components/Workbench/AppRouter.jsx +++ b/client/src/components/Workbench/AppRouter.jsx @@ -1,7 +1,7 @@ import React, { useEffect } from 'react'; import { useDispatch, useSelector } from 'react-redux'; import { useSystems } from 'hooks/datafiles'; -import { BrowserRouter as Router, Route } from 'react-router-dom'; +import { BrowserRouter as Router, Route, Redirect } from 'react-router-dom'; import Workbench from './Workbench'; import * as ROUTES from '../../constants/routes'; import TicketStandaloneCreate from '../Tickets/TicketStandaloneCreate'; @@ -9,6 +9,9 @@ import PublicData from '../PublicData/PublicData'; import RequestAccess from '../RequestAccess/RequestAccess'; import GoogleDrivePrivacyPolicy from '../ManageAccount/GoogleDrivePrivacyPolicy'; import SiteSearch from '../SiteSearch'; +import PublicationsPublicView from '../Publications/PublicationsPublicView'; +import PublicationDetailPublicView from '../Publications/PublicationDetailPublicView'; + function AppRouter() { const dispatch = useDispatch(); @@ -45,6 +48,16 @@ function AppRouter() { + + + + + { + return ; + }} + /> { const { canEditDataset, canRequestPublication, canReviewPublication } = useSelector((state) => { const { members } = state.projects.metadata; - const { username } = state.authenticatedUser.user; + const { username } = state.authenticatedUser?.user ?? {}; const currentUser = members.find( (member) => member.user?.username === username ); diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx index a1acd41c65..24b01ded05 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataTitleAddon/DataFilesProjectFileListingMetadataTitleAddon.jsx @@ -23,7 +23,7 @@ const DataFilesProjectFileListingMetadataTitleAddon = ({ const userAccess = state.projects.metadata.members .filter((member) => member.user - ? member.user.username === state.authenticatedUser.user.username + ? member.user.username === state.authenticatedUser?.user?.username : { access: null } ) .map((currentUser) => { diff --git a/server/conf/nginx/nginx.conf b/server/conf/nginx/nginx.conf index 2ca304cb1b..23631de4b4 100644 --- a/server/conf/nginx/nginx.conf +++ b/server/conf/nginx/nginx.conf @@ -80,7 +80,7 @@ http { alias /srv/www/portal/server/docs; } - location ~ ^/(core|auth|workbench|tickets|googledrive-privacy-policy|public-data|request-access|accounts|api|login|webhooks|search) { + location ~ ^/(core|auth|workbench|tickets|googledrive-privacy-policy|public-data|publications|request-access|accounts|api|login|webhooks|search) { proxy_pass http://portal_core; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Host $host; diff --git a/server/portal/apps/datafiles/views.py b/server/portal/apps/datafiles/views.py index c3630ded17..76f8a6a7ee 100644 --- a/server/portal/apps/datafiles/views.py +++ b/server/portal/apps/datafiles/views.py @@ -55,7 +55,7 @@ def get(self, request): response['default_host'] = system_def.host response['default_system'] = system_id else: - response['system_list'] = [sys for sys in portal_systems if sys['scheme'] == 'public'] + response['system_list'] = [sys for sys in portal_systems if sys['scheme'] == 'public' or sys['system'] == settings.PORTAL_PROJECTS_PUBLISHED_ROOT_SYSTEM_NAME] return JsonResponse(response) @@ -71,6 +71,8 @@ def get(self, request, systemId): public_sys = next((sys for sys in settings.PORTAL_DATAFILES_STORAGE_SYSTEMS if sys['scheme'] == 'public'), None) if public_sys and public_sys['system'] == systemId: client = service_account() + if systemId.startswith(settings.PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX): + client = service_account() else: return JsonResponse({'message': 'Unauthorized'}, status=401) system_def = client.systems.getSystem(systemId=systemId) @@ -92,6 +94,8 @@ def get(self, request, operation=None, scheme=None, system=None, path='/'): public_sys = next((sys for sys in settings.PORTAL_DATAFILES_STORAGE_SYSTEMS if sys['scheme'] == 'public'), None) if public_sys and public_sys['system'] == system and path.startswith(public_sys['homeDir'].strip('/')): client = service_account() + if system and system.startswith(settings.PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX): + client = service_account() else: return JsonResponse( {'message': 'This data requires authentication to view.'}, diff --git a/server/portal/apps/projects/views.py b/server/portal/apps/projects/views.py index 8350cdcd3a..18bd9a1551 100644 --- a/server/portal/apps/projects/views.py +++ b/server/portal/apps/projects/views.py @@ -10,6 +10,7 @@ from django.conf import settings from django.http import JsonResponse from django.utils.decorators import method_decorator +from portal.libs.agave.utils import service_account from portal.utils.decorators import agave_jwt_login from portal.exceptions.api import ApiException from portal.views.base import BaseApiView @@ -157,7 +158,6 @@ def post(self, request): # pylint: disable=no-self-use @method_decorator(agave_jwt_login, name='dispatch') -@method_decorator(login_required, name='dispatch') class ProjectInstanceApiView(BaseApiView): """Project Instance API view. @@ -178,8 +178,11 @@ def get(self, request, project_id=None, system_id=None): # Based on url mapping, either system_id or project_id is always available. if system_id is not None: project_id = system_id.split(f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.")[1] - - client = request.user.tapis_oauth.client + + if system_id and system_id.startswith(settings.PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX): + client = service_account() + else: + client = request.user.tapis_oauth.client prj = get_project(client, project_id) diff --git a/server/portal/urls.py b/server/portal/urls.py index f702a8c12d..fa8294bae8 100644 --- a/server/portal/urls.py +++ b/server/portal/urls.py @@ -103,6 +103,7 @@ namespace='googledrive-privacy-policy')), path('workbench/', include('portal.apps.workbench.urls', namespace='workbench')), path('public-data/', include('portal.apps.public_data.urls', namespace='public')), + path('publications/', include('portal.apps.public_data.urls', namespace='publications')), path('request-access/', include('portal.apps.request_access.urls', namespace='request_access')), path('search/', include('portal.apps.site_search.urls', namespace='site_search')), From bf07c6ad91d4ab555ad80f344deca57a3ae31352 Mon Sep 17 00:00:00 2001 From: Jake Rosenberg Date: Tue, 18 Feb 2025 13:49:55 -0600 Subject: [PATCH 160/328] Task/WC-120: Datacite operations and pipeline integration (#1037) * Rebase datacite operations on latest DRP branch * integrate datacite utils in pipeline --------- Co-authored-by: Shayan Khan --- .../datacite_operations.py | 130 ++++++++++++++++++ .../project_publish_operations.py | 11 +- server/portal/settings/settings.py | 15 ++ server/portal/settings/settings_default.py | 5 + .../settings/settings_secret.example.py | 3 + 5 files changed, 163 insertions(+), 1 deletion(-) create mode 100644 server/portal/apps/projects/workspace_operations/datacite_operations.py diff --git a/server/portal/apps/projects/workspace_operations/datacite_operations.py b/server/portal/apps/projects/workspace_operations/datacite_operations.py new file mode 100644 index 0000000000..218e4b11cd --- /dev/null +++ b/server/portal/apps/projects/workspace_operations/datacite_operations.py @@ -0,0 +1,130 @@ +import datetime +from typing import Optional +import json +import requests +import networkx as nx +from django.conf import settings + + +def get_datacite_json(pub_graph: nx.DiGraph): + """ + Generate datacite payload for a publishable entity. `pub_graph` is the output of + either `get_publication_subtree` or `get_publication_full_tree`. + """ + + datacite_json = {} + + base_meta_node = "NODE_ROOT" + + base_meta = pub_graph.nodes[base_meta_node]["value"] + + author_attr = [] + institutions = [] + for author in base_meta.get("authors", []): + author_attr.append( + { + "nameType": "Personal", + "givenName": author.get("first_name", ""), + "familyName": author.get("last_name", ""), + } + ) + institutions.append(author.get("inst", "")) + + datacite_json["contributors"] = [ + { + "contributorType": "HostingInstitution", + "nameType": "Organizational", + "name": institution, + } + for institution in list(set(institutions)) + ] + datacite_json["creators"] = author_attr + datacite_json["titles"] = [{"title": base_meta["title"]}] + + datacite_json["publisher"] = "Digital Rocks Portal" + + datacite_json["publicationYear"] = datetime.datetime.now().year + + project_id = base_meta["projectId"] + datacite_url = f"{settings.PORTAL_PUBLICATION_DATACITE_URL_PREFIX}/{project_id}" + + datacite_json["url"] = datacite_url + datacite_json["prefix"] = settings.PORTAL_PUBLICATION_DATACITE_SHOULDER + + return datacite_json + + +def upsert_datacite_json(datacite_json: dict, doi: Optional[str] = None): + """ + Create a draft DOI in datacite with the specified metadata. If a DOI is specified, + the metadata for that DOI is updated instead. + """ + if doi: + datacite_json.pop("publicationYear", None) + + datacite_payload = { + "data": { + "type": "dois", + "relationships": { + "client": {"data": {"type": "clients", "id": "tdl.tacc"}} + }, + "attributes": datacite_json, + } + } + if not doi: + res = requests.post( + f"{settings.DATACITE_URL.strip('/')}/dois", + auth=(settings.DATACITE_USER, settings.DATACITE_PASS), + data=json.dumps(datacite_payload), + headers={"Content-Type": "application/vnd.api+json"}, + timeout=30, + ) + else: + res = requests.put( + f"{settings.DATACITE_URL.strip('/')}/dois/{doi}", + auth=(settings.DATACITE_USER, settings.DATACITE_PASS), + data=json.dumps(datacite_payload), + headers={"Content-Type": "application/vnd.api+json"}, + timeout=30, + ) + + return res.json() + + +def publish_datacite_doi(doi: str): + """ + Set a DOI's status to `Findable` in Datacite. + """ + payload = {"data": {"type": "dois", "attributes": {"event": "publish"}}} + + res = requests.put( + f"{settings.DATACITE_URL.strip('/')}/dois/{doi}", + auth=(settings.DATACITE_USER, settings.DATACITE_PASS), + data=json.dumps(payload), + headers={"Content-Type": "application/vnd.api+json"}, + timeout=30, + ) + return res.json() + + +def hide_datacite_doi(doi: str): + """ + Remove a Datacite DOI from public consumption. + """ + payload = {"data": {"type": "dois", "attributes": {"event": "hide"}}} + + res = requests.put( + f"{settings.DATACITE_URL.strip('/')}/dois/{doi}", + auth=(settings.DATACITE_USER, settings.DATACITE_PASS), + data=json.dumps(payload), + headers={"Content-Type": "application/vnd.api+json"}, + timeout=30, + ) + return res.json() + + +def get_doi_publication_date(doi: str) -> str: + """Look up the publication date for a DOI""" + res = requests.get(f"{settings.DATACITE_URL.strip('/')}/dois/{doi}", timeout=30) + res.raise_for_status() + return res.json()["data"]["attributes"]["created"] diff --git a/server/portal/apps/projects/workspace_operations/project_publish_operations.py b/server/portal/apps/projects/workspace_operations/project_publish_operations.py index 1cd37ac710..b25d6dfd6a 100644 --- a/server/portal/apps/projects/workspace_operations/project_publish_operations.py +++ b/server/portal/apps/projects/workspace_operations/project_publish_operations.py @@ -8,6 +8,7 @@ from portal.apps._custom.drp import constants from portal.libs.agave.utils import user_account, service_account from portal.apps.publications.models import Publication, PublicationRequest +from portal.apps.projects.workspace_operations.datacite_operations import get_datacite_json, upsert_datacite_json, publish_datacite_doi from django.db import transaction from portal.apps.projects.workspace_operations.graph_operations import remove_trash_nodes from portal.apps.search.tasks import index_publication @@ -118,7 +119,12 @@ def publish_project(self, project_id: str, version: Optional[int] = 1): value=nx.node_link_data(publication_tree), ) - doi = 'test_doi' # Replace with actual DOI retrieval logic + # Mint a DataCite DOI + existing_doi = source_project.value.get("doi", None) + + datacite_json = get_datacite_json(publication_tree) + datacite_resp = upsert_datacite_json(datacite_json, doi=existing_doi) + doi = datacite_resp["data"]["id"] # Update project metadata with datacite doi source_project_id = f'{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}' @@ -140,6 +146,9 @@ def publish_project(self, project_id: str, version: Optional[int] = 1): defaults={"value": published_project.value, "tree": nx.node_link_data(pub_tree), "version": version}, ) + if not settings.DEBUG: + publish_datacite_doi(doi) + index_publication(project_id) # transfer files diff --git a/server/portal/settings/settings.py b/server/portal/settings/settings.py index 8daa52252c..edab9df086 100644 --- a/server/portal/settings/settings.py +++ b/server/portal/settings/settings.py @@ -592,6 +592,21 @@ PORTAL_PUBLICATION_REVIEWERS_GROUP_NAME = settings_custom.\ _PORTAL_PUBLICATION_REVIEWERS_GROUP_NAME +PORTAL_PUBLICATION_DATACITE_SHOULDER = settings_custom.\ + _PORTAL_PUBLICATION_DATACITE_SHOULDER + +PORTAL_PUBLICATION_DATACITE_URL_PREFIX = settings_custom.\ + _PORTAL_PUBLICATION_DATACITE_URL_PREFIX + +DATACITE_URL = settings_custom.\ + _DATACITE_URL + +DATACITE_USER = settings_secret.\ + _DATACITE_USER + +DATACITE_PASS = settings_secret.\ + _DATACITE_PASS + PORTAL_PROJECTS_PRIVATE_KEY = settings_secret.\ _PORTAL_PROJECTS_PRIVATE_KEY diff --git a/server/portal/settings/settings_default.py b/server/portal/settings/settings_default.py index 6ba54927fc..977988dcce 100644 --- a/server/portal/settings/settings_default.py +++ b/server/portal/settings/settings_default.py @@ -223,6 +223,11 @@ _PORTAL_PUBLICATION_REVIEWERS_GROUP_NAME = 'PROJECT_REVIEWER' +# Datacite +_PORTAL_PUBLICATION_DATACITE_SHOULDER = "10.80023" +_PORTAL_PUBLICATION_DATACITE_URL_PREFIX = "https://cep.test/data/tapis/projects/drp.project.published.test" +_DATACITE_URL = "https://api.test.datacite.org/" + ######################## # Custom Portal Template Assets # Asset path root is static files output dir. diff --git a/server/portal/settings/settings_secret.example.py b/server/portal/settings/settings_secret.example.py index 5c3d20d927..5260edc1bb 100644 --- a/server/portal/settings/settings_secret.example.py +++ b/server/portal/settings/settings_secret.example.py @@ -116,3 +116,6 @@ "directory": "external-resources" } } + +_DATACITE_USER = "tdl.tacc" +_DATACITE_PASS = "CHANGEME" From 1a7a491b590ff34201d673387a6bc38ab357dca8 Mon Sep 17 00:00:00 2001 From: Jake Rosenberg Date: Tue, 18 Feb 2025 15:04:54 -0600 Subject: [PATCH 161/328] Fix workbench publications linking to public area (#1060) --- .../DataFilesProjectFileListing.jsx | 10 ++++++++-- .../Publications/PublicationDetailPublicView.jsx | 3 ++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx b/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx index 95698a10a4..cad7f104d6 100644 --- a/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx +++ b/client/src/components/DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing.jsx @@ -16,10 +16,16 @@ import { import DataFilesListing from '../DataFilesListing/DataFilesListing'; import styles from './DataFilesProjectFileListing.module.scss'; -const DataFilesProjectFileListing = ({ rootSystem, system, path }) => { +const DataFilesProjectFileListing = ({ + rootSystem, + system, + path, + basePath, +}) => { const dispatch = useDispatch(); const { fetchListing } = useFileListing('FilesListing'); const { isPublicationSystem, isReviewSystem } = useSystems(); + if (!basePath) basePath = '/workbench/data'; // logic to render addonComponents for DRP const portalName = useSelector((state) => state.workbench.portalName); @@ -172,7 +178,7 @@ const DataFilesProjectFileListing = ({ rootSystem, system, path }) => { scheme="projects" system={system} path={path || '/'} - basePath="/publications" + basePath={basePath} rootSystem={rootSystem} /> diff --git a/client/src/components/Publications/PublicationDetailPublicView.jsx b/client/src/components/Publications/PublicationDetailPublicView.jsx index 8db841f096..c7eb1f1672 100644 --- a/client/src/components/Publications/PublicationDetailPublicView.jsx +++ b/client/src/components/Publications/PublicationDetailPublicView.jsx @@ -8,7 +8,7 @@ import DataFilesShowPathModal from '../DataFiles/DataFilesModals/DataFilesShowPa import DataFilesViewDataModal from '../DataFiles/DataFilesModals/DataFilesViewDataModal'; import DataFilesProjectFileListing from '../DataFiles/DataFilesProjectFileListing/DataFilesProjectFileListing'; -function PublicationDetailPublicView({params}) { +function PublicationDetailPublicView({ params }) { return (
From 7add4a0757bc475e0719b25bc8cb5f3947e1f5d6 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Fri, 21 Feb 2025 16:02:55 -0600 Subject: [PATCH 162/328] fix publish bug --- .../workspace_operations/project_publish_operations.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/portal/apps/projects/workspace_operations/project_publish_operations.py b/server/portal/apps/projects/workspace_operations/project_publish_operations.py index b25d6dfd6a..a28768504f 100644 --- a/server/portal/apps/projects/workspace_operations/project_publish_operations.py +++ b/server/portal/apps/projects/workspace_operations/project_publish_operations.py @@ -119,6 +119,9 @@ def publish_project(self, project_id: str, version: Optional[int] = 1): value=nx.node_link_data(publication_tree), ) + source_project_id = f'{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}' + source_project = ProjectMetadata.get_project_by_id(source_project_id) + # Mint a DataCite DOI existing_doi = source_project.value.get("doi", None) @@ -127,8 +130,6 @@ def publish_project(self, project_id: str, version: Optional[int] = 1): doi = datacite_resp["data"]["id"] # Update project metadata with datacite doi - source_project_id = f'{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}' - source_project = ProjectMetadata.get_project_by_id(source_project_id) source_project.value['doi'] = doi source_project.value['publicationDate'] = published_project.created source_project.save() From 056b1847e09f65d0643b43b94369218d1331f954 Mon Sep 17 00:00:00 2001 From: Shayan Khan Date: Mon, 24 Feb 2025 10:18:39 -0600 Subject: [PATCH 163/328] task/WC-21 - Digital Rocks Cover Image Upload (#1063) * Added cover image upload functionality - User can choose a cover image to upload when creating a project - Added file processing function for cover image - Updated project post and patch requests to accept multipart/form-data - File is served to user as a tapis postit * Added cover image support for review and publication systems * update cover image listing logic * set cover image postits to expire after 24 hours --- .../_common/Form/DynamicForm/DynamicForm.jsx | 1 + ...aFilesProjectEditDescriptionModalAddon.jsx | 3 + ...taFilesProjectFileListingMetadataAddon.jsx | 9 ++- .../ProjectDescription.jsx | 8 ++ .../drp/utils/DataDisplay/DataDisplay.jsx | 19 ++++- client/src/redux/sagas/projects.sagas.js | 49 ++++++++++-- server/portal/apps/_custom/drp/models.py | 1 + server/portal/apps/projects/views.py | 76 ++++++++++++++++--- .../project_publish_operations.py | 27 +++++++ server/portal/libs/files/file_processing.py | 39 +++++++++- 10 files changed, 208 insertions(+), 24 deletions(-) diff --git a/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx b/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx index f7f74f177e..16f0766f77 100644 --- a/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx +++ b/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx @@ -300,6 +300,7 @@ const DynamicForm = ({ initialFormFields, onChange }) => { name={field.name} label={field.label} type="file" + accept={field?.validation?.accept} description={field?.description} required={field?.validation?.required} onChange={(event) => { diff --git a/client/src/components/_custom/drp/DataFilesProjectEditDescriptionModalAddon/DataFilesProjectEditDescriptionModalAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectEditDescriptionModalAddon/DataFilesProjectEditDescriptionModalAddon.jsx index 0e9e897446..3865c141d5 100644 --- a/client/src/components/_custom/drp/DataFilesProjectEditDescriptionModalAddon/DataFilesProjectEditDescriptionModalAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectEditDescriptionModalAddon/DataFilesProjectEditDescriptionModalAddon.jsx @@ -35,6 +35,9 @@ const DataFilesProjectEditDescriptionModalAddon = ({ setValidationSchema }) => { }); }); } else { + if (field.type === 'file') { + return; + } setFieldValue(field.name, metadata[field.name]); } } diff --git a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx index e64b4318db..c9a6653424 100644 --- a/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectFileListingMetadataAddon/DataFilesProjectFileListingMetadataAddon.jsx @@ -15,6 +15,8 @@ const excludeKeys = [ 'sample', 'digital_dataset', 'file_objs', + 'cover_image', + 'cover_image_url', ]; const DataFilesProjectFileListingMetadataAddon = ({ @@ -33,6 +35,8 @@ const DataFilesProjectFileListingMetadataAddon = ({ license, doi, keywords, + cover_image, + cover_image_url, }) => { const dateOptions = { month: 'long', day: 'numeric', year: 'numeric' }; const dateLabel = publication_date ? 'Publication Date' : 'Created'; @@ -45,6 +49,8 @@ const DataFilesProjectFileListingMetadataAddon = ({ license: license ?? 'None', ...(doi && { doi }), ...(keywords && { keywords }), + ...(cover_image && { cover_image }), + ...(cover_image_url && { cover_image_url }), }; }; @@ -105,8 +111,9 @@ const DataFilesProjectFileListingMetadataAddon = ({ ))} diff --git a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx index 99c7a593bb..da3ce08d35 100644 --- a/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx +++ b/client/src/components/_custom/drp/DataFilesProjectPublish/DataFilesProjectPublishWizardSteps/ProjectDescription.jsx @@ -44,6 +44,14 @@ const ProjectDescription = ({ project }) => { License: project.license ?? 'None', }; + if (project.cover_image) { + projectData['Cover Image'] = ( + + {project.cover_image.split('/').pop()} + + ); + } + if (project.keywords) { projectData['Keywords'] = project.keywords; } diff --git a/client/src/components/_custom/drp/utils/DataDisplay/DataDisplay.jsx b/client/src/components/_custom/drp/utils/DataDisplay/DataDisplay.jsx index 8059132262..2e6205fbfc 100644 --- a/client/src/components/_custom/drp/utils/DataDisplay/DataDisplay.jsx +++ b/client/src/components/_custom/drp/utils/DataDisplay/DataDisplay.jsx @@ -3,7 +3,6 @@ import PropTypes from 'prop-types'; import { Section, SectionContent, LoadingSpinner, Button } from '_common'; import { useLocation, Link } from 'react-router-dom'; import styles from './DataDisplay.module.scss'; -import { useFileListing } from 'hooks/datafiles'; import { useDispatch } from 'react-redux'; // Function to format the dict key from snake_case to Label Case i.e. data_type -> Data Type @@ -80,9 +79,17 @@ const processModalViewableData = (data) => { })); }; -const DataDisplay = ({ data, path, excludeKeys, modalData }) => { - const location = useLocation(); - +const processCoverImage = (data) => { + return [{ + label: 'Cover Image', + value: + + {data.cover_image.split('/').pop()} + + }] +} + +const DataDisplay = ({ data, path, excludeKeys, modalData, coverImage }) => { //filter out empty values and unwanted keys let processedData = Object.entries(data) .filter(([key, value]) => value !== '' && !excludeKeys.includes(key)) @@ -91,6 +98,10 @@ const DataDisplay = ({ data, path, excludeKeys, modalData }) => { value: typeof value === 'string' ? formatLabel(value) : value, })); + if (coverImage) { + processedData.unshift(...processCoverImage(data)); + } + if (path) { processedData.unshift(...processSampleAndOriginData(data, path)); } diff --git a/client/src/redux/sagas/projects.sagas.js b/client/src/redux/sagas/projects.sagas.js index ae4edc6daf..2cb307dbeb 100644 --- a/client/src/redux/sagas/projects.sagas.js +++ b/client/src/redux/sagas/projects.sagas.js @@ -1,6 +1,7 @@ import { put, takeLatest, call } from 'redux-saga/effects'; import queryStringParser from 'query-string'; import { fetchUtil } from 'utils/fetchUtil'; +import Cookies from 'js-cookie'; export async function fetchProjectsListing(queryString, rootSystem) { const q = queryStringParser.stringify({ query_string: queryString }); @@ -63,14 +64,30 @@ export function* showSharedWorkspaces(action) { } export async function fetchCreateProject(project) { + + const formData = new FormData(); + + const { file, ...projectMetadata } = project.metadata; // Exclude the file + formData.append('metadata', JSON.stringify(projectMetadata)); + + if (file) { + formData.append('cover_image', file); + } + + Object.entries(project) + .filter(([key, value]) => value != null && key !== 'metadata') + .forEach(([key, value]) => { + formData.append(key, value); + }); + const result = await fetchUtil({ url: `/api/projects/`, method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify(project), + headers: { 'X-CSRFToken': Cookies.get('csrftoken') }, + credentials: 'same-origin', + body: formData, }); + return result.response; } @@ -166,14 +183,30 @@ export function* setMember(action) { } export async function setTitleDescriptionUtil(projectId, data) { + + const formData = new FormData(); + + const { file, ...projectMetadata } = data.metadata; // Exclude the file + formData.append('metadata', JSON.stringify(projectMetadata)); + + if (file) { + formData.append('cover_image', file); + } + + Object.entries(data) + .filter(([key, value]) => value != null && key !== 'metadata') + .forEach(([key, value]) => { + formData.append(key, value); + }); + const result = await fetchUtil({ url: `/api/projects/${projectId}/`, method: 'PATCH', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify(data), + headers: { 'X-CSRFToken': Cookies.get('csrftoken') }, + credentials: 'same-origin', + body: formData, }); + return result.response; } diff --git a/server/portal/apps/_custom/drp/models.py b/server/portal/apps/_custom/drp/models.py index 095fe853c3..10a7efd608 100644 --- a/server/portal/apps/_custom/drp/models.py +++ b/server/portal/apps/_custom/drp/models.py @@ -135,6 +135,7 @@ class DrpProjectMetadata(DrpMetadataModel): is_review_project: Optional[bool] = None is_published_project: Optional[bool] = None guest_users: list[DrpGuestUser] = [] + cover_image: Optional[str] = None class DrpDatasetMetadata(DrpMetadataModel): """Model for Base DRP Dataset Metadata""" diff --git a/server/portal/apps/projects/views.py b/server/portal/apps/projects/views.py index 18bd9a1551..a17e843da4 100644 --- a/server/portal/apps/projects/views.py +++ b/server/portal/apps/projects/views.py @@ -35,6 +35,9 @@ from portal.apps._custom.drp import constants from portal.apps.projects.workspace_operations.graph_operations import add_node_to_project, initialize_project_graph, get_node_from_path from portal.apps.projects.tasks import process_file, sync_files_without_metadata +from portal.libs.files.file_processing import resize_cover_image +from portal.libs.agave.utils import service_account +from django.http.multipartparser import MultiPartParser LOGGER = logging.getLogger(__name__) @@ -133,22 +136,35 @@ def get(self, request, root_system=None): @transaction.atomic def post(self, request): # pylint: disable=no-self-use """POST handler.""" - data = json.loads(request.body) - title = data['title'] - description = data['description'] - metadata = data['metadata'] + title = request.POST.get('title') + description = request.POST.get('description') + metadata = request.POST.get('metadata') + cover_image = request.FILES.get('cover_image') workspace_number = increment_workspace_count() - workspace_id = f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{settings.PORTAL_PROJECTS_ID_PREFIX}-{workspace_number}" + system_id = f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{settings.PORTAL_PROJECTS_ID_PREFIX}-{workspace_number}" if metadata is not None: - metadata["projectId"] = workspace_id + metadata = json.loads(metadata) + + if cover_image: + metadata['cover_image'] = f'media/{settings.PORTAL_PROJECTS_ID_PREFIX}-{workspace_number}/cover_image/{cover_image.name}' + + metadata["projectId"] = system_id project_meta = create_project_metadata(metadata) initialize_project_graph(project_meta.project_id) client = request.user.tapis_oauth.client system_id = create_shared_workspace(client, title, request.user.username, description, workspace_number) + # Upload cover image to media folder + if cover_image: + service_client = service_account() + resized_file = resize_cover_image(cover_image) + service_client.files.insert(systemId=settings.PORTAL_PROJECTS_ROOT_SYSTEM_NAME, + path=f'media/{settings.PORTAL_PROJECTS_ID_PREFIX}-{workspace_number}/cover_image/{cover_image.name}', + file=resized_file) + return JsonResponse( { 'status': 200, @@ -191,6 +207,20 @@ def get(self, request, project_id=None, system_id=None): prj.update(get_ordered_value(project.name, project.value)) prj["projectId"] = project_id + if prj["cover_image"] is not None: + service_client = service_account() + + if prj["is_published_project"]: + root_system = settings.PORTAL_PROJECTS_PUBLISHED_ROOT_SYSTEM_NAME + elif prj["is_review_project"]: + root_system = settings.PORTAL_PROJECTS_REVIEW_ROOT_SYSTEM_NAME + else: + root_system = settings.PORTAL_PROJECTS_ROOT_SYSTEM_NAME + + postit = service_client.files.createPostIt(systemId=root_system, path=prj['cover_image'], allowedUses=-1, + validSeconds=86400) + prj["cover_image_url"] = postit.redeemUrl + if not getattr(prj, 'is_review_project', False) and not getattr(prj, 'is_published_project', False): sync_files_without_metadata.delay(client.access_token.access_token, f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}") except: @@ -236,17 +266,43 @@ def patch( :param request: Request object :param str project_id: Project Id. """ - data = json.loads(request.body) - metadata = data['metadata'] + query_dict, multi_value_dict = MultiPartParser(request.META, request, + request.upload_handlers).parse() + + title = query_dict.get('title') + description = query_dict.get('description') + metadata = query_dict.get('metadata') + cover_image = multi_value_dict.get('cover_image') + project_id_full = f"{settings.PORTAL_PROJECTS_SYSTEM_PREFIX}.{project_id}" client = request.user.tapis_oauth.client - workspace_def = update_project(client, project_id, data['title'], data['description']) + workspace_def = update_project(client, project_id, title, description) if metadata is not None: - entity = patch_project_entity(project_id_full, metadata) + metadata = json.loads(metadata) + + if cover_image: + metadata['cover_image'] = f'media/{project_id}/cover_image/{cover_image.name}' + + entity = patch_project_entity(project_id_full, metadata) workspace_def.update(get_ordered_value(entity.name, entity.value)) workspace_def["projectId"] = project_id + + # Upload cover image to media folder + if cover_image: + service_client = service_account() + resized_file = resize_cover_image(cover_image) + service_client.files.insert(systemId=settings.PORTAL_PROJECTS_ROOT_SYSTEM_NAME, + path=f'media/{project_id}/cover_image/{cover_image.name}', + file=resized_file) + + # Get the postit for the cover image + postit = service_client.files.createPostIt(systemId=settings.PORTAL_PROJECTS_ROOT_SYSTEM_NAME, + path=f'media/{project_id}/cover_image/{cover_image.name}', + allowedUses=-1, + validSeconds=86400) + workspace_def["cover_image_url"] = postit.redeemUrl return JsonResponse( { diff --git a/server/portal/apps/projects/workspace_operations/project_publish_operations.py b/server/portal/apps/projects/workspace_operations/project_publish_operations.py index a28768504f..284d1d251f 100644 --- a/server/portal/apps/projects/workspace_operations/project_publish_operations.py +++ b/server/portal/apps/projects/workspace_operations/project_publish_operations.py @@ -39,6 +39,27 @@ def _transfer_files(client, source_system_id, dest_system_id): transfer = service_client.files.createTransferTask(elements=transfer_elements) return transfer +def _transfer_cover_image(source_system_id, dest_system_id, cover_image_path): + + if not cover_image_path: + logger.info('No cover image found for project, skipping transfer.') + return None + + service_client = service_account() + + # Transfer the cover image to the destination system + transfer_elements = [ + { + 'sourceURI': f'tapis://{source_system_id}/{cover_image_path}', + 'destinationURI': f'tapis://{dest_system_id}/{cover_image_path}' + } + ] + + transfer = service_client.files.createTransferTask(elements=transfer_elements) + logger.info(f"Transfer task created for cover image: {transfer.uuid}") + return transfer + + def _check_transfer_status(service_client, transfer_task_id): transfer_details = service_client.files.getTransferTask(transferTaskId=transfer_task_id) return transfer_details.status @@ -155,6 +176,9 @@ def publish_project(self, project_id: str, version: Optional[int] = 1): # transfer files client = service_account() transfer = _transfer_files(client, review_system_id, published_system_id) + cover_image_transfer = _transfer_cover_image(settings.PORTAL_PROJECTS_ROOT_REVIEW_SYSTEM_NAME, + settings.PORTAL_PROJECTS_PUBLISHED_ROOT_SYSTEM_NAME, + project_meta.value.get("coverImage", None)) poll_tapis_file_transfer.apply_async( args=(transfer.uuid, False), @@ -180,6 +204,9 @@ def copy_graph_and_files_for_review_system(self, user_access_token, source_works client = user_account(user_access_token) transfer = _transfer_files(client, source_system_id, review_system_id) + cover_image_trasnfer = _transfer_cover_image(settings.PORTAL_PROJECTS_ROOT_SYSTEM_NAME, + settings.PORTAL_PROJECTS_ROOT_REVIEW_SYSTEM_NAME, + review_project.value.get("coverImage", None)) logger.info(f'Transfer task submmited with id {transfer.uuid}') diff --git a/server/portal/libs/files/file_processing.py b/server/portal/libs/files/file_processing.py index c86e128ece..2143b1a590 100644 --- a/server/portal/libs/files/file_processing.py +++ b/server/portal/libs/files/file_processing.py @@ -1,11 +1,13 @@ import numpy as np import io +import os import logging from matplotlib import pyplot as plt import csv import matplotlib.animation as anim import tempfile import tifffile as tiff +from PIL import Image logger = logging.getLogger(__name__) @@ -180,4 +182,39 @@ def save_to_tempfile(self): gif_binary_data = animated_gif.save_to_tempfile() logger.debug('Animated Gif Created') - return gif_binary_data \ No newline at end of file + return gif_binary_data + +def resize_cover_image(img): + + max_size = 500 + image = Image.open(img) + (width, height) = image.size + + _, ext = os.path.splitext(img.name) + + if width > max_size or height > max_size: + # Calculate the resizing modifier + modifier = max_size / width if width > height else max_size / height + resized_width = width * modifier + resized_height = height * modifier + size = (round(resized_width), round(resized_height)) + + # Resize the image + image = image.resize(size, Image.Resampling.LANCZOS) + + format_map = { + '.jpg': 'JPEG', + '.jpeg': 'JPEG', + '.png': 'PNG', + '.gif': 'GIF', + } + + # Save the resized image to a binary stream + buffer = io.BytesIO() + image.save(buffer, format=format_map[ext]) # Preserve the original format + buffer.seek(0) # Reset the stream's position to the beginning + + # Clean up + image.close() + + return buffer.getvalue() From 6a9e8e9e1d63c8526a5d0bbd26af18f56753ce75 Mon Sep 17 00:00:00 2001 From: Shayan Khan Date: Mon, 24 Feb 2025 10:21:21 -0600 Subject: [PATCH 164/328] task/WC-171 - Digital Rocks Migration Scripts (#1061) * migration utils for drp * add exception handling * add settings for drp mysql db * update poetry lock file * update drp sql utils * fixes for pub date and string bug * bug fix for sample path and added field sanitization * fix for analysis data and pub doi errors --- server/poetry.lock | 233 +++++---- .../apps/_custom/drp/metadata_mappings.py | 61 +++ server/portal/apps/_custom/drp/models.py | 2 + .../commands/migrate_drp_project_files.py | 99 ++++ .../commands/migrate_drp_project_metadata.py | 473 ++++++++++++++++++ .../projects/migration_utils/sql_db_utils.py | 82 +++ server/portal/apps/publications/views.py | 2 +- server/portal/settings/settings.py | 8 + server/pyproject.toml | 1 + 9 files changed, 852 insertions(+), 109 deletions(-) create mode 100644 server/portal/apps/_custom/drp/metadata_mappings.py create mode 100644 server/portal/apps/projects/management/commands/migrate_drp_project_files.py create mode 100644 server/portal/apps/projects/management/commands/migrate_drp_project_metadata.py create mode 100644 server/portal/apps/projects/migration_utils/sql_db_utils.py diff --git a/server/poetry.lock b/server/poetry.lock index fdfd41b117..838f4499f1 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -956,61 +956,61 @@ pyflakes = ">=3.1.0,<3.2.0" [[package]] name = "fonttools" -version = "4.55.3" +version = "4.56.0" description = "Tools to manipulate font files" optional = false python-versions = ">=3.8" files = [ - {file = "fonttools-4.55.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:1dcc07934a2165ccdc3a5a608db56fb3c24b609658a5b340aee4ecf3ba679dc0"}, - {file = "fonttools-4.55.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f7d66c15ba875432a2d2fb419523f5d3d347f91f48f57b8b08a2dfc3c39b8a3f"}, - {file = "fonttools-4.55.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27e4ae3592e62eba83cd2c4ccd9462dcfa603ff78e09110680a5444c6925d841"}, - {file = "fonttools-4.55.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:62d65a3022c35e404d19ca14f291c89cc5890032ff04f6c17af0bd1927299674"}, - {file = "fonttools-4.55.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d342e88764fb201286d185093781bf6628bbe380a913c24adf772d901baa8276"}, - {file = "fonttools-4.55.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:dd68c87a2bfe37c5b33bcda0fba39b65a353876d3b9006fde3adae31f97b3ef5"}, - {file = "fonttools-4.55.3-cp310-cp310-win32.whl", hash = "sha256:1bc7ad24ff98846282eef1cbeac05d013c2154f977a79886bb943015d2b1b261"}, - {file = "fonttools-4.55.3-cp310-cp310-win_amd64.whl", hash = "sha256:b54baf65c52952db65df39fcd4820668d0ef4766c0ccdf32879b77f7c804d5c5"}, - {file = "fonttools-4.55.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:8c4491699bad88efe95772543cd49870cf756b019ad56294f6498982408ab03e"}, - {file = "fonttools-4.55.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5323a22eabddf4b24f66d26894f1229261021dacd9d29e89f7872dd8c63f0b8b"}, - {file = "fonttools-4.55.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5480673f599ad410695ca2ddef2dfefe9df779a9a5cda89503881e503c9c7d90"}, - {file = "fonttools-4.55.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da9da6d65cd7aa6b0f806556f4985bcbf603bf0c5c590e61b43aa3e5a0f822d0"}, - {file = "fonttools-4.55.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e894b5bd60d9f473bed7a8f506515549cc194de08064d829464088d23097331b"}, - {file = "fonttools-4.55.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:aee3b57643827e237ff6ec6d28d9ff9766bd8b21e08cd13bff479e13d4b14765"}, - {file = "fonttools-4.55.3-cp311-cp311-win32.whl", hash = "sha256:eb6ca911c4c17eb51853143624d8dc87cdcdf12a711fc38bf5bd21521e79715f"}, - {file = "fonttools-4.55.3-cp311-cp311-win_amd64.whl", hash = "sha256:6314bf82c54c53c71805318fcf6786d986461622dd926d92a465199ff54b1b72"}, - {file = "fonttools-4.55.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:f9e736f60f4911061235603a6119e72053073a12c6d7904011df2d8fad2c0e35"}, - {file = "fonttools-4.55.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7a8aa2c5e5b8b3bcb2e4538d929f6589a5c6bdb84fd16e2ed92649fb5454f11c"}, - {file = "fonttools-4.55.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:07f8288aacf0a38d174445fc78377a97fb0b83cfe352a90c9d9c1400571963c7"}, - {file = "fonttools-4.55.3-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8d5e8916c0970fbc0f6f1bece0063363bb5857a7f170121a4493e31c3db3314"}, - {file = "fonttools-4.55.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ae3b6600565b2d80b7c05acb8e24d2b26ac407b27a3f2e078229721ba5698427"}, - {file = "fonttools-4.55.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:54153c49913f45065c8d9e6d0c101396725c5621c8aee744719300f79771d75a"}, - {file = "fonttools-4.55.3-cp312-cp312-win32.whl", hash = "sha256:827e95fdbbd3e51f8b459af5ea10ecb4e30af50221ca103bea68218e9615de07"}, - {file = "fonttools-4.55.3-cp312-cp312-win_amd64.whl", hash = "sha256:e6e8766eeeb2de759e862004aa11a9ea3d6f6d5ec710551a88b476192b64fd54"}, - {file = "fonttools-4.55.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a430178ad3e650e695167cb53242dae3477b35c95bef6525b074d87493c4bf29"}, - {file = "fonttools-4.55.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:529cef2ce91dc44f8e407cc567fae6e49a1786f2fefefa73a294704c415322a4"}, - {file = "fonttools-4.55.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e75f12c82127486fac2d8bfbf5bf058202f54bf4f158d367e41647b972342ca"}, - {file = "fonttools-4.55.3-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:859c358ebf41db18fb72342d3080bce67c02b39e86b9fbcf1610cca14984841b"}, - {file = "fonttools-4.55.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:546565028e244a701f73df6d8dd6be489d01617863ec0c6a42fa25bf45d43048"}, - {file = "fonttools-4.55.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:aca318b77f23523309eec4475d1fbbb00a6b133eb766a8bdc401faba91261abe"}, - {file = "fonttools-4.55.3-cp313-cp313-win32.whl", hash = "sha256:8c5ec45428edaa7022f1c949a632a6f298edc7b481312fc7dc258921e9399628"}, - {file = "fonttools-4.55.3-cp313-cp313-win_amd64.whl", hash = "sha256:11e5de1ee0d95af4ae23c1a138b184b7f06e0b6abacabf1d0db41c90b03d834b"}, - {file = "fonttools-4.55.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:caf8230f3e10f8f5d7593eb6d252a37caf58c480b19a17e250a63dad63834cf3"}, - {file = "fonttools-4.55.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b586ab5b15b6097f2fb71cafa3c98edfd0dba1ad8027229e7b1e204a58b0e09d"}, - {file = "fonttools-4.55.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a8c2794ded89399cc2169c4d0bf7941247b8d5932b2659e09834adfbb01589aa"}, - {file = "fonttools-4.55.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cf4fe7c124aa3f4e4c1940880156e13f2f4d98170d35c749e6b4f119a872551e"}, - {file = "fonttools-4.55.3-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:86721fbc389ef5cc1e2f477019e5069e8e4421e8d9576e9c26f840dbb04678de"}, - {file = "fonttools-4.55.3-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:89bdc5d88bdeec1b15af790810e267e8332d92561dce4f0748c2b95c9bdf3926"}, - {file = "fonttools-4.55.3-cp38-cp38-win32.whl", hash = "sha256:bc5dbb4685e51235ef487e4bd501ddfc49be5aede5e40f4cefcccabc6e60fb4b"}, - {file = "fonttools-4.55.3-cp38-cp38-win_amd64.whl", hash = "sha256:cd70de1a52a8ee2d1877b6293af8a2484ac82514f10b1c67c1c5762d38073e56"}, - {file = "fonttools-4.55.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:bdcc9f04b36c6c20978d3f060e5323a43f6222accc4e7fcbef3f428e216d96af"}, - {file = "fonttools-4.55.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c3ca99e0d460eff46e033cd3992a969658c3169ffcd533e0a39c63a38beb6831"}, - {file = "fonttools-4.55.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22f38464daa6cdb7b6aebd14ab06609328fe1e9705bb0fcc7d1e69de7109ee02"}, - {file = "fonttools-4.55.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed63959d00b61959b035c7d47f9313c2c1ece090ff63afea702fe86de00dbed4"}, - {file = "fonttools-4.55.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:5e8d657cd7326eeaba27de2740e847c6b39dde2f8d7cd7cc56f6aad404ddf0bd"}, - {file = "fonttools-4.55.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:fb594b5a99943042c702c550d5494bdd7577f6ef19b0bc73877c948a63184a32"}, - {file = "fonttools-4.55.3-cp39-cp39-win32.whl", hash = "sha256:dc5294a3d5c84226e3dbba1b6f61d7ad813a8c0238fceea4e09aa04848c3d851"}, - {file = "fonttools-4.55.3-cp39-cp39-win_amd64.whl", hash = "sha256:aedbeb1db64496d098e6be92b2e63b5fac4e53b1b92032dfc6988e1ea9134a4d"}, - {file = "fonttools-4.55.3-py3-none-any.whl", hash = "sha256:f412604ccbeee81b091b420272841e5ec5ef68967a9790e80bffd0e30b8e2977"}, - {file = "fonttools-4.55.3.tar.gz", hash = "sha256:3983313c2a04d6cc1fe9251f8fc647754cf49a61dac6cb1e7249ae67afaafc45"}, + {file = "fonttools-4.56.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:331954d002dbf5e704c7f3756028e21db07097c19722569983ba4d74df014000"}, + {file = "fonttools-4.56.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8d1613abd5af2f93c05867b3a3759a56e8bf97eb79b1da76b2bc10892f96ff16"}, + {file = "fonttools-4.56.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:705837eae384fe21cee5e5746fd4f4b2f06f87544fa60f60740007e0aa600311"}, + {file = "fonttools-4.56.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc871904a53a9d4d908673c6faa15689874af1c7c5ac403a8e12d967ebd0c0dc"}, + {file = "fonttools-4.56.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:38b947de71748bab150259ee05a775e8a0635891568e9fdb3cdd7d0e0004e62f"}, + {file = "fonttools-4.56.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:86b2a1013ef7a64d2e94606632683f07712045ed86d937c11ef4dde97319c086"}, + {file = "fonttools-4.56.0-cp310-cp310-win32.whl", hash = "sha256:133bedb9a5c6376ad43e6518b7e2cd2f866a05b1998f14842631d5feb36b5786"}, + {file = "fonttools-4.56.0-cp310-cp310-win_amd64.whl", hash = "sha256:17f39313b649037f6c800209984a11fc256a6137cbe5487091c6c7187cae4685"}, + {file = "fonttools-4.56.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ef04bc7827adb7532be3d14462390dd71287644516af3f1e67f1e6ff9c6d6df"}, + {file = "fonttools-4.56.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ffda9b8cd9cb8b301cae2602ec62375b59e2e2108a117746f12215145e3f786c"}, + {file = "fonttools-4.56.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2e993e8db36306cc3f1734edc8ea67906c55f98683d6fd34c3fc5593fdbba4c"}, + {file = "fonttools-4.56.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:003548eadd674175510773f73fb2060bb46adb77c94854af3e0cc5bc70260049"}, + {file = "fonttools-4.56.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bd9825822e7bb243f285013e653f6741954d8147427aaa0324a862cdbf4cbf62"}, + {file = "fonttools-4.56.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b23d30a2c0b992fb1c4f8ac9bfde44b5586d23457759b6cf9a787f1a35179ee0"}, + {file = "fonttools-4.56.0-cp311-cp311-win32.whl", hash = "sha256:47b5e4680002ae1756d3ae3b6114e20aaee6cc5c69d1e5911f5ffffd3ee46c6b"}, + {file = "fonttools-4.56.0-cp311-cp311-win_amd64.whl", hash = "sha256:14a3e3e6b211660db54ca1ef7006401e4a694e53ffd4553ab9bc87ead01d0f05"}, + {file = "fonttools-4.56.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d6f195c14c01bd057bc9b4f70756b510e009c83c5ea67b25ced3e2c38e6ee6e9"}, + {file = "fonttools-4.56.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fa760e5fe8b50cbc2d71884a1eff2ed2b95a005f02dda2fa431560db0ddd927f"}, + {file = "fonttools-4.56.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d54a45d30251f1d729e69e5b675f9a08b7da413391a1227781e2a297fa37f6d2"}, + {file = "fonttools-4.56.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:661a8995d11e6e4914a44ca7d52d1286e2d9b154f685a4d1f69add8418961563"}, + {file = "fonttools-4.56.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9d94449ad0a5f2a8bf5d2f8d71d65088aee48adbe45f3c5f8e00e3ad861ed81a"}, + {file = "fonttools-4.56.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f59746f7953f69cc3290ce2f971ab01056e55ddd0fb8b792c31a8acd7fee2d28"}, + {file = "fonttools-4.56.0-cp312-cp312-win32.whl", hash = "sha256:bce60f9a977c9d3d51de475af3f3581d9b36952e1f8fc19a1f2254f1dda7ce9c"}, + {file = "fonttools-4.56.0-cp312-cp312-win_amd64.whl", hash = "sha256:300c310bb725b2bdb4f5fc7e148e190bd69f01925c7ab437b9c0ca3e1c7cd9ba"}, + {file = "fonttools-4.56.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f20e2c0dfab82983a90f3d00703ac0960412036153e5023eed2b4641d7d5e692"}, + {file = "fonttools-4.56.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f36a0868f47b7566237640c026c65a86d09a3d9ca5df1cd039e30a1da73098a0"}, + {file = "fonttools-4.56.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62b4c6802fa28e14dba010e75190e0e6228513573f1eeae57b11aa1a39b7e5b1"}, + {file = "fonttools-4.56.0-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a05d1f07eb0a7d755fbe01fee1fd255c3a4d3730130cf1bfefb682d18fd2fcea"}, + {file = "fonttools-4.56.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0073b62c3438cf0058488c002ea90489e8801d3a7af5ce5f7c05c105bee815c3"}, + {file = "fonttools-4.56.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e2cad98c94833465bcf28f51c248aaf07ca022efc6a3eba750ad9c1e0256d278"}, + {file = "fonttools-4.56.0-cp313-cp313-win32.whl", hash = "sha256:d0cb73ccf7f6d7ca8d0bc7ea8ac0a5b84969a41c56ac3ac3422a24df2680546f"}, + {file = "fonttools-4.56.0-cp313-cp313-win_amd64.whl", hash = "sha256:62cc1253827d1e500fde9dbe981219fea4eb000fd63402283472d38e7d8aa1c6"}, + {file = "fonttools-4.56.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:3fd3fccb7b9adaaecfa79ad51b759f2123e1aba97f857936ce044d4f029abd71"}, + {file = "fonttools-4.56.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:193b86e9f769320bc98ffdb42accafb5d0c8c49bd62884f1c0702bc598b3f0a2"}, + {file = "fonttools-4.56.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6e81c1cc80c1d8bf071356cc3e0e25071fbba1c75afc48d41b26048980b3c771"}, + {file = "fonttools-4.56.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e9270505a19361e81eecdbc2c251ad1e1a9a9c2ad75fa022ccdee533f55535dc"}, + {file = "fonttools-4.56.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:53f5e9767978a4daf46f28e09dbeb7d010319924ae622f7b56174b777258e5ba"}, + {file = "fonttools-4.56.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:9da650cb29bc098b8cfd15ef09009c914b35c7986c8fa9f08b51108b7bc393b4"}, + {file = "fonttools-4.56.0-cp38-cp38-win32.whl", hash = "sha256:965d0209e6dbdb9416100123b6709cb13f5232e2d52d17ed37f9df0cc31e2b35"}, + {file = "fonttools-4.56.0-cp38-cp38-win_amd64.whl", hash = "sha256:654ac4583e2d7c62aebc6fc6a4c6736f078f50300e18aa105d87ce8925cfac31"}, + {file = "fonttools-4.56.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca7962e8e5fc047cc4e59389959843aafbf7445b6c08c20d883e60ced46370a5"}, + {file = "fonttools-4.56.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a1af375734018951c31c0737d04a9d5fd0a353a0253db5fbed2ccd44eac62d8c"}, + {file = "fonttools-4.56.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:442ad4122468d0e47d83bc59d0e91b474593a8c813839e1872e47c7a0cb53b10"}, + {file = "fonttools-4.56.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3cf4f8d2a30b454ac682e12c61831dcb174950c406011418e739de592bbf8f76"}, + {file = "fonttools-4.56.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:96a4271f63a615bcb902b9f56de00ea225d6896052c49f20d0c91e9f43529a29"}, + {file = "fonttools-4.56.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:6c1d38642ca2dddc7ae992ef5d026e5061a84f10ff2b906be5680ab089f55bb8"}, + {file = "fonttools-4.56.0-cp39-cp39-win32.whl", hash = "sha256:2d351275f73ebdd81dd5b09a8b8dac7a30f29a279d41e1c1192aedf1b6dced40"}, + {file = "fonttools-4.56.0-cp39-cp39-win_amd64.whl", hash = "sha256:d6ca96d1b61a707ba01a43318c9c40aaf11a5a568d1e61146fafa6ab20890793"}, + {file = "fonttools-4.56.0-py3-none-any.whl", hash = "sha256:1088182f68c303b50ca4dc0c82d42083d176cba37af1937e1a976a31149d4d14"}, + {file = "fonttools-4.56.0.tar.gz", hash = "sha256:a114d1567e1a1586b7e9e7fc2ff686ca542a82769a296cef131e4c4af51e58f4"}, ] [package.extras] @@ -1995,6 +1995,23 @@ files = [ {file = "msgpack-1.0.7.tar.gz", hash = "sha256:572efc93db7a4d27e404501975ca6d2d9775705c2d922390d878fcf768d92c87"}, ] +[[package]] +name = "mysqlclient" +version = "2.2.7" +description = "Python interface to MySQL" +optional = false +python-versions = ">=3.8" +files = [ + {file = "mysqlclient-2.2.7-cp310-cp310-win_amd64.whl", hash = "sha256:2e3c11f7625029d7276ca506f8960a7fd3c5a0a0122c9e7404e6a8fe961b3d22"}, + {file = "mysqlclient-2.2.7-cp311-cp311-win_amd64.whl", hash = "sha256:a22d99d26baf4af68ebef430e3131bb5a9b722b79a9fcfac6d9bbf8a88800687"}, + {file = "mysqlclient-2.2.7-cp312-cp312-win_amd64.whl", hash = "sha256:4b4c0200890837fc64014cc938ef2273252ab544c1b12a6c1d674c23943f3f2e"}, + {file = "mysqlclient-2.2.7-cp313-cp313-win_amd64.whl", hash = "sha256:201a6faa301011dd07bca6b651fe5aaa546d7c9a5426835a06c3172e1056a3c5"}, + {file = "mysqlclient-2.2.7-cp39-cp39-win_amd64.whl", hash = "sha256:199dab53a224357dd0cb4d78ca0e54018f9cee9bf9ec68d72db50e0a23569076"}, + {file = "mysqlclient-2.2.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:92af368ed9c9144737af569c86d3b6c74a012a6f6b792eb868384787b52bb585"}, + {file = "mysqlclient-2.2.7-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:977e35244fe6ef44124e9a1c2d1554728a7b76695598e4b92b37dc2130503069"}, + {file = "mysqlclient-2.2.7.tar.gz", hash = "sha256:24ae22b59416d5fcce7e99c9d37548350b4565baac82f95e149cac6ce4163845"}, +] + [[package]] name = "networkx" version = "3.3" @@ -2015,66 +2032,66 @@ test = ["pytest (>=7.2)", "pytest-cov (>=4.0)"] [[package]] name = "numpy" -version = "2.2.1" +version = "2.2.3" description = "Fundamental package for array computing in Python" optional = false python-versions = ">=3.10" files = [ - {file = "numpy-2.2.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5edb4e4caf751c1518e6a26a83501fda79bff41cc59dac48d70e6d65d4ec4440"}, - {file = "numpy-2.2.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:aa3017c40d513ccac9621a2364f939d39e550c542eb2a894b4c8da92b38896ab"}, - {file = "numpy-2.2.1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:61048b4a49b1c93fe13426e04e04fdf5a03f456616f6e98c7576144677598675"}, - {file = "numpy-2.2.1-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:7671dc19c7019103ca44e8d94917eba8534c76133523ca8406822efdd19c9308"}, - {file = "numpy-2.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4250888bcb96617e00bfa28ac24850a83c9f3a16db471eca2ee1f1714df0f957"}, - {file = "numpy-2.2.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a7746f235c47abc72b102d3bce9977714c2444bdfaea7888d241b4c4bb6a78bf"}, - {file = "numpy-2.2.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:059e6a747ae84fce488c3ee397cee7e5f905fd1bda5fb18c66bc41807ff119b2"}, - {file = "numpy-2.2.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f62aa6ee4eb43b024b0e5a01cf65a0bb078ef8c395e8713c6e8a12a697144528"}, - {file = "numpy-2.2.1-cp310-cp310-win32.whl", hash = "sha256:48fd472630715e1c1c89bf1feab55c29098cb403cc184b4859f9c86d4fcb6a95"}, - {file = "numpy-2.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:b541032178a718c165a49638d28272b771053f628382d5e9d1c93df23ff58dbf"}, - {file = "numpy-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:40f9e544c1c56ba8f1cf7686a8c9b5bb249e665d40d626a23899ba6d5d9e1484"}, - {file = "numpy-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f9b57eaa3b0cd8db52049ed0330747b0364e899e8a606a624813452b8203d5f7"}, - {file = "numpy-2.2.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:bc8a37ad5b22c08e2dbd27df2b3ef7e5c0864235805b1e718a235bcb200cf1cb"}, - {file = "numpy-2.2.1-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:9036d6365d13b6cbe8f27a0eaf73ddcc070cae584e5ff94bb45e3e9d729feab5"}, - {file = "numpy-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:51faf345324db860b515d3f364eaa93d0e0551a88d6218a7d61286554d190d73"}, - {file = "numpy-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:38efc1e56b73cc9b182fe55e56e63b044dd26a72128fd2fbd502f75555d92591"}, - {file = "numpy-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:31b89fa67a8042e96715c68e071a1200c4e172f93b0fbe01a14c0ff3ff820fc8"}, - {file = "numpy-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4c86e2a209199ead7ee0af65e1d9992d1dce7e1f63c4b9a616500f93820658d0"}, - {file = "numpy-2.2.1-cp311-cp311-win32.whl", hash = "sha256:b34d87e8a3090ea626003f87f9392b3929a7bbf4104a05b6667348b6bd4bf1cd"}, - {file = "numpy-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:360137f8fb1b753c5cde3ac388597ad680eccbbbb3865ab65efea062c4a1fd16"}, - {file = "numpy-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:694f9e921a0c8f252980e85bce61ebbd07ed2b7d4fa72d0e4246f2f8aa6642ab"}, - {file = "numpy-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3683a8d166f2692664262fd4900f207791d005fb088d7fdb973cc8d663626faa"}, - {file = "numpy-2.2.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:780077d95eafc2ccc3ced969db22377b3864e5b9a0ea5eb347cc93b3ea900315"}, - {file = "numpy-2.2.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:55ba24ebe208344aa7a00e4482f65742969a039c2acfcb910bc6fcd776eb4355"}, - {file = "numpy-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9b1d07b53b78bf84a96898c1bc139ad7f10fda7423f5fd158fd0f47ec5e01ac7"}, - {file = "numpy-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5062dc1a4e32a10dc2b8b13cedd58988261416e811c1dc4dbdea4f57eea61b0d"}, - {file = "numpy-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:fce4f615f8ca31b2e61aa0eb5865a21e14f5629515c9151850aa936c02a1ee51"}, - {file = "numpy-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:67d4cda6fa6ffa073b08c8372aa5fa767ceb10c9a0587c707505a6d426f4e046"}, - {file = "numpy-2.2.1-cp312-cp312-win32.whl", hash = "sha256:32cb94448be47c500d2c7a95f93e2f21a01f1fd05dd2beea1ccd049bb6001cd2"}, - {file = "numpy-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:ba5511d8f31c033a5fcbda22dd5c813630af98c70b2661f2d2c654ae3cdfcfc8"}, - {file = "numpy-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f1d09e520217618e76396377c81fba6f290d5f926f50c35f3a5f72b01a0da780"}, - {file = "numpy-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3ecc47cd7f6ea0336042be87d9e7da378e5c7e9b3c8ad0f7c966f714fc10d821"}, - {file = "numpy-2.2.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:f419290bc8968a46c4933158c91a0012b7a99bb2e465d5ef5293879742f8797e"}, - {file = "numpy-2.2.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:5b6c390bfaef8c45a260554888966618328d30e72173697e5cabe6b285fb2348"}, - {file = "numpy-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:526fc406ab991a340744aad7e25251dd47a6720a685fa3331e5c59fef5282a59"}, - {file = "numpy-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f74e6fdeb9a265624ec3a3918430205dff1df7e95a230779746a6af78bc615af"}, - {file = "numpy-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:53c09385ff0b72ba79d8715683c1168c12e0b6e84fb0372e97553d1ea91efe51"}, - {file = "numpy-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f3eac17d9ec51be534685ba877b6ab5edc3ab7ec95c8f163e5d7b39859524716"}, - {file = "numpy-2.2.1-cp313-cp313-win32.whl", hash = "sha256:9ad014faa93dbb52c80d8f4d3dcf855865c876c9660cb9bd7553843dd03a4b1e"}, - {file = "numpy-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:164a829b6aacf79ca47ba4814b130c4020b202522a93d7bff2202bfb33b61c60"}, - {file = "numpy-2.2.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:4dfda918a13cc4f81e9118dea249e192ab167a0bb1966272d5503e39234d694e"}, - {file = "numpy-2.2.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:733585f9f4b62e9b3528dd1070ec4f52b8acf64215b60a845fa13ebd73cd0712"}, - {file = "numpy-2.2.1-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:89b16a18e7bba224ce5114db863e7029803c179979e1af6ad6a6b11f70545008"}, - {file = "numpy-2.2.1-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:676f4eebf6b2d430300f1f4f4c2461685f8269f94c89698d832cdf9277f30b84"}, - {file = "numpy-2.2.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27f5cdf9f493b35f7e41e8368e7d7b4bbafaf9660cba53fb21d2cd174ec09631"}, - {file = "numpy-2.2.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1ad395cf254c4fbb5b2132fee391f361a6e8c1adbd28f2cd8e79308a615fe9d"}, - {file = "numpy-2.2.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:08ef779aed40dbc52729d6ffe7dd51df85796a702afbf68a4f4e41fafdc8bda5"}, - {file = "numpy-2.2.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:26c9c4382b19fcfbbed3238a14abf7ff223890ea1936b8890f058e7ba35e8d71"}, - {file = "numpy-2.2.1-cp313-cp313t-win32.whl", hash = "sha256:93cf4e045bae74c90ca833cba583c14b62cb4ba2cba0abd2b141ab52548247e2"}, - {file = "numpy-2.2.1-cp313-cp313t-win_amd64.whl", hash = "sha256:bff7d8ec20f5f42607599f9994770fa65d76edca264a87b5e4ea5629bce12268"}, - {file = "numpy-2.2.1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7ba9cc93a91d86365a5d270dee221fdc04fb68d7478e6bf6af650de78a8339e3"}, - {file = "numpy-2.2.1-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:3d03883435a19794e41f147612a77a8f56d4e52822337844fff3d4040a142964"}, - {file = "numpy-2.2.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4511d9e6071452b944207c8ce46ad2f897307910b402ea5fa975da32e0102800"}, - {file = "numpy-2.2.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5c5cc0cbabe9452038ed984d05ac87910f89370b9242371bd9079cb4af61811e"}, - {file = "numpy-2.2.1.tar.gz", hash = "sha256:45681fd7128c8ad1c379f0ca0776a8b0c6583d2f69889ddac01559dfe4390918"}, + {file = "numpy-2.2.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:cbc6472e01952d3d1b2772b720428f8b90e2deea8344e854df22b0618e9cce71"}, + {file = "numpy-2.2.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cdfe0c22692a30cd830c0755746473ae66c4a8f2e7bd508b35fb3b6a0813d787"}, + {file = "numpy-2.2.3-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:e37242f5324ffd9f7ba5acf96d774f9276aa62a966c0bad8dae692deebec7716"}, + {file = "numpy-2.2.3-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:95172a21038c9b423e68be78fd0be6e1b97674cde269b76fe269a5dfa6fadf0b"}, + {file = "numpy-2.2.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5b47c440210c5d1d67e1cf434124e0b5c395eee1f5806fdd89b553ed1acd0a3"}, + {file = "numpy-2.2.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0391ea3622f5c51a2e29708877d56e3d276827ac5447d7f45e9bc4ade8923c52"}, + {file = "numpy-2.2.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f6b3dfc7661f8842babd8ea07e9897fe3d9b69a1d7e5fbb743e4160f9387833b"}, + {file = "numpy-2.2.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:1ad78ce7f18ce4e7df1b2ea4019b5817a2f6a8a16e34ff2775f646adce0a5027"}, + {file = "numpy-2.2.3-cp310-cp310-win32.whl", hash = "sha256:5ebeb7ef54a7be11044c33a17b2624abe4307a75893c001a4800857956b41094"}, + {file = "numpy-2.2.3-cp310-cp310-win_amd64.whl", hash = "sha256:596140185c7fa113563c67c2e894eabe0daea18cf8e33851738c19f70ce86aeb"}, + {file = "numpy-2.2.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:16372619ee728ed67a2a606a614f56d3eabc5b86f8b615c79d01957062826ca8"}, + {file = "numpy-2.2.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5521a06a3148686d9269c53b09f7d399a5725c47bbb5b35747e1cb76326b714b"}, + {file = "numpy-2.2.3-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:7c8dde0ca2f77828815fd1aedfdf52e59071a5bae30dac3b4da2a335c672149a"}, + {file = "numpy-2.2.3-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:77974aba6c1bc26e3c205c2214f0d5b4305bdc719268b93e768ddb17e3fdd636"}, + {file = "numpy-2.2.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d42f9c36d06440e34226e8bd65ff065ca0963aeecada587b937011efa02cdc9d"}, + {file = "numpy-2.2.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2712c5179f40af9ddc8f6727f2bd910ea0eb50206daea75f58ddd9fa3f715bb"}, + {file = "numpy-2.2.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c8b0451d2ec95010d1db8ca733afc41f659f425b7f608af569711097fd6014e2"}, + {file = "numpy-2.2.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d9b4a8148c57ecac25a16b0e11798cbe88edf5237b0df99973687dd866f05e1b"}, + {file = "numpy-2.2.3-cp311-cp311-win32.whl", hash = "sha256:1f45315b2dc58d8a3e7754fe4e38b6fce132dab284a92851e41b2b344f6441c5"}, + {file = "numpy-2.2.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f48ba6f6c13e5e49f3d3efb1b51c8193215c42ac82610a04624906a9270be6f"}, + {file = "numpy-2.2.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:12c045f43b1d2915eca6b880a7f4a256f59d62df4f044788c8ba67709412128d"}, + {file = "numpy-2.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:87eed225fd415bbae787f93a457af7f5990b92a334e346f72070bf569b9c9c95"}, + {file = "numpy-2.2.3-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:712a64103d97c404e87d4d7c47fb0c7ff9acccc625ca2002848e0d53288b90ea"}, + {file = "numpy-2.2.3-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:a5ae282abe60a2db0fd407072aff4599c279bcd6e9a2475500fc35b00a57c532"}, + {file = "numpy-2.2.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5266de33d4c3420973cf9ae3b98b54a2a6d53a559310e3236c4b2b06b9c07d4e"}, + {file = "numpy-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b787adbf04b0db1967798dba8da1af07e387908ed1553a0d6e74c084d1ceafe"}, + {file = "numpy-2.2.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:34c1b7e83f94f3b564b35f480f5652a47007dd91f7c839f404d03279cc8dd021"}, + {file = "numpy-2.2.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4d8335b5f1b6e2bce120d55fb17064b0262ff29b459e8493d1785c18ae2553b8"}, + {file = "numpy-2.2.3-cp312-cp312-win32.whl", hash = "sha256:4d9828d25fb246bedd31e04c9e75714a4087211ac348cb39c8c5f99dbb6683fe"}, + {file = "numpy-2.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:83807d445817326b4bcdaaaf8e8e9f1753da04341eceec705c001ff342002e5d"}, + {file = "numpy-2.2.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7bfdb06b395385ea9b91bf55c1adf1b297c9fdb531552845ff1d3ea6e40d5aba"}, + {file = "numpy-2.2.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:23c9f4edbf4c065fddb10a4f6e8b6a244342d95966a48820c614891e5059bb50"}, + {file = "numpy-2.2.3-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:a0c03b6be48aaf92525cccf393265e02773be8fd9551a2f9adbe7db1fa2b60f1"}, + {file = "numpy-2.2.3-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:2376e317111daa0a6739e50f7ee2a6353f768489102308b0d98fcf4a04f7f3b5"}, + {file = "numpy-2.2.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8fb62fe3d206d72fe1cfe31c4a1106ad2b136fcc1606093aeab314f02930fdf2"}, + {file = "numpy-2.2.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:52659ad2534427dffcc36aac76bebdd02b67e3b7a619ac67543bc9bfe6b7cdb1"}, + {file = "numpy-2.2.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1b416af7d0ed3271cad0f0a0d0bee0911ed7eba23e66f8424d9f3dfcdcae1304"}, + {file = "numpy-2.2.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1402da8e0f435991983d0a9708b779f95a8c98c6b18a171b9f1be09005e64d9d"}, + {file = "numpy-2.2.3-cp313-cp313-win32.whl", hash = "sha256:136553f123ee2951bfcfbc264acd34a2fc2f29d7cdf610ce7daf672b6fbaa693"}, + {file = "numpy-2.2.3-cp313-cp313-win_amd64.whl", hash = "sha256:5b732c8beef1d7bc2d9e476dbba20aaff6167bf205ad9aa8d30913859e82884b"}, + {file = "numpy-2.2.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:435e7a933b9fda8126130b046975a968cc2d833b505475e588339e09f7672890"}, + {file = "numpy-2.2.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:7678556eeb0152cbd1522b684dcd215250885993dd00adb93679ec3c0e6e091c"}, + {file = "numpy-2.2.3-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:2e8da03bd561504d9b20e7a12340870dfc206c64ea59b4cfee9fceb95070ee94"}, + {file = "numpy-2.2.3-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:c9aa4496fd0e17e3843399f533d62857cef5900facf93e735ef65aa4bbc90ef0"}, + {file = "numpy-2.2.3-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4ca91d61a4bf61b0f2228f24bbfa6a9facd5f8af03759fe2a655c50ae2c6610"}, + {file = "numpy-2.2.3-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:deaa09cd492e24fd9b15296844c0ad1b3c976da7907e1c1ed3a0ad21dded6f76"}, + {file = "numpy-2.2.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:246535e2f7496b7ac85deffe932896a3577be7af8fb7eebe7146444680297e9a"}, + {file = "numpy-2.2.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:daf43a3d1ea699402c5a850e5313680ac355b4adc9770cd5cfc2940e7861f1bf"}, + {file = "numpy-2.2.3-cp313-cp313t-win32.whl", hash = "sha256:cf802eef1f0134afb81fef94020351be4fe1d6681aadf9c5e862af6602af64ef"}, + {file = "numpy-2.2.3-cp313-cp313t-win_amd64.whl", hash = "sha256:aee2512827ceb6d7f517c8b85aa5d3923afe8fc7a57d028cffcd522f1c6fd082"}, + {file = "numpy-2.2.3-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:3c2ec8a0f51d60f1e9c0c5ab116b7fc104b165ada3f6c58abf881cb2eb16044d"}, + {file = "numpy-2.2.3-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:ed2cf9ed4e8ebc3b754d398cba12f24359f018b416c380f577bbae112ca52fc9"}, + {file = "numpy-2.2.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39261798d208c3095ae4f7bc8eaeb3481ea8c6e03dc48028057d3cbdbdb8937e"}, + {file = "numpy-2.2.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:783145835458e60fa97afac25d511d00a1eca94d4a8f3ace9fe2043003c678e4"}, + {file = "numpy-2.2.3.tar.gz", hash = "sha256:dbdc15f0c81611925f382dfa97b3bd0bc2c1ce19d4fe50482cb0ddc12ba30020"}, ] [[package]] @@ -3629,4 +3646,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "031b706da68ac6b9eacdb2c7febde1918019a91c79477dcff2fdd78d219d1fd5" +content-hash = "76cec3c482f91d34b7b9fd2cb4a50089307d942f5024220a4feef8a26217cee7" diff --git a/server/portal/apps/_custom/drp/metadata_mappings.py b/server/portal/apps/_custom/drp/metadata_mappings.py new file mode 100644 index 0000000000..401af3d31b --- /dev/null +++ b/server/portal/apps/_custom/drp/metadata_mappings.py @@ -0,0 +1,61 @@ +"""Mapping of metadata fields from old DRP database to new DRP metadata model.""" + +SAMPLE_POROUS_MEDIA_TYPE_MAPPINGS = { + 'Beads': 'beads', + 'BEAD': 'beads', + 'Sandstone': 'sandstone', + 'SAND': 'sandstone', + 'CARB': 'carbonate', + 'SOIL': 'soil', + 'FIBR': 'fibrous_media', + 'GRAN': 'granite', + 'COAL': 'coal', + 'Other': 'other', + 'OTHE': 'other', +} + +SAMPLE_SOURCE_MAPPINGS = { + 'Artificial': 'artificial', + 'A': 'artificial', + 'Natural': 'natural', + 'N': 'natural', +} + +ORIGIN_DATA_IS_SEGMENTED_MAPPING = { + 1: 'yes', + 2: 'no' +} + +ORIGIN_DATA_VOXEL_UNIT_MAPPING = { + 'micrometer': 'micrometer', + 'um': 'micrometer', + 'mm': 'millimeter', + 'nm': 'nanometer', + 'other': 'other' +} + +ANALYSIS_DATA_TYPE_MAPPING = { + 'Simulation': 'simulation', + 'GeometricAnalysis': 'geometric_analysis', + 'Other': 'other', +} + +FILE_IMAGE_TYPE_MAPPING = { + '8-bit': '8_bit', + '64-bit Real': '64_bit_real', + '16-bit Unsigned': '16_bit_unsigned', + '32-bit Real': '32_bit_real', + '32-bit Signed': '32_bit_signed', + '24-bit RGB': '24_bit_rgb', + '32-bit Unsigned': '32_bit_unsigned', +} + +FILE_BYTE_ORDER_MAPPING = { + 'little-endian': 'little_endian', + 'big-endian': 'big_endian', +} + +FILE_USE_BINARY_CORRECTION_MAPPING = { + 1: True, + 0: False, +} \ No newline at end of file diff --git a/server/portal/apps/_custom/drp/models.py b/server/portal/apps/_custom/drp/models.py index 10a7efd608..e8d2170290 100644 --- a/server/portal/apps/_custom/drp/models.py +++ b/server/portal/apps/_custom/drp/models.py @@ -49,6 +49,7 @@ class FileObj(DrpMetadataModel): system: str name: str path: str + legacy_path: Optional[str] = None type: Literal["file", "dir"] length: Optional[int] = None last_modified: Optional[str] = None @@ -97,6 +98,7 @@ class DrpProjectRelatedPublications(DrpMetadataModel): publication_title: str publication_author: str + publication_doi: str publication_date_of_publication: str publication_publisher: str publication_description: Optional[str] = None diff --git a/server/portal/apps/projects/management/commands/migrate_drp_project_files.py b/server/portal/apps/projects/management/commands/migrate_drp_project_files.py new file mode 100644 index 0000000000..f0506865ca --- /dev/null +++ b/server/portal/apps/projects/management/commands/migrate_drp_project_files.py @@ -0,0 +1,99 @@ +from django.core.management.base import BaseCommand +from django.conf import settings +import networkx as nx +from portal.apps.projects.migration_utils.sql_db_utils import get_project_by_id, query_projects, query_published_projects +from portal.apps.publications.models import Publication +from portal.libs.agave.utils import service_account +from portal.apps.projects.workspace_operations.project_publish_operations import _add_values_to_tree + +class Command(BaseCommand): + help = "Migrate DRP project files to the new location." + + def add_arguments(self, parser): + parser.add_argument( + '--dry-run', + action='store_true', + help="Run the command without saving changes." + ) + + parser.add_argument( + '--project-id', + type=str, + help="The ID of the project to migrate." + ) + + parser.add_argument( + '--publication', + action='store_true', + help="Moves files to a published project location" + ) + + parser.add_argument( + '--project', + action='store_true', + help="Moves files to a regular project location" + ) + + def handle(self, *args, **options): + self.dry_run = options['dry_run'] + + if not options['project'] and not options['publication']: + print("Please specify either --project or --publication") + return + + if options['project'] and options['publication']: + print("Please specify either --project or --publication, not both") + return + + self.publication = options['publication'] + self.project = options['project'] + + if options['project_id']: + projects = get_project_by_id(options['project_id']) + elif self.publication: + projects = query_published_projects() + else: + projects = query_projects() + + client = service_account() + + for project in projects: + try: + pub_id = f"{settings.PORTAL_PROJECTS_ID_PREFIX}-{project['id']}" + project_prefix = settings.PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX if self.publication else settings.PORTAL_PROJECTS_SYSTEM_PREFIX + project_id = f'{project_prefix}.{pub_id}' + + + if self.publication: + pub = Publication.objects.get(project_id=pub_id) + project_graph = pub.tree + else: + project_graph = nx.node_link_data(_add_values_to_tree(project_id)) + + file_mapping = {} + + for node in project_graph.get('nodes', []): + file_objs = node.get("value", {}).get("fileObjs", []) + for file_obj in file_objs: + legacy_path = file_obj.get("legacyPath") + path = file_obj.get("path") + if legacy_path and path: + file_mapping[legacy_path] = path + + transfer_elements = [] + for legacy_path, new_path in file_mapping.items(): + + transfer_elements.append( + { + 'sourceURI': f'tapis://cloud.data/corral-repl/utexas/pge-nsf/media/{legacy_path.strip("/")}', + 'destinationURI': f'tapis://{project_id}/{new_path.strip("/")}' + }) + + if not self.dry_run: + transfer = client.files.createTransferTask(elements=transfer_elements) + print(f"Transfer started for {len(file_mapping)} files: {transfer}") + else: + print(f"Dry run complete for project {project['id']} with {len(file_mapping)} files to transfer. No changes made.") + except Exception as e: + print(f"Error processing project {project['id']}: {e}") + continue diff --git a/server/portal/apps/projects/management/commands/migrate_drp_project_metadata.py b/server/portal/apps/projects/management/commands/migrate_drp_project_metadata.py new file mode 100644 index 0000000000..4e0b449ec3 --- /dev/null +++ b/server/portal/apps/projects/management/commands/migrate_drp_project_metadata.py @@ -0,0 +1,473 @@ +import os +import networkx as nx +from pathlib import Path +from django.conf import settings +from django.db import transaction +import dateutil.parser +from django.utils import timezone +from django.core.management.base import BaseCommand +from portal.apps.projects.migration_utils.sql_db_utils import (get_project_by_id, query_advanced_file_metadata, query_analysis_data, + query_files, query_origin_data, query_projects, query_published_projects, + query_related_publications, query_authors, query_samples, query_user) +from portal.apps._custom.drp.models import DrpProjectRelatedPublications +from portal.apps import SCHEMA_MAPPING +from portal.apps._custom.drp import constants, metadata_mappings +from portal.apps.projects.workspace_operations.project_meta_operations import (add_file_associations, create_entity_metadata, + create_file_obj, create_project_metadata) +from portal.apps.projects.workspace_operations.shared_workspace_operations import create_workspace_dir, create_workspace_system, set_workspace_acls +from portal.libs.agave.utils import service_account +from portal.libs.agave.operations import mkdir +from portal.apps.projects.workspace_operations.graph_operations import add_node_to_project, get_node_from_uuid, get_root_node, initialize_project_graph +from portal.apps.projects.workspace_operations.project_publish_operations import _add_values_to_tree +from portal.apps.publications.models import Publication +from portal.apps.search.tasks import index_publication +from portal.apps.projects.models.project_metadata import ProjectMetadata +class Command(BaseCommand): + help = "Migrate DRP project metadata to the new schema." + + def add_arguments(self, parser): + parser.add_argument( + '--dry-run', + action='store_true', + help="Run the command without saving changes." + ) + + parser.add_argument( + '--project-id', + type=str, + help="Specify a project ID to migrate." + ) + + parser.add_argument( + '--publication', + action='store_true', + help="Creates a published project and publication using existing published projects" + ) + + parser.add_argument( + '--project', + action='store_true', + help="Creates a regular project using existing unpublished projects" + ) + + def make_directories(self, project_id, mappings): + + client = service_account() + + paths = [data['path'] for data in mappings.values()] + + for path in paths: + parent_path, name = os.path.split(path) + mkdir(client, project_id, parent_path, name) + print(f"Created directory at path: {path} with name: {name}") + + def update_project_and_create_publication(self, published_project_id): + + client = service_account() + project_id = published_project_id.split(f'{settings.PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX}.')[-1] + + pub_tree = _add_values_to_tree(published_project_id) + published_project = ProjectMetadata.get_project_by_id(published_project_id) + published_project.project_graph.value = nx.node_link_data(pub_tree) + published_project.save() + + pub_date_str = published_project.value.get('publicationDate', None) + + pub_defaults = { + 'value': published_project.value, + 'tree': nx.node_link_data(pub_tree), + 'version': 1, + } + + if pub_date_str: + pub_date = dateutil.parser.parse(pub_date_str) + pub_date = timezone.make_aware(pub_date) + pub_defaults['created'] = pub_date + + pub_metadata = Publication.objects.update_or_create( + project_id=project_id, + defaults=pub_defaults + ) + + print(f'Created publication for {published_project_id}') + + client.systems.shareSystemPublic(systemId=published_project_id) + + index_publication(project_id) + + def handle(self, *args, **options): + self.dry_run = options['dry_run'] + + if not options['project'] and not options['publication']: + print("Please specify either --project or --publication") + return + + if options['project'] and options['publication']: + print("Please specify either --project or --publication, not both") + return + + self.publication = options['publication'] + self.project = options['project'] + + if options['project_id']: + projects = get_project_by_id(options['project_id']) + elif (self.publication): + projects = query_published_projects() + else: + projects = query_projects() + + print(f"Found {len(projects)} projects to migrate") + + projects_with_error = [] + + for project in projects: + + try: + with transaction.atomic(): + print(f"Processing project {project['id']}") + + project_mapping = self.migrate_project(project) + + sample_mappings = self.migrate_sample(project_mapping) + + origin_data_mappings = self.migrate_origin_data(project_mapping, sample_mappings) + + analysis_data_mappings = self.migrate_analysis_dataset(project_mapping, sample_mappings, origin_data_mappings) + + if not self.dry_run: + _, new_project_id = project_mapping + self.make_directories(new_project_id, sample_mappings) + self.make_directories(new_project_id, origin_data_mappings) + self.make_directories(new_project_id, analysis_data_mappings) + + if self.publication: + self.update_project_and_create_publication(new_project_id) + else: + print(f"Dry run success for project {project['id']}: No changes made to the database.") + except Exception as e: + print(f"Error processing project {project['id']}: {e}") + projects_with_error.append({project['id']: str(e)}) + continue + + if projects_with_error: + print(f'Migration completed with errors for projects: {projects_with_error}') + else: + print(f'Migration completed successfully for all projects.') + + def get_project_users(self, project_id, user_id): + + user_list = [] + + owner_data = query_user(user_id) + if owner_data: + owner = owner_data[0] + user_list.append({ + 'first_name': owner['first_name'], + 'last_name': owner['last_name'], + 'email': owner['email'] + }) + + collaborators = query_authors(project_id) + + for collaborator in collaborators: + user_data = query_user(collaborator['user_id']) + if user_data: + user = user_data[0] + user_info = { + 'first_name': user['first_name'], + 'last_name': user['last_name'], + 'email': user['email'] + } + if user_info not in user_list: # Avoid duplicates + user_list.append(user_info) + + return user_list + + def get_related_publications(self, project_id): + """Fetch related publications.""" + return [ + DrpProjectRelatedPublications( + publication_title=pub['title'], + publication_author=pub['author'], + publication_doi=pub['doi'].split("doi:")[-1] if pub['doi'] else '', + publication_date_of_publication=pub['publication_year'], + publication_publisher=pub['publisher'], + publication_description=pub['abstract'], + publication_link=pub['url'] + ) for pub in query_related_publications(project_id) + ] + + def create_project(self, legacy_project_id, project_metadata_value): + + service_client = service_account() + portal_admin_username = settings.PORTAL_ADMIN_USERNAME + + workspace_id = f'{settings.PORTAL_PROJECTS_ID_PREFIX}-{legacy_project_id}' + + system_prefix = settings.PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX if self.publication else settings.PORTAL_PROJECTS_SYSTEM_PREFIX + + system_id = f'{system_prefix}.{workspace_id}' + + root_system_name = settings.PORTAL_PROJECTS_PUBLISHED_ROOT_SYSTEM_NAME if self.publication else settings.PORTAL_PROJECTS_ROOT_SYSTEM_NAME + + root_dir = settings.PORTAL_PROJECTS_PUBLISHED_ROOT_DIR if self.publication else settings.PORTAL_PROJECTS_ROOT_DIR + + project_data = { + **project_metadata_value, + "project_id": system_id, + } + + print(f"Creating project {project_data['title']}") + + new_project = create_project_metadata(project_data) + new_project.save() + + create_workspace_dir(workspace_id, root_system_name) + + set_workspace_acls(service_client, root_system_name, workspace_id, portal_admin_username, "add", "writer") + + create_workspace_system( + service_client, workspace_id, project_data['title'], project_data['description'], None, + f"{system_id}", + f"{root_dir}/{workspace_id}" + ) + + print(f"Created project {project_data['title']} with ID {system_id}") + + return system_id + + def migrate_project(self, project): + """Migrates a project using `create_entity_metadata` and supports dry-run mode.""" + + project_data = { + "title": project['name'], + "description": project['description'], + "license": 'ODC-BY 1.0' if project['license'] and project['license'] == 1 else None, + "doi": project['doi'].split("doi:")[-1] if project['doi'] else None, + "authors": self.get_project_users(project['id'], project['user_id']), + "is_review_project": False, + "is_published_project": True if self.publication else False, + "publication_date": str(project['creation_date']) if self.publication else None, + "related_publications": self.get_related_publications(project['id']) + } + + if self.dry_run: + project_id = 'dry_run_project_id' + project_data['project_id'] = project_id + validated_project = SCHEMA_MAPPING[constants.PROJECT].model_validate(project_data) + # print(f"Dry run: Successfully created project with legacy id: {project['id']}") + else: + project_id = self.create_project(project['id'], project_data) + initialize_project_graph(project_id) + + return project['id'], project_id + + def migrate_sample(self, project_mapping): + + legacy_project_id, new_project_id = project_mapping + + samples = query_samples(legacy_project_id) + sample_mappings = {} + + for sample in samples: + porous_media_type = metadata_mappings.SAMPLE_POROUS_MEDIA_TYPE_MAPPINGS.get(sample['porous_media_type']) + source = metadata_mappings.SAMPLE_SOURCE_MAPPINGS.get(sample['source']) + + sample_data = { + "name": sample['name'], + "description": sample['description'], + "data_type": "sample", + "porous_media_type": porous_media_type, + "porous_media_other_description": sample.get('porous_media_other_description'), + "source": source, + "grain_size_min": sample.get('grain_size_min'), + "grain_size_max": sample.get('grain_size_max'), + "grain_size_avg": sample.get('grain_size_avg'), + "porosity": sample.get('porosity'), + "identifier": sample.get('identifier'), + "geographic_origin": sample.get('geographic_origin'), + "geographical_location": sample.get('location'), + } + + if self.dry_run: + validated_sample = SCHEMA_MAPPING[constants.SAMPLE].model_validate(sample_data) + sample_uuid = f'dry_run_sample_id_{sample["id"]}' + # print(f"Dry run: Successfully created sample with legacy id: {sample['id']}") + else: + sample_entity = create_entity_metadata(new_project_id, constants.SAMPLE, sample_data) + sample_uuid = sample_entity.uuid + parent_node = get_root_node(new_project_id) + add_node_to_project(new_project_id, parent_node['id'], sample_uuid, constants.SAMPLE, sample_entity.value['name']) + + + sample_mappings[sample['id']] = {'sample_uuid': sample_uuid, 'path': f'{sample["name"]}'} + + return sample_mappings + + def get_file_objs(self, files, path, project_id): + + file_objs = [] + client = service_account() + + directory_cache = {} + + for file in files: + + file_name = Path(file['file']).name + + if file['isAdvancedImageFile'] == 1: + advanced_image_file = query_advanced_file_metadata(file['id'])[0] + + value = { + 'data_type': 'file', + 'name': file_name, + 'image_type': metadata_mappings.FILE_IMAGE_TYPE_MAPPING.get(advanced_image_file['image_type']), + 'height': advanced_image_file['height'], + 'width': advanced_image_file['width'], + 'number_of_images': advanced_image_file['numberOfImages'], + 'offset_to_first_image': advanced_image_file['offsetToFirstImage'], + 'gap_between_images': advanced_image_file['gapBetweenImages'], + 'byte_order': metadata_mappings.FILE_BYTE_ORDER_MAPPING.get(advanced_image_file['byteOrder']), + 'use_binary_correction': metadata_mappings.FILE_USE_BINARY_CORRECTION_MAPPING.get(advanced_image_file['use_binary_correction']), + } + + file_obj = create_file_obj(project_id, file_name, None, f'{path}/{file_name}', value) + file_obj.legacy_path = file['file'] + file_objs.append(file_obj) + + # Find and add the generated image files for this advanced file + file_parent_path = Path(file['file']).parent + + if file_parent_path not in directory_cache: + try: + directory_cache[file_parent_path] = client.files.listFiles(systemId='cloud.data', + path = f'/corral-repl/utexas/pge-nsf/media/{file_parent_path}') + except Exception as e: + print(f"Error listing files in directory {file_parent_path}: {e}") + continue + + files_in_parent_path = directory_cache[file_parent_path] + + generated_files = [gen_file for gen_file in files_in_parent_path + if gen_file.name.startswith(file_name) and gen_file.name != file_name] + + for gen_file in generated_files: + gen_file_obj = create_file_obj(project_id, gen_file.name, None, f'{path}/{gen_file.name}', { 'data_type': 'file' }) + gen_file_obj.legacy_path = f'{file_parent_path}/{gen_file.name}' + file_objs.append(gen_file_obj) + + else: + file_obj = create_file_obj(project_id, file_name, None, f'{path}/{file_name}', { 'data_type': 'file' }) + file_obj.legacy_path = file['file'] + file_objs.append(file_obj) + + return file_objs + + def migrate_origin_data(self, project_mapping, sample_mappings): + """Migrates origin datasets and adds them to the project graph.""" + + legacy_project_id, new_project_id = project_mapping + + origin_data_mappings = {} + + for legacy_sample_id, new_sample in sample_mappings.items(): + for origin in query_origin_data(legacy_project_id, legacy_sample_id): + + is_segmented = metadata_mappings.ORIGIN_DATA_IS_SEGMENTED_MAPPING.get(origin['is_segmented']) + voxel_unit = metadata_mappings.ORIGIN_DATA_VOXEL_UNIT_MAPPING.get(origin['voxel_units']) + + new_sample_uuid = new_sample['sample_uuid'] + new_sample_path = new_sample['path'] + + files = query_files(origin['id'], None) + + file_objs = self.get_file_objs(files, f"{new_sample_path}/{origin['name']}", new_project_id) + + origin_data = { + "name": origin['name'], + "description": origin.get('provenance', ''), + "data_type": "digital_dataset", + "is_segmented": is_segmented, + "sample": new_sample_uuid, + "voxel_x": origin.get('voxel_x'), + "voxel_y": origin.get('voxel_y'), + "voxel_z": origin.get('voxel_z'), + "voxel_units": voxel_unit, + "external_uri": origin.get('external_url'), + "file_objs": file_objs, + } + + if self.dry_run: + validated_origin = SCHEMA_MAPPING[constants.DIGITAL_DATASET].model_validate(origin_data) + origin_uuid = f'dry_run_origin_id_{origin["id"]}' + # print(f"Dry run: Successfully created origin data with legacy id: {origin['id']}") + else: + new_origin = create_entity_metadata( + project_id=new_project_id, + name=constants.DIGITAL_DATASET, + value=origin_data + ) + origin_uuid = new_origin.uuid + parent_node = get_node_from_uuid(new_project_id, new_sample_uuid) + add_node_to_project(new_project_id, parent_node['id'], origin_uuid, constants.DIGITAL_DATASET, new_origin.value['name']) + add_file_associations(origin_uuid, file_objs) + + origin_data_mappings[origin['id']] = {'origin_data_uuid': origin_uuid, 'path': f'{new_sample_path}/{origin["name"]}', + 'sample_id': legacy_sample_id} + + return origin_data_mappings + + def migrate_analysis_dataset(self, project_mapping, sample_mappings, origin_data_mappings): + + legacy_project_id, new_project_id = project_mapping + analysis_data_mappings = {} + + for analysis in query_analysis_data(legacy_project_id): + new_sample = sample_mappings.get(analysis['sample_id']) + new_origin_data = origin_data_mappings.get(analysis['base_origin_data_id']) + + if not new_sample: + if new_origin_data: + new_sample = sample_mappings.get(new_origin_data['sample_id']) + else: + print(f'No sample found for analysis {analysis["id"]}. Skipping.') + continue + + dataset_type = metadata_mappings.ANALYSIS_DATA_TYPE_MAPPING.get(analysis['type']) + + files = query_files(None, analysis['id']) + + analysis_data_path = f"{new_origin_data['path']}/{analysis['name']}" if new_origin_data else f"{new_sample['path']}/{analysis['name']}" + + file_objs = self.get_file_objs(files, analysis_data_path, new_project_id) + + analysis_data = { + "name": analysis['name'], + "description": analysis.get('description', ''), + "data_type": "analysis_data", + "is_segmented": "no", + "dataset_type": dataset_type, + "sample": new_sample['sample_uuid'], + "digital_dataset": new_origin_data['origin_data_uuid'] if new_origin_data else None, + "external_uri": analysis.get('external_url'), + "file_objs": file_objs, + } + + if self.dry_run: + validated_analysis = SCHEMA_MAPPING[constants.ANALYSIS_DATA].model_validate(analysis_data) + analysis_uuid = f'dry_run_analysis_id_{analysis["id"]}' + # print(f"Dry run: Successfully created analysis data with legacy id: {analysis['id']}") + else: + new_analysis = create_entity_metadata( + project_id=new_project_id, + name=constants.ANALYSIS_DATA, + value=analysis_data + ) + analysis_uuid = new_analysis.uuid + parent_node = get_node_from_uuid(new_project_id, new_origin_data['origin_data_uuid'] if new_origin_data else new_sample['sample_uuid']) + add_node_to_project(new_project_id, parent_node['id'], analysis_uuid, constants.ANALYSIS_DATA, new_analysis.value['name']) + add_file_associations(analysis_uuid, file_objs) + + analysis_data_mappings[analysis['id']] = {'analysis_data_uuid': analysis_uuid, 'path': analysis_data_path} + + return analysis_data_mappings \ No newline at end of file diff --git a/server/portal/apps/projects/migration_utils/sql_db_utils.py b/server/portal/apps/projects/migration_utils/sql_db_utils.py new file mode 100644 index 0000000000..19ba22ac5f --- /dev/null +++ b/server/portal/apps/projects/migration_utils/sql_db_utils.py @@ -0,0 +1,82 @@ +from django.db import connections + +def run_query(query, params=None): + with connections['drp_mysql'].cursor() as cursor: + if params: + cursor.execute(query, params) + else: + cursor.execute(query) + columns = [col[0] for col in cursor.description] + rows = cursor.fetchall() + + result = [] + for row in rows: + row_dict = dict(zip(columns, row)) + # Trim all string fields + for key, value in row_dict.items(): + if isinstance(value, str): + row_dict[key] = value.strip() + result.append(row_dict) + return result + +def query_projects(): + query = "SELECT * FROM upload_project WHERE access = %s;" + params = [2] + return run_query(query, params) + +def query_published_projects(): + query = "SELECT * FROM upload_project WHERE access = %s;" + params = [1] + return run_query(query, params) + +def get_project_by_id(project_id): + query = "SELECT * FROM upload_project WHERE id = %s;" + params = [project_id] + return run_query(query, params) + +def query_related_publications(project_id): + query = "SELECT * FROM upload_publication WHERE project_id = %s;" + params = [project_id] + return run_query(query, params) + +def query_authors(project_id): + query = "SELECT * FROM upload_collaborator WHERE project_id = %s;" + params = [project_id] + return run_query(query, params) + +def query_user(user_id): + query = "SELECT * FROM upload_myuser WHERE id = %s;" + params = [user_id] + return run_query(query, params) + +def query_samples(project_id): + query = "SELECT * FROM upload_sample WHERE project_id = %s;" + params = [project_id] + return run_query(query, params) + +def query_origin_data(project_id, sample_id): + query = "SELECT * FROM upload_origin_data WHERE project_id = %s AND sample_id = %s;" + params = [project_id, sample_id] + return run_query(query, params) + +def query_analysis_data(project_id): + query = "SELECT * FROM upload_analysis_data WHERE project_id = %s;" + params = [project_id] + return run_query(query, params) + +def query_files(origin_data_id, analysis_data_id): + if analysis_data_id is None: + query = "SELECT * FROM upload_datafile WHERE origin_data_id = %s AND analysis_data_id IS NULL;" + params = [origin_data_id] + elif origin_data_id is None: + query = "SELECT * FROM upload_datafile WHERE origin_data_id IS NULL AND analysis_data_id = %s;" + params = [analysis_data_id] + else: + query = "SELECT * FROM upload_datafile WHERE origin_data_id = %s AND analysis_data_id = %s;" + params = [origin_data_id, analysis_data_id] + return run_query(query, params) + +def query_advanced_file_metadata(file_id): + query = "SELECT * FROM upload_advancedimagefile WHERE datafile_ptr_id = %s;" + params = [file_id] + return run_query(query, params) diff --git a/server/portal/apps/publications/views.py b/server/portal/apps/publications/views.py index f4f17397c8..b14d52451d 100644 --- a/server/portal/apps/publications/views.py +++ b/server/portal/apps/publications/views.py @@ -199,7 +199,7 @@ def get(self, request): 'description': publication.value.get('description'), 'keywords': publication.value.get('keywords'), 'authors': publication.value.get('authors'), - 'publication_date': publication.last_updated, + 'publication_date': publication.created, } for publication in publications ] diff --git a/server/portal/settings/settings.py b/server/portal/settings/settings.py index edab9df086..528aa96063 100644 --- a/server/portal/settings/settings.py +++ b/server/portal/settings/settings.py @@ -258,6 +258,14 @@ 'PASSWORD': settings_secret._DJANGO_DB_PASSWORD, 'HOST': settings_secret._DJANGO_DB_HOST, 'PORT': settings_secret._DJANGO_DB_PORT + }, + 'drp_mysql': { + 'ENGINE': settings_secret._DJANGO_DRP_DB_ENGINE, + 'NAME': settings_secret._DJANGO_DRP_DB_NAME, + 'USER': settings_secret._DJANGO_DRP_DB_USER, + 'PASSWORD': settings_secret._DJANGO_DRP_DB_PASSWORD, + 'HOST': settings_secret._DJANGO_DRP_DB_HOST, + 'PORT': settings_secret._DJANGO_DRP_DB_PORT } } diff --git a/server/pyproject.toml b/server/pyproject.toml index 3944c5e4d3..4fe524db0a 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -44,6 +44,7 @@ networkx = "^3.2.1" numpy = "^2.2.1" matplotlib = "^3.10.0" tifffile = "^2025.1.10" +mysqlclient = "^2.2.7" [tool.poetry.group.dev.dependencies] pytest = "^7.3.1" From 9e6ea019a75d6a77dce2a160868368cbe78098f5 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Mon, 24 Feb 2025 10:43:25 -0600 Subject: [PATCH 165/328] update image generation notification message --- server/portal/apps/projects/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/portal/apps/projects/tasks.py b/server/portal/apps/projects/tasks.py index 79a7d6b3b0..a53cfb4d2f 100644 --- a/server/portal/apps/projects/tasks.py +++ b/server/portal/apps/projects/tasks.py @@ -257,7 +257,7 @@ def process_file(self, project_id: str, path: str, user_access_token: str, encod Notification.EVENT_TYPE: 'default', Notification.STATUS: Notification.INFO, Notification.USER: username, - Notification.MESSAGE: f'Generating Images', + Notification.MESSAGE: f'Generating Images for {Path(path).name}', }) logger.info(f'Processing file {path} in project {project_id}') From 4b1df921a6ebb898f876f932d78c1b7502b22b12 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Mon, 24 Feb 2025 10:49:38 -0600 Subject: [PATCH 166/328] update file processing --- server/portal/apps/projects/tasks.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/server/portal/apps/projects/tasks.py b/server/portal/apps/projects/tasks.py index a53cfb4d2f..c1a74e64dc 100644 --- a/server/portal/apps/projects/tasks.py +++ b/server/portal/apps/projects/tasks.py @@ -275,17 +275,21 @@ def process_file(self, project_id: str, path: str, user_access_token: str, encod file_obj: FileObj = get_file_obj(project_id, path) - if file and file_obj: + if file and file_obj: value = get_ordered_value(constants.FILE, file_obj.get('value')) file_name = file_obj.get('name') _, file_ext = os.path.splitext(file_obj.get('name')) - if file_ext in ['.tif', '.tiff']: - adv_image = conf_tiff(file) - else: - adv_image = conf_raw(value, file) + try: + if file_ext in ['.tif', '.tiff']: + adv_image = conf_tiff(file) + else: + adv_image = conf_raw(value, file) + except Exception as e: + logger.error(f'Could not generate advanced image for {file_name} due to error: {e}') + return try: if value.get('use_binary_correction'): From b6b03999b9a8cd8d8d785197da695e4f6be160d6 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Mon, 24 Feb 2025 19:06:42 -0600 Subject: [PATCH 167/328] added migration of project cover image --- .../commands/migrate_drp_project_files.py | 17 ++++++++++++++++- .../commands/migrate_drp_project_metadata.py | 8 +++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/server/portal/apps/projects/management/commands/migrate_drp_project_files.py b/server/portal/apps/projects/management/commands/migrate_drp_project_files.py index f0506865ca..4ed25f96d2 100644 --- a/server/portal/apps/projects/management/commands/migrate_drp_project_files.py +++ b/server/portal/apps/projects/management/commands/migrate_drp_project_files.py @@ -1,6 +1,7 @@ from django.core.management.base import BaseCommand from django.conf import settings import networkx as nx +from pathlib import Path from portal.apps.projects.migration_utils.sql_db_utils import get_project_by_id, query_projects, query_published_projects from portal.apps.publications.models import Publication from portal.libs.agave.utils import service_account @@ -33,6 +34,16 @@ def add_arguments(self, parser): action='store_true', help="Moves files to a regular project location" ) + + def transfer_cover_image(self, client, workspace_id, cover_pic): + root_system = settings.PORTAL_PROJECTS_PUBLISHED_ROOT_SYSTEM_NAME if self.publication else settings.PORTAL_PROJECTS_ROOT_SYSTEM_NAME + cover_image_transfer_elements = [{ + 'sourceURI': f'tapis://cloud.data/corral-repl/utexas/pge-nsf/media/{cover_pic.strip("/")}', + 'destinationURI': f'tapis://{root_system}/media/{workspace_id}/cover_image/{Path(cover_pic).name}' + }] + cover_image_transfer = client.files.createTransferTask(elements=cover_image_transfer_elements) + print(f"Cover image transfer started: {cover_image_transfer}") + def handle(self, *args, **options): self.dry_run = options['dry_run'] @@ -63,7 +74,6 @@ def handle(self, *args, **options): project_prefix = settings.PORTAL_PROJECTS_PUBLISHED_SYSTEM_PREFIX if self.publication else settings.PORTAL_PROJECTS_SYSTEM_PREFIX project_id = f'{project_prefix}.{pub_id}' - if self.publication: pub = Publication.objects.get(project_id=pub_id) project_graph = pub.tree @@ -92,8 +102,13 @@ def handle(self, *args, **options): if not self.dry_run: transfer = client.files.createTransferTask(elements=transfer_elements) print(f"Transfer started for {len(file_mapping)} files: {transfer}") + + # transfer the cover image + if project["cover_pic"]: + self.transfer_cover_image(client, pub_id, project["cover_pic"]) else: print(f"Dry run complete for project {project['id']} with {len(file_mapping)} files to transfer. No changes made.") except Exception as e: print(f"Error processing project {project['id']}: {e}") continue + diff --git a/server/portal/apps/projects/management/commands/migrate_drp_project_metadata.py b/server/portal/apps/projects/management/commands/migrate_drp_project_metadata.py index 4e0b449ec3..fc266cc8d3 100644 --- a/server/portal/apps/projects/management/commands/migrate_drp_project_metadata.py +++ b/server/portal/apps/projects/management/commands/migrate_drp_project_metadata.py @@ -215,6 +215,7 @@ def create_project(self, legacy_project_id, project_metadata_value): project_data = { **project_metadata_value, "project_id": system_id, + "cover_image": f"media/{workspace_id}/cover_image/{Path(project_metadata_value['cover_image']).name}" if project_metadata_value.get('cover_image') else None, } print(f"Creating project {project_data['title']}") @@ -248,12 +249,17 @@ def migrate_project(self, project): "is_review_project": False, "is_published_project": True if self.publication else False, "publication_date": str(project['creation_date']) if self.publication else None, - "related_publications": self.get_related_publications(project['id']) + "related_publications": self.get_related_publications(project['id']), + "cover_image": project['cover_pic'] if project['cover_pic'] else None, } if self.dry_run: project_id = 'dry_run_project_id' project_data['project_id'] = project_id + if project_data.get('cover_image'): + project_data['cover_image'] = f"media/{project_id}/cover_image/{Path(project_data['cover_image']).name}" + else: + project_data['cover_image'] = None validated_project = SCHEMA_MAPPING[constants.PROJECT].model_validate(project_data) # print(f"Dry run: Successfully created project with legacy id: {project['id']}") else: From ba5bef90527bfa416f45df552b6b5dbaa1efe6a0 Mon Sep 17 00:00:00 2001 From: Van Go <35277477+van-go@users.noreply.github.com> Date: Tue, 25 Feb 2025 10:04:06 -0600 Subject: [PATCH 168/328] Wc 129 dataset form redirect fix (#1006) * refactor: remove unnecessary comment in drp.sagas.js * Refactor path generation logic in DataFilesFormModal * Refactor path generation logic in DataFilesFormModal and executeOperation function in drp.sagas.js * Refactor reloadPage function to simplify path handling in DataFilesFormModal * fix incorrect redirect bug --------- Co-authored-by: Jake Rosenberg Co-authored-by: Shayan Khan --- .../DataFilesModals/DataFilesFormModal.jsx | 21 ++++++++----------- client/src/redux/sagas/_custom/drp.sagas.js | 3 ++- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx index 29b5be8e6a..090b8148c3 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesFormModal.jsx @@ -15,19 +15,16 @@ const DataFilesFormModal = () => { const history = useHistory(); const location = useLocation(); - const reloadPage = (updatedPath = '') => { - // Updated regex to capture the URL up until the last project segment - let projectUrl = location.pathname.replace( - /(\/projects\/[^/]+\/[^/]+)\/?.*/, - '$1' - ); + const reloadPage = (updatedPath = '') => { + const match = location.pathname.match(/^\/workbench\/data\/tapis\/[^\/]+\/[^\/]+\/[^\/]+/); + if (!match) return; - if (projectUrl.endsWith('/')) { - projectUrl = projectUrl.slice(0, -1); - } - - const path = updatedPath ? `${projectUrl}/${updatedPath}` : `${projectUrl}`; - history.replace(path); + const projectUrl = match[0]; + + const cleanProjectUrl = projectUrl.replace(/\/$/, ''); + const cleanUpdatedPath = updatedPath.replace(/^\/+/, ''); + + history.replace(`${cleanProjectUrl}/${cleanUpdatedPath}`); }; const { form, selectedFile, formName, additionalData, useReloadCallback } = diff --git a/client/src/redux/sagas/_custom/drp.sagas.js b/client/src/redux/sagas/_custom/drp.sagas.js index dee4f5f241..1644053aa9 100644 --- a/client/src/redux/sagas/_custom/drp.sagas.js +++ b/client/src/redux/sagas/_custom/drp.sagas.js @@ -50,9 +50,10 @@ function* executeOperation( ? `${path}/${file.path.split('/').pop()}` : path; + // Check if the file name has changed. If not, keep the same path const reloadPath = isEdit && file.name !== values.name - ? newPath.replace(file.name, values.name) + ? newPath.replace(`/${file.name}`, `/${values.name}`) : newPath; yield call(reloadCallback, reloadPath); From 0cfa7c7e2e4b52f737940f963476f7b0956dc7e7 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Tue, 25 Feb 2025 12:33:04 -0600 Subject: [PATCH 169/328] run file processing for files with additional metadata --- server/portal/apps/projects/tasks.py | 8 ++++++++ server/portal/apps/projects/views.py | 3 ++- server/portal/libs/agave/operations.py | 7 ++++--- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/server/portal/apps/projects/tasks.py b/server/portal/apps/projects/tasks.py index c1a74e64dc..e2291859d6 100644 --- a/server/portal/apps/projects/tasks.py +++ b/server/portal/apps/projects/tasks.py @@ -289,6 +289,14 @@ def process_file(self, project_id: str, path: str, user_access_token: str, encod adv_image = conf_raw(value, file) except Exception as e: logger.error(f'Could not generate advanced image for {file_name} due to error: {e}') + + Notification.objects.create(**{ + Notification.EVENT_TYPE: 'default', + Notification.STATUS: Notification.INFO, + Notification.USER: username, + Notification.MESSAGE: f'Failed to Generate Images for {Path(path).name}', + }) + return try: diff --git a/server/portal/apps/projects/views.py b/server/portal/apps/projects/views.py index a17e843da4..8cf90153df 100644 --- a/server/portal/apps/projects/views.py +++ b/server/portal/apps/projects/views.py @@ -457,7 +457,8 @@ def patch(self, request: HttpRequest, project_id: str): if value['data_type'] == 'file': try: patch_file_obj_entity(client, project_id, value, path) - process_file.delay(project_id, path.lstrip("/"), client.access_token.access_token) + if (len(value) > 1): + process_file.delay(project_id, path.lstrip("/"), client.access_token.access_token) except Exception as exc: raise ApiException("Error updating file metadata", status=500) from exc else: diff --git a/server/portal/libs/agave/operations.py b/server/portal/libs/agave/operations.py index 66129947fa..3e25c4ba62 100644 --- a/server/portal/libs/agave/operations.py +++ b/server/portal/libs/agave/operations.py @@ -561,9 +561,10 @@ def upload(client, system, path, uploaded_file, metadata=None): add_file_associations(root_node['uuid'], [file_obj]) # additional processing for files - encoded_file = base64.b64encode(uploaded_file.read()).decode('utf-8') - uploaded_file.seek(0) - transaction.on_commit(lambda: process_file.delay(file_obj.system, file_obj.path, client.access_token.access_token, encoded_file)) + if len(metadata) > 1: + encoded_file = base64.b64encode(uploaded_file.read()).decode('utf-8') + uploaded_file.seek(0) + transaction.on_commit(lambda: process_file.delay(file_obj.system, file_obj.path, client.access_token.access_token, encoded_file)) response_json = client.files.insert(systemId=system, path=dest_path, file=uploaded_file) tapis_indexer.apply_async(kwargs={'access_token': client.access_token.access_token, From b357d90f9bbda618a0364d395ec6b36e9dd9f8a5 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Tue, 25 Feb 2025 13:02:19 -0600 Subject: [PATCH 170/328] fix cover image url bug --- server/portal/apps/projects/views.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/server/portal/apps/projects/views.py b/server/portal/apps/projects/views.py index 8cf90153df..d70fe09298 100644 --- a/server/portal/apps/projects/views.py +++ b/server/portal/apps/projects/views.py @@ -36,7 +36,6 @@ from portal.apps.projects.workspace_operations.graph_operations import add_node_to_project, initialize_project_graph, get_node_from_path from portal.apps.projects.tasks import process_file, sync_files_without_metadata from portal.libs.files.file_processing import resize_cover_image -from portal.libs.agave.utils import service_account from django.http.multipartparser import MultiPartParser LOGGER = logging.getLogger(__name__) @@ -210,9 +209,9 @@ def get(self, request, project_id=None, system_id=None): if prj["cover_image"] is not None: service_client = service_account() - if prj["is_published_project"]: + if prj.get("is_published_project", False): root_system = settings.PORTAL_PROJECTS_PUBLISHED_ROOT_SYSTEM_NAME - elif prj["is_review_project"]: + elif prj.get("is_review_project", False): root_system = settings.PORTAL_PROJECTS_REVIEW_ROOT_SYSTEM_NAME else: root_system = settings.PORTAL_PROJECTS_ROOT_SYSTEM_NAME From 599bf671868eac8f2b7ca92003da4b6e155ec922 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Tue, 25 Feb 2025 14:51:38 -0600 Subject: [PATCH 171/328] review project cover image bug fix --- server/portal/apps/projects/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/portal/apps/projects/views.py b/server/portal/apps/projects/views.py index d70fe09298..76b6852a0d 100644 --- a/server/portal/apps/projects/views.py +++ b/server/portal/apps/projects/views.py @@ -212,7 +212,7 @@ def get(self, request, project_id=None, system_id=None): if prj.get("is_published_project", False): root_system = settings.PORTAL_PROJECTS_PUBLISHED_ROOT_SYSTEM_NAME elif prj.get("is_review_project", False): - root_system = settings.PORTAL_PROJECTS_REVIEW_ROOT_SYSTEM_NAME + root_system = settings.PORTAL_PROJECTS_ROOT_REVIEW_SYSTEM_NAME else: root_system = settings.PORTAL_PROJECTS_ROOT_SYSTEM_NAME From 292ccc05e8c03c5512b9b68d32ecb14bdedab7d6 Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Tue, 25 Feb 2025 15:18:08 -0600 Subject: [PATCH 172/328] fix duplicate publication namespace warning --- server/portal/urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/portal/urls.py b/server/portal/urls.py index fa8294bae8..990bdc7330 100644 --- a/server/portal/urls.py +++ b/server/portal/urls.py @@ -91,7 +91,7 @@ path('api/projects/', include('portal.apps.projects.urls', namespace='projects')), path('api/site-search/', include('portal.apps.site_search.api.urls', namespace='site_search_api')), path('api/forms/', include('portal.apps.forms.urls', namespace='forms')), - path('api/publications/', include('portal.apps.publications.urls', namespace='publications')), + path('api/publications/', include('portal.apps.publications.urls', namespace='publications_api')), # webhooks path('webhooks/', include('portal.apps.webhooks.urls', namespace='webhooks')), From 857036015b8db59ccea863d8ed49a25e0bca0cfa Mon Sep 17 00:00:00 2001 From: Shayan Khan Date: Tue, 25 Feb 2025 15:40:54 -0600 Subject: [PATCH 173/328] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c4068503a2..831e749f09 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ The base Portal code for TACC WMA Workspace Portals # Local Development Setup -## Prerequisites for running the portal application +## Prerequisites for running the portal application: * Docker > 20.10.7 * Docker Compose > 1.29.x From 14b212762acdcd6f6c3d3dc38b793c08c4a3e50d Mon Sep 17 00:00:00 2001 From: shayanaijaz Date: Tue, 25 Feb 2025 15:44:25 -0600 Subject: [PATCH 174/328] update publication url namespace --- server/portal/apps/publications/urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/portal/apps/publications/urls.py b/server/portal/apps/publications/urls.py index 41c4aee40a..f2ba371362 100644 --- a/server/portal/apps/publications/urls.py +++ b/server/portal/apps/publications/urls.py @@ -3,7 +3,7 @@ from portal.apps.publications import views from django.urls import path -app_name = 'publications' +app_name = 'publications_api' urlpatterns = [ path('publication-request/', views.PublicationRequestView.as_view(), name='publication_request'), path('publication-request//', views.PublicationRequestView.as_view(), name='publication_request_detail'), From 1ee184b1438ed9170020cc29e3f8fa3dc98318eb Mon Sep 17 00:00:00 2001 From: Van Go <35277477+van-go@users.noreply.github.com> Date: Fri, 28 Feb 2025 10:45:50 -0600 Subject: [PATCH 175/328] Requested changes to Analysis Dataset view (#1056) * Requested changes to Analysis Dataset view * Remove console logs from DynamicForm component and streamline option filtering logic * removing a comment --------- Co-authored-by: Shayan Khan Co-authored-by: shayanaijaz --- .../_common/Form/DynamicForm/DynamicForm.jsx | 11 ++++++----- .../_custom/drp/utils/hooks/useDrpDatasetModals.js | 5 +++-- server/portal/apps/_custom/drp/models.py | 3 ++- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx b/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx index 16f0766f77..e80d82bc79 100644 --- a/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx +++ b/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx @@ -23,17 +23,18 @@ const DynamicForm = ({ initialFormFields, onChange }) => { modifiedField ) => { const { dependency } = field; - - const filteredOptions = field.options.filter( - (option) => option.dependentId == values[dependency.name] - ); + const filteredOptions = field.options.filter(option => { + if (option.value === 'other') { + return true; + } + return option.dependentId === values[dependency.name]; + }); const updatedOptions = [{ value: '', label: '' }, ...filteredOptions]; // Only update the field value if the modified field is the dependency field if (modifiedField && modifiedField.name === dependency.name) { setFieldValue(field.name, updatedOptions[0].value); } - return { ...field, hidden: false, diff --git a/client/src/components/_custom/drp/utils/hooks/useDrpDatasetModals.js b/client/src/components/_custom/drp/utils/hooks/useDrpDatasetModals.js index 2ba2ecab0d..ec61a5407c 100644 --- a/client/src/components/_custom/drp/utils/hooks/useDrpDatasetModals.js +++ b/client/src/components/_custom/drp/utils/hooks/useDrpDatasetModals.js @@ -100,7 +100,7 @@ const useDrpDatasetModals = ( }; }) ); - } else if (field.name === 'base_origin_data') { + } else if (field.name === 'digital_dataset') { field.options.push( ...originDatasets.map((originData) => { return { @@ -108,7 +108,8 @@ const useDrpDatasetModals = ( label: originData.value.name, dependentId: originData.value.sample, }; - }) + }), + { value: 'other', label: 'Other (Specify Below)' } ); } }); diff --git a/server/portal/apps/_custom/drp/models.py b/server/portal/apps/_custom/drp/models.py index e8d2170290..fc6cd0f5da 100644 --- a/server/portal/apps/_custom/drp/models.py +++ b/server/portal/apps/_custom/drp/models.py @@ -147,7 +147,7 @@ class DrpDatasetMetadata(DrpMetadataModel): ) name: str - description: str + description: Optional[str] = None data_type: Literal[ "sample", "origin_data", @@ -235,3 +235,4 @@ class DrpAnalysisDatasetMetadata(DrpDatasetMetadata): sample: str # base_origin_data: Optional[str] = None digital_dataset: Optional[str] = None + digital_dataset_other_information: Optional[str] = None From 7cac920dd2358382294a0bc17ad8556eac5afd1e Mon Sep 17 00:00:00 2001 From: Van Go <35277477+van-go@users.noreply.github.com> Date: Fri, 28 Feb 2025 18:27:00 -0600 Subject: [PATCH 176/328] Requested changes to Edit Dataset view (#1059) * Requested changes to Edit Dataset view * make certain unused field optional * refactor data fetching in DataFilesProjectEditDescriptionModalAddon and add publication_type field to DrpProjectRelatedPublications model * update to use title_field and bug fix * small update * fix to exclude null values in project metadata response --------- Co-authored-by: shayanaijaz --- .../DataFilesProjectEditDescriptionModal.jsx | 4 ++-- .../_common/Form/DynamicForm/DynamicForm.jsx | 7 +++++-- .../Form/DynamicForm/DynamicForm.module.scss | 8 ++++++++ .../src/components/_common/Form/FormField.jsx | 3 +-- .../src/components/_common/Form/FormField.scss | 5 +++++ ...ataFilesProjectEditDescriptionModalAddon.jsx | 17 +++++++++-------- ...ProjectEditDescriptionModalAddon.module.scss | 5 +++++ server/portal/apps/_custom/drp/models.py | 13 +++++++++---- .../project_meta_operations.py | 4 +++- 9 files changed, 47 insertions(+), 19 deletions(-) create mode 100644 client/src/components/_custom/drp/DataFilesProjectEditDescriptionModalAddon/DataFilesProjectEditDescriptionModalAddon.module.scss diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesProjectEditDescriptionModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesProjectEditDescriptionModal.jsx index 03b5aa54c6..cd6b780c3a 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesProjectEditDescriptionModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesProjectEditDescriptionModal.jsx @@ -98,7 +98,7 @@ const DataFilesProjectEditDescriptionModal = () => { aria-label="title" label={
- Workspace Title{' '} + Dataset Title{' '} (Maximum 150 characters) @@ -110,7 +110,7 @@ const DataFilesProjectEditDescriptionModal = () => { aria-label="description" label={
- Workspace Description{' '} + Dataset Description{' '} (Maximum 800 characters) diff --git a/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx b/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx index e80d82bc79..ee48c20446 100644 --- a/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx +++ b/client/src/components/_common/Form/DynamicForm/DynamicForm.jsx @@ -217,6 +217,9 @@ const DynamicForm = ({ initialFormFields, onChange }) => {

{field.label}

+ {field.description && ( +

{field.description}

+ )}
{values[field.name]?.map((_, index) => (
{ div:first-child { padding-bottom: 20px; } + +.array-field-description { + font-size: 12px; + font-style: italic; + color: #666; + margin-top: -5px; /* Reduce space between title and description */ + margin-bottom: 10px; /* Add space before form fields */ +} diff --git a/client/src/components/_common/Form/FormField.jsx b/client/src/components/_common/Form/FormField.jsx index d4b7c1d688..8ca13b71ee 100644 --- a/client/src/components/_common/Form/FormField.jsx +++ b/client/src/components/_common/Form/FormField.jsx @@ -121,6 +121,7 @@ const FormField = ({ {label && hasAddon ? : null} {label && !hasAddon ? : null} + {!hasAddon ? : null} {tapisFile ? ( <> - + + + ); +}; + +export default DataFilesLargeDownloadModal; diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesLargeDownloadModal.module.scss b/client/src/components/DataFiles/DataFilesModals/DataFilesLargeDownloadModal.module.scss new file mode 100644 index 0000000000..d59f869a07 --- /dev/null +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesLargeDownloadModal.module.scss @@ -0,0 +1,10 @@ +/* Rules placed for consistent styling */ +.info { + margin-top: 1rem; + margin-bottom: 0; +} + +/* Enlarges the button to accommodate its text */ +.linkButton { + max-width: unset; +} diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesModalTables/DataFilesModalListingTable.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesModalTables/DataFilesModalListingTable.jsx index 93f318597b..6072ef4e1f 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesModalTables/DataFilesModalListingTable.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesModalTables/DataFilesModalListingTable.jsx @@ -120,7 +120,7 @@ const DataFilesModalButtonCell = ({