diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index e611106..0000000 Binary files a/.DS_Store and /dev/null differ diff --git a/.gitignore b/.gitignore index e2101d0..50cafae 100644 --- a/.gitignore +++ b/.gitignore @@ -139,6 +139,7 @@ celerybeat.pid .env .envrc .venv +.myvenv/ env/ venv/ ENV/ @@ -206,3 +207,6 @@ cython_debug/ marimo/_static/ marimo/_lsp/ __marimo__/ + +#macOS metadata +.DS_Store diff --git a/.myvenv/bin/Activate.ps1 b/.myvenv/bin/Activate.ps1 deleted file mode 100644 index 2fb3852..0000000 --- a/.myvenv/bin/Activate.ps1 +++ /dev/null @@ -1,241 +0,0 @@ -<# -.Synopsis -Activate a Python virtual environment for the current PowerShell session. - -.Description -Pushes the python executable for a virtual environment to the front of the -$Env:PATH environment variable and sets the prompt to signify that you are -in a Python virtual environment. Makes use of the command line switches as -well as the `pyvenv.cfg` file values present in the virtual environment. - -.Parameter VenvDir -Path to the directory that contains the virtual environment to activate. The -default value for this is the parent of the directory that the Activate.ps1 -script is located within. - -.Parameter Prompt -The prompt prefix to display when this virtual environment is activated. By -default, this prompt is the name of the virtual environment folder (VenvDir) -surrounded by parentheses and followed by a single space (ie. '(.venv) '). - -.Example -Activate.ps1 -Activates the Python virtual environment that contains the Activate.ps1 script. - -.Example -Activate.ps1 -Verbose -Activates the Python virtual environment that contains the Activate.ps1 script, -and shows extra information about the activation as it executes. - -.Example -Activate.ps1 -VenvDir C:\Users\MyUser\Common\.venv -Activates the Python virtual environment located in the specified location. - -.Example -Activate.ps1 -Prompt "MyPython" -Activates the Python virtual environment that contains the Activate.ps1 script, -and prefixes the current prompt with the specified string (surrounded in -parentheses) while the virtual environment is active. - -.Notes -On Windows, it may be required to enable this Activate.ps1 script by setting the -execution policy for the user. You can do this by issuing the following PowerShell -command: - -PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser - -For more information on Execution Policies: -https://go.microsoft.com/fwlink/?LinkID=135170 - -#> -Param( - [Parameter(Mandatory = $false)] - [String] - $VenvDir, - [Parameter(Mandatory = $false)] - [String] - $Prompt -) - -<# Function declarations --------------------------------------------------- #> - -<# -.Synopsis -Remove all shell session elements added by the Activate script, including the -addition of the virtual environment's Python executable from the beginning of -the PATH variable. - -.Parameter NonDestructive -If present, do not remove this function from the global namespace for the -session. - -#> -function global:deactivate ([switch]$NonDestructive) { - # Revert to original values - - # The prior prompt: - if (Test-Path -Path Function:_OLD_VIRTUAL_PROMPT) { - Copy-Item -Path Function:_OLD_VIRTUAL_PROMPT -Destination Function:prompt - Remove-Item -Path Function:_OLD_VIRTUAL_PROMPT - } - - # The prior PYTHONHOME: - if (Test-Path -Path Env:_OLD_VIRTUAL_PYTHONHOME) { - Copy-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME -Destination Env:PYTHONHOME - Remove-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME - } - - # The prior PATH: - if (Test-Path -Path Env:_OLD_VIRTUAL_PATH) { - Copy-Item -Path Env:_OLD_VIRTUAL_PATH -Destination Env:PATH - Remove-Item -Path Env:_OLD_VIRTUAL_PATH - } - - # Just remove the VIRTUAL_ENV altogether: - if (Test-Path -Path Env:VIRTUAL_ENV) { - Remove-Item -Path env:VIRTUAL_ENV - } - - # Just remove the _PYTHON_VENV_PROMPT_PREFIX altogether: - if (Get-Variable -Name "_PYTHON_VENV_PROMPT_PREFIX" -ErrorAction SilentlyContinue) { - Remove-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Scope Global -Force - } - - # Leave deactivate function in the global namespace if requested: - if (-not $NonDestructive) { - Remove-Item -Path function:deactivate - } -} - -<# -.Description -Get-PyVenvConfig parses the values from the pyvenv.cfg file located in the -given folder, and returns them in a map. - -For each line in the pyvenv.cfg file, if that line can be parsed into exactly -two strings separated by `=` (with any amount of whitespace surrounding the =) -then it is considered a `key = value` line. The left hand string is the key, -the right hand is the value. - -If the value starts with a `'` or a `"` then the first and last character is -stripped from the value before being captured. - -.Parameter ConfigDir -Path to the directory that contains the `pyvenv.cfg` file. -#> -function Get-PyVenvConfig( - [String] - $ConfigDir -) { - Write-Verbose "Given ConfigDir=$ConfigDir, obtain values in pyvenv.cfg" - - # Ensure the file exists, and issue a warning if it doesn't (but still allow the function to continue). - $pyvenvConfigPath = Join-Path -Resolve -Path $ConfigDir -ChildPath 'pyvenv.cfg' -ErrorAction Continue - - # An empty map will be returned if no config file is found. - $pyvenvConfig = @{ } - - if ($pyvenvConfigPath) { - - Write-Verbose "File exists, parse `key = value` lines" - $pyvenvConfigContent = Get-Content -Path $pyvenvConfigPath - - $pyvenvConfigContent | ForEach-Object { - $keyval = $PSItem -split "\s*=\s*", 2 - if ($keyval[0] -and $keyval[1]) { - $val = $keyval[1] - - # Remove extraneous quotations around a string value. - if ("'""".Contains($val.Substring(0, 1))) { - $val = $val.Substring(1, $val.Length - 2) - } - - $pyvenvConfig[$keyval[0]] = $val - Write-Verbose "Adding Key: '$($keyval[0])'='$val'" - } - } - } - return $pyvenvConfig -} - - -<# Begin Activate script --------------------------------------------------- #> - -# Determine the containing directory of this script -$VenvExecPath = Split-Path -Parent $MyInvocation.MyCommand.Definition -$VenvExecDir = Get-Item -Path $VenvExecPath - -Write-Verbose "Activation script is located in path: '$VenvExecPath'" -Write-Verbose "VenvExecDir Fullname: '$($VenvExecDir.FullName)" -Write-Verbose "VenvExecDir Name: '$($VenvExecDir.Name)" - -# Set values required in priority: CmdLine, ConfigFile, Default -# First, get the location of the virtual environment, it might not be -# VenvExecDir if specified on the command line. -if ($VenvDir) { - Write-Verbose "VenvDir given as parameter, using '$VenvDir' to determine values" -} -else { - Write-Verbose "VenvDir not given as a parameter, using parent directory name as VenvDir." - $VenvDir = $VenvExecDir.Parent.FullName.TrimEnd("\\/") - Write-Verbose "VenvDir=$VenvDir" -} - -# Next, read the `pyvenv.cfg` file to determine any required value such -# as `prompt`. -$pyvenvCfg = Get-PyVenvConfig -ConfigDir $VenvDir - -# Next, set the prompt from the command line, or the config file, or -# just use the name of the virtual environment folder. -if ($Prompt) { - Write-Verbose "Prompt specified as argument, using '$Prompt'" -} -else { - Write-Verbose "Prompt not specified as argument to script, checking pyvenv.cfg value" - if ($pyvenvCfg -and $pyvenvCfg['prompt']) { - Write-Verbose " Setting based on value in pyvenv.cfg='$($pyvenvCfg['prompt'])'" - $Prompt = $pyvenvCfg['prompt']; - } - else { - Write-Verbose " Setting prompt based on parent's directory's name. (Is the directory name passed to venv module when creating the virutal environment)" - Write-Verbose " Got leaf-name of $VenvDir='$(Split-Path -Path $venvDir -Leaf)'" - $Prompt = Split-Path -Path $venvDir -Leaf - } -} - -Write-Verbose "Prompt = '$Prompt'" -Write-Verbose "VenvDir='$VenvDir'" - -# Deactivate any currently active virtual environment, but leave the -# deactivate function in place. -deactivate -nondestructive - -# Now set the environment variable VIRTUAL_ENV, used by many tools to determine -# that there is an activated venv. -$env:VIRTUAL_ENV = $VenvDir - -if (-not $Env:VIRTUAL_ENV_DISABLE_PROMPT) { - - Write-Verbose "Setting prompt to '$Prompt'" - - # Set the prompt to include the env name - # Make sure _OLD_VIRTUAL_PROMPT is global - function global:_OLD_VIRTUAL_PROMPT { "" } - Copy-Item -Path function:prompt -Destination function:_OLD_VIRTUAL_PROMPT - New-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Description "Python virtual environment prompt prefix" -Scope Global -Option ReadOnly -Visibility Public -Value $Prompt - - function global:prompt { - Write-Host -NoNewline -ForegroundColor Green "($_PYTHON_VENV_PROMPT_PREFIX) " - _OLD_VIRTUAL_PROMPT - } -} - -# Clear PYTHONHOME -if (Test-Path -Path Env:PYTHONHOME) { - Copy-Item -Path Env:PYTHONHOME -Destination Env:_OLD_VIRTUAL_PYTHONHOME - Remove-Item -Path Env:PYTHONHOME -} - -# Add the venv to the PATH -Copy-Item -Path Env:PATH -Destination Env:_OLD_VIRTUAL_PATH -$Env:PATH = "$VenvExecDir$([System.IO.Path]::PathSeparator)$Env:PATH" diff --git a/.myvenv/bin/activate b/.myvenv/bin/activate deleted file mode 100644 index 7906667..0000000 --- a/.myvenv/bin/activate +++ /dev/null @@ -1,66 +0,0 @@ -# This file must be used with "source bin/activate" *from bash* -# you cannot run it directly - -deactivate () { - # reset old environment variables - if [ -n "${_OLD_VIRTUAL_PATH:-}" ] ; then - PATH="${_OLD_VIRTUAL_PATH:-}" - export PATH - unset _OLD_VIRTUAL_PATH - fi - if [ -n "${_OLD_VIRTUAL_PYTHONHOME:-}" ] ; then - PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}" - export PYTHONHOME - unset _OLD_VIRTUAL_PYTHONHOME - fi - - # This should detect bash and zsh, which have a hash command that must - # be called to get it to forget past commands. Without forgetting - # past commands the $PATH changes we made may not be respected - if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then - hash -r 2> /dev/null - fi - - if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then - PS1="${_OLD_VIRTUAL_PS1:-}" - export PS1 - unset _OLD_VIRTUAL_PS1 - fi - - unset VIRTUAL_ENV - if [ ! "${1:-}" = "nondestructive" ] ; then - # Self destruct! - unset -f deactivate - fi -} - -# unset irrelevant variables -deactivate nondestructive - -VIRTUAL_ENV="/Users/ashmit/CMIL/fusion-packages/.myvenv" -export VIRTUAL_ENV - -_OLD_VIRTUAL_PATH="$PATH" -PATH="$VIRTUAL_ENV/bin:$PATH" -export PATH - -# unset PYTHONHOME if set -# this will fail if PYTHONHOME is set to the empty string (which is bad anyway) -# could use `if (set -u; : $PYTHONHOME) ;` in bash -if [ -n "${PYTHONHOME:-}" ] ; then - _OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}" - unset PYTHONHOME -fi - -if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then - _OLD_VIRTUAL_PS1="${PS1:-}" - PS1="(.myvenv) ${PS1:-}" - export PS1 -fi - -# This should detect bash and zsh, which have a hash command that must -# be called to get it to forget past commands. Without forgetting -# past commands the $PATH changes we made may not be respected -if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then - hash -r 2> /dev/null -fi diff --git a/.myvenv/bin/activate.csh b/.myvenv/bin/activate.csh deleted file mode 100644 index 990edbb..0000000 --- a/.myvenv/bin/activate.csh +++ /dev/null @@ -1,25 +0,0 @@ -# This file must be used with "source bin/activate.csh" *from csh*. -# You cannot run it directly. -# Created by Davide Di Blasi . -# Ported to Python 3.3 venv by Andrew Svetlov - -alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; test "\!:*" != "nondestructive" && unalias deactivate' - -# Unset irrelevant variables. -deactivate nondestructive - -setenv VIRTUAL_ENV "/Users/ashmit/CMIL/fusion-packages/.myvenv" - -set _OLD_VIRTUAL_PATH="$PATH" -setenv PATH "$VIRTUAL_ENV/bin:$PATH" - - -set _OLD_VIRTUAL_PROMPT="$prompt" - -if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then - set prompt = "(.myvenv) $prompt" -endif - -alias pydoc python -m pydoc - -rehash diff --git a/.myvenv/bin/activate.fish b/.myvenv/bin/activate.fish deleted file mode 100644 index 77e8e83..0000000 --- a/.myvenv/bin/activate.fish +++ /dev/null @@ -1,64 +0,0 @@ -# This file must be used with "source /bin/activate.fish" *from fish* -# (https://fishshell.com/); you cannot run it directly. - -function deactivate -d "Exit virtual environment and return to normal shell environment" - # reset old environment variables - if test -n "$_OLD_VIRTUAL_PATH" - set -gx PATH $_OLD_VIRTUAL_PATH - set -e _OLD_VIRTUAL_PATH - end - if test -n "$_OLD_VIRTUAL_PYTHONHOME" - set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME - set -e _OLD_VIRTUAL_PYTHONHOME - end - - if test -n "$_OLD_FISH_PROMPT_OVERRIDE" - functions -e fish_prompt - set -e _OLD_FISH_PROMPT_OVERRIDE - functions -c _old_fish_prompt fish_prompt - functions -e _old_fish_prompt - end - - set -e VIRTUAL_ENV - if test "$argv[1]" != "nondestructive" - # Self-destruct! - functions -e deactivate - end -end - -# Unset irrelevant variables. -deactivate nondestructive - -set -gx VIRTUAL_ENV "/Users/ashmit/CMIL/fusion-packages/.myvenv" - -set -gx _OLD_VIRTUAL_PATH $PATH -set -gx PATH "$VIRTUAL_ENV/bin" $PATH - -# Unset PYTHONHOME if set. -if set -q PYTHONHOME - set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME - set -e PYTHONHOME -end - -if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" - # fish uses a function instead of an env var to generate the prompt. - - # Save the current fish_prompt function as the function _old_fish_prompt. - functions -c fish_prompt _old_fish_prompt - - # With the original prompt function renamed, we can override with our own. - function fish_prompt - # Save the return status of the last command. - set -l old_status $status - - # Output the venv prompt; color taken from the blue of the Python logo. - printf "%s%s%s" (set_color 4B8BBE) "(.myvenv) " (set_color normal) - - # Restore the return status of the previous command. - echo "exit $old_status" | . - # Output the original/"old" prompt. - _old_fish_prompt - end - - set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV" -end diff --git a/.myvenv/bin/debugpy b/.myvenv/bin/debugpy deleted file mode 100755 index e3abd74..0000000 --- a/.myvenv/bin/debugpy +++ /dev/null @@ -1,8 +0,0 @@ -#!/Users/ashmit/CMIL/fusion-packages/.myvenv/bin/python -# -*- coding: utf-8 -*- -import re -import sys -from debugpy.server.cli import main -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) diff --git a/.myvenv/bin/debugpy-adapter b/.myvenv/bin/debugpy-adapter deleted file mode 100755 index 17d5fd0..0000000 --- a/.myvenv/bin/debugpy-adapter +++ /dev/null @@ -1,8 +0,0 @@ -#!/Users/ashmit/CMIL/fusion-packages/.myvenv/bin/python -# -*- coding: utf-8 -*- -import re -import sys -from debugpy.adapter.__main__ import main -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) diff --git a/.myvenv/bin/ipython b/.myvenv/bin/ipython deleted file mode 100755 index 4932643..0000000 --- a/.myvenv/bin/ipython +++ /dev/null @@ -1,8 +0,0 @@ -#!/Users/ashmit/CMIL/fusion-packages/.myvenv/bin/python -# -*- coding: utf-8 -*- -import re -import sys -from IPython import start_ipython -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(start_ipython()) diff --git a/.myvenv/bin/ipython3 b/.myvenv/bin/ipython3 deleted file mode 100755 index 4932643..0000000 --- a/.myvenv/bin/ipython3 +++ /dev/null @@ -1,8 +0,0 @@ -#!/Users/ashmit/CMIL/fusion-packages/.myvenv/bin/python -# -*- coding: utf-8 -*- -import re -import sys -from IPython import start_ipython -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(start_ipython()) diff --git a/.myvenv/bin/jupyter b/.myvenv/bin/jupyter deleted file mode 100755 index f99ebdf..0000000 --- a/.myvenv/bin/jupyter +++ /dev/null @@ -1,8 +0,0 @@ -#!/Users/ashmit/CMIL/fusion-packages/.myvenv/bin/python -# -*- coding: utf-8 -*- -import re -import sys -from jupyter_core.command import main -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) diff --git a/.myvenv/bin/jupyter-kernel b/.myvenv/bin/jupyter-kernel deleted file mode 100755 index d2428ec..0000000 --- a/.myvenv/bin/jupyter-kernel +++ /dev/null @@ -1,8 +0,0 @@ -#!/Users/ashmit/CMIL/fusion-packages/.myvenv/bin/python -# -*- coding: utf-8 -*- -import re -import sys -from jupyter_client.kernelapp import main -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) diff --git a/.myvenv/bin/jupyter-kernelspec b/.myvenv/bin/jupyter-kernelspec deleted file mode 100755 index b281205..0000000 --- a/.myvenv/bin/jupyter-kernelspec +++ /dev/null @@ -1,8 +0,0 @@ -#!/Users/ashmit/CMIL/fusion-packages/.myvenv/bin/python -# -*- coding: utf-8 -*- -import re -import sys -from jupyter_client.kernelspecapp import KernelSpecApp -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(KernelSpecApp.launch_instance()) diff --git a/.myvenv/bin/jupyter-migrate b/.myvenv/bin/jupyter-migrate deleted file mode 100755 index de4b676..0000000 --- a/.myvenv/bin/jupyter-migrate +++ /dev/null @@ -1,8 +0,0 @@ -#!/Users/ashmit/CMIL/fusion-packages/.myvenv/bin/python -# -*- coding: utf-8 -*- -import re -import sys -from jupyter_core.migrate import main -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) diff --git a/.myvenv/bin/jupyter-run b/.myvenv/bin/jupyter-run deleted file mode 100755 index 64d3cd0..0000000 --- a/.myvenv/bin/jupyter-run +++ /dev/null @@ -1,8 +0,0 @@ -#!/Users/ashmit/CMIL/fusion-packages/.myvenv/bin/python -# -*- coding: utf-8 -*- -import re -import sys -from jupyter_client.runapp import RunApp -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(RunApp.launch_instance()) diff --git a/.myvenv/bin/jupyter-troubleshoot b/.myvenv/bin/jupyter-troubleshoot deleted file mode 100755 index f302f4b..0000000 --- a/.myvenv/bin/jupyter-troubleshoot +++ /dev/null @@ -1,8 +0,0 @@ -#!/Users/ashmit/CMIL/fusion-packages/.myvenv/bin/python -# -*- coding: utf-8 -*- -import re -import sys -from jupyter_core.troubleshoot import main -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) diff --git a/.myvenv/bin/pip b/.myvenv/bin/pip deleted file mode 100755 index 3793c8d..0000000 --- a/.myvenv/bin/pip +++ /dev/null @@ -1,8 +0,0 @@ -#!/Users/ashmit/CMIL/fusion-packages/.myvenv/bin/python3 -# -*- coding: utf-8 -*- -import re -import sys -from pip._internal.cli.main import main -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) diff --git a/.myvenv/bin/pip3 b/.myvenv/bin/pip3 deleted file mode 100755 index 3793c8d..0000000 --- a/.myvenv/bin/pip3 +++ /dev/null @@ -1,8 +0,0 @@ -#!/Users/ashmit/CMIL/fusion-packages/.myvenv/bin/python3 -# -*- coding: utf-8 -*- -import re -import sys -from pip._internal.cli.main import main -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) diff --git a/.myvenv/bin/pip3.9 b/.myvenv/bin/pip3.9 deleted file mode 100755 index 3793c8d..0000000 --- a/.myvenv/bin/pip3.9 +++ /dev/null @@ -1,8 +0,0 @@ -#!/Users/ashmit/CMIL/fusion-packages/.myvenv/bin/python3 -# -*- coding: utf-8 -*- -import re -import sys -from pip._internal.cli.main import main -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) diff --git a/.myvenv/bin/pygmentize b/.myvenv/bin/pygmentize deleted file mode 100755 index 041904e..0000000 --- a/.myvenv/bin/pygmentize +++ /dev/null @@ -1,8 +0,0 @@ -#!/Users/ashmit/CMIL/fusion-packages/.myvenv/bin/python -# -*- coding: utf-8 -*- -import re -import sys -from pygments.cmdline import main -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) diff --git a/.myvenv/bin/python b/.myvenv/bin/python deleted file mode 120000 index b8a0adb..0000000 --- a/.myvenv/bin/python +++ /dev/null @@ -1 +0,0 @@ -python3 \ No newline at end of file diff --git a/.myvenv/bin/python3 b/.myvenv/bin/python3 deleted file mode 120000 index f25545f..0000000 --- a/.myvenv/bin/python3 +++ /dev/null @@ -1 +0,0 @@ -/Library/Developer/CommandLineTools/usr/bin/python3 \ No newline at end of file diff --git a/.myvenv/bin/python3.9 b/.myvenv/bin/python3.9 deleted file mode 120000 index b8a0adb..0000000 --- a/.myvenv/bin/python3.9 +++ /dev/null @@ -1 +0,0 @@ -python3 \ No newline at end of file diff --git a/.myvenv/pyvenv.cfg b/.myvenv/pyvenv.cfg deleted file mode 100644 index 4760c1f..0000000 --- a/.myvenv/pyvenv.cfg +++ /dev/null @@ -1,3 +0,0 @@ -home = /Library/Developer/CommandLineTools/usr/bin -include-system-site-packages = false -version = 3.9.6 diff --git a/.myvenv/share/jupyter/kernels/python3/kernel.json b/.myvenv/share/jupyter/kernels/python3/kernel.json deleted file mode 100644 index cca38a4..0000000 --- a/.myvenv/share/jupyter/kernels/python3/kernel.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "argv": [ - "python", - "-m", - "ipykernel_launcher", - "-f", - "{connection_file}" - ], - "display_name": "Python 3 (ipykernel)", - "language": "python", - "metadata": { - "debugger": true - } -} \ No newline at end of file diff --git a/.myvenv/share/jupyter/kernels/python3/logo-32x32.png b/.myvenv/share/jupyter/kernels/python3/logo-32x32.png deleted file mode 100644 index be81330..0000000 Binary files a/.myvenv/share/jupyter/kernels/python3/logo-32x32.png and /dev/null differ diff --git a/.myvenv/share/jupyter/kernels/python3/logo-64x64.png b/.myvenv/share/jupyter/kernels/python3/logo-64x64.png deleted file mode 100644 index eebbff6..0000000 Binary files a/.myvenv/share/jupyter/kernels/python3/logo-64x64.png and /dev/null differ diff --git a/.myvenv/share/jupyter/kernels/python3/logo-svg.svg b/.myvenv/share/jupyter/kernels/python3/logo-svg.svg deleted file mode 100644 index 467b07b..0000000 --- a/.myvenv/share/jupyter/kernels/python3/logo-svg.svg +++ /dev/null @@ -1,265 +0,0 @@ - - - - - - - - image/svg+xml - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/.myvenv/share/man/man1/ipython.1 b/.myvenv/share/man/man1/ipython.1 deleted file mode 100644 index 0f4a191..0000000 --- a/.myvenv/share/man/man1/ipython.1 +++ /dev/null @@ -1,60 +0,0 @@ -.\" Hey, EMACS: -*- nroff -*- -.\" First parameter, NAME, should be all caps -.\" Second parameter, SECTION, should be 1-8, maybe w/ subsection -.\" other parameters are allowed: see man(7), man(1) -.TH IPYTHON 1 "July 15, 2011" -.\" Please adjust this date whenever revising the manpage. -.\" -.\" Some roff macros, for reference: -.\" .nh disable hyphenation -.\" .hy enable hyphenation -.\" .ad l left justify -.\" .ad b justify to both left and right margins -.\" .nf disable filling -.\" .fi enable filling -.\" .br insert line break -.\" .sp insert n+1 empty lines -.\" for manpage-specific macros, see man(7) and groff_man(7) -.\" .SH section heading -.\" .SS secondary section heading -.\" -.\" -.\" To preview this page as plain text: nroff -man ipython.1 -.\" -.SH NAME -ipython \- Tools for Interactive Computing in Python. -.SH SYNOPSIS -.B ipython -.RI [ options ] " files" ... - -.B ipython subcommand -.RI [ options ] ... - -.SH DESCRIPTION -An interactive Python shell with automatic history (input and output), dynamic -object introspection, easier configuration, command completion, access to the -system shell, integration with numerical and scientific computing tools, -web notebook, Qt console, and more. - -For more information on how to use IPython, see 'ipython \-\-help', -or 'ipython \-\-help\-all' for all available command\(hyline options. - -.SH "ENVIRONMENT VARIABLES" -.sp -.PP -\fIIPYTHONDIR\fR -.RS 4 -This is the location where IPython stores all its configuration files. The default -is $HOME/.ipython if IPYTHONDIR is not defined. - -You can see the computed value of IPYTHONDIR with `ipython locate`. - -.SH FILES - -IPython uses various configuration files stored in profiles within IPYTHONDIR. -To generate the default configuration files and start configuring IPython, -do 'ipython profile create', and edit '*_config.py' files located in -IPYTHONDIR/profile_default. - -.SH AUTHORS -IPython is written by the IPython Development Team . diff --git a/fusion/plugins/plugins.py b/fusion/plugins/plugins.py index ec66499..b957f5c 100644 --- a/fusion/plugins/plugins.py +++ b/fusion/plugins/plugins.py @@ -7,7 +7,10 @@ import shutil import time import anndata +import shlex import pandas as pd +import socket +import json # Session store for submitted Slurm jobs _SLURM_JOBS = {} # job_id -> {'name': str, 'log': str, 'start': float} @@ -40,6 +43,11 @@ def _print_new_log_lines(log_path, last_pos): return new_pos except Exception: return last_pos + +def _is_hipergator(): + """Check if running on HiPerGator.""" + hostname = socket.gethostname() + return "ufhpc" in hostname def _stream_slurm_log(job_id, job_name, log_path, poll_interval=10): """Stream live log output. Blocks until job finishes or Ctrl+C.""" @@ -250,8 +258,10 @@ def _track_slurm_job(job_id, job_name, log_filename, poll_interval=10): def _get_jupyter_slurm_resources(analysis_type=None): """Return fixed Slurm resource limits for analysis jobs.""" if analysis_type=='label_transfer': - return "03:00:00", "64GB", 4 - return "03:00:00", "16gb", 2 + return "03:00:00", "64GB", 4, 0, None + if analysis_type in ('frozen_glom_segmentation', 'multicompartment_segmentation'): + return "03:00:00", "16gb", 2, 1, 'hpg-turin' + return "03:00:00", "16gb", 2, 0, None def get_hive_workspace_root(): """ @@ -282,6 +292,66 @@ def get_hive_workspace_root(): print(f"Warning: 'user-workspaces' structure not found in {current_path}.") return current_path +def _resolve_user_path(path, must_exist=True): + """ + Resolve a user-entered path to a real absolute path. + + Fixes: + cwd = /home/user/fusion_demo_notebooks + input = fusion_demo_notebooks/datasets/data_1 + final = /home/user/fusion_demo_notebooks/datasets/data_1 + + Also keeps /blue, /orange, and /home paths real. + """ + raw_path = str(path).strip().strip('"').strip("'") + raw_path = os.path.expanduser(raw_path) + + if os.path.isabs(raw_path): + resolved = os.path.abspath(raw_path) + if must_exist and not os.path.exists(resolved): + raise FileNotFoundError(f"Path not found: {resolved}") + return resolved + + cwd = os.getcwd() + normalized = raw_path.replace("\\", "/") + parts = [p for p in normalized.split("/") if p] + + candidates = [] + + if parts: + cur = cwd + while cur and cur != os.path.dirname(cur): + if os.path.basename(cur) == parts[0]: + candidates.append(os.path.abspath(os.path.join(os.path.dirname(cur), raw_path))) + cur = os.path.dirname(cur) + + if parts[0] in {"home", "blue", "orange"}: + candidates.append(os.path.abspath(os.path.join(os.sep, raw_path))) + + candidates.append(os.path.abspath(os.path.join(cwd, raw_path))) + + for candidate in dict.fromkeys(candidates): + if os.path.exists(candidate): + return candidate + + if must_exist: + checked = "\n".join(f" - {c}" for c in dict.fromkeys(candidates)) + raise FileNotFoundError(f"Path not found: {raw_path}\nChecked:\n{checked}") + + return candidates[0] + + +def _get_bind_root(path): + abs_path = os.path.abspath(path) + parts = abs_path.strip(os.sep).split(os.sep) + if not parts or not parts[0]: + return os.sep + return os.path.join(os.sep, parts[0]) + + +def _quote_path(path): + return shlex.quote(str(path)) + def sanitize_h5ad_obsm(h5ad_path): """ @@ -316,6 +386,81 @@ def sanitize_h5ad_obsm(h5ad_path): #print(f" Overwrote: {h5ad_path}") return True +CONFIG_PATH = os.path.expanduser("~/.fusion_config.json") + +def _get_apptainer_cache_lines(): + """Resolve the Apptainer cache directory and return shell export lines.""" + config = {} + + if os.path.exists(CONFIG_PATH): + try: + with open(CONFIG_PATH, "r") as f: + config = json.load(f) + except (OSError, json.JSONDecodeError): + config = {} + + saved_path = config.get("apptainer_cache") + + print("\nApptainer Cache Directory:") + print("-" * 40) + print(" [1] Default (home directory)") + print(" [2] Custom path") + if saved_path: + print(f" [3] Saved path: {saved_path}") + + valid_choices = {"1", "2"} + if saved_path: + valid_choices.add("3") + + while True: + options = "/".join(sorted(valid_choices)) + choice = input(f"Enter choice ({options}): ").strip() + if choice in valid_choices: + break + print("Please enter a valid choice.") + + if choice == "1": + return "" + + if choice == "3": + cache_path = saved_path + else: + while True: + entered_path = input("Enter cache directory path: ").strip() + if not entered_path: + print("Path cannot be empty.") + continue + + cache_path = os.path.abspath(os.path.expanduser(entered_path)) + + try: + os.makedirs(cache_path, exist_ok=True) + break + except OSError as e: + print(f"Could not create directory: {e}") + + config["apptainer_cache"] = cache_path + try: + with open(CONFIG_PATH, "w") as f: + json.dump(config, f, indent=2) + print("Cache path saved for future runs.") + except OSError as e: + print(f"Warning: Could not save cache path: {e}") + + try: + os.makedirs(cache_path, exist_ok=True) + tmp_path = os.path.join(cache_path, "tmp") + os.makedirs(tmp_path, exist_ok=True) + except OSError as e: + raise RuntimeError(f"Could not prepare Apptainer cache directory: {e}") from e + + print(f"Cache will be stored at: {cache_path}") + + return ( + f"export APPTAINER_CACHEDIR={_quote_path(cache_path)}\n" + f"export APPTAINER_TMPDIR={_quote_path(tmp_path)}" + ) + def run_apptainer_analysis(): """ Interactive function to generate and submit analysis tasks using Apptainer containers via Slurm. @@ -424,15 +569,18 @@ def run_apptainer_analysis(): while True: value = input(f"Enter {param} path: ").strip() if value: - user_params[param] = value - break + try: + user_params[param] = _resolve_user_path(value, must_exist=True) + break + except FileNotFoundError as e: + print(f"\nError: {e}\n") else: print(f"{param} is required. Please enter a value.") else: print(f"\nNo additional parameters required for {analysis_type.replace('_', ' ').title()}") if analysis_type == "label_transfer" and "counts_file" in user_params: - h5ad_abs = os.path.join(workspace_path, user_params["counts_file"].lstrip(os.sep)) + h5ad_abs = user_params["counts_file"] #print("\nChecking h5ad file for non-numeric 'DeepScence' obsm columns...") #print("─" * 55) try: @@ -446,14 +594,12 @@ def run_apptainer_analysis(): # Going up input_depth levels reaches the dataset root folder. primary = config.get('primary_input') if primary and primary in user_params: - rel_input = user_params[primary].lstrip(os.sep).lstrip('.') + input_abs = user_params[primary] input_depth = config.get('input_depth', 2) - - # Validate that the path has enough components for the configured depth. - # e.g. depth=2 requires at least "a/b/file" (2 separators). - path_parts = [p for p in rel_input.replace('\\', '/').split('/') if p] + + path_parts = [p for p in input_abs.replace('\\', '/').split('/') if p] if len(path_parts) <= input_depth: - print(f"\nWarning: '{rel_input}' is too shallow for this analysis.") + print(f"\nWarning: '{input_abs}' is too shallow for this analysis.") print(f" Expected a path at least {input_depth + 1} levels deep, e.g.:") if input_depth == 2: print(f" fusion_demo_notebooks/datasets/HBM355.CWFF.355/ometiff-pyramids/file.tif") @@ -461,16 +607,18 @@ def run_apptainer_analysis(): print(f" fusion_demo_notebooks/datasets/HBM355.CWFF.355/expr.h5ad") print("Please re-run and enter the correct path.\n") return None - - dataset_root = rel_input + + dataset_root = input_abs for _ in range(input_depth): dataset_root = os.path.dirname(dataset_root) - user_params['output_dir'] = f"{dataset_root}/{config['output_subdir']}" + + user_params['output_dir'] = os.path.join(dataset_root, config['output_subdir']) + if 'annotations_subdir' in config: - user_params['annotations_dir'] = f"{dataset_root}/{config['annotations_subdir']}" + user_params['annotations_dir'] = os.path.join(dataset_root, config['annotations_subdir']) if analysis_type == "spot_annotation": import glob as _glob - files_dir = os.path.join(workspace_path, dataset_root, "Files") + files_dir = os.path.join(dataset_root, "Files") matches = _glob.glob(os.path.join(files_dir, "*_integrated.rds")) if len(matches) == 0: print(f"\nError: No *_integrated.rds file found in {files_dir}") @@ -482,12 +630,12 @@ def run_apptainer_analysis(): print(f" {m}") print("Please ensure only one integrated RDS exists.\n") return None - rds_rel = os.path.relpath(matches[0], workspace_path) - user_params['input_file'] = rds_rel - print(f"Found integrated RDS: {rds_rel}") + rds_path = matches[0] + user_params['input_file'] = rds_path + print(f"Found integrated RDS: {rds_path}") if analysis_type == "spatial_aggregation": - segmented_ftu_dir = os.path.join(workspace_path, dataset_root, "Segmented_FTU") + segmented_ftu_dir = os.path.join(dataset_root, "Segmented_FTU") supported_exts = {'.json', '.geojson', '.xml', '.csv', '.parquet'} if os.path.isdir(segmented_ftu_dir): ann_files = sorted([ @@ -515,8 +663,8 @@ def run_apptainer_analysis(): valid = True for idx in indices: if 1 <= idx <= len(ann_files): - rel_path = os.path.join(dataset_root, "Segmented_FTU", ann_files[idx - 1]) - selected_paths.append(rel_path) + ann_path = os.path.join(dataset_root, "Segmented_FTU", ann_files[idx - 1]) + selected_paths.append(ann_path) else: print(f" {idx} is out of range (1-{len(ann_files)}). Please try again.") valid = False @@ -532,39 +680,57 @@ def run_apptainer_analysis(): job_name = analysis_type # Match resources to the current JupyterHub Slurm session - time_limit, mem_limit, cpus = _get_jupyter_slurm_resources(analysis_type) - + time_limit, mem_limit, cpus, gpus, partition = _get_jupyter_slurm_resources(analysis_type) + + cache_lines = _get_apptainer_cache_lines() + path_params = config.get('path_params', set()) + bind_roots = set() + + for param_name, param_value in user_params.items(): + if param_name in path_params and param_value is not None: + values = param_value if isinstance(param_value, list) else [param_value] + for value in values: + bind_roots.add(_get_bind_root(value)) + + bind_args = " ".join( + f"-B {_quote_path(root)}:{_quote_path(root)}" + for root in sorted(bind_roots) + ) + + if analysis_type in ( + "frozen_glom_segmentation", + "multicompartment_segmentation"): + NV='--nv' + else: + NV="" + apptainer_cmd_parts = [ - "apptainer exec", - f"-B {workspace_path}:{mount_point}", + f"apptainer exec {NV} {bind_args}", f"docker://{config['image']}", f"python {config['script']}" ] # Add user-provided and auto-derived parameters to the apptainer command. - # path_params are prefixed with the container mount point (/data). - # skip_script_params are used locally for derivation but not passed to the script. - path_params = config.get('path_params', set()) + # Paths are already absolute host paths, and the same roots are bound into the container. skip_script_params = config.get('skip_script_params', set()) + for param_name, param_value in user_params.items(): if param_name in skip_script_params: continue + if param_value is not None: - if param_name in path_params: - if isinstance(param_value, list): - paths_str = " ".join(f"{mount_point}/{p}" for p in param_value) - apptainer_cmd_parts.append(f"--{param_name} {paths_str}") - else: - apptainer_cmd_parts.append(f"--{param_name} {mount_point}/{param_value}") + if isinstance(param_value, list): + paths_str = " ".join(_quote_path(p) for p in param_value) + apptainer_cmd_parts.append(f"--{param_name} {paths_str}") else: - apptainer_cmd_parts.append(f"--{param_name} {param_value}") + apptainer_cmd_parts.append(f"--{param_name} {_quote_path(param_value)}") else: apptainer_cmd_parts.append(f"--{param_name}") - - # Add fixed (hardcoded) params — quote values that contain spaces. - for param_name, param_value in config.get('fixed_params', {}).items(): - quoted = f"'{param_value}'" if ' ' in str(param_value) else str(param_value) - apptainer_cmd_parts.append(f"--{param_name} {quoted}") + + # Add fixed (hardcoded) params — quote values that contain spaces. + for param_name, param_value in config.get('fixed_params', {}).items(): + quoted = _quote_path(param_value) + apptainer_cmd_parts.append(f"--{param_name} {quoted}") apptainer_command = " \\\n ".join(apptainer_cmd_parts) @@ -575,15 +741,8 @@ def run_apptainer_analysis(): primary = config.get('primary_input', 'input_file') if primary in user_params: - relative_input_path = user_params[primary] - clean_relative_path = relative_input_path.lstrip(os.sep).lstrip('.') - - # Walk up input_depth levels to reach the dataset root - dataset_rel = clean_relative_path - for _ in range(config.get('input_depth', 2)): - dataset_rel = os.path.dirname(dataset_rel) - - logs_dir = os.path.join(workspace_path, dataset_rel, 'logs') + dataset_rel = dataset_root + logs_dir = os.path.join(dataset_root, 'logs') os.makedirs(logs_dir, exist_ok=True) target_dir = logs_dir @@ -595,17 +754,17 @@ def run_apptainer_analysis(): cleanup_cmd = "" pre_cmd = "" if analysis_type == "label_transfer" and primary in user_params: - abs_dataset_root = os.path.join(workspace_path, dataset_rel) - cleanup_cmd = f"\nrm -rf {abs_dataset_root}/expr_components" + abs_dataset_root = dataset_rel + cleanup_cmd = f"\nrm -rf {_quote_path(os.path.join(abs_dataset_root, 'expr_components'))}" elif analysis_type == "spot_annotation" and primary in user_params: - abs_files_dir = os.path.join(workspace_path, dataset_rel, "Files") + abs_files_dir = os.path.join(dataset_rel, "Files") cleanup_cmd = f"\nmv {abs_files_dir}/*_annotations.json {abs_files_dir}/Spots.json" elif analysis_type == "spatial_aggregation" and 'agg_annotations' in user_params: import json as _json - abs_output_dir = os.path.join(workspace_path, user_params['output_dir']) + abs_output_dir = user_params['output_dir'] mkdir_lines = [] for ann_path in user_params['agg_annotations']: - abs_ann_path = os.path.join(workspace_path, ann_path) + abs_ann_path = ann_path try: with open(abs_ann_path) as _f: ann_data = _json.load(_f) @@ -613,7 +772,7 @@ def run_apptainer_analysis(): or ann_data.get('name', '')) if '/' in ann_name: subdir = os.path.join(abs_output_dir, os.path.dirname(ann_name)) - mkdir_lines.append(f"mkdir -p {subdir}") + mkdir_lines.append(f"mkdir -p {_quote_path(subdir)}") except Exception: pass if mkdir_lines: @@ -628,6 +787,10 @@ def run_apptainer_analysis(): done find {abs_output_dir} -mindepth 1 -type d -empty -delete""" + gpu_lines = "" + if gpus > 0 and partition and _is_hipergator(): + gpu_lines = f"#SBATCH --partition={partition}\n#SBATCH --gpus={gpus}" + # create a submission script content slurm_script_content = f"""#!/bin/bash #SBATCH --job-name={job_name} @@ -636,6 +799,8 @@ def run_apptainer_analysis(): #SBATCH --mem={mem_limit} #SBATCH --ntasks=1 #SBATCH --cpus-per-task={cpus} +{gpu_lines} +{cache_lines} echo "Starting job on $(hostname)" module load apptainer 2>/dev/null || echo "Apptainer module load skipped or failed" @@ -1037,4 +1202,4 @@ def run_analysis(backend, gc=None, user_name=None, hubmap_id=None, file_path=Non dir_path=dir_path, ) else: - raise ValueError(f"Unknown backend '{backend}'. Choose 'notebook' or 'fusion'.") + raise ValueError(f"Unknown backend '{backend}'. Choose 'notebook' or 'fusion'.") \ No newline at end of file diff --git a/fusion/utilities/hubmap_data_transfer.py b/fusion/utilities/hubmap_data_transfer.py index 978ef57..f109f1b 100644 --- a/fusion/utilities/hubmap_data_transfer.py +++ b/fusion/utilities/hubmap_data_transfer.py @@ -1,32 +1,30 @@ """ -transfer.py ----------- -Simple Globus Connect Personal (GCP) + HuBMAP transfer script. - -Important HiPerGator behavior: -- GCP allowed path is forced to the user's current working directory. -- So run this script from the /blue/... or /orange/... folder you want GCP to access. - -Workflow: -1. Ensure GCP, globus CLI, and hubmap-clt are available. -2. Add GCP and current Python environment bin directory to PATH. -3. Run GCP setup if needed. -4. Set GCP config-paths to current working directory. -5. Run Globus CLI login using: globus login --no-local-server -6. Check globus whoami and ask user to confirm account. -7. Run HuBMAP login using: hubmap-clt login --no-browser -8. Build or use manifest. -9. Stop previous GCP, start GCP again, wait until connected. -10. Run hubmap-clt transfer. +Globus Connect Personal (GCP) + HuBMAP transfer script. + +Simple workflow: + +1. Check that Globus CLI, HuBMAP CLT, and Globus Connect Personal are available. +2. Make sure the user is logged into Globus and HuBMAP. +3. Ask how the destination path should be interpreted: + - Option 1: use destination as a full absolute path. + - Option 2: append destination to the current working directory. +4. If the final destination is inside the home directory, download there directly. +5. If the final destination is outside home, download temporarily inside home first. +6. Generate or use the HuBMAP manifest file. +7. Start Globus Connect Personal and submit the HuBMAP transfer. +8. Wait for the Globus transfer task to finish. +9. If a temporary home download was used, copy the files to the final destination. +10. Clean up temporary files from the home directory. """ import glob import os +import re import shutil import subprocess import sys import time -import re from typing import Union @@ -37,44 +35,43 @@ def _home() -> str: - return os.path.expanduser("~") + return os.path.abspath(os.path.expanduser("~")) + + +def _run(cmd: list, **kwargs) -> subprocess.CompletedProcess: + return subprocess.run(cmd, check=True, **kwargs) def _add_to_path(path: str) -> None: - current_path = os.environ.get("PATH", "") - paths = current_path.split(os.pathsep) if current_path else [] + current = os.environ.get("PATH", "") + paths = current.split(os.pathsep) if current else [] if path not in paths: - os.environ["PATH"] = path + os.pathsep + current_path + os.environ["PATH"] = path + os.pathsep + current -def _command_exists(command: str) -> bool: +def _exists(command: str) -> bool: return shutil.which(command) is not None -def _run(cmd: list, **kwargs) -> subprocess.CompletedProcess: - return subprocess.run(cmd, check=True, **kwargs) - +def _ensure_cli(command: str, package: str) -> None: + _add_to_path(os.path.dirname(sys.executable)) -def _ensure_python_cli(command: str, package: str) -> None: - venv_bin = os.path.dirname(sys.executable) - _add_to_path(venv_bin) - - if _command_exists(command): + if _exists(command): return print(f"[setup] {command} not found. Installing {package}...") _run([sys.executable, "-m", "pip", "install", package]) - if not _command_exists(command): + if not _exists(command): raise RuntimeError( f"[setup] Installed {package}, but {command} is still not found. " - f"Check PATH. Python executable: {sys.executable}" + f"Python executable: {sys.executable}" ) def _ensure_required_tools() -> None: - _ensure_python_cli("globus", "globus-cli") - _ensure_python_cli("hubmap-clt", "atlas-consortia-clt") + _ensure_cli("globus", "globus-cli") + _ensure_cli("hubmap-clt", "atlas-consortia-clt") print("[setup] Required CLIs available:") print(f" globus: {shutil.which('globus')}") @@ -83,9 +80,10 @@ def _ensure_required_tools() -> None: def _get_gcp_dir() -> Union[str, None]: matches = [ - path for path in glob.glob(os.path.join(_home(), "globusconnectpersonal-*")) - if os.path.isdir(path) + p for p in glob.glob(os.path.join(_home(), "globusconnectpersonal-*")) + if os.path.isdir(p) ] + matches.sort() return matches[0] if matches else None @@ -99,10 +97,6 @@ def _get_gcp_binary() -> Union[str, None]: return binary if os.path.isfile(binary) else None -def _is_gcp_installed() -> bool: - return _get_gcp_binary() is not None - - def _is_gcp_configured() -> bool: lta_dir = os.path.join(_home(), ".globusonline", "lta") if not os.path.isdir(lta_dir): @@ -116,11 +110,11 @@ def _is_gcp_configured() -> bool: def _download_file(url: str, output_path: str) -> None: - if _command_exists("wget"): + if _exists("wget"): _run(["wget", "-q", "--show-progress", "-O", output_path, url]) return - if _command_exists("curl"): + if _exists("curl"): _run(["curl", "-L", url, "-o", output_path]) return @@ -149,11 +143,11 @@ def _install_gcp() -> None: try: with open(bashrc, "r") as f: - bashrc_text = f.read() + text = f.read() except FileNotFoundError: - bashrc_text = "" + text = "" - if gcp_dir not in bashrc_text: + if gcp_dir not in text: with open(bashrc, "a") as f: f.write(f"\n# Added by transfer.py\n{path_line}\n") print(f"[gcp-setup] Added GCP to ~/.bashrc: {path_line}") @@ -162,7 +156,7 @@ def _install_gcp() -> None: def _setup_gcp_once() -> None: - if not _is_gcp_installed(): + if _get_gcp_binary() is None: _install_gcp() if _is_gcp_configured(): @@ -175,26 +169,13 @@ def _setup_gcp_once() -> None: print("\n[gcp-setup] One-time GCP setup required.") print("[gcp-setup] A URL may appear. Open it, log in, paste the code, then enter endpoint name.") - print("[gcp-setup] Example endpoint name: ashmit-hipergator-gcp\n") - _run([binary, "-setup", "--no-gui"]) print("[gcp-setup] GCP setup complete.\n") -def _set_gcp_allowed_path_to_cwd() -> None: - """ - Set GCP allowed path to the current working directory. - - Original code relied on the default: - ~,0,1 - - This function overwrites ~/.globusonline/lta/config-paths with: - ,0,1 - - So run from /blue/... or /orange/... before calling transfer(). - """ - cwd = os.getcwd() - config_dir = os.path.join(_home(), ".globusonline", "lta") +def _set_gcp_allowed_path() -> None: + home = _home() + config_dir = os.path.join(home, ".globusonline", "lta") config_path = os.path.join(config_dir, "config-paths") if not os.path.isdir(config_dir): @@ -204,18 +185,63 @@ def _set_gcp_allowed_path_to_cwd() -> None: ) with open(config_path, "w") as f: - f.write(f"{cwd},0,1\n") + f.write(f"{home},0,1\n") - print("[gcp-setup] GCP allowed path set to current working directory:") - print(f" {cwd}") + print(f"[gcp-setup] GCP allowed path set to: {home}") -def _globus_whoami() -> Union[str, None]: - result = subprocess.run( - ["globus", "whoami"], - capture_output=True, - text=True, +def _start_gcp(wait_seconds: int = 120) -> None: + binary = _get_gcp_binary() + if binary is None: + raise RuntimeError("[gcp] GCP binary not found.") + + print("[gcp] Restarting Globus Connect Personal...") + subprocess.run([binary, "-stop"], capture_output=True, text=True) + time.sleep(3) + + subprocess.Popen( + [binary, "-start"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + + deadline = time.time() + wait_seconds + last_status = "" + + while time.time() < deadline: + result = subprocess.run([binary, "-status"], capture_output=True, text=True) + status = ((result.stdout or "") + "\n" + (result.stderr or "")).strip() + last_status = status + lower = status.lower() + + connected = ( + "globus online:" in lower + and "connected" in lower + and "no globus connect personal connected" not in lower + ) + + if connected: + print("[gcp] Globus Connect Personal is connected.") + print("[gcp] Please wait a few seconds...") + time.sleep(15) + return + + print("[gcp] Waiting for GCP to connect...") + time.sleep(5) + + raise RuntimeError( + "[gcp] GCP did not connect.\n" + f"Last status:\n{last_status}\n\n" + "Try manually:\n" + f" {binary} -stop\n" + f" {binary} -start -debug\n" + f" {binary} -status" ) + + +def _globus_whoami() -> Union[str, None]: + result = subprocess.run(["globus", "whoami"], capture_output=True, text=True) if result.returncode != 0: return None @@ -223,13 +249,35 @@ def _globus_whoami() -> Union[str, None]: return account if account else None +def _globus_login_no_local_server() -> None: + result = subprocess.run( + ["globus", "login", "--no-local-server"], + input="\n", + text=True, + capture_output=True, + ) + print((result.stdout or "") + "\n" + (result.stderr or "")) + + auth_code = input("[globus] Paste the Authorization Code here: ").strip() + + result2 = subprocess.run( + ["globus", "login", "--no-local-server"], + input=auth_code + "\n", + text=True, + capture_output=True, + ) + print((result2.stdout or "") + "\n" + (result2.stderr or "")) + + if result2.returncode != 0: + raise RuntimeError("[globus] Globus login failed.") + + def _ensure_globus_login() -> None: account = _globus_whoami() if account is None: print("\n[globus] Globus CLI login required.") - print("[globus] Open the URL in your browser, then paste the auth code here.\n") - _run(["globus", "login", "--no-local-server"]) + _globus_login_no_local_server() account = _globus_whoami() if account is None: @@ -242,8 +290,15 @@ def _ensure_globus_login() -> None: return print("[globus] Logging out. Please login with the correct account.") - _run(["globus", "logout"]) - _run(["globus", "login", "--no-local-server"]) + + subprocess.run( + ["globus", "logout"], + input="y\n", + text=True, + check=True, + ) + + _globus_login_no_local_server() account = _globus_whoami() if account is None: @@ -262,90 +317,39 @@ def _ensure_globus_login() -> None: def _ensure_hubmap_login() -> None: - print("\n[hubmap] Running HuBMAP login.") - print("[hubmap] If a URL appears, open it in your browser and paste the auth code here.\n") - - _run(["hubmap-clt", "login", "--no-browser"]) - print("[hubmap] HuBMAP login step complete.\n") - - -def _start_gcp(wait_seconds: int = 120) -> None: - binary = _get_gcp_binary() - if binary is None: - raise RuntimeError("[gcp] GCP binary not found.") - - print("[gcp] Stopping any previous Globus Connect Personal process...") - subprocess.run([binary, "-stop"], capture_output=True, text=True) - time.sleep(3) - - print("[gcp] Starting Globus Connect Personal...") - subprocess.Popen( - [binary, "-start"], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - start_new_session=True, + result = subprocess.run( + ["hubmap-clt", "whoami"], + capture_output=True, + text=True, ) - deadline = time.time() + wait_seconds - last_status = "" - - while time.time() < deadline: - result = subprocess.run( - [binary, "-status"], - capture_output=True, - text=True, - ) + output = (result.stdout or "") + "\n" + (result.stderr or "") - status = ((result.stdout or "") + "\n" + (result.stderr or "")).strip() - last_status = status - - print("[gcp] Status:") - print(status) - - lower_status = status.lower() - is_connected = ( - "globus online:" in lower_status - and "connected" in lower_status - and "no globus connect personal connected" not in lower_status - ) - - if is_connected: - print("[gcp] Globus Connect Personal is connected.") - print("[gcp] Waiting 15 seconds for Globus Transfer API to recognize endpoint...") - time.sleep(15) - return - - print("[gcp] Waiting for GCP to connect...") - time.sleep(5) + if result.returncode == 0 and "not logged" not in output.lower(): + print("[hubmap] HuBMAP login already active.") + return - raise RuntimeError( - "[gcp] GCP did not connect.\n" - f"Last status:\n{last_status}\n\n" - "Try manually:\n" - f" {binary} -stop\n" - f" {binary} -start -debug\n" - f" {binary} -status" - ) + print("\n[hubmap] Running HuBMAP login.") + _run(["hubmap-clt", "login"]) + print("[hubmap] HuBMAP login step complete.\n") -def _handle_session_reauth(output: str) -> bool: +def _handle_session_reauth(output: str) -> None: match = re.search(r"globus session update (\S+)", output) if not match: - raise RuntimeError("Session reauth required but could not extract identity ID from output.") - + raise RuntimeError("Session reauth required but identity ID could not be extracted.") + identity_id = match.group(1) print(f"[globus] Session expired. Re-authenticating with identity: {identity_id}") - # Get the auth URL first result = subprocess.run( ["globus", "session", "update", "--no-local-server", identity_id], - input="\n", # send empty input to get the URL printed + input="\n", text=True, capture_output=True, ) print(result.stdout) - # Let the user paste the code via Python input() auth_code = input("Paste the Authorization Code here: ").strip() result2 = subprocess.run( @@ -354,11 +358,130 @@ def _handle_session_reauth(output: str) -> bool: text=True, capture_output=True, ) + if result2.returncode != 0: raise RuntimeError("[globus] Re-authentication failed.") - + print("[globus] Re-authentication successful.") - return True + + +def _can_write(path: str) -> bool: + try: + path = os.path.abspath(os.path.expanduser(path)) + os.makedirs(path, exist_ok=True) + + test_file = os.path.join(path, ".fusion_write_test") + with open(test_file, "w") as f: + f.write("test") + os.remove(test_file) + + return True + + except Exception as e: + print(f"[transfer] Cannot write to destination: {path}") + print(f"[transfer] Reason: {e}") + return False + + +def _collapse_adjacent_duplicates(path: str) -> str: + path = os.path.abspath(os.path.expanduser(path)) + absolute = path.startswith(os.sep) + + parts = [p for p in path.split(os.sep) if p not in {"", "."}] + cleaned = [] + + for part in parts: + if cleaned and cleaned[-1] == part: + continue + cleaned.append(part) + + result = os.path.join(*cleaned) if cleaned else "" + if absolute: + result = os.sep + result + + return os.path.abspath(result) + + +def _append_to_cwd(destination: str) -> str: + cwd = os.getcwd() + parts = [ + p for p in os.path.expanduser(destination).split(os.sep) + if p not in {"", "."} + ] + + cwd_last = os.path.basename(os.path.abspath(cwd)) + + if parts and parts[0] == cwd_last: + parts = parts[1:] + + joined = os.path.join(cwd, os.path.join(*parts) if parts else "") + return _collapse_adjacent_duplicates(joined) + + +def _ask_destination(destination: str) -> tuple: + while True: + print("\n[transfer] How should destination be interpreted?") + print("[transfer] 1 = Use destination as absolute/full path") + print("[transfer] 2 = Append destination to current working directory") + print(f"[transfer] destination entered: {destination}") + print(f"[transfer] current working directory: {os.getcwd()}") + + choice = input("[transfer] Choose 1 or 2: ").strip() + + if choice == "1": + if not os.path.isabs(os.path.expanduser(destination)): + print("[transfer] You selected absolute/full path, but destination is not absolute.") + destination = input("[transfer] Enter corrected absolute/full destination path: ").strip() + continue + + resolved = _collapse_adjacent_duplicates(destination) + + if _can_write(resolved): + print(f"[transfer] Using absolute/full destination: {resolved}") + return resolved, "absolute" + + destination = input("[transfer] Enter corrected absolute/full destination path: ").strip() + continue + + if choice == "2": + resolved = _append_to_cwd(destination) + + if _can_write(resolved): + print(f"[transfer] Using cwd-appended destination: {resolved}") + return resolved, "cwd" + + destination = input("[transfer] Enter corrected destination path: ").strip() + continue + + print("[transfer] Invalid choice. Please enter 1 or 2.") + + +def _resolve_download_plan(final_destination: str, mode: str) -> tuple: + home = _home() + + if mode == "cwd": + if os.path.commonpath([home, final_destination]) != home: + raise ValueError( + "[transfer] CWD-appended destination resolved outside home.\n" + f"[transfer] Home: {home}\n" + f"[transfer] Destination: {final_destination}" + ) + + return final_destination, os.path.relpath(final_destination, home), False + + if mode == "absolute": + if os.path.commonpath([home, final_destination]) == home: + return final_destination, os.path.relpath(final_destination, home), False + + name = os.path.basename(os.path.abspath(final_destination)) + timestamp = time.strftime("%Y%m%d_%H%M%S") + temp = os.path.join(home, "hubmap_temp_transfer", f"{name}_{timestamp}") + + os.makedirs(temp, exist_ok=True) + + return temp, os.path.relpath(temp, home), True + + raise ValueError(f"Unknown destination mode: {mode}") def _build_manifest(hubmap_ids: list, manifest_dir: str) -> str: @@ -367,12 +490,20 @@ def _build_manifest(hubmap_ids: list, manifest_dir: str) -> str: timestamp = time.strftime("%Y%m%d_%H%M%S") manifest_path = os.path.join(manifest_dir, f"hubmap_manifest_{timestamp}.txt") + clean_ids = [] + with open(manifest_path, "w") as f: for hubmap_id in hubmap_ids: - f.write(f"{hubmap_id.strip()} /\n") + clean_id = str(hubmap_id).strip() + if clean_id: + clean_ids.append(clean_id) + f.write(f"{clean_id} /\n") + + if not clean_ids: + raise ValueError("No valid HuBMAP IDs were provided.") print(f"[gcp] Manifest generated: {manifest_path}") - print(f"[gcp] IDs included: {', '.join(hubmap_ids)}") + print(f"[gcp] IDs included: {', '.join(clean_ids)}") return manifest_path @@ -382,11 +513,6 @@ def _resolve_manifest( manifest_path: Union[str, None], manifest_dir: str, ) -> str: - - print("params in _resolv_manifest") - print("hubmap_id = ", hubmap_id) - print("manifest_path = ", manifest_path) - if hubmap_id is not None and manifest_path is not None: raise ValueError("Provide either hubmap_id or manifest_path, not both.") @@ -394,6 +520,7 @@ def _resolve_manifest( manifest_path = os.path.abspath(os.path.expanduser(manifest_path)) if not os.path.isfile(manifest_path): raise FileNotFoundError(f"Manifest not found: {manifest_path}") + print(f"[gcp] Using manifest: {manifest_path}") return manifest_path @@ -401,27 +528,81 @@ def _resolve_manifest( ids = [hubmap_id] if isinstance(hubmap_id, str) else list(hubmap_id) if not ids: raise ValueError("hubmap_id list is empty.") + return _build_manifest(ids, manifest_dir) raise ValueError("Provide either hubmap_id or manifest_path.") +def _wait_for_task(task_id: str) -> None: + print(f"[gcp] Waiting for Globus task to finish: {task_id}") + + result = subprocess.run( + ["globus", "task", "wait", task_id], + capture_output=True, + text=True, + ) + + if result.returncode != 0: + print((result.stdout or "") + "\n" + (result.stderr or "")) + raise RuntimeError( + "[gcp] Globus task did not finish successfully. " + "Check the task in Globus activity before copying." + ) + + print("[gcp] Globus task finished successfully.") + + +def _copy_contents(source_dir: str, destination_dir: str) -> None: + source_dir = os.path.abspath(os.path.expanduser(source_dir)) + destination_dir = os.path.abspath(os.path.expanduser(destination_dir)) + + os.makedirs(destination_dir, exist_ok=True) + + print(f"[transfer] Copying downloaded data to: {destination_dir}") + + for item in os.listdir(source_dir): + src = os.path.join(source_dir, item) + dst = os.path.join(destination_dir, item) + + if os.path.isdir(src): + shutil.copytree(src, dst, dirs_exist_ok=True) + else: + shutil.copy2(src, dst) + + print("[transfer] Copy complete.") + + +def _clean_temp_download(download_destination: str) -> None: + shutil.rmtree(download_destination, ignore_errors=True) + + temp_root = os.path.join(_home(), "hubmap_temp_transfer") + + if os.path.isdir(temp_root): + for root, dirs, files in os.walk(temp_root, topdown=False): + if not dirs and not files: + try: + os.rmdir(root) + except OSError: + pass + + def setup() -> None: _ensure_required_tools() _setup_gcp_once() - _set_gcp_allowed_path_to_cwd() + _set_gcp_allowed_path() _ensure_globus_login() _ensure_hubmap_login() + print("[setup] Setup complete. You can now call transfer().") def transfer( - destination: str, + destination: Union[str, None] = None, hubmap_id: Union[str, list, None] = None, manifest_path: Union[str, None] = None, protected: bool = False, ) -> None: - if destination is None: if isinstance(hubmap_id, str): destination = f"./{hubmap_id}" @@ -432,30 +613,41 @@ def transfer( else: raise ValueError("destination could not be inferred. Provide hubmap_id or manifest_path.") - destination = os.path.abspath(os.path.expanduser(destination)) - os.makedirs(destination, exist_ok=True) + final_destination, mode = _ask_destination(destination) + + download_destination, destination_for_hubmap, copy_after_download = _resolve_download_plan( + final_destination, + mode, + ) + + print(f"[transfer] Final destination: {final_destination}") + if copy_after_download: + print(f"[transfer] Temporary download location: {download_destination}") + + os.makedirs(download_destination, exist_ok=True) _ensure_required_tools() _setup_gcp_once() - _set_gcp_allowed_path_to_cwd() + _set_gcp_allowed_path() _ensure_globus_login() _ensure_hubmap_login() - + manifest = _resolve_manifest( hubmap_id=hubmap_id, manifest_path=manifest_path, - manifest_dir=destination, + manifest_dir=final_destination, ) _start_gcp() - cmd = ["hubmap-clt", "transfer", manifest, "--destination", destination] + cmd = ["hubmap-clt", "transfer", manifest, "-d", destination_for_hubmap] if protected: cmd.append("--from-protected-space") print(f"[gcp] Starting {'protected' if protected else 'public'} transfer...") - print("[gcp] Command:", " ".join(cmd)) + print("[gcp] This may take a few minutes. Please wait...") + #print("[gcp] Command:", " ".join(cmd)) error_indicators = [ "Globus CLI Error", @@ -473,19 +665,39 @@ def transfer( output = (result.stdout or "") + "\n" + (result.stderr or "") print(output) - if "Session reauthentication required" in output: if attempt == 0: _handle_session_reauth(output) continue - else: - raise RuntimeError("[globus] Re-authentication succeeded but transfer still failed.") - if result.returncode != 0 or any(error in output for error in error_indicators): + raise RuntimeError("[globus] Re-authentication succeeded but transfer still failed.") + + failed = result.returncode != 0 or any(err in output for err in error_indicators) + + if failed: raise RuntimeError( "[gcp] Transfer failed. See output above. " "Check GCP allowed path, endpoint connectivity, and dataset access." ) - print(f"[gcp] Transfer complete. Data should be in: {destination}") - break + task_match = re.search(r"Task ID:\s*([a-fA-F0-9-]+)", output) + task_id = task_match.group(1) if task_match else None + + if copy_after_download: + if task_id is None: + raise RuntimeError( + "[gcp] Transfer was accepted, but Task ID could not be found. " + "Cannot safely copy to final destination until transfer completion is confirmed." + ) + + _wait_for_task(task_id) + _copy_contents(download_destination, final_destination) + _clean_temp_download(download_destination) + + print("[gcp] Transfer and copy complete.") + print(f"[gcp] Final data location: {final_destination}") + else: + print(f"[gcp] Transfer initiated. Data should be in: {final_destination}") + print("[gcp] If files are still moving, check Globus activity.") + + break \ No newline at end of file diff --git a/fusion/utilities/utility.py b/fusion/utilities/utility.py index 28d41aa..f0c2fd5 100644 --- a/fusion/utilities/utility.py +++ b/fusion/utilities/utility.py @@ -8,8 +8,69 @@ import shutil import mimetypes +def _resolve_local_path(path, must_exist=False): + """ + Resolve local user-entered paths safely. + + Fixes cases like: + cwd = /home/user/fusion_demo_notebooks + input = fusion_demo_notebooks/datasets/data_1 + final = /home/user/fusion_demo_notebooks/datasets/data_1 + + Also handles: + blue/... -> /blue/... + orange/... -> /orange/... + home/... -> /home/... + """ + if path is None: + return None + + raw_path = str(path).strip().strip('"').strip("'") + raw_path = os.path.expanduser(raw_path) + + if os.path.isabs(raw_path): + resolved = os.path.abspath(raw_path) + if must_exist and not os.path.exists(resolved): + raise FileNotFoundError(f"Path not found: {resolved}") + return resolved + + cwd = os.getcwd() + normalized = raw_path.replace("\\", "/") + parts = [p for p in normalized.split("/") if p] + + candidates = [] + + # Fix duplicated current-folder prefix. + # Example: + # cwd = /home/user/fusion_demo_notebooks + # raw = fusion_demo_notebooks/datasets/... + # candidate = /home/user/fusion_demo_notebooks/datasets/... + if parts: + cur = cwd + while cur and cur != os.path.dirname(cur): + if os.path.basename(cur) == parts[0]: + candidates.append(os.path.abspath(os.path.join(os.path.dirname(cur), raw_path))) + cur = os.path.dirname(cur) + + # Handle missing leading slash for known HPC roots. + if parts[0] in {"home", "blue", "orange"}: + candidates.append(os.path.abspath(os.path.join(os.sep, raw_path))) + + # Normal cwd-relative fallback. + candidates.append(os.path.abspath(os.path.join(cwd, raw_path))) + + for candidate in dict.fromkeys(candidates): + if os.path.exists(candidate): + return candidate + + if must_exist: + checked = "\n".join(f" - {c}" for c in dict.fromkeys(candidates)) + raise FileNotFoundError(f"Path not found: {raw_path}\nChecked:\n{checked}") + + return candidates[0] def download_folder_zip_from_fusion_backend(gc, folder_id, output_dir="."): + output_dir = _resolve_local_path(output_dir, must_exist=False) os.makedirs(output_dir, exist_ok=True) # Get the folder info so we can name the zip file correctly @@ -40,13 +101,15 @@ def download_folder_zip_from_fusion_backend(gc, folder_id, output_dir="."): def download_from_fusion_to_workspace(gc, resource_id, resource_type="file", output_dir="."): + output_dir = _resolve_local_path(output_dir, must_exist=False) + if resource_type == "file": info = gc.get(f"/file/{resource_id}") file_name = info["name"] # Use the file name as the folder name folder_name = os.path.splitext(file_name)[0] - folder_path = f"./datasets/{folder_name}/image" + folder_path = os.path.join(output_dir, "datasets", folder_name, "image") os.makedirs(folder_path, exist_ok=True) # Set the final output path inside the new folder @@ -385,7 +448,7 @@ def optimize_workspace_wsi(source, delete_originals=False): Returns: dict: Optimization results and statistics. """ - source_path = Path(source) + source_path = Path(_resolve_local_path(source, must_exist=False)) results = { "source": source, @@ -426,7 +489,7 @@ def optimize_workspace_wsi(source, delete_originals=False): else: # Assume it's a hubmap_id - base_path = Path.cwd() / "datasets" / source + base_path = Path(_resolve_local_path(Path("datasets") / str(source), must_exist=False)) if not base_path.exists(): print(f"Error: Directory 'datasets/{source}' does not exist and '{source}' is not a valid file/folder path.") @@ -780,7 +843,18 @@ def upload_to_fusion_backend(gc, hubmap_id=None, user=None, file_path=None, file tif_item_id = None if hubmap_id: temp_download = True + if file_path: + file_path = _resolve_local_path(file_path, must_exist=True) + + if file_paths: + file_paths = [ + _resolve_local_path(path, must_exist=True) + for path in file_paths + ] + if dir_path: + dir_path = _resolve_local_path(dir_path, must_exist=True) + temp_folder = None should_upload = True @@ -807,12 +881,24 @@ def upload_to_fusion_backend(gc, hubmap_id=None, user=None, file_path=None, file # If user chose to use existing folder without uploading, return early if not should_upload: + existing_tif_item_id = None + + if is_analysis_result: + items = gc.get('/item', parameters={'folderId': folder_id}) + + for item in items: + item_details = gc.get(f'/item/{item["_id"]}') + if 'largeImage' in item_details: + existing_tif_item_id = item['_id'] + break + return { - "folder_id": folder_id, - "uploaded_files": [], - "failed_files": [], - "total_uploaded": 0, - "total_failed": 0 + "folder_id": folder_id, + "uploaded_files": [], + "failed_files": [], + "total_uploaded": 0, + "total_failed": 0, + "tif_item_id": existing_tif_item_id } # 1: Upload from HubMAP ID @@ -1253,6 +1339,7 @@ def download_reference_files_from_fusion(gc): ] } for output_dir, file_ids in assets.items(): + output_dir = _resolve_local_path(output_dir, must_exist=False) print(f"\nDownloading {output_dir} to {output_dir}/") os.makedirs(output_dir, exist_ok=True) for file_id in file_ids: @@ -1270,6 +1357,7 @@ def download_model_files_from_fusion(gc): ] } for output_dir, file_ids in assets.items(): + output_dir = _resolve_local_path(output_dir, must_exist=False) print(f"\nDownloading {output_dir} to {output_dir}/") os.makedirs(output_dir, exist_ok=True) for file_id in file_ids: @@ -1280,6 +1368,7 @@ def download_model_files_from_fusion(gc): print(f" Done.") def download_file_from_fusion(gc, file_id, output_dir): + output_dir = _resolve_local_path(output_dir, must_exist=False) os.makedirs(output_dir, exist_ok=True) file_info = gc.get(f"/file/{file_id}") out_path = os.path.join(output_dir, file_info["name"])