Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
103 changes: 64 additions & 39 deletions components/admin/meta-packages/SPECS/meta-packages.spec
Original file line number Diff line number Diff line change
Expand Up @@ -121,23 +121,25 @@ Collection of compute node base packages

%package -n %{PROJ_NAME}-%{compiler_family}-io-libs
Summary: OpenHPC IO libraries for GNU
%ifnarch ppc64le
Requires: adios2-%{compiler_family}-mpich%{PROJ_DELIM}
Requires: adios2-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: hdf5-%{compiler_family}%{PROJ_DELIM}
Requires: netcdf-cxx-%{compiler_family}%{PROJ_DELIM}
Requires: netcdf-cxx-%{compiler_family}-mpich%{PROJ_DELIM}
Requires: netcdf-cxx-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: netcdf-fortran-%{compiler_family}%{PROJ_DELIM}
Requires: netcdf-fortran-%{compiler_family}-mpich%{PROJ_DELIM}
Requires: netcdf-fortran-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: netcdf-%{compiler_family}%{PROJ_DELIM}
Requires: netcdf-%{compiler_family}-mpich%{PROJ_DELIM}
Requires: netcdf-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: pnetcdf-%{compiler_family}-mpich%{PROJ_DELIM}
Requires: pnetcdf-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: phdf5-%{compiler_family}-mpich%{PROJ_DELIM}
%endif
Requires: adios2-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: hdf5-%{compiler_family}%{PROJ_DELIM}
Requires: netcdf-cxx-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: netcdf-fortran-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: netcdf-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: pnetcdf-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: phdf5-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
%ifnarch aarch64
%ifnarch aarch64 ppc64le
Requires: adios2-%{compiler_family}-mvapich2%{PROJ_DELIM}
Requires: netcdf-cxx-%{compiler_family}-mvapich2%{PROJ_DELIM}
Requires: netcdf-fortran-%{compiler_family}-mvapich2%{PROJ_DELIM}
Expand All @@ -148,6 +150,7 @@ Requires: phdf5-%{compiler_family}-mvapich2%{PROJ_DELIM}
%description -n %{PROJ_NAME}-%{compiler_family}-io-libs
Collection of IO library builds for use with GNU compiler toolchain

%ifnarch ppc64le
%package -n %{PROJ_NAME}-%{compiler_family}-mpich-io-libs
Summary: OpenHPC IO libraries for GNU and MPICH
Requires: adios2-%{compiler_family}-mpich%{PROJ_DELIM}
Expand All @@ -159,6 +162,7 @@ Requires: phdf5-%{compiler_family}-mpich%{PROJ_DELIM}
Requires: hdf5-%{compiler_family}%{PROJ_DELIM}
%description -n %{PROJ_NAME}-%{compiler_family}-mpich-io-libs
Collection of IO library builds for use with GNU compiler toolchain and the MPICH runtime
%endif

%package -n %{PROJ_NAME}-%{compiler_family}-%{mpi_family}-io-libs
Summary: OpenHPC IO libraries for GNU and OpenMPI
Expand All @@ -174,29 +178,31 @@ Collection of IO library builds for use with GNU compiler toolchain and the Open

%package -n %{PROJ_NAME}-%{compiler_family}-parallel-libs
Summary: OpenHPC parallel libraries for GNU
%ifnarch ppc64le
Requires: boost-%{compiler_family}-mpich%{PROJ_DELIM}
Requires: boost-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: fftw-%{compiler_family}-mpich%{PROJ_DELIM}
Requires: fftw-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: hypre-%{compiler_family}-mpich%{PROJ_DELIM}
Requires: hypre-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: mfem-%{compiler_family}-mpich%{PROJ_DELIM}
Requires: mfem-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: mumps-%{compiler_family}-mpich%{PROJ_DELIM}
Requires: mumps-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: petsc-%{compiler_family}-mpich%{PROJ_DELIM}
Requires: petsc-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: scalapack-%{compiler_family}-mpich%{PROJ_DELIM}
Requires: scalapack-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: slepc-%{compiler_family}-mpich%{PROJ_DELIM}
Requires: slepc-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: ptscotch-%{compiler_family}-mpich%{PROJ_DELIM}
Requires: ptscotch-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: superlu_dist-%{compiler_family}-mpich%{PROJ_DELIM}
Requires: superlu_dist-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: trilinos-%{compiler_family}-mpich%{PROJ_DELIM}
%endif
Requires: boost-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: fftw-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: hypre-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: mfem-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: mumps-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: petsc-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: scalapack-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: slepc-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: ptscotch-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: superlu_dist-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: trilinos-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
%ifnarch aarch64
%ifnarch aarch64 ppc64le
Requires: boost-%{compiler_family}-mvapich2%{PROJ_DELIM}
Requires: fftw-%{compiler_family}-mvapich2%{PROJ_DELIM}
Requires: hypre-%{compiler_family}-mvapich2%{PROJ_DELIM}
Expand All @@ -212,6 +218,7 @@ Requires: trilinos-%{compiler_family}-mvapich2%{PROJ_DELIM}
%description -n %{PROJ_NAME}-%{compiler_family}-parallel-libs
Collection of parallel library builds for use with GNU compiler toolchain

%ifnarch ppc64le
%package -n %{PROJ_NAME}-%{compiler_family}-mpich-parallel-libs
Summary: OpenHPC parallel libraries for GNU and MPICH
Requires: boost-%{compiler_family}-mpich%{PROJ_DELIM}
Expand All @@ -227,6 +234,7 @@ Requires: superlu_dist-%{compiler_family}-mpich%{PROJ_DELIM}
Requires: trilinos-%{compiler_family}-mpich%{PROJ_DELIM}
%description -n %{PROJ_NAME}-%{compiler_family}-mpich-parallel-libs
Collection of parallel library builds for use with GNU compiler toolchain and the MPICH runtime
%endif

%package -n %{PROJ_NAME}-%{compiler_family}-%{mpi_family}-parallel-libs
Summary: OpenHPC parallel libraries for GNU and OpenMPI
Expand All @@ -246,22 +254,28 @@ Collection of parallel library builds for use with GNU compiler toolchain and th

%package -n %{PROJ_NAME}-%{compiler_family}-perf-tools
Summary: OpenHPC performance tools for GNU
%ifnarch ppc64le
Requires: dimemas-%{compiler_family}-mpich%{PROJ_DELIM}
Requires: dimemas-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: extrae-%{compiler_family}-mpich%{PROJ_DELIM}
Requires: extrae-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: imb-%{compiler_family}-mpich%{PROJ_DELIM}
Requires: imb-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: omb-%{compiler_family}-mpich%{PROJ_DELIM}
Requires: omb-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: paraver%{PROJ_DELIM}
Requires: tau-%{compiler_family}-mpich%{PROJ_DELIM}
Requires: tau-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: scalasca-%{compiler_family}-mpich%{PROJ_DELIM}
Requires: scalasca-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: scorep-%{compiler_family}-mpich%{PROJ_DELIM}
%endif
Requires: dimemas-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: extrae-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: imb-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: omb-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: tau-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: scalasca-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: scorep-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
%ifnarch aarch64
%ifarch ppc64le
Requires: likwid-%{compiler_family}%{PROJ_DELIM}
Requires: papi%{PROJ_DELIM}
%endif
%ifnarch aarch64 ppc64le
Requires: dimemas-%{compiler_family}-mvapich2%{PROJ_DELIM}
Requires: extrae-%{compiler_family}-mvapich2%{PROJ_DELIM}
Requires: imb-%{compiler_family}-mvapich2%{PROJ_DELIM}
Expand All @@ -275,6 +289,7 @@ Requires: scorep-%{compiler_family}-mvapich2%{PROJ_DELIM}
%description -n %{PROJ_NAME}-%{compiler_family}-perf-tools
Collection of performance tool builds for use with GNU compiler toolchain

%ifnarch ppc64le
%package -n %{PROJ_NAME}-%{compiler_family}-mpich-perf-tools
Summary: OpenHPC performance tools for GNU and MPICH
Requires: dimemas-%{compiler_family}-mpich%{PROJ_DELIM}
Expand All @@ -288,6 +303,7 @@ Requires: scorep-%{compiler_family}-mpich%{PROJ_DELIM}
Requires: papi%{PROJ_DELIM}
%description -n %{PROJ_NAME}-%{compiler_family}-mpich-perf-tools
Collection of performance tool builds for use with GNU compiler toolchain and the MPICH runtime
%endif

%package -n %{PROJ_NAME}-%{compiler_family}-%{mpi_family}-perf-tools
Summary: OpenHPC performance tools for GNU and OpenMPI
Expand All @@ -296,7 +312,9 @@ Requires: extrae-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: imb-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: likwid-%{compiler_family}%{PROJ_DELIM}
Requires: omb-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
%ifnarch ppc64le
Requires: paraver%{PROJ_DELIM}
%endif
Requires: tau-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: scalasca-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: scorep-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Expand All @@ -312,10 +330,12 @@ Collection of python related library builds for use with GNU compiler toolchain

%package -n %{PROJ_NAME}-%{compiler_family}-python3-libs
Summary: OpenHPC python3 libraries for GNU
%ifnarch ppc64le
Requires: %{python_prefix}-mpi4py-%{compiler_family}-mpich%{PROJ_DELIM}
%endif
Requires: %{python_prefix}-mpi4py-%{compiler_family}-%{mpi_family}%{PROJ_DELIM}
Requires: %{python_prefix}-numpy-%{compiler_family}%{PROJ_DELIM}
%ifnarch aarch64
%ifnarch aarch64 ppc64le
Requires: %{python_prefix}-mpi4py-%{compiler_family}-mvapich2%{PROJ_DELIM}
%endif
%description -n %{PROJ_NAME}-%{compiler_family}-python3-libs
Expand All @@ -339,7 +359,9 @@ Requires: metis-%{compiler_family}%{PROJ_DELIM}
Requires: openblas-%{compiler_family}%{PROJ_DELIM}
Requires: plasma-%{compiler_family}%{PROJ_DELIM}
Requires: R-%{compiler_family}%{PROJ_DELIM}
%ifnarch ppc64le
Requires: scotch-%{compiler_family}%{PROJ_DELIM}
%endif
Requires: superlu-%{compiler_family}%{PROJ_DELIM}
%description -n %{PROJ_NAME}-%{compiler_family}-serial-libs
Collection of serial library builds for use with GNU compiler toolchain
Expand Down Expand Up @@ -371,21 +393,15 @@ Collection of server packages for SLURM

%package -n %{PROJ_NAME}-warewulf
Summary: OpenHPC base packages for Warewulf
Requires: warewulf-cluster%{PROJ_DELIM}
Requires: warewulf-common%{PROJ_DELIM}
Requires: warewulf-common%{PROJ_DELIM}-localdb
Requires: warewulf-ipmi%{PROJ_DELIM}
Requires: warewulf-ipmi%{PROJ_DELIM}-initramfs-%{_arch}
Requires: warewulf-provision%{PROJ_DELIM}-initramfs-%{_arch}
Requires: warewulf-provision%{PROJ_DELIM}
Requires: warewulf-provision%{PROJ_DELIM}-server
Requires: warewulf-provision%{PROJ_DELIM}-server-ipxe-%{_arch}
Requires: warewulf-vnfs%{PROJ_DELIM}
# Warewulf 4.x ships as a single package; the warewulf3-style split packages
# (warewulf-cluster/common/ipmi/provision/vnfs) are no longer built and are
# Conflicted by warewulf >= 4, so require the unified package on every arch.
Requires: warewulf%{PROJ_DELIM}
%description -n %{PROJ_NAME}-warewulf
Collection of base packages for Warewulf provisioning

# x86_64 specific groups
%ifnarch aarch64
%ifnarch aarch64 ppc64le

%package -n %{PROJ_NAME}-%{compiler_family}-mvapich2-parallel-libs
Summary: OpenHPC parallel libraries for GNU and MVAPICH2
Expand Down Expand Up @@ -708,6 +724,7 @@ Collection of serial library builds for use with Intel(R) oneAPI Toolkit
%endif
%else
# aarch64 specific groups
%ifarch aarch64
%if !0%{?openEuler}

%package -n %{PROJ_NAME}-arm1-serial-libs
Expand Down Expand Up @@ -802,6 +819,7 @@ Collection of parallel library builds for use with the Arm Compiler for Linux an

%endif
%endif
%endif


%prep
Expand All @@ -819,13 +837,19 @@ Collection of parallel library builds for use with the Arm Compiler for Linux an
%files -n %{PROJ_NAME}-base
%files -n %{PROJ_NAME}-base-compute
%files -n %{PROJ_NAME}-%{compiler_family}-io-libs
%ifnarch ppc64le
%files -n %{PROJ_NAME}-%{compiler_family}-mpich-io-libs
%endif
%files -n %{PROJ_NAME}-%{compiler_family}-%{mpi_family}-io-libs
%files -n %{PROJ_NAME}-%{compiler_family}-parallel-libs
%ifnarch ppc64le
%files -n %{PROJ_NAME}-%{compiler_family}-mpich-parallel-libs
%endif
%files -n %{PROJ_NAME}-%{compiler_family}-%{mpi_family}-parallel-libs
%files -n %{PROJ_NAME}-%{compiler_family}-perf-tools
%ifnarch ppc64le
%files -n %{PROJ_NAME}-%{compiler_family}-mpich-perf-tools
%endif
%files -n %{PROJ_NAME}-%{compiler_family}-%{mpi_family}-perf-tools
%files -n %{PROJ_NAME}-%{compiler_family}-python-libs
%files -n %{PROJ_NAME}-%{compiler_family}-python3-libs
Expand All @@ -835,7 +859,7 @@ Collection of parallel library builds for use with the Arm Compiler for Linux an
%files -n %{PROJ_NAME}-slurm-server
%files -n %{PROJ_NAME}-warewulf
# x86_64 specific groups
%ifnarch aarch64
%ifnarch aarch64 ppc64le
%files -n %{PROJ_NAME}-%{compiler_family}-mvapich2-io-libs
%files -n %{PROJ_NAME}-%{compiler_family}-mvapich2-perf-tools
%files -n %{PROJ_NAME}-%{compiler_family}-mvapich2-parallel-libs
Expand All @@ -859,6 +883,7 @@ Collection of parallel library builds for use with the Arm Compiler for Linux an
%endif
%else
# aarch64 specific groups
%ifarch aarch64
%if !0%{?openEuler}
%files -n %{PROJ_NAME}-arm1-serial-libs
%files -n %{PROJ_NAME}-arm1-io-libs
Expand All @@ -869,4 +894,4 @@ Collection of parallel library builds for use with the Arm Compiler for Linux an
%files -n %{PROJ_NAME}-arm1-%{mpi_family}-parallel-libs
%endif
%endif

%endif
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ Version %{version}
whatis("Name: GNU Compiler Collection")
whatis("Version: %{version}")
whatis("Category: compiler, runtime support")
whatis("Description: GNU Compiler Family (C/C++/Fortran for x86_64)")
whatis("Description: GNU Compiler Family (C/C++/Fortran)")
whatis("URL: http://gcc.gnu.org/")

local version = "%{version}"
Expand Down
34 changes: 33 additions & 1 deletion components/dev-tools/numpy/SPECS/python-numpy.spec
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,10 @@ BuildRequires: python3-meson-python
BuildRequires: %{python_prefix}-pip
BuildRequires: ninja-build
BuildRequires: pkg-config
BuildRequires: fdupes gcc
BuildRequires: gcc
%if 0%{?suse_version}
BuildRequires: fdupes
%endif
#!BuildIgnore: post-build-checks

# Default library install path
Expand All @@ -57,6 +60,35 @@ basic linear algebra and random number generation.
%setup -q -n %{pname}-%{version}
# Convert PEP 639 license string to old-style dict for older meson-python
sed -i "s|^license = '\(.*\)'|license = {text = '\1'}|" pyproject.toml
%ifarch ppc64le
# Fix VSX3 intrinsics used when building with the -mcpu=power9 baseline (numpy PR #29627).
# Applied as asserted in-place edits rather than a context patch: each substitution is
# checked so a numpy %{version} layout drift fails the build loudly instead of silently
# miscompiling the VSX3 half-precision paths (a no-op sed would otherwise pass unnoticed).
_assert_patched() {
# $1 = file, $2 = literal string that MUST be present after the edit
if ! grep -qF -- "$2" "$1"; then
echo "ERROR: ppc64le numpy VSX3 fix did not apply to $1 (numpy %{version} drift?)" >&2
exit 1
fi
}

sed -i "s|args: {'val': '-mcpu=power8', 'match': '.*vsx'}|args: ['-mcpu=power8', '-mvsx']|" meson_cpu/ppc64/meson.build
_assert_patched meson_cpu/ppc64/meson.build "args: ['-mcpu=power8', '-mvsx']"

sed -i 's|defined(NPY_HAVE_VSX3) && defined(NPY_HAVE_VSX_ASM)|defined(NPY_HAVE_VSX3) \&\& defined(NPY_HAVE_VSX_ASM) \&\& defined(NPY__CPU_TARGET_VSX3)|g' numpy/_core/src/common/half.hpp
sed -i 's|defined(NPY_HAVE_VSX3) && defined(vec_extract_fp_from_shorth)|defined(NPY_HAVE_VSX3) \&\& defined(vec_extract_fp_from_shorth) \&\& defined(NPY__CPU_TARGET_VSX3)|g' numpy/_core/src/common/half.hpp
sed -i 's|defined(NPY_HAVE_VSX3) && defined(NPY_HAVE_VSX3_HALF_DOUBLE)|defined(NPY_HAVE_VSX3) \&\& defined(NPY_HAVE_VSX3_HALF_DOUBLE) \&\& defined(NPY__CPU_TARGET_VSX3)|g' numpy/_core/src/common/half.hpp
_assert_patched numpy/_core/src/common/half.hpp "&& defined(NPY__CPU_TARGET_VSX3)"

# numpy/distutils/ccompiler_opt.py ships in some numpy layouts (incl. 2.4.4) but is not
# guaranteed across versions; patch it only when this tree provides it so a future
# relayout degrades to a no-op instead of a hard sed error.
if [ -f numpy/distutils/ccompiler_opt.py ]; then
sed -i "s|flags=\"-mcpu=power8\", implies_detect=False|flags=\"-mcpu=power8 -mvsx\", implies_detect=False|" numpy/distutils/ccompiler_opt.py
_assert_patched numpy/distutils/ccompiler_opt.py 'flags="-mcpu=power8 -mvsx"'
fi
%endif

%build
# OpenHPC compiler/mpi designation
Expand Down
16 changes: 16 additions & 0 deletions components/parallel-libs/boost/SPECS/boost.spec
Original file line number Diff line number Diff line change
Expand Up @@ -124,10 +124,22 @@ using python : %{python3_version} : %{__python3} : /usr/include/python%{python3_
import os ;
local RPM_OPT_FLAGS = [ os.environ RPM_OPT_FLAGS ] ;
local RPM_LD_FLAGS = [ os.environ RPM_LD_FLAGS ] ;
local MPI_DIR = [ os.environ MPI_DIR ] ;
%if "%{compiler_family}" == "gnu14" || "%{compiler_family}" == "gnu15"
using gcc : : : <compileflags>$(RPM_OPT_FLAGS) <linkflags>$(RPM_LD_FLAGS) ;
%endif
%if ("%{mpi_family}" == "mpich" || "%{mpi_family}" == "mvapich2") && "%{?OHPC_USE_CCACHE}" == "yes"
# MPICH-derived wrappers built with ccache report "ccache g++ ..."; Boost.MPI strips
# only one executable token when parsing wrapper output, so configure it
# explicitly for these MPI families.
using mpi : :
<include>$(MPI_DIR)/include
<library-path>$(MPI_DIR)/lib
<find-shared-library>mpicxx
<find-shared-library>mpi ;
%else
using mpi : $MPICXX ;
%endif
EOF

# Generate b2
Expand All @@ -142,8 +154,12 @@ EOF
address-model="64" \
%ifarch aarch64 %{arm}
architecture="arm" \
%else
%ifarch ppc64le
architecture="power" \
%else
architecture="x86" \
%endif
%endif
threading="multi" \
link="shared" \
Expand Down
3 changes: 3 additions & 0 deletions components/parallel-libs/fftw/SPECS/fftw.spec
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,9 @@ for i in %{precision_list} ; do
%endif
%ifarch aarch64
LOOPBASEFLAGS="${LOOPBASEFLAGS} --enable-neon"
%endif
%ifarch ppc64le
LOOPBASEFLAGS="${LOOPBASEFLAGS} --enable-vsx"
%endif
fi
mkdir ${i}
Expand Down
Loading
Loading