diff --git a/Changes.md b/Changes.md index aab3f7f522e..fe3966cbb9e 100644 --- a/Changes.md +++ b/Changes.md @@ -5,6 +5,10 @@ Features -------- - Cycles : Updated to version 5.0.0. +- FileList : Added node for listing matching files. +- DeleteFiles : Added node for deleting files. +- CopyFiles : Added node for copying files. +- RenameFiles : Added node for renaming files. Improvements ------------ diff --git a/include/GafferDispatch/CopyFiles.h b/include/GafferDispatch/CopyFiles.h new file mode 100644 index 00000000000..8fd10b6dac8 --- /dev/null +++ b/include/GafferDispatch/CopyFiles.h @@ -0,0 +1,83 @@ +////////////////////////////////////////////////////////////////////////// +// +// Copyright (c) 2025, Cinesite VFX Ltd. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above +// copyright notice, this list of conditions and the following +// disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided with +// the distribution. +// +// * Neither the name of John Haddon nor the names of +// any other contributors to this software may be used to endorse or +// promote products derived from this software without specific prior +// written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include "GafferDispatch/TaskNode.h" + +#include "Gaffer/StringPlug.h" +#include "Gaffer/TypedObjectPlug.h" + +namespace GafferDispatch +{ + +class GAFFERDISPATCH_API CopyFiles : public TaskNode +{ + + public : + + explicit CopyFiles( const std::string &name=defaultName() ); + ~CopyFiles() override; + + GAFFER_NODE_DECLARE_TYPE( GafferDispatch::CopyFiles, CopyFilesTypeId, TaskNode ); + + Gaffer::StringVectorDataPlug *filesPlug(); + const Gaffer::StringVectorDataPlug *filesPlug() const; + + Gaffer::StringPlug *destinationPlug(); + const Gaffer::StringPlug *destinationPlug() const; + + Gaffer::BoolPlug *overwritePlug(); + const Gaffer::BoolPlug *overwritePlug() const; + + Gaffer::BoolPlug *deleteSourcePlug(); + const Gaffer::BoolPlug *deleteSourcePlug() const; + + protected : + + IECore::MurmurHash hash( const Gaffer::Context *context ) const override; + void execute() const override; + + private : + + static size_t g_firstPlugIndex; + + // Friendship for the bindings + friend struct GafferDispatchBindings::Detail::TaskNodeAccessor; + +}; + +} // namespace GafferDispatch diff --git a/include/GafferDispatch/DeleteFiles.h b/include/GafferDispatch/DeleteFiles.h new file mode 100644 index 00000000000..9aa6fa18189 --- /dev/null +++ b/include/GafferDispatch/DeleteFiles.h @@ -0,0 +1,76 @@ +////////////////////////////////////////////////////////////////////////// +// +// Copyright (c) 2025, Cinesite VFX Ltd. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above +// copyright notice, this list of conditions and the following +// disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided with +// the distribution. +// +// * Neither the name of John Haddon nor the names of +// any other contributors to this software may be used to endorse or +// promote products derived from this software without specific prior +// written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include "GafferDispatch/TaskNode.h" + +#include "Gaffer/TypedObjectPlug.h" + +namespace GafferDispatch +{ + +class GAFFERDISPATCH_API DeleteFiles : public TaskNode +{ + + public : + + explicit DeleteFiles( const std::string &name=defaultName() ); + ~DeleteFiles() override; + + GAFFER_NODE_DECLARE_TYPE( GafferDispatch::DeleteFiles, DeleteFilesTypeId, TaskNode ); + + Gaffer::StringVectorDataPlug *filesPlug(); + const Gaffer::StringVectorDataPlug *filesPlug() const; + + Gaffer::BoolPlug *deleteDirectoriesPlug(); + const Gaffer::BoolPlug *deleteDirectoriesPlug() const; + + protected : + + IECore::MurmurHash hash( const Gaffer::Context *context ) const override; + void execute() const override; + + private : + + static size_t g_firstPlugIndex; + + // Friendship for the bindings + friend struct GafferDispatchBindings::Detail::TaskNodeAccessor; + +}; + +} // namespace GafferDispatch diff --git a/include/GafferDispatch/FileList.h b/include/GafferDispatch/FileList.h new file mode 100644 index 00000000000..4bb65b081b3 --- /dev/null +++ b/include/GafferDispatch/FileList.h @@ -0,0 +1,117 @@ +////////////////////////////////////////////////////////////////////////// +// +// Copyright (c) 2025, Cinesite VFX Ltd. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above +// copyright notice, this list of conditions and the following +// disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided with +// the distribution. +// +// * Neither the name of John Haddon nor the names of +// any other contributors to this software may be used to endorse or +// promote products derived from this software without specific prior +// written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include "GafferDispatch/Export.h" +#include "GafferDispatch/TypeIds.h" + +#include "Gaffer/ComputeNode.h" +#include "Gaffer/NumericPlug.h" +#include "Gaffer/StringPlug.h" +#include "Gaffer/TypedObjectPlug.h" + +namespace GafferDispatch +{ + +class GAFFERDISPATCH_API FileList : public Gaffer::ComputeNode +{ + + public : + + explicit FileList( const std::string &name=defaultName() ); + ~FileList() override; + + GAFFER_NODE_DECLARE_TYPE( GafferDispatch::FileList, FileListTypeId, Gaffer::ComputeNode ); + + Gaffer::BoolPlug *enabledPlug() override; + const Gaffer::BoolPlug *enabledPlug() const override; + + Gaffer::StringPlug *directoryPlug(); + const Gaffer::StringPlug *directoryPlug() const; + + Gaffer::IntPlug *refreshCountPlug(); + const Gaffer::IntPlug *refreshCountPlug() const; + + Gaffer::StringPlug *inclusionsPlug(); + const Gaffer::StringPlug *inclusionsPlug() const; + + Gaffer::StringPlug *exclusionsPlug(); + const Gaffer::StringPlug *exclusionsPlug() const; + + Gaffer::StringPlug *extensionsPlug(); + const Gaffer::StringPlug *extensionsPlug() const; + + Gaffer::BoolPlug *searchSubdirectoriesPlug(); + const Gaffer::BoolPlug *searchSubdirectoriesPlug() const; + + Gaffer::BoolPlug *absolutePlug(); + const Gaffer::BoolPlug *absolutePlug() const; + + enum class SequenceMode + { + // Lists all files individually, even if they belong to a sequence. + Files, + // Collects files into frame sequences, listing only the sequences. + // Files not in a sequence are ommitted. + Sequences, + // Outputs sequences where possible, with non-sequence files listed + // individually. + FilesAndSequences, + }; + + Gaffer::IntPlug *sequenceModePlug(); + const Gaffer::IntPlug *sequenceModePlug() const; + + Gaffer::StringVectorDataPlug *outPlug(); + const Gaffer::StringVectorDataPlug *outPlug() const; + + void affects( const Gaffer::Plug *input, AffectedPlugsContainer &outputs ) const override; + + protected : + + void hash( const Gaffer::ValuePlug *output, const Gaffer::Context *context, IECore::MurmurHash &h ) const override; + void compute( Gaffer::ValuePlug *output, const Gaffer::Context *context ) const override; + + Gaffer::ValuePlug::CachePolicy computeCachePolicy( const Gaffer::ValuePlug *output ) const override; + + private : + + static size_t g_firstPlugIndex; + +}; + +} // namespace GafferDispatch diff --git a/include/GafferDispatch/RenameFiles.h b/include/GafferDispatch/RenameFiles.h new file mode 100644 index 00000000000..e166acb11aa --- /dev/null +++ b/include/GafferDispatch/RenameFiles.h @@ -0,0 +1,107 @@ +////////////////////////////////////////////////////////////////////////// +// +// Copyright (c) 2025, Cinesite VFX Ltd. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above +// copyright notice, this list of conditions and the following +// disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided with +// the distribution. +// +// * Neither the name of John Haddon nor the names of +// any other contributors to this software may be used to endorse or +// promote products derived from this software without specific prior +// written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include "GafferDispatch/TaskNode.h" + +#include "Gaffer/StringPlug.h" +#include "Gaffer/TypedObjectPlug.h" + +namespace GafferDispatch +{ + +class GAFFERDISPATCH_API RenameFiles : public TaskNode +{ + + public : + + explicit RenameFiles( const std::string &name=defaultName() ); + ~RenameFiles() override; + + GAFFER_NODE_DECLARE_TYPE( GafferDispatch::RenameFiles, RenameFilesTypeId, TaskNode ); + + Gaffer::StringVectorDataPlug *filesPlug(); + const Gaffer::StringVectorDataPlug *filesPlug() const; + + Gaffer::StringPlug *namePlug(); + const Gaffer::StringPlug *namePlug() const; + + Gaffer::StringPlug *deletePrefixPlug(); + const Gaffer::StringPlug *deletePrefixPlug() const; + + Gaffer::StringPlug *deleteSuffixPlug(); + const Gaffer::StringPlug *deleteSuffixPlug() const; + + Gaffer::StringPlug *findPlug(); + const Gaffer::StringPlug *findPlug() const; + + Gaffer::StringPlug *replacePlug(); + const Gaffer::StringPlug *replacePlug() const; + + Gaffer::BoolPlug *useRegularExpressionsPlug(); + const Gaffer::BoolPlug *useRegularExpressionsPlug() const; + + Gaffer::StringPlug *addPrefixPlug(); + const Gaffer::StringPlug *addPrefixPlug() const; + + Gaffer::StringPlug *addSuffixPlug(); + const Gaffer::StringPlug *addSuffixPlug() const; + + Gaffer::BoolPlug *replaceExtensionPlug(); + const Gaffer::BoolPlug *replaceExtensionPlug() const; + + Gaffer::StringPlug *extensionPlug(); + const Gaffer::StringPlug *extensionPlug() const; + + Gaffer::BoolPlug *overwritePlug(); + const Gaffer::BoolPlug *overwritePlug() const; + + protected : + + IECore::MurmurHash hash( const Gaffer::Context *context ) const override; + void execute() const override; + + private : + + static size_t g_firstPlugIndex; + + // Friendship for the bindings + friend struct GafferDispatchBindings::Detail::TaskNodeAccessor; + +}; + +} // namespace GafferDispatch diff --git a/include/GafferDispatch/TypeIds.h b/include/GafferDispatch/TypeIds.h index 042a064385d..ece94fd3408 100644 --- a/include/GafferDispatch/TypeIds.h +++ b/include/GafferDispatch/TypeIds.h @@ -47,6 +47,10 @@ enum TypeId DispatcherTypeId = 118802, TaskListTypeId = 118803, FrameMaskTypeId = 118804, + FileListTypeId = 118805, + DeleteFilesTypeId = 118806, + CopyFilesTypeId = 118807, + RenameFilesTypeId = 118808, LastTypeId = 118999 diff --git a/python/GafferDispatchTest/CopyFilesTest.py b/python/GafferDispatchTest/CopyFilesTest.py new file mode 100644 index 00000000000..ec6fa3534ef --- /dev/null +++ b/python/GafferDispatchTest/CopyFilesTest.py @@ -0,0 +1,233 @@ +########################################################################## +# +# Copyright (c) 2025, Cinesite VFX Ltd. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided with +# the distribution. +# +# * Neither the name of John Haddon nor the names of +# any other contributors to this software may be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +########################################################################## + +import os +import pathlib +import subprocess +import shutil +import sys +import unittest + +import IECore + +import GafferDispatch +import GafferTest + +class CopyFilesTest( GafferTest.TestCase ) : + + @classmethod + def setUpClass( cls ) : + + GafferTest.TestCase.setUpClass() + + if sys.platform == "darwin" : + cls.__ramDisk = pathlib.Path( "/Volumes/GafferTest" ) + assert( not cls.__ramDisk.exists() ) + image = subprocess.check_output( [ "hdiutil", "attach", "-nomount", "ram://1024" ] ).strip() + subprocess.check_call( [ "diskutil", "erasevolume", "HFS+", "GafferTest", image ] ) + elif sys.platform == "linux" : + cls.__ramDisk = pathlib.Path( "/dev/shm/GafferTest" ) + assert( not cls.__ramDisk.exists() ) + cls.__ramDisk.mkdir() + else : + cls.__ramDisk = None + + @classmethod + def tearDownClass( cls ) : + + GafferTest.TestCase.tearDownClass() + + if sys.platform == "darwin" : + subprocess.check_call( [ "hdiutil", "detach", cls.__ramDisk ] ) + elif sys.platform == "linux" : + shutil.rmtree( cls.__ramDisk ) + + def setUp( self ) : + + GafferTest.TestCase.setUp( self ) + + if self.__ramDisk is not None : + self.__temporaryRAMDirectory = pathlib.Path( self.__ramDisk ) / "copyFilesTest" + self.__temporaryRAMDirectory.mkdir() + else : + self.__temporaryRAMDirectory = None + + def tearDown( self ) : + + GafferTest.TestCase.tearDown( self ) + + if self.__temporaryRAMDirectory is not None : + shutil.rmtree( self.__temporaryRAMDirectory ) + + def __sourceDirectories( self ) : + + # We run every test with two different source directories, with one + # of them being on a RAM-based filesystem. This gives us test coverage + # for the inability of `filesystem::rename()` to move files between file + # systems. + return [ self.temporaryDirectory() ] + [ self.__temporaryRAMDirectory ] if self.__temporaryRAMDirectory is not None else [] + + def testMissingDestinationDirectory( self ) : + + for sourceDirectory in self.__sourceDirectories() : + + with self.subTest( sourceDirectory = sourceDirectory ) : + + source = sourceDirectory / "source.txt" + source.write_text( "Test" ) + + destination = self.temporaryDirectory() / "destination" + + node = GafferDispatch.CopyFiles() + node["files"].setValue( IECore.StringVectorData( [ str( source ) ] ) ) + node["destination"].setValue( destination ) + node["task"].execute() + + self.assertTrue( destination.is_dir() ) + self.assertEqual( ( destination / "source.txt" ).read_text(), "Test" ) + + shutil.rmtree( destination ) + + def testDeleteSource( self ) : + + for sourceDirectory in self.__sourceDirectories() : + + with self.subTest( sourceDirectory = sourceDirectory ) : + + source = sourceDirectory / "source.txt" + source.write_text( "Test" ) + + destination = self.temporaryDirectory() / "destination" + node = GafferDispatch.CopyFiles() + + node["files"].setValue( IECore.StringVectorData( [ str( source ) ] ) ) + node["destination"].setValue( destination ) + node["deleteSource"].setValue( True ) + node["task"].execute() + + self.assertTrue( destination.is_dir() ) + self.assertEqual( ( destination / "source.txt" ).read_text(), "Test" ) + self.assertFalse( source.exists() ) + + shutil.rmtree( destination ) + + def testOverwrite( self ) : + + for sourceDirectory in self.__sourceDirectories() : + + with self.subTest( sourceDirectory = sourceDirectory ) : + + source = sourceDirectory / "file.txt" + source.write_text( "Test" ) + + destinationDir = self.temporaryDirectory() / "destination" + destinationDir.mkdir() + + destinationFile = destinationDir / source.name + destinationFile.touch() + + node = GafferDispatch.CopyFiles() + node["files"].setValue( IECore.StringVectorData( [ str( source ) ] ) ) + node["destination"].setValue( destinationDir ) + + self.assertFalse( node["overwrite"].getValue() ) + + with self.assertRaisesRegex( RuntimeError, "File exists" ) : + node["task"].execute() + self.assertEqual( destinationFile.read_text(), "" ) + + node["overwrite"].setValue( True ) + node["task"].execute() + + self.assertEqual( destinationFile.read_text(), "Test" ) + + shutil.rmtree( destinationDir ) + + def testSourceNotDeletedIfCopyFails( self ) : + + for sourceDirectory in self.__sourceDirectories() : + + with self.subTest( sourceDirectory = sourceDirectory ) : + + source = sourceDirectory / "file.txt" + source.write_text( "Test" ) + + destinationDir = self.temporaryDirectory() / "destination" + destinationDir.mkdir( mode = os.O_RDONLY, exist_ok = True ) + + node = GafferDispatch.CopyFiles() + node["files"].setValue( IECore.StringVectorData( [ str( source ) ] ) ) + node["destination"].setValue( destinationDir ) + node["overwrite"].setValue( True ) + node["deleteSource"].setValue( True ) + + with self.assertRaisesRegex( RuntimeError, "Permission denied" ) : + node["task"].execute() + + self.assertTrue( source.exists() ) + self.assertEqual( source.read_text(), "Test" ) + + def testCopyDirectory( self ) : + + for sourceDirectory in self.__sourceDirectories() : + + with self.subTest( sourceDirectory = sourceDirectory ) : + + sourceDirectory = sourceDirectory / "myFiles" + sourceDirectory.mkdir() + ( sourceDirectory / "a.txt" ).write_text( "Test A" ) + ( sourceDirectory / "b.txt" ).write_text( "Test B" ) + ( sourceDirectory / "subDir" ).mkdir() + ( sourceDirectory / "subDir" / "c.txt" ).write_text( "Test C" ) + + destinationDir = self.temporaryDirectory() / "destination" + + node = GafferDispatch.CopyFiles() + node["files"].setValue( IECore.StringVectorData( [ str( sourceDirectory ) ] ) ) + node["destination"].setValue( destinationDir ) + node["deleteSource"].setValue( True ) + node["task"].execute() + + self.assertEqual( ( destinationDir / "myFiles" / "a.txt" ).read_text(), "Test A" ) + self.assertEqual( ( destinationDir / "myFiles" / "b.txt" ).read_text(), "Test B" ) + self.assertEqual( ( destinationDir / "myFiles" / "subDir" / "c.txt" ).read_text(), "Test C" ) + + self.assertFalse( sourceDirectory.exists() ) + + shutil.rmtree( destinationDir ) + +if __name__ == "__main__": + unittest.main() diff --git a/python/GafferDispatchTest/DeleteFilesTest.py b/python/GafferDispatchTest/DeleteFilesTest.py new file mode 100644 index 00000000000..14e635400ba --- /dev/null +++ b/python/GafferDispatchTest/DeleteFilesTest.py @@ -0,0 +1,112 @@ +########################################################################## +# +# Copyright (c) 2025, Cinesite VFX Ltd. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided with +# the distribution. +# +# * Neither the name of John Haddon nor the names of +# any other contributors to this software may be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +########################################################################## + +import pathlib +import unittest + +import IECore + +import Gaffer +import GafferDispatch +import GafferTest + +class DeleteFilesTest( GafferTest.TestCase ) : + + def setUp( self ) : + + GafferTest.TestCase.setUp( self ) + + # Make a little directory structure to test against. + + for path in [ + "a", + "b", + "d/a", + "d/b", + ] : + path = self.temporaryDirectory() / path + path.parent.mkdir( parents = True, exist_ok = True ) + path.touch() + + def testDeleteFiles( self ) : + + node = GafferDispatch.DeleteFiles() + node["files"].setValue( + IECore.StringVectorData( [ + str( self.temporaryDirectory() / "a" ), + str( self.temporaryDirectory() / "d" / "a" ), + ] ) + ) + + for file in node["files"].getValue() : + self.assertTrue( pathlib.Path( file ).is_file() ) + + node["task"].execute() + for file in node["files"].getValue() : + self.assertFalse( pathlib.Path( file ).exists() ) + + self.assertTrue( ( self.temporaryDirectory() / "b" ).is_file() ) + self.assertTrue( ( self.temporaryDirectory() / "d" / "b" ).is_file() ) + + def testDeleteDirectories( self ) : + + dir = self.temporaryDirectory() / "d" + + node = GafferDispatch.DeleteFiles() + node["files"].setValue( IECore.StringVectorData( [ str( dir ) ] ) ) + + self.assertTrue( dir.is_dir() ) + + with self.assertRaisesRegex( Gaffer.ProcessException, "(Directory not empty|The directory is not empty)" ) : + node["task"].execute() + + self.assertTrue( dir.is_dir() ) + + node["deleteDirectories"].setValue( True ) + node["task"].execute() + + self.assertFalse( dir.exists() ) + + def testHash( self ) : + + node = GafferDispatch.DeleteFiles() + self.assertEqual( node["task"].hash(), IECore.MurmurHash() ) + + node["files"].setValue( IECore.StringVectorData( [ "a" ] ) ) + self.assertNotEqual( node["task"].hash(), IECore.MurmurHash() ) + +if __name__ == "__main__": + unittest.main() diff --git a/python/GafferDispatchTest/FileListTest.py b/python/GafferDispatchTest/FileListTest.py new file mode 100644 index 00000000000..d4bd411ccb3 --- /dev/null +++ b/python/GafferDispatchTest/FileListTest.py @@ -0,0 +1,243 @@ +########################################################################## +# +# Copyright (c) 2025, Cinesite VFX Ltd. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided with +# the distribution. +# +# * Neither the name of John Haddon nor the names of +# any other contributors to this software may be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +########################################################################## + +import pathlib +import unittest + +import IECore + +import GafferDispatch +import GafferTest + +class FileListTest( GafferTest.TestCase ) : + + def setUp( self ) : + + GafferTest.TestCase.setUp( self ) + + # Make a little directory structure to test against. + + for path in [ + "a1", + "a2", + "b1", + "b2", + "d1/a", + "d1/b", + "d2/sd1/a", + "d2/sd1/b", + "d2/sd2/a", + "e", + "e.tiff", + "f.TIFF", + "g.jpg", + ] : + path = self.temporaryDirectory() / path + path.parent.mkdir( parents = True, exist_ok = True ) + path.touch() + + def testUnspecifiedDirectory( self ) : + + node = GafferDispatch.FileList() + self.assertEqual( node["directory"].getValue(), "" ) + self.assertEqual( node["out"].getValue(), IECore.StringVectorData() ) + + def testSimpleInclusions( self ) : + + node = GafferDispatch.FileList() + node["directory"].setValue( self.temporaryDirectory() ) + node["absolute"].setValue( False ) + + self.assertEqual( + node["out"].getValue(), + IECore.StringVectorData( sorted( [ + p.name for p in self.temporaryDirectory().glob( "*" ) + ] ) ) + ) + + node["inclusions"].setValue( "a*" ) + self.assertEqual( + node["out"].getValue(), + IECore.StringVectorData( sorted( [ + p.name for p in self.temporaryDirectory().glob( "a*" ) + ] ) ) + ) + + node["inclusions"].setValue( "a* d*" ) + self.assertEqual( + node["out"].getValue(), + IECore.StringVectorData( sorted( [ + p.name for p in self.temporaryDirectory().glob( "a*" ) + ] + [ + p.name for p in self.temporaryDirectory().glob( "d*" ) + ] ) ) + ) + + def testRecursiveInclusions( self ) : + + node = GafferDispatch.FileList() + node["directory"].setValue( self.temporaryDirectory() ) + node["absolute"].setValue( False ) + + node["inclusions"].setValue( ".../a*" ) + self.assertEqual( + node["out"].getValue(), + IECore.StringVectorData( [ "a1", "a2", "d1/a", "d2/sd1/a", "d2/sd2/a" ] ) + ) + + def testExclusions( self ) : + + node = GafferDispatch.FileList() + node["directory"].setValue( self.temporaryDirectory() ) + node["inclusions"].setValue( "..." ) + node["exclusions"].setValue( "d*" ) + node["absolute"].setValue( False ) + self.assertEqual( + node["out"].getValue(), + IECore.StringVectorData( [ "a1", "a2", "b1", "b2", "e", "e.tiff", "f.TIFF", "g.jpg" ] ) + ) + + def testSearchSubdirectories( self ) : + + node = GafferDispatch.FileList() + node["directory"].setValue( self.temporaryDirectory() ) + node["absolute"].setValue( False ) + + node["inclusions"].setValue( "a*" ) + node["searchSubdirectories"].setValue( True ) + self.assertEqual( + node["out"].getValue(), + IECore.StringVectorData( [ "a1", "a2", "d1/a", "d2/sd1/a", "d2/sd2/a" ] ) + ) + + def testDisable( self ) : + + node = GafferDispatch.FileList() + node["directory"].setValue( self.temporaryDirectory() ) + self.assertTrue( len( node["out"].getValue() ) ) + + node["enabled"].setValue( False ) + self.assertEqual( node["out"].getValue(), IECore.StringVectorData() ) + + def testExtensions( self ) : + + node = GafferDispatch.FileList() + node["directory"].setValue( self.temporaryDirectory() ) + node["extensions"].setValue( "tiff" ) + node["absolute"].setValue( False ) + self.assertEqual( node["out"].getValue(), IECore.StringVectorData( [ "e.tiff", "f.TIFF" ] ) ) + + node["extensions"].setValue( "tiff jpg" ) + self.assertEqual( node["out"].getValue(), IECore.StringVectorData( [ "e.tiff", "f.TIFF", "g.jpg" ] ) ) + + def testFilePatternExtensions( self ) : + + node = GafferDispatch.FileList() + node["directory"].setValue( self.temporaryDirectory() ) + node["inclusions"].setValue( "*.tiff *.jpg" ) + node["absolute"].setValue( False ) + self.assertEqual( node["out"].getValue(), IECore.StringVectorData( [ "e.tiff", "g.jpg" ] ) ) + + def testRefresh( self ) : + + node = GafferDispatch.FileList() + node["directory"].setValue( self.temporaryDirectory() ) + node["absolute"].setValue( False ) + self.assertIn( "a1", node["out"].getValue() ) + + ( self.temporaryDirectory() / "a1" ).unlink() + self.assertIn( "a1", node["out"].getValue() ) + + node["refreshCount"].setValue( 1 ) + self.assertNotIn( "a1", node["out"].getValue() ) + + def testAbsolute( self ) : + + node = GafferDispatch.FileList() + node["directory"].setValue( self.temporaryDirectory() ) + self.assertTrue( node["absolute"].getValue() ) + + absolutePaths = [ pathlib.Path( p ) for p in node["out"] ] + for path in absolutePaths : + self.assertTrue( path.is_absolute() ) + + node["absolute"].setValue( False ) + relativePaths = [ pathlib.Path( p ) for p in node["out"] ] + for path in relativePaths : + self.assertFalse( path.is_absolute() ) + + for absolutePath, relativePath in zip( absolutePaths, relativePaths ) : + self.assertEqual( absolutePath, relativePath.absolute() ) + + def testSequenceMode( self ) : + + directory = self.temporaryDirectory() / "sequenceMode" + directory.mkdir() + + sequencePath = directory / f"test.####.exr" + + sequenceFilePaths = [] + for i in range( 0, 5 ) : + sequenceFilePath = directory / f"test.{i:04}.exr" + sequenceFilePath.touch() + sequenceFilePaths.append( sequenceFilePath ) + + filePath = directory / "test.exr" + filePath.touch() + + node = GafferDispatch.FileList() + node["directory"].setValue( directory ) + + self.assertEqual( node["sequenceMode"].getValue(), node.SequenceMode.Files ) + self.assertEqual( + node["out"].getValue(), + IECore.StringVectorData( [ x.as_posix() for x in sequenceFilePaths + [ filePath ] ] ) + ) + + node["sequenceMode"].setValue( node.SequenceMode.Sequences ) + self.assertEqual( + node["out"].getValue(), + IECore.StringVectorData( [ sequencePath.as_posix() ] ) + ) + + node["sequenceMode"].setValue( node.SequenceMode.FilesAndSequences ) + self.assertEqual( + node["out"].getValue(), + IECore.StringVectorData( [ filePath.as_posix(), sequencePath.as_posix() ] ) + ) + +if __name__ == "__main__": + unittest.main() diff --git a/python/GafferDispatchTest/RenameFilesTest.py b/python/GafferDispatchTest/RenameFilesTest.py new file mode 100644 index 00000000000..64c685fc085 --- /dev/null +++ b/python/GafferDispatchTest/RenameFilesTest.py @@ -0,0 +1,263 @@ +########################################################################## +# +# Copyright (c) 2025, Cinesite VFX Ltd. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided with +# the distribution. +# +# * Neither the name of John Haddon nor the names of +# any other contributors to this software may be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +########################################################################## + +import unittest + +import IECore + +import Gaffer +import GafferDispatch +import GafferTest + +class RenameFilesTest( GafferTest.TestCase ) : + + def testSourceVariable( self ) : + + path1 = self.temporaryDirectory() / "test1" + path2 = self.temporaryDirectory() / "test2" + path1.touch() + path2.touch() + + spreadsheet = Gaffer.Spreadsheet() + spreadsheet["selector"].setValue( "${source}" ) + spreadsheet["rows"].addColumn( Gaffer.StringPlug( "name" ) ) + spreadsheet["rows"].addRows( 2 ) + spreadsheet["rows"][0]["name"].setValue( str( path1 ) ) + spreadsheet["rows"][0]["cells"]["name"]["value"].setValue( "apple" ) + spreadsheet["rows"][1]["name"].setValue( str( path2 ) ) + spreadsheet["rows"][1]["cells"]["name"]["value"].setValue( "pear" ) + + rename = GafferDispatch.RenameFiles() + rename["files"].setValue( IECore.StringVectorData( [ str( path1 ), str( path2 ) ] ) ) + rename["name"].setInput( spreadsheet["out"]["name"] ) + rename["task"].execute() + + self.assertEqual( + { p.name for p in self.temporaryDirectory().glob( "*" ) }, + { "apple", "pear" } + ) + + def testStemAndExtensionVariables( self ) : + + path = self.temporaryDirectory() / "test.tar" + path.touch() + + rename = GafferDispatch.RenameFiles() + rename["files"].setValue( IECore.StringVectorData( [ str( path ) ] ) ) + rename["name"].setValue( "${source:stem}Suffix${source:extension}.gz" ) + rename["task"].execute() + + self.assertEqual( + [ p.name for p in self.temporaryDirectory().glob( "*" ) ], + [ "testSuffix.tar.gz" ] + ) + + def testChangeAffixes( self ) : + + path = self.temporaryDirectory() / "prefixMiddleSuffix.ext" + path.touch() + + rename = GafferDispatch.RenameFiles() + rename["files"].setValue( IECore.StringVectorData( [ str( path ) ] ) ) + rename["deletePrefix"].setValue( "prefix" ) + rename["deleteSuffix"].setValue( "Suffix" ) + rename["addPrefix"].setValue( "newBeginning" ) + rename["addSuffix"].setValue( "NewEnding" ) + rename["task"].execute() + + self.assertEqual( + [ p.name for p in self.temporaryDirectory().glob( "*" ) ], + [ "newBeginningMiddleNewEnding.ext" ] + ) + + def testFindReplace( self ) : + + path = self.temporaryDirectory() / "aBaBaBA.abc" + path.touch() + + rename = GafferDispatch.RenameFiles() + rename["files"].setValue( IECore.StringVectorData( [ str( path ) ] ) ) + rename["find"].setValue( "a" ) + rename["replace"].setValue( "C" ) + rename["task"].execute() + + self.assertEqual( + [ p.name for p in self.temporaryDirectory().glob( "*" ) ], + [ "CBCBCBA.abc" ] + ) + + def testRegularExpressions( self ) : + + path = self.temporaryDirectory() / "abc123.ext" + path.touch() + + rename = GafferDispatch.RenameFiles() + rename["files"].setValue( IECore.StringVectorData( [ str( path ) ] ) ) + rename["find"].setValue( "a*c" ) + rename["replace"].setValue( "{0}d" ) + rename["useRegularExpressions"].setValue( True ) + rename["task"].execute() + + self.assertEqual( + [ p.name for p in self.temporaryDirectory().glob( "*" ) ], + [ "abcd123.ext" ] + ) + + def testNameOverridesEverything( self ) : + + path = self.temporaryDirectory() / "prefixMiddleSuffix.ext" + path.touch() + + rename = GafferDispatch.RenameFiles() + rename["files"].setValue( IECore.StringVectorData( [ str( path ) ] ) ) + rename["name"].setValue( "newName.abc" ) + rename["deletePrefix"].setValue( "prefix" ) + rename["deleteSuffix"].setValue( "Suffix" ) + rename["find"].setValue( "Middle" ) + rename["replace"].setValue( "NewMiddle" ) + rename["addPrefix"].setValue( "newPrefix" ) + rename["addSuffix"].setValue( "NewSuffix" ) + rename["task"].execute() + + self.assertEqual( + [ p.name for p in self.temporaryDirectory().glob( "*" ) ], + [ "newName.abc" ] + ) + + def testConflictingNames( self ) : + + source1 = self.temporaryDirectory() / "path1" + source1.write_text( "1" ) + + source2 = self.temporaryDirectory() / "path2" + source2.write_text( "2" ) + + destination = self.temporaryDirectory() / "path" + + rename = GafferDispatch.RenameFiles() + rename["files"].setValue( IECore.StringVectorData( [ str( source1 ), str( source2 ) ] ) ) + rename["name"].setValue( "path" ) + + with self.assertRaisesRegex( RuntimeError, f'.*Destination ".*/{destination.name}" has multiple source files : ".*/{source1.name}" and ".*/{source2.name}"' ) : + rename["task"].execute() + + self.assertFalse( destination.exists() ) + self.assertEqual( source1.read_text(), "1" ) + self.assertEqual( source2.read_text(), "2" ) + + def testMissingSourceFile( self ) : + + source = self.temporaryDirectory() / "missing" + + rename = GafferDispatch.RenameFiles() + rename["files"].setValue( IECore.StringVectorData( [ str( source ) ] ) ) + rename["name"].setValue( "renamed" ) + + with self.assertRaisesRegex( RuntimeError, f'.*(No such file|cannot find the file).*missing' ) : + rename["task"].execute() + + def testOneSourceOverwritingAnother( self ) : + + fileA = self.temporaryDirectory() / "fileA" + fileA.write_text( "A" ) + + fileB = self.temporaryDirectory() / "fileB" + fileB.write_text( "B" ) + + rename = GafferDispatch.RenameFiles() + rename["files"].setValue( IECore.StringVectorData( [ str( fileA ), str( fileB ) ] ) ) + rename["deleteSuffix"].setValue( "A" ) + rename["addSuffix"].setValue( "B" ) + + with self.assertRaisesRegex( RuntimeError, f'.*Renaming of ".*/{fileA.name}" would overwrite source ".*/{fileB.name}"' ) : + rename["task"].execute() + + self.assertEqual( fileA.read_text(), "A" ) + self.assertEqual( fileB.read_text(), "B" ) + + def testOverwriteExistingFile( self ) : + + fileA = self.temporaryDirectory() / "fileA" + fileA.write_text( "A" ) + + fileB = self.temporaryDirectory() / "fileB" + fileB.write_text( "B" ) + + rename = GafferDispatch.RenameFiles() + rename["files"].setValue( IECore.StringVectorData( [ str( fileA ) ] ) ) + rename["name"].setValue( "fileB" ) + + with self.assertRaisesRegex( RuntimeError, f'.*Can not overwrite destination ".*/{fileB.name}" unless `overwrite` plug is set.' ) : + rename["task"].execute() + + self.assertEqual( fileA.read_text(), "A" ) + self.assertEqual( fileB.read_text(), "B" ) + + rename["overwrite"].setValue( True ) + rename["task"].execute() + + self.assertFalse( fileA.exists() ) + self.assertEqual( fileB.read_text(), "A" ) + + def testReplaceExtension( self ) : + + file = self.temporaryDirectory() / "test.tiff" + file.touch() + + rename = GafferDispatch.RenameFiles() + rename["files"].setValue( IECore.StringVectorData( [ str( file ) ] ) ) + rename["replaceExtension"].setValue( True ) + rename["extension"].setValue( "jpeg" ) + rename["task"].execute() + + self.assertFalse( file.exists() ) + self.assertTrue( file.with_suffix( ".jpeg" ).exists() ) + + def testRemoveExtension( self ) : + + file = self.temporaryDirectory() / "test.txt" + file.touch() + + rename = GafferDispatch.RenameFiles() + rename["files"].setValue( IECore.StringVectorData( [ str( file ) ] ) ) + rename["replaceExtension"].setValue( True ) + rename["task"].execute() + + self.assertFalse( file.exists() ) + self.assertTrue( file.with_suffix( "" ).exists() ) + +if __name__ == "__main__": + unittest.main() diff --git a/python/GafferDispatchTest/__init__.py b/python/GafferDispatchTest/__init__.py index b12d1463404..c8e5775e611 100644 --- a/python/GafferDispatchTest/__init__.py +++ b/python/GafferDispatchTest/__init__.py @@ -58,6 +58,10 @@ from .DispatchApplicationTest import DispatchApplicationTest from .ModuleTest import ModuleTest from .StatsApplicationTest import StatsApplicationTest +from .FileListTest import FileListTest +from .DeleteFilesTest import DeleteFilesTest +from .CopyFilesTest import CopyFilesTest +from .RenameFilesTest import RenameFilesTest if __name__ == "__main__": import unittest diff --git a/python/GafferDispatchUI/CopyFilesUI.py b/python/GafferDispatchUI/CopyFilesUI.py new file mode 100644 index 00000000000..c363efe15f6 --- /dev/null +++ b/python/GafferDispatchUI/CopyFilesUI.py @@ -0,0 +1,97 @@ +########################################################################## +# +# Copyright (c) 2025, Cinesite VFX Ltd. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided with +# the distribution. +# +# * Neither the name of John Haddon nor the names of +# any other contributors to this software may be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +########################################################################## + +import Gaffer +import GafferDispatch + +Gaffer.Metadata.registerNode( + + GafferDispatch.CopyFiles, + + "description", + """ + Copies or moves files into a destination directory. + """, + + plugs = { + + "files" : { + + "description" : + """ + The files to be copied. + """, + + "ui:acceptsFileList" : True, + + }, + + "destination" : { + + "description" : + """ + The destination directory to copy the files to. If + this does not exist yet, it will be created automatically. + """, + + "plugValueWidget:type" : "GafferUI.FileSystemPathPlugValueWidget", + + }, + + "overwrite" : { + + "description" : + """ + If a file already exists at the destination, then overwrites + it rather than erroring. Defaults to off, to reduce the chance + of accidental data loss. + """, + + }, + + "deleteSource" : { + + "description" : + """ + Turns the operation into a move/rename by deleting the source files + after they are copied to the destination. + """, + + }, + + }, + +) diff --git a/python/GafferDispatchUI/DeleteFilesUI.py b/python/GafferDispatchUI/DeleteFilesUI.py new file mode 100644 index 00000000000..3e4c20367e1 --- /dev/null +++ b/python/GafferDispatchUI/DeleteFilesUI.py @@ -0,0 +1,75 @@ +########################################################################## +# +# Copyright (c) 2025, Cinesite VFX Ltd. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided with +# the distribution. +# +# * Neither the name of John Haddon nor the names of +# any other contributors to this software may be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +########################################################################## + +import Gaffer +import GafferDispatch + +Gaffer.Metadata.registerNode( + + GafferDispatch.DeleteFiles, + + "description", + """ + Deletes files from the filesystem. + """, + + plugs = { + + "files" : { + + "description" : + """ + The files to be deleted. It is common for this to be driven + by a FileList node. + """, + + "ui:acceptsFileList" : True, + + }, + + "deleteDirectories" : { + + "description" : + """ + Enables deletion of non-empty directories. Defaults off, as a + safeguard against accidental deletion of large amounts of data. + """, + + }, + + }, + +) diff --git a/python/GafferDispatchUI/FileListUI.py b/python/GafferDispatchUI/FileListUI.py new file mode 100644 index 00000000000..d8de1725258 --- /dev/null +++ b/python/GafferDispatchUI/FileListUI.py @@ -0,0 +1,235 @@ +########################################################################## +# +# Copyright (c) 2025, Cinesite VFX Ltd. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided with +# the distribution. +# +# * Neither the name of John Haddon nor the names of +# any other contributors to this software may be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +########################################################################## + +import functools + +import Gaffer +import GafferUI +import GafferDispatch + +Gaffer.Metadata.registerNode( + + GafferDispatch.FileList, + + "description", + """ + Searches the filesystem for files matching certain criteria, + and outputs a list of the file paths. + """, + + "nodeGadget:type", "GafferUI::AuxiliaryNodeGadget", + "auxiliaryNodeGadget:label", "f", + "nodeGadget:focusGadgetVisible", False, + + plugs = { + + "*" : { + + "nodule:type" : "", + + }, + + "directory" : { + + "description" : + """ + The directory in which to search for files. By default, all files + in the directory will be returned. Use the `filePatterns` and + `fileExtensions` plugs to filter the files. + """, + + "plugValueWidget:type" : "GafferUI.FileSystemPathPlugValueWidget", + + }, + + "refreshCount" : { + + "description" : + """ + May be incremented to force a fresh search, taking into account + any changes to the filesystem that have occurred since the + previous one. + """, + + "plugValueWidget:type" : "GafferUI.RefreshPlugValueWidget", + "layout:label" : "", + "layout:accessory" : True, + + }, + + "inclusions" : { + + "description" : + """ + A space-separated list of patterns used to filter the file + list. Only filenames matching the patterns are included in + the output. The following wildcards are available : + + - `*` : Matches any sequence of characters, except `/`. + - `?` : Matches any single character. + - `[A-Z]` : Matches any single character in the specified range. + - `...` : Matches any number of subdirectories. + + Examples : + + - `*` : Matches all files in the directory. + - `test*` : Matches all files starting with `test`. + - `imageFiles/*` : Matches all files in an `imageFiles` subdirectory. + - `imageFiles/.../*` : Matches recursively, finding all files in all + subdirectories of `imageFiles`. + """, + + }, + + "exclusions" : { + + "description" : + """ + A space-separated list of patterns used to exclude files from the + list. Supports the same wildcards as `inclusions`. + """, + + }, + + "extensions" : { + + "description" : + """ + A list of file extensions to filter on. Extension comparison + is case-insensitive. + """, + + "preset:All" : "*", + + "plugValueWidget:type" : "GafferUI.PresetsPlugValueWidget", + "presetsPlugValueWidget:allowCustom" : True, + + }, + + "searchSubdirectories" : { + + "description" : + """ + Extends the search to all subdirectories of `directory`. + Equivalent to prefixing all `inclusions` and `exclusions` + with `.../`. + """, + + }, + + "absolute" : { + + "description" : + """ + Outputs absolute paths. When off, the output paths are relative + to the `directory`. + """, + + }, + + "sequenceMode" : { + + "description" : + """ + Defines how frame sequences are treated. A frame sequence is a group of files + whose names differ only in their frame number. + + - Files : All files are listed. Files from frame sequences are listed individually. + - Sequences : Files from each frame sequence are listed as a single path containing + `#` characters representing the frame numbers. Files not from a frame sequence are omitted. + - FilesAndSequences : As for Sequences mode, but also including files not in any frame sequence. + """, + + "plugValueWidget:type" : "GafferUI.PresetsPlugValueWidget", + "preset:Files" : GafferDispatch.FileList.SequenceMode.Files, + "preset:Sequences" : GafferDispatch.FileList.SequenceMode.Sequences, + "preset:Files And Sequences" : GafferDispatch.FileList.SequenceMode.FilesAndSequences, + + }, + + "out" : { + + "description" : + """ + The list of all files found by the node. + """, + + "layout:section" : "Files", + "nodule:type" : "GafferUI::StandardNodule", + + } + + } + +) + +def __createFileList( plugs ) : + + parentNode = next( iter( plugs ) ).node().parent() + + with Gaffer.UndoScope( parentNode.scriptNode() ) : + + fileList = GafferDispatch.FileList() + parentNode.addChild( fileList ) + + for plug in plugs : + plug.setInput( fileList["out"] ) + + GafferUI.NodeEditor.acquire( fileList ) + +def __plugPopupMenu( menuDefinition, plugValueWidget ) : + + if not len( plugValueWidget.getPlugs() ) : + return + + for plug in plugValueWidget.getPlugs() : + if not Gaffer.Metadata.value( plug, "ui:acceptsFileList" ) : + return + if plug.getInput() is not None : + return + node = plug.node() + if node is None or node.parent() is None : + return + + menuDefinition.prepend( "/FileListDivider", { "divider" : True } ) + menuDefinition.prepend( + "/Create File List...", + { + "command" : functools.partial( __createFileList, plugValueWidget.getPlugs() ) + } + ) + +GafferUI.PlugValueWidget.popupMenuSignal().connect( __plugPopupMenu ) diff --git a/python/GafferDispatchUI/RenameFilesUI.py b/python/GafferDispatchUI/RenameFilesUI.py new file mode 100644 index 00000000000..a8270c137c9 --- /dev/null +++ b/python/GafferDispatchUI/RenameFilesUI.py @@ -0,0 +1,258 @@ +########################################################################## +# +# Copyright (c) 2025, Cinesite VFX Ltd. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided with +# the distribution. +# +# * Neither the name of John Haddon nor the names of +# any other contributors to this software may be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +########################################################################## + +import Gaffer +import GafferDispatch + +Gaffer.Metadata.registerNode( + + GafferDispatch.RenameFiles, + + "description", + """ + Renames files, with options for changing prefixes and suffixes, searching & replacing, + replacing extensions and defining arbitrary names using expressions or spreadsheets. + + > Tip : Use the CopyFiles node to copy or move files between directories. + """, + + "layout:activator:nameIsSetToDefault", lambda node : node["name"].isSetToDefault(), + + "ui:spreadsheet:enabledRowNamesConnection", "files", + "ui:spreadsheet:selectorValue", "${source}", + + plugs = { + + "files" : { + + "description" : + """ + The list of files to be renamed. It is common for this to be driven + by a FileList node. + """, + + "ui:acceptsFileList" : True, + "layout:divider" : True, + + }, + + "name" : { + + "description" : + """ + The new name for the file. If this name is non-empty then it + takes precedence, and all other renaming operations are ignored. + + > Tip : The following context variables are defined when evaluating + > this plug : + > + > `source` : The full path to the file being renamed. + > `source:stem` : The name of the file being renamed, excluding the extension. + > `source:extension` : The extension of the file being renamed. + """, + + "layout:divider" : True, + "ui:spreadsheet:selectorValue" : "${source}", + + }, + + "deletePrefix" : { + + "description" : + """ + A prefix to remove from the start of the original name. Prefixes are removed + before the suffixes and before the find and replace operation is + performed. + """, + + "layout:activator" : "nameIsSetToDefault", + + }, + + "deleteSuffix" : { + + "description" : + """ + A suffix to remove from the start of the original name. Suffixes are removed + before the find and replace operation is performed. + """, + + "layout:activator" : "nameIsSetToDefault", + "layout:divider" : True, + + }, + + "find" : { + + "description" : + """ + A string to search for within the original name. All occurrences of this string + will be replaced with the value of `replace`. When `useRegularExpressions` + is on, the search string is treated as a regular expression, with the + following syntax : + + Matching + -------- + + - `.` : Matches any character. + - `[aef]` : Matches any character in the set. + - `[^aef]` : Matches any character not in the set. + - `[a-z]` : Matches any character in the specified range. + - `[[:digit:]]` : Matches any numeric digit. + - `[[:space:]]` : Matches any whitespace character. + + Repetition + ---------- + + - `*` : Matches the preceding pattern any number of times (including none). + - `+` : Matches the preceding pattern 1 or more times. + - `{N}` : Matches the preceding pattern N times. + - `{M,N}` : Matches the preceding pattern between M and N times. + + Alternatives + ------------ + + - `A|B` : Matches either pattern A or pattern B. + + Captures + -------- + + - `()` : Captures the subgroup of the pattern within the brackets, + allowing it to be referenced by `{}` in the `replace` string. + + """, + + "layout:activator" : "nameIsSetToDefault", + + }, + + "replace" : { + + "description" : + """ + The replacement for strings matched by the `find` plug. + When `useRegularExpressions` is on, this can refer to + captured patterns using Python's standard string formatting + syntax : + + - `{0}` : The entire string matched by the regular expresion. + - `{1}` : The 1st subgroup captured within `()` brackets by the `find` string. + - `{N}` : The Nth subgroup captured within `()` brackets by the `find` string. + - `{1:0>4}` : The 1st subgroup, aligned to the right and padded to width 4. + """, + + "layout:activator" : "nameIsSetToDefault", + + }, + + "useRegularExpressions" : { + + "description" : + """ + When on, the `find` string is treated as a regular expression, + allowing it to perform complex pattern matching and to capture sections + of the match to be referenced by the `replace` string. + """, + + "layout:activator" : "nameIsSetToDefault", + "layout:divider" : True, + + }, + + "addPrefix" : { + + "description" : + """ + A string to add at the start of the name. Prefixes are + added last, after the find and replace operation has + been performed. + """, + + "layout:activator" : "nameIsSetToDefault", + + }, + + "addSuffix" : { + + "description" : + """ + A string to add at the end of the name. Suffixes are + added last, after the find and replace operation has + been performed. + """, + + "layout:activator" : "nameIsSetToDefault", + "layout:divider" : True, + + }, + + "replaceExtension" : { + + "description" : + """ + Replaces the file extension with the one specified + by the `extension` plug. + """, + + "layout:activator" : "nameIsSetToDefault", + + }, + + "extension" : { + + "description" : + """ + The new extension to be used when `replaceExtension` is on. + """, + + "layout:activator" : lambda plug : plug.parent()["name"].isSetToDefault() and plug.parent()["replaceExtension"].getValue(), + "layout:divider" : True, + + }, + + "overwrite" : { + + "description" : + """ + If a file already exists at the destination, then overwrites + it rather than erroring. Defaults to off, to reduce the chance + of accidental data loss. + """, + + }, + + } +) diff --git a/python/GafferDispatchUI/WedgeUI.py b/python/GafferDispatchUI/WedgeUI.py index 5e613188587..f54b5c12b25 100644 --- a/python/GafferDispatchUI/WedgeUI.py +++ b/python/GafferDispatchUI/WedgeUI.py @@ -281,6 +281,7 @@ """, "layout:visibilityActivator" : "modeIsStringList", + "ui:acceptsFileList" : True, }, diff --git a/python/GafferDispatchUI/__init__.py b/python/GafferDispatchUI/__init__.py index d029d1dd411..91aa0e8da50 100644 --- a/python/GafferDispatchUI/__init__.py +++ b/python/GafferDispatchUI/__init__.py @@ -49,5 +49,9 @@ from . import PythonCommandUI from . import FrameMaskUI from .LocalJobs import LocalJobs +from . import FileListUI +from . import DeleteFilesUI +from . import CopyFilesUI +from . import RenameFilesUI __import__( "IECore" ).loadConfig( "GAFFER_STARTUP_PATHS", subdirectory = "GafferDispatchUI" ) diff --git a/python/GafferImage/ContactSheet.gfr b/python/GafferImage/ContactSheet.gfr index 7ac68cf10d8..4e50c116f12 100644 --- a/python/GafferImage/ContactSheet.gfr +++ b/python/GafferImage/ContactSheet.gfr @@ -1092,6 +1092,7 @@ Gaffer.Metadata.registerValue( __children["ContactSheet"]["tileNames"], 'layout: Gaffer.Metadata.registerValue( __children["ContactSheet"]["tileNames"], 'layout:index', 5 ) Gaffer.Metadata.registerValue( __children["ContactSheet"]["tileNames"], 'layout:visibilityActivator', 'modeIsCollect' ) Gaffer.Metadata.registerValue( __children["ContactSheet"]["tileNames"], 'description', 'The values to use for the `tileNameVariable` when in `Collect` mode. Each value defines a tile in the contact sheet. Use the tile variables in upstream nodes to vary the source images.' ) +Gaffer.Metadata.registerValue( __children["ContactSheet"]["tileNames"], 'ui:acceptsFileList', True ) Gaffer.Metadata.registerValue( __children["ContactSheet"]["tileNameVariable"], 'nodule:type', '' ) Gaffer.Metadata.registerValue( __children["ContactSheet"]["tileNameVariable"], 'layout:section', 'Settings' ) Gaffer.Metadata.registerValue( __children["ContactSheet"]["tileNameVariable"], 'layout:index', 6 ) diff --git a/python/GafferImageUI/ImageReaderUI.py b/python/GafferImageUI/ImageReaderUI.py index 9aeb440c7bb..97ac1db8f60 100644 --- a/python/GafferImageUI/ImageReaderUI.py +++ b/python/GafferImageUI/ImageReaderUI.py @@ -37,6 +37,7 @@ import IECore import Gaffer +import GafferDispatch import GafferUI import GafferImage @@ -291,6 +292,11 @@ ) +Gaffer.Metadata.registerValue( + GafferDispatch.FileList, "extensions", "preset:Image Files", + " ".join( GafferImage.ImageReader.supportedExtensions() ) +) + # Augments PresetsPlugValueWidget label with the computed colour space # when preset is "Automatic". Since this involves opening the file to # read metadata, we do the work in the background via an auxiliary plug diff --git a/python/GafferSceneUI/SceneReaderUI.py b/python/GafferSceneUI/SceneReaderUI.py index e43ac08861c..00e83b84b50 100644 --- a/python/GafferSceneUI/SceneReaderUI.py +++ b/python/GafferSceneUI/SceneReaderUI.py @@ -39,6 +39,7 @@ import IECoreScene import Gaffer +import GafferDispatch import GafferUI import GafferScene @@ -118,6 +119,11 @@ ) +Gaffer.Metadata.registerValue( + GafferDispatch.FileList, "extensions", "preset:Scene Files", + " ".join( GafferScene.SceneReader.supportedExtensions() ) +) + ########################################################################## # Right click menu for tags ########################################################################## diff --git a/src/GafferDispatch/CopyFiles.cpp b/src/GafferDispatch/CopyFiles.cpp new file mode 100644 index 00000000000..a0ea6325c00 --- /dev/null +++ b/src/GafferDispatch/CopyFiles.cpp @@ -0,0 +1,171 @@ +////////////////////////////////////////////////////////////////////////// +// +// Copyright (c) 2025, Cinesite VFX Ltd. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above +// copyright notice, this list of conditions and the following +// disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided with +// the distribution. +// +// * Neither the name of John Haddon nor the names of +// any other contributors to this software may be used to endorse or +// promote products derived from this software without specific prior +// written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +////////////////////////////////////////////////////////////////////////// + +#include "GafferDispatch/CopyFiles.h" + +#include + +using namespace std; +using namespace IECore; +using namespace Gaffer; +using namespace GafferDispatch; + +GAFFER_NODE_DEFINE_TYPE( CopyFiles ); + +size_t CopyFiles::g_firstPlugIndex = 0; + +CopyFiles::CopyFiles( const std::string &name ) + : TaskNode( name ) +{ + storeIndexOfNextChild( g_firstPlugIndex ); + addChild( new StringVectorDataPlug( "files" ) ); + addChild( new StringPlug( "destination" ) ); + addChild( new BoolPlug( "overwrite" ) ); + addChild( new BoolPlug( "deleteSource" ) ); +} + +CopyFiles::~CopyFiles() +{ +} + +Gaffer::StringVectorDataPlug *CopyFiles::filesPlug() +{ + return getChild( g_firstPlugIndex ); +} + +const Gaffer::StringVectorDataPlug *CopyFiles::filesPlug() const +{ + return getChild( g_firstPlugIndex ); +} + +Gaffer::StringPlug *CopyFiles::destinationPlug() +{ + return getChild( g_firstPlugIndex + 1 ); +} + +const Gaffer::StringPlug *CopyFiles::destinationPlug() const +{ + return getChild( g_firstPlugIndex + 1 ); +} + +Gaffer::BoolPlug *CopyFiles::overwritePlug() +{ + return getChild( g_firstPlugIndex + 2 ); +} + +const Gaffer::BoolPlug *CopyFiles::overwritePlug() const +{ + return getChild( g_firstPlugIndex + 2 ); +} + +Gaffer::BoolPlug *CopyFiles::deleteSourcePlug() +{ + return getChild( g_firstPlugIndex + 3 ); +} + +const Gaffer::BoolPlug *CopyFiles::deleteSourcePlug() const +{ + return getChild( g_firstPlugIndex + 3 ); +} + +IECore::MurmurHash CopyFiles::hash( const Gaffer::Context *context ) const +{ + ConstStringVectorDataPtr filesData = filesPlug()->getValue(); + const std::string destination = destinationPlug()->getValue(); + if( filesData->readable().empty() || destination.empty() ) + { + return IECore::MurmurHash(); + } + + IECore::MurmurHash h = TaskNode::hash( context ); + filesData->hash( h ); + h.append( destination ); + overwritePlug()->hash( h ); + deleteSourcePlug()->hash( h ); + return h; +} + +void CopyFiles::execute() const +{ + ConstStringVectorDataPtr filesData = filesPlug()->getValue(); + filesystem::path destination = destinationPlug()->getValue(); + if( filesData->readable().empty() || destination.empty() ) + { + return; + } + + filesystem::path destinationPath( destination ); + filesystem::create_directories( destinationPath ); + + const bool deleteSource = deleteSourcePlug()->getValue(); + const bool overwrite = overwritePlug()->getValue(); + + filesystem::copy_options options = filesystem::copy_options::recursive; + if( overwrite ) + { + options |= filesystem::copy_options::overwrite_existing; + } + + for( const auto &file : filesData->readable() ) + { + const filesystem::path filePath = file; + const filesystem::path destinationFilePath = destinationPath / filePath.filename(); + if( deleteSource && ( overwrite || !filesystem::exists( destinationFilePath ) ) ) + { + try + { + // If we can simply rename the file, then that will be faster. + std::filesystem::rename( file, destinationFilePath ); + continue; + } + catch( const filesystem::filesystem_error & ) + { + // Couldn't rename. This could be because source and destination + // are on different filesystems, in which case we suppress the + // exception and fall through to copy/remove which should succeed. + // Or it could be due to another problem such as permissions, in + // which case we still suppress and fall through, expecting to + // throw again. + } + } + filesystem::copy( filePath, destinationFilePath, options ); + if( deleteSource ) + { + filesystem::remove_all( filePath ); + } + } + +} diff --git a/src/GafferDispatch/DeleteFiles.cpp b/src/GafferDispatch/DeleteFiles.cpp new file mode 100644 index 00000000000..1785d327753 --- /dev/null +++ b/src/GafferDispatch/DeleteFiles.cpp @@ -0,0 +1,113 @@ +////////////////////////////////////////////////////////////////////////// +// +// Copyright (c) 2025, Cinesite VFX Ltd. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above +// copyright notice, this list of conditions and the following +// disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided with +// the distribution. +// +// * Neither the name of John Haddon nor the names of +// any other contributors to this software may be used to endorse or +// promote products derived from this software without specific prior +// written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +////////////////////////////////////////////////////////////////////////// + +#include "GafferDispatch/DeleteFiles.h" + +#include "Gaffer/Context.h" + +#include + +using namespace std; +using namespace IECore; +using namespace Gaffer; +using namespace GafferDispatch; + +GAFFER_NODE_DEFINE_TYPE( DeleteFiles ); + +size_t DeleteFiles::g_firstPlugIndex = 0; + +DeleteFiles::DeleteFiles( const std::string &name ) + : TaskNode( name ) +{ + storeIndexOfNextChild( g_firstPlugIndex ); + addChild( new StringVectorDataPlug( "files" ) ); + addChild( new BoolPlug( "deleteDirectories" ) ); +} + +DeleteFiles::~DeleteFiles() +{ +} + +Gaffer::StringVectorDataPlug *DeleteFiles::filesPlug() +{ + return getChild( g_firstPlugIndex ); +} + +const Gaffer::StringVectorDataPlug *DeleteFiles::filesPlug() const +{ + return getChild( g_firstPlugIndex ); +} + +Gaffer::BoolPlug *DeleteFiles::deleteDirectoriesPlug() +{ + return getChild( g_firstPlugIndex + 1 ); +} + +const Gaffer::BoolPlug *DeleteFiles::deleteDirectoriesPlug() const +{ + return getChild( g_firstPlugIndex + 1 ); +} + +IECore::MurmurHash DeleteFiles::hash( const Gaffer::Context *context ) const +{ + ConstStringVectorDataPtr filesData = filesPlug()->getValue(); + if( filesData->readable().empty() ) + { + return IECore::MurmurHash(); + } + + IECore::MurmurHash h = TaskNode::hash( context ); + filesData->hash( h ); + deleteDirectoriesPlug()->hash( h ); + return h; +} + +void DeleteFiles::execute() const +{ + const bool deleteDirectories = deleteDirectoriesPlug()->getValue(); + ConstStringVectorDataPtr filesData = filesPlug()->getValue(); + for( const auto &file : filesData->readable() ) + { + if( deleteDirectories ) + { + filesystem::remove_all( filesystem::path( file ) ); + } + else + { + filesystem::remove( filesystem::path( file ) ); + } + } +} diff --git a/src/GafferDispatch/FileList.cpp b/src/GafferDispatch/FileList.cpp new file mode 100644 index 00000000000..cfd215fa795 --- /dev/null +++ b/src/GafferDispatch/FileList.cpp @@ -0,0 +1,385 @@ +////////////////////////////////////////////////////////////////////////// +// +// Copyright (c) 2025, Cinesite VFX Ltd. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above +// copyright notice, this list of conditions and the following +// disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided with +// the distribution. +// +// * Neither the name of John Haddon nor the names of +// any other contributors to this software may be used to endorse or +// promote products derived from this software without specific prior +// written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +////////////////////////////////////////////////////////////////////////// + +#include "GafferDispatch/FileList.h" + +#include "IECore/FileSequenceFunctions.h" +#include "IECore/PathMatcher.h" +#include "IECore/StringAlgo.h" + +#include "boost/algorithm/string/case_conv.hpp" + +#include + +using namespace std; +using namespace IECore; +using namespace Gaffer; +using namespace GafferDispatch; + +namespace +{ + +void fileListWalk( const filesystem::path fileSystemPath, const vector &path, const IECore::PathMatcher &inclusions, const IECore::PathMatcher &exclusions, vector &pathList ) +{ + const unsigned exclusionsMatch = exclusions.match( path ); + if( exclusionsMatch & PathMatcher::ExactMatch ) + { + return; + } + + const unsigned inclusionsMatch = inclusions.match( path ); + if( ( inclusionsMatch & PathMatcher::ExactMatch ) && path.size() ) + { + pathList.push_back( fileSystemPath ); + } + + if( ( inclusionsMatch & PathMatcher::DescendantMatch ) && filesystem::is_directory( fileSystemPath ) ) + { + vector childPath = path; childPath.push_back( InternedString() ); + for( auto directoryEntry : filesystem::directory_iterator( fileSystemPath ) ) + { + childPath.back() = InternedString( directoryEntry.path().filename().string() ); + fileListWalk( directoryEntry.path(), childPath, inclusions, exclusions, pathList ); + } + } +} + +PathMatcher pathMatcher( const string &patterns, bool recurse ) +{ + PathMatcher result; + + std::vector patternsSplit; + IECore::StringAlgo::tokenize( patterns, ' ', patternsSplit ); + for( const auto &pattern : patternsSplit ) + { + result.addPath( pattern ); + } + + if( recurse ) + { + PathMatcher recursiveResult; + recursiveResult.addPaths( result, { "..." } ); + result = recursiveResult; + } + + return result; +} + +StringVectorDataPtr fileList( const std::filesystem::path &directory, + const string &inclusions, const string &exclusions, const string &fileExtensions, + bool recurse, bool absolute, FileList::SequenceMode sequenceMode +) +{ + if( directory.empty() ) + { + return new StringVectorData; + } + + const PathMatcher inclusionsPathMatcher = pathMatcher( inclusions, recurse ); + const PathMatcher exclusionsPathMatcher = pathMatcher( exclusions, recurse ); + + vector pathList; + fileListWalk( directory, {}, inclusionsPathMatcher, exclusionsPathMatcher, pathList ); + std::sort( pathList.begin(), pathList.end() ); + + const string fileExtensionsLower = boost::algorithm::to_lower_copy( fileExtensions ); + + vector fileList; fileList.reserve( pathList.size() ); + for( const auto &path : pathList ) + { + const string extension = boost::algorithm::to_lower_copy( path.extension().string() ); + const char *extensionWithoutDot = extension.size() ? extension.c_str() + 1 : extension.c_str(); + if( !IECore::StringAlgo::matchMultiple( extensionWithoutDot, fileExtensionsLower ) ) + { + continue; + } + + if( absolute ) + { + fileList.push_back( filesystem::absolute( path ).generic_string() ); + } + else + { + fileList.push_back( filesystem::relative( path, directory ).generic_string() ); + } + } + + if( sequenceMode != FileList::SequenceMode::Files ) + { + vector sequences; + IECore::findSequences( fileList, sequences ); + + if( sequenceMode == FileList::SequenceMode::Sequences ) + { + fileList.clear(); + } + else + { + // Remove files that are not in a sequence. + vector sequenceFiles; + for( const auto &sequence : sequences ) + { + sequence->fileNames( sequenceFiles ); + } + unordered_set sequenceFilesSet( sequenceFiles.begin(), sequenceFiles.end() ); + fileList.erase( + std::remove_if( + fileList.begin(), + fileList.end(), + [&] ( const string &x ) -> bool { + return sequenceFilesSet.count( x ); + } + ), + fileList.end() + ); + } + + for( const auto &sequence : sequences ) + { + fileList.push_back( sequence->getFileName() ); + } + } + + return new StringVectorData( std::move( fileList ) ); +} + +} // namespace + +GAFFER_NODE_DEFINE_TYPE( FileList ); + +size_t FileList::g_firstPlugIndex = 0; + +FileList::FileList( const std::string &name ) + : ComputeNode( name ) +{ + storeIndexOfNextChild( g_firstPlugIndex ); + + addChild( new BoolPlug( "enabled", Plug::In, true ) ); + addChild( new StringPlug( "directory" ) ); + addChild( new IntPlug( "refreshCount" ) ); + addChild( new StringPlug( "inclusions", Plug::In, "*" ) ); + addChild( new StringPlug( "exclusions", Plug::In, "" ) ); + addChild( new StringPlug( "extensions", Plug::In, "*" ) ); + addChild( new BoolPlug( "searchSubdirectories" ) ); + addChild( new BoolPlug( "absolute", Plug::In, true ) ); + addChild( new IntPlug( "sequenceMode", Plug::In, (int)SequenceMode::Files, (int)SequenceMode::Files, (int)SequenceMode::FilesAndSequences ) ); + addChild( new StringVectorDataPlug( "out", Plug::Out ) ); +} + +FileList::~FileList() +{ +} + +Gaffer::BoolPlug *FileList::enabledPlug() +{ + return getChild( g_firstPlugIndex ); +} + +const Gaffer::BoolPlug *FileList::enabledPlug() const +{ + return getChild( g_firstPlugIndex ); +} + +Gaffer::StringPlug *FileList::directoryPlug() +{ + return getChild( g_firstPlugIndex + 1 ); +} + +const Gaffer::StringPlug *FileList::directoryPlug() const +{ + return getChild( g_firstPlugIndex + 1 ); +} + +Gaffer::IntPlug *FileList::refreshCountPlug() +{ + return getChild( g_firstPlugIndex + 2 ); +} + +const Gaffer::IntPlug *FileList::refreshCountPlug() const +{ + return getChild( g_firstPlugIndex + 2 ); +} + +Gaffer::StringPlug *FileList::inclusionsPlug() +{ + return getChild( g_firstPlugIndex + 3 ); +} + +const Gaffer::StringPlug *FileList::inclusionsPlug() const +{ + return getChild( g_firstPlugIndex + 3 ); +} + +Gaffer::StringPlug *FileList::exclusionsPlug() +{ + return getChild( g_firstPlugIndex + 4 ); +} + +const Gaffer::StringPlug *FileList::exclusionsPlug() const +{ + return getChild( g_firstPlugIndex + 4 ); +} + +Gaffer::StringPlug *FileList::extensionsPlug() +{ + return getChild( g_firstPlugIndex + 5 ); +} + +const Gaffer::StringPlug *FileList::extensionsPlug() const +{ + return getChild( g_firstPlugIndex + 5 ); +} + +Gaffer::BoolPlug *FileList::searchSubdirectoriesPlug() +{ + return getChild( g_firstPlugIndex + 6 ); +} + +const Gaffer::BoolPlug *FileList::searchSubdirectoriesPlug() const +{ + return getChild( g_firstPlugIndex + 6 ); +} + +Gaffer::BoolPlug *FileList::absolutePlug() +{ + return getChild( g_firstPlugIndex + 7 ); +} + +const Gaffer::BoolPlug *FileList::absolutePlug() const +{ + return getChild( g_firstPlugIndex + 7 ); +} + +Gaffer::IntPlug *FileList::sequenceModePlug() +{ + return getChild( g_firstPlugIndex + 8 ); +} + +const Gaffer::IntPlug *FileList::sequenceModePlug() const +{ + return getChild( g_firstPlugIndex + 8 ); +} + +Gaffer::StringVectorDataPlug *FileList::outPlug() +{ + return getChild( g_firstPlugIndex + 9 ); +} + +const Gaffer::StringVectorDataPlug *FileList::outPlug() const +{ + return getChild( g_firstPlugIndex + 9 ); +} + +void FileList::affects( const Plug *input, AffectedPlugsContainer &outputs ) const +{ + ComputeNode::affects( input, outputs ); + + if( + input == enabledPlug() || + input == directoryPlug() || + input == refreshCountPlug() || + input == inclusionsPlug() || + input == exclusionsPlug() || + input == extensionsPlug() || + input == searchSubdirectoriesPlug() || + input == absolutePlug() || + input == sequenceModePlug() + ) + { + outputs.push_back( outPlug() ); + } +} + +void FileList::hash( const ValuePlug *output, const Context *context, IECore::MurmurHash &h ) const +{ + if( output == outPlug() ) + { + ComputeNode::hash( output, context, h ); + if( enabledPlug()->getValue() ) + { + directoryPlug()->hash( h ); + refreshCountPlug()->hash( h ); + inclusionsPlug()->hash( h ); + exclusionsPlug()->hash( h ); + extensionsPlug()->hash( h ); + searchSubdirectoriesPlug()->hash( h ); + absolutePlug()->hash( h ); + sequenceModePlug()->hash( h ); + } + } + else + { + ComputeNode::hash( output, context, h ); + } +} + +void FileList::compute( ValuePlug *output, const Context *context ) const +{ + if( output == outPlug() ) + { + if( enabledPlug()->getValue() ) + { + static_cast( output )->setValue( + fileList( + directoryPlug()->getValue(), + inclusionsPlug()->getValue(), + exclusionsPlug()->getValue(), + extensionsPlug()->getValue(), + searchSubdirectoriesPlug()->getValue(), + absolutePlug()->getValue(), + (SequenceMode)sequenceModePlug()->getValue() + ) + ); + } + else + { + output->setToDefault(); + } + } + else + { + ComputeNode::compute( output, context ); + } +} + +ValuePlug::CachePolicy FileList::computeCachePolicy( const ValuePlug *output ) const +{ + if( output == outPlug() ) + { + return ValuePlug::CachePolicy::TaskCollaboration; + } + return ComputeNode::computeCachePolicy( output ); +} diff --git a/src/GafferDispatch/RenameFiles.cpp b/src/GafferDispatch/RenameFiles.cpp new file mode 100644 index 00000000000..c0b01bb7847 --- /dev/null +++ b/src/GafferDispatch/RenameFiles.cpp @@ -0,0 +1,413 @@ +////////////////////////////////////////////////////////////////////////// +// +// Copyright (c) 2025, Cinesite VFX Ltd. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above +// copyright notice, this list of conditions and the following +// disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided with +// the distribution. +// +// * Neither the name of John Haddon nor the names of +// any other contributors to this software may be used to endorse or +// promote products derived from this software without specific prior +// written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +////////////////////////////////////////////////////////////////////////// + +#include "GafferDispatch/RenameFiles.h" + +#include "boost/algorithm/string/predicate.hpp" +#include "boost/algorithm/string/replace.hpp" + +#include "fmt/args.h" +#include "fmt/format.h" +#include "fmt/std.h" + +#include +#include + +using namespace std; +using namespace IECore; +using namespace Gaffer; +using namespace GafferDispatch; + +////////////////////////////////////////////////////////////////////////// +// Internal utilities +////////////////////////////////////////////////////////////////////////// + +namespace +{ + +/// \todo This is copied from `GafferScene/Rename.cpp`. We should probably +/// find a shared home for it at some point. +string regexReplace( const std::string &s, const std::regex &r, const std::string &f ) +{ + // Iterator for all regex matches within `s`. + sregex_iterator matchIt( s.begin(), s.end(), r ); + sregex_iterator matchEnd; + + if( matchIt == matchEnd ) + { + // No matches + return s; + } + + ssub_match suffix; + std::string result; + for( ; matchIt != matchEnd; ++matchIt ) + { + // Add any unmatched text from before this match. + result.insert( result.end(), matchIt->prefix().first, matchIt->prefix().second ); + + // Format this match using the format string provided, and + // add it to our result. + fmt::dynamic_format_arg_store store; + for( const auto &subMatch : *matchIt ) + { + store.push_back( subMatch.str() ); + } + + try + { + result += fmt::vformat( f, store ); + } + catch( fmt::format_error &e ) + { + // Augment the error with a little bit more information, to + // give people half a chance of figuring out the problem. + throw IECore::Exception( + fmt::format( "Error applying replacement `{}` : {}", f, e.what() ) + ); + } + + suffix = matchIt->suffix(); + } + // The suffix for one match is the same as the prefix for the next + // match. So we only need to add the suffix from the last match. + result.insert( result.end(), suffix.first, suffix.second ); + + return result; +} + +const IECore::InternedString g_sourceVariable( "source" ); +const IECore::InternedString g_sourceStemVariable( "source:stem" ); +const IECore::InternedString g_sourceExtensionVariable( "source:extension" ); + +} // namespace + +////////////////////////////////////////////////////////////////////////// +// RenameFiles +////////////////////////////////////////////////////////////////////////// + +GAFFER_NODE_DEFINE_TYPE( RenameFiles ); + +size_t RenameFiles::g_firstPlugIndex = 0; + +RenameFiles::RenameFiles( const std::string &name ) + : TaskNode( name ) +{ + storeIndexOfNextChild( g_firstPlugIndex ); + addChild( new StringVectorDataPlug( "files" ) ); + addChild( new StringPlug( "name" ) ); + addChild( new StringPlug( "deletePrefix" ) ); + addChild( new StringPlug( "deleteSuffix" ) ); + addChild( new StringPlug( "find" ) ); + addChild( new StringPlug( "replace" ) ); + addChild( new BoolPlug( "useRegularExpressions" ) ); + addChild( new StringPlug( "addPrefix" ) ); + addChild( new StringPlug( "addSuffix" ) ); + addChild( new BoolPlug( "replaceExtension" ) ); + addChild( new StringPlug( "extension" ) ); + addChild( new BoolPlug( "overwrite" ) ); +} + +RenameFiles::~RenameFiles() +{ +} + +Gaffer::StringVectorDataPlug *RenameFiles::filesPlug() +{ + return getChild( g_firstPlugIndex ); +} + +const Gaffer::StringVectorDataPlug *RenameFiles::filesPlug() const +{ + return getChild( g_firstPlugIndex ); +} + +Gaffer::StringPlug *RenameFiles::namePlug() +{ + return getChild( g_firstPlugIndex + 1 ); +} + +const Gaffer::StringPlug *RenameFiles::namePlug() const +{ + return getChild( g_firstPlugIndex + 1 ); +} + +Gaffer::StringPlug *RenameFiles::deletePrefixPlug() +{ + return getChild( g_firstPlugIndex + 2 ); +} + +const Gaffer::StringPlug *RenameFiles::deletePrefixPlug() const +{ + return getChild( g_firstPlugIndex + 2 ); +} + +Gaffer::StringPlug *RenameFiles::deleteSuffixPlug() +{ + return getChild( g_firstPlugIndex + 3 ); +} + +const Gaffer::StringPlug *RenameFiles::deleteSuffixPlug() const +{ + return getChild( g_firstPlugIndex + 3 ); +} + +Gaffer::StringPlug *RenameFiles::findPlug() +{ + return getChild( g_firstPlugIndex + 4 ); +} + +const Gaffer::StringPlug *RenameFiles::findPlug() const +{ + return getChild( g_firstPlugIndex + 4 ); +} + +Gaffer::StringPlug *RenameFiles::replacePlug() +{ + return getChild( g_firstPlugIndex + 5 ); +} +const Gaffer::StringPlug *RenameFiles::replacePlug() const +{ + return getChild( g_firstPlugIndex + 5 ); +} + +Gaffer::BoolPlug *RenameFiles::useRegularExpressionsPlug() +{ + return getChild( g_firstPlugIndex + 6 ); +} + +const Gaffer::BoolPlug *RenameFiles::useRegularExpressionsPlug() const +{ + return getChild( g_firstPlugIndex + 6 ); +} + +Gaffer::StringPlug *RenameFiles::addPrefixPlug() +{ + return getChild( g_firstPlugIndex + 7 ); +} + +const Gaffer::StringPlug *RenameFiles::addPrefixPlug() const +{ + return getChild( g_firstPlugIndex + 7 ); +} + +Gaffer::StringPlug *RenameFiles::addSuffixPlug() +{ + return getChild( g_firstPlugIndex + 8 ); +} +const Gaffer::StringPlug *RenameFiles::addSuffixPlug() const +{ + return getChild( g_firstPlugIndex + 8 ); +} + +Gaffer::BoolPlug *RenameFiles::replaceExtensionPlug() +{ + return getChild( g_firstPlugIndex + 9 ); +} + +const Gaffer::BoolPlug *RenameFiles::replaceExtensionPlug() const +{ + return getChild( g_firstPlugIndex + 9 ); +} + +Gaffer::StringPlug *RenameFiles::extensionPlug() +{ + return getChild( g_firstPlugIndex + 10 ); +} + +const Gaffer::StringPlug *RenameFiles::extensionPlug() const +{ + return getChild( g_firstPlugIndex + 10 ); +} + +Gaffer::BoolPlug *RenameFiles::overwritePlug() +{ + return getChild( g_firstPlugIndex + 11 ); +} + +const Gaffer::BoolPlug *RenameFiles::overwritePlug() const +{ + return getChild( g_firstPlugIndex + 11 ); +} + +IECore::MurmurHash RenameFiles::hash( const Gaffer::Context *context ) const +{ + ConstStringVectorDataPtr filesData = filesPlug()->getValue(); + if( filesData->readable().empty() ) + { + return IECore::MurmurHash(); + } + + IECore::MurmurHash h = TaskNode::hash( context ); + filesData->hash( h ); + namePlug()->hash( h ); + deletePrefixPlug()->hash( h ); + deleteSuffixPlug()->hash( h ); + findPlug()->hash( h ); + replacePlug()->hash( h ); + useRegularExpressionsPlug()->hash( h ); + addPrefixPlug()->hash( h ); + addSuffixPlug()->hash( h ); + replaceExtensionPlug()->hash( h ); + extensionPlug()->hash( h ); + overwritePlug()->hash( h ); + return h; +} + +void RenameFiles::execute() const +{ + ConstStringVectorDataPtr filesData = filesPlug()->getValue(); + const string deletePrefix = deletePrefixPlug()->getValue(); + const string deleteSuffix = deleteSuffixPlug()->getValue(); + const string find = findPlug()->getValue(); + const string replace = replacePlug()->getValue(); + const bool useRegularExpressions = useRegularExpressionsPlug()->getValue(); + const string addPrefix = addPrefixPlug()->getValue(); + const string addSuffix = addSuffixPlug()->getValue(); + std::optional extension; + if( replaceExtensionPlug()->getValue() ) + { + extension = extensionPlug()->getValue(); + if( extension->size() && (*extension)[0] != '.' ) + { + extension->insert( 0, "." ); + } + } + + // Build a map from destination path to source path. This allows + // us to sanity check the operation before committing to doing it. + std::map destinationToSource; + + Context::EditableScope context( Context::current() ); + for( const auto &file : filesData->readable() ) + { + context.set( g_sourceVariable, &file ); + const filesystem::path sourceFilePath = filesystem::canonical( file ); + + const string sourceStem = sourceFilePath.stem().string(); + context.set( g_sourceStemVariable, &sourceStem ); + + const string sourceExtension = sourceFilePath.extension().string(); + context.set( g_sourceExtensionVariable, &sourceExtension ); + + string name = namePlug()->getValue(); + if( !name.size() ) + { + string stem = sourceFilePath.stem().string(); + + if( boost::starts_with( stem, deletePrefix ) ) + { + stem.erase( 0, deletePrefix.size() ); + } + + if( boost::ends_with( stem, deleteSuffix ) ) + { + stem.erase( stem.size() - deleteSuffix.size() ); + } + + if( find.size() ) + { + if( useRegularExpressions ) + { + stem = regexReplace( stem, regex( find ), replace ); + } + else + { + boost::replace_all( stem, find, replace ); + } + } + + stem.insert( 0, addPrefixPlug()->getValue() ); + stem.insert( stem.size(), addSuffixPlug()->getValue() ); + + name = stem; + if( extension ) + { + name += *extension; + } + else + { + name += sourceFilePath.extension().string(); + } + } + + filesystem::path destinationFilePath = sourceFilePath; + destinationFilePath.replace_filename( name ); + + const auto [it, inserted] = destinationToSource.insert( { destinationFilePath, sourceFilePath } ); + if( !inserted ) + { + throw IECore::Exception( + fmt::format( + "Destination \"{}\" has multiple source files : \"{}\" and \"{}\"", + destinationFilePath.generic_string(), it->second.generic_string(), sourceFilePath.generic_string() + ) + ); + } + } + + // Check that we're not writing over any source files. + + for( const auto &[destinationFilePath, sourceFilePath] : destinationToSource ) + { + auto it = destinationToSource.find( sourceFilePath ); + if( it != destinationToSource.end() ) + { + throw IECore::Exception( + fmt::format( + "Renaming of \"{}\" would overwrite source \"{}\"", + it->second.generic_string(), sourceFilePath.generic_string() + ) + ); + } + } + + // Finally do the work. + + const bool overwrite = overwritePlug()->getValue(); + for( const auto &[destinationFilePath, sourceFilePath] : destinationToSource ) + { + if( !overwrite && filesystem::exists( destinationFilePath ) ) + { + throw IECore::Exception( + fmt::format( + "Can not overwrite destination \"{}\" unless `overwrite` plug is set.", destinationFilePath.generic_string() + ) + ); + } + filesystem::rename( sourceFilePath, destinationFilePath ); + } +} diff --git a/src/GafferDispatchModule/FileNodeBinding.cpp b/src/GafferDispatchModule/FileNodeBinding.cpp new file mode 100644 index 00000000000..7a8f2da566f --- /dev/null +++ b/src/GafferDispatchModule/FileNodeBinding.cpp @@ -0,0 +1,70 @@ +////////////////////////////////////////////////////////////////////////// +// +// Copyright (c) 2025, Cinesite VFX Ltd. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above +// copyright notice, this list of conditions and the following +// disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided with +// the distribution. +// +// * Neither the name of John Haddon nor the names of +// any other contributors to this software may be used to endorse or +// promote products derived from this software without specific prior +// written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +////////////////////////////////////////////////////////////////////////// + +#include "boost/python.hpp" + +#include "FileNodeBinding.h" + +#include "GafferDispatch/CopyFiles.h" +#include "GafferDispatch/DeleteFiles.h" +#include "GafferDispatch/FileList.h" +#include "GafferDispatch/RenameFiles.h" + +#include "GafferDispatchBindings/TaskNodeBinding.h" + +#include "GafferBindings/DependencyNodeBinding.h" + +using namespace Gaffer; +using namespace GafferBindings; +using namespace GafferDispatch; +using namespace GafferDispatchBindings; + +void GafferDispatchModule::bindFileNodes() +{ + { + boost::python::scope s = DependencyNodeClass(); + + boost::python::enum_( "SequenceMode" ) + .value( "Files", FileList::SequenceMode::Files ) + .value( "Sequences", FileList::SequenceMode::Sequences ) + .value( "FilesAndSequences", FileList::SequenceMode::FilesAndSequences ) + ; + } + + TaskNodeClass(); + TaskNodeClass(); + TaskNodeClass(); +} diff --git a/src/GafferDispatchModule/FileNodeBinding.h b/src/GafferDispatchModule/FileNodeBinding.h new file mode 100644 index 00000000000..21c99c1aaaa --- /dev/null +++ b/src/GafferDispatchModule/FileNodeBinding.h @@ -0,0 +1,44 @@ +////////////////////////////////////////////////////////////////////////// +// +// Copyright (c) 2025, Cinesite VFX Ltd. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above +// copyright notice, this list of conditions and the following +// disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided with +// the distribution. +// +// * Neither the name of John Haddon nor the names of +// any other contributors to this software may be used to endorse or +// promote products derived from this software without specific prior +// written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +////////////////////////////////////////////////////////////////////////// + +#pragma once + +namespace GafferDispatchModule +{ + +void bindFileNodes(); + +} // namespace GafferDispatchModule diff --git a/src/GafferDispatchModule/GafferDispatchModule.cpp b/src/GafferDispatchModule/GafferDispatchModule.cpp index d43755df682..024dd6809f2 100644 --- a/src/GafferDispatchModule/GafferDispatchModule.cpp +++ b/src/GafferDispatchModule/GafferDispatchModule.cpp @@ -37,6 +37,7 @@ #include "boost/python.hpp" #include "DispatcherBinding.h" +#include "FileNodeBinding.h" #include "TaskNodeBinding.h" using namespace boost::python; @@ -47,5 +48,6 @@ BOOST_PYTHON_MODULE( _GafferDispatch ) bindTaskNode(); bindDispatcher(); + bindFileNodes(); } diff --git a/startup/gui/menus.py b/startup/gui/menus.py index 9a8048d00aa..7c8055f60d0 100644 --- a/startup/gui/menus.py +++ b/startup/gui/menus.py @@ -581,6 +581,9 @@ def __usdLightCreator( lightType ) : nodeMenu.append( "/Dispatch/Wedge", GafferDispatch.Wedge ) nodeMenu.append( "/Dispatch/Frame Mask", GafferDispatch.FrameMask, searchText = "FrameMask" ) nodeMenu.append( "/Dispatch/Local Dispatcher", GafferDispatch.LocalDispatcher, searchText = "LocalDispatcher" ) +nodeMenu.append( "/Dispatch/Delete Files", GafferDispatch.DeleteFiles, searchText = "DeleteFiles" ) +nodeMenu.append( "/Dispatch/Copy Files", GafferDispatch.CopyFiles, searchText = "CopyFiles" ) +nodeMenu.append( "/Dispatch/Rename Files", GafferDispatch.RenameFiles, searchText = "RenameFiles" ) # ML nodes @@ -618,6 +621,7 @@ def __usdLightCreator( lightType ) : nodeMenu.append( "/Utility/Context Query", Gaffer.ContextQuery, searchText = "ContextQuery" ) nodeMenu.append( "/Utility/Collect", Gaffer.Collect ) nodeMenu.append( "/Utility/Pattern Match", Gaffer.PatternMatch, searchText = "PatternMatch" ) +nodeMenu.append( "/Utility/File List", GafferDispatch.FileList, searchText = "FileList" ) ## Miscellaneous UI ###########################################################################