diff --git a/Command/WarmCacheCommand.php b/Command/WarmCacheCommand.php new file mode 100644 index 000000000..133c18a49 --- /dev/null +++ b/Command/WarmCacheCommand.php @@ -0,0 +1,90 @@ + + */ +class WarmCacheCommand extends ContainerAwareCommand +{ + protected function configure() + { + $this + ->setName('liip:imagine:cache:warm') + ->setDescription('Warms cache for paths provided by given warmers (or all warmers, if run w/o params)') + ->addOption('chunk-size', 'c', InputOption::VALUE_REQUIRED, 'Chunk size', 100) + ->addOption('force', 'f', InputOption::VALUE_NONE, 'Force cache warm up for already cached images') + ->addArgument('warmers', InputArgument::OPTIONAL|InputArgument::IS_ARRAY, 'Warmers names') + ->setHelp(<<%command.name% command warms up cache by specified parameters. + +A warmer can be configured for one or more filter set. A warmer should return a list of paths. +This command gets the paths from warmer and create cache (i.e. filtered image) for each filter configured for given warmer. + +Warmers should be separated by spaces: +php app/console %command.name% warmer1 warmer2 +All cache for a given `warmers` will be warmed up + +php app/console %command.name% +Cache for all warmers will be warmed up when executing this command without parameters. + +Note, that --force option will force regeneration of the cache only if warmer returns the path. +Generally, there should be NO need to use this option, instead, use liip:imagine:cache:remove command to clear cache. +Then run this command to warm-up the cache +EOF + ); + } + + protected function execute(InputInterface $input, OutputInterface $output) + { + $warmers = $input->getArgument('warmers'); + + /** @var CacheWarmer $cacheWarmer */ + $cacheWarmer = $this->getContainer()->get('liip_imagine.cache.warmer'); + $cacheWarmer->setLoggerClosure($this->getLoggerClosure($output)); + + if ($chunkSize = $input->getOption('chunk-size')) { + $cacheWarmer->setChunkSize($chunkSize); + } + + $force = false; + if ($input->getOption('force')) { + $force = true; + } + + $cacheWarmer->warm($force, $warmers); + } + + /** + * Returns Logger Closure + * + * @return callable + */ + protected function getLoggerClosure(OutputInterface $output) + { + $loggerClosure = function ($message, $msgType = 'info') use ($output) { + $time = date('Y-m-d G:i:s'); + $message = sprintf( + '%s | Mem cur/peak: %dm/%dm | <' . $msgType . '>%s', + $time, + round(memory_get_usage(true) / 1024 / 1024, 1), + round(memory_get_peak_usage(true) / 1024 / 1024, 1), + $message + ); + $output->writeln($message); + }; + return $loggerClosure; + } +} diff --git a/DependencyInjection/Compiler/CacheWarmersCompilerPass.php b/DependencyInjection/Compiler/CacheWarmersCompilerPass.php new file mode 100644 index 000000000..7bfc2fd00 --- /dev/null +++ b/DependencyInjection/Compiler/CacheWarmersCompilerPass.php @@ -0,0 +1,26 @@ +findTaggedServiceIds('liip_imagine.cache.warmer'); + + if (count($tags) > 0 && $container->hasDefinition('liip_imagine.cache.warmer')) { + $warmer = $container->getDefinition('liip_imagine.cache.warmer'); + + foreach ($tags as $id => $tag) { + $warmer->addMethodCall('addWarmer', array($tag[0]['warmer'], new Reference($id))); + } + } + } +} diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index cd4701bb7..663a2c8a8 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -191,6 +191,12 @@ public function getConfigTreeBuilder() ->useAttributeAsKey('name') ->prototype('variable')->end() ->end() + ->end() + ->arrayNode('warmers') + ->defaultValue(array()) + ->useAttributeAsKey('name') + ->prototype('scalar') + ->end() ->end() ->end() ->end() diff --git a/DependencyInjection/Factory/Resolver/FormatResolverFactory.php b/DependencyInjection/Factory/Resolver/FormatResolverFactory.php new file mode 100644 index 000000000..fac185fe0 --- /dev/null +++ b/DependencyInjection/Factory/Resolver/FormatResolverFactory.php @@ -0,0 +1,42 @@ +replaceArgument(2, $config['web_root']); + $resolverDefinition->replaceArgument(3, $config['cache_prefix']); + $resolverDefinition->addTag('liip_imagine.cache.resolver', array( + 'resolver' => $resolverName + )); + $resolverId = 'liip_imagine.cache.resolver.'.$resolverName; + + $container->setDefinition($resolverId, $resolverDefinition); + + return $resolverId; + } + + /** + * {@inheritDoc} + */ + public function getName() + { + return 'format'; + } +} diff --git a/Imagine/Cache/CacheManager.php b/Imagine/Cache/CacheManager.php index 389127ba7..f46d508fb 100644 --- a/Imagine/Cache/CacheManager.php +++ b/Imagine/Cache/CacheManager.php @@ -54,6 +54,11 @@ class CacheManager */ protected $defaultResolver; + /** + * @var CacheWarmer + */ + protected $cacheWarmer; + /** * @var bool */ @@ -95,6 +100,18 @@ public function addResolver($filter, ResolverInterface $resolver) } } + /** + * @param \Liip\ImagineBundle\Imagine\Cache\CacheWarmer $cacheWarmer + * + * @return CacheManager + */ + public function setCacheWarmer($cacheWarmer) + { + $this->cacheWarmer = $cacheWarmer; + + return $this; + } + /** * Gets filtered path for rendering in the browser. * It could be the cached one or an url of filter action. @@ -238,6 +255,8 @@ public function remove($paths = null, $filters = null) $paths = array_filter($paths); $filters = array_filter($filters); + $this->cacheWarmer->clearWarmed($paths, $filters); + $mapping = new \SplObjectStorage(); foreach ($filters as $filter) { $resolver = $this->getResolver($filter, null); diff --git a/Imagine/Cache/CacheWarmer.php b/Imagine/Cache/CacheWarmer.php new file mode 100644 index 000000000..25b253080 --- /dev/null +++ b/Imagine/Cache/CacheWarmer.php @@ -0,0 +1,250 @@ + + */ +class CacheWarmer +{ + /** + * @var WarmerInterface[] + */ + protected $warmers = []; + + /** + * Chunk size to query warmer in one step + * + * @var int + */ + protected $chunkSize = 100; + + /** + * @var CacheManager + */ + protected $cacheManager; + + /** + * @var DataManager + */ + protected $dataManager; + + /** + * @var FilterManager + */ + protected $filterManager; + + /** + * @var callable + */ + protected $loggerClosure; + + public function __construct(DataManager $dataManager, FilterManager $filterManager) + { + $this->dataManager = $dataManager; + $this->filterManager = $filterManager; + } + + /** + * @param int $chunkSize + * + * @return CacheWarmer + */ + public function setChunkSize($chunkSize) + { + $this->chunkSize = $chunkSize; + + return $this; + } + + /** + * Sets logger closure - a callable which will be passed verbose messages + * + * @param callable $loggerClosure + * + * @return CacheWarmer + */ + public function setLoggerClosure($loggerClosure) + { + $this->loggerClosure = $loggerClosure; + + return $this; + } + + /** + * @param \Liip\ImagineBundle\Imagine\Cache\CacheManager $cacheManager + * + * @return CacheWarmer + */ + public function setCacheManager($cacheManager) + { + $this->cacheManager = $cacheManager; + + return $this; + } + + public function addWarmer($name, WarmerInterface $warmer) + { + $this->warmers[$name] = $warmer; + } + + /** + * @param bool $force If set to true, cache is warmed up for paths already stored in cached (regenerate thumbs) + * @param array|null $selectedWarmers An optional array of warmers to process, if null - all warmers will be processed + * + * @throws \InvalidArgumentException + * + * @return void + */ + public function warm($force = false, $selectedWarmers = null) + { + $filtersByWarmer = $this->getFiltersByWarmers(); + if (!$filtersByWarmer) { + $this->log('No warmes are configured - add some as `warmers` param in your filter sets'); + + return; + } + + foreach ($filtersByWarmer as $warmerName => $filters) { + if (isset($selectedWarmers) && !empty($selectedWarmers) && !in_array($warmerName, $selectedWarmers, true)) { + $this->log( + sprintf( + 'Skipping warmer %s as it\'s not listed in selected warmers: [%s]', + $warmerName, + implode(', ', $selectedWarmers) + ) + ); + continue; + } + if (!isset($this->warmers[$warmerName])) { + throw new \InvalidArgumentException(sprintf('Could not find warmer "%s"', $warmerName)); + } + + $this->log(sprintf('Processing warmer "%s"', $warmerName)); + $start = 0; + $warmer = $this->warmers[$warmerName]; + while ($paths = $warmer->getPaths($start, $this->chunkSize)) { + $this->log( + sprintf( + 'Processing chunk %d - %d for warmer "%s"', + $start, + $start + $this->chunkSize, + $warmerName + ) + ); + $warmedPaths = $this->warmPaths($paths, $filters, $force); + $warmer->setWarmed($warmedPaths); + $start += count($paths) - count($warmedPaths); + } + $this->log(sprintf('Finished processing warmer "%s"', $warmerName)); + } + } + + public function clearWarmed($paths, $filters) + { + $filtersByWarmer = $this->getFiltersByWarmers(); + foreach ($filtersByWarmer as $warmerName => $warmerFilters) { + if (array_intersect($filters, $warmerFilters)) { + if (!isset($this->warmers[$warmerName])) { + throw new \InvalidArgumentException(sprintf('Could not find warmer "%s"', $warmerName)); + } + $warmer = $this->warmers[$warmerName]; + $warmer->clearWarmed($paths); + } + } + } + + protected function getFiltersByWarmers() + { + $all = $this->filterManager->getFilterConfiguration()->all(); + $warmers = []; + foreach ($all as $filterSet => $config) { + if (isset($config['warmers']) && $config['warmers']) { + foreach ($config['warmers'] as $warmer) { + if (!isset($warmers[$warmer])) { + $warmers[$warmer] = [$filterSet]; + } else { + $warmers[$warmer][] = $filterSet; + } + } + } + } + + return $warmers; + } + + /** + * @param array $paths + * @param array $filters + * @param bool $force + * + * @return array + */ + protected function warmPaths($paths, $filters, $force) + { + $successfulWarmedPaths = []; + foreach ($paths as $pathData) { + $aPath = $pathData['path']; + $binaries = []; + foreach ($filters as $filter) { + $this->log(sprintf('Warming up path "%s" for filter "%s"', $aPath, $filter)); + + $isStored = $this->cacheManager->isStored($aPath, $filter); + if ($force || !$isStored) { + // this is to avoid loading binary with the same loader for multiple filters + $loader = $this->dataManager->getLoader($filter); + $isStored = false; + + try { + $hash = spl_object_hash($loader); + if (!isset($binaries[$hash])) { + // if NotLoadable is thrown - it will just bubble up + // everything returned by Warmer should be loadable + $binaries[$hash] = $this->dataManager->find($filter, $aPath); + } + $this->cacheManager->store( + $this->filterManager->applyFilter($binaries[$hash], $filter), + $aPath, + $filter + ); + + $isStored = true; + } catch (\RuntimeException $e) { + $message = sprintf('Unable to warm cache for filter "%s", due to - "%s"', + $filter, $e->getMessage()); + $this->log($message, 'error'); + } + } + + if ($isStored) { + $successfulWarmedPaths[] = $pathData; + } + } + } + + return $successfulWarmedPaths; + } + + protected function log($message, $type = 'info') + { + if (is_callable($this->loggerClosure)) { + $loggerClosure = $this->loggerClosure; + $loggerClosure($message, $type); + } + } +} diff --git a/Imagine/Cache/Resolver/FormatResolver.php b/Imagine/Cache/Resolver/FormatResolver.php new file mode 100644 index 000000000..4c4e11a0f --- /dev/null +++ b/Imagine/Cache/Resolver/FormatResolver.php @@ -0,0 +1,85 @@ +filterManager = $filterManager; + } + + /** + * {@inheritDoc} + */ + protected function getFilePath($path, $filter) + { + return $this->webRoot.'/'.$this->getFileUrl($this->replaceImageFileExtension($path, $filter), $filter); + } + + /** + * {@inheritDoc} + */ + protected function getFileUrl($path, $filter) + { + return $this->cachePrefix.'/'.$filter.'/'.ltrim($this->replaceImageFileExtension($path, $filter), '/'); + } + + /** + * Replaces original image file extension to conversion format extension + * + * @param string $path + * @param string $filter + */ + protected function replaceImageFileExtension($path, $filter) + { + $newExtension = $this->getImageFormat($filter); + if (null !== $newExtension) { + $path = preg_replace('/\.[^.]+$/', '.'.$newExtension, $path); + } + + return $path; + } + + /** + * Returns image conversion format + * + * @param $filterName + */ + protected function getImageFormat($filterName) + { + $filterConfig = $this->filterManager->getFilterConfiguration(); + $currentFilterConfig = $filterConfig->get($filterName); + + return $currentFilterConfig['format']; + } +} diff --git a/Imagine/Cache/Warmer/WarmerInterface.php b/Imagine/Cache/Warmer/WarmerInterface.php new file mode 100644 index 000000000..ebc0a4c61 --- /dev/null +++ b/Imagine/Cache/Warmer/WarmerInterface.php @@ -0,0 +1,50 @@ +addCompilerPass(new PostProcessorsCompilerPass()); $container->addCompilerPass(new ResolversCompilerPass()); $container->addCompilerPass(new MetadataReaderCompilerPass()); + $container->addCompilerPass(new CacheWarmersCompilerPass()); if (class_exists(AddTopicMetaPass::class)) { $container->addCompilerPass(AddTopicMetaPass::create() @@ -65,5 +68,7 @@ public function build(ContainerBuilder $container) $extension->addLoaderFactory(new FileSystemLoaderFactory()); $extension->addLoaderFactory(new FlysystemLoaderFactory()); $extension->addLoaderFactory(new ChainLoaderFactory()); + + $extension->addResolverFactory(new FormatResolverFactory()); } } diff --git a/Resources/config/imagine.xml b/Resources/config/imagine.xml index c56db6cd2..7d98c87e4 100644 --- a/Resources/config/imagine.xml +++ b/Resources/config/imagine.xml @@ -127,6 +127,9 @@ %liip_imagine.cache.resolver.default% %liip_imagine.webp.generate% + + + @@ -360,6 +363,14 @@ + + + + + + + + @@ -413,5 +424,14 @@ + + + + + + + + + diff --git a/Service/FilterService.php b/Service/FilterService.php index 86149e293..cb44b03b3 100644 --- a/Service/FilterService.php +++ b/Service/FilterService.php @@ -161,6 +161,10 @@ private function createFilteredBinary(FilterPathContainer $filterPathContainer, { $binary = $this->dataManager->find($filter, $filterPathContainer->getSource()); + if ($this->isSvg($binary)) { + return $binary; + } + try { return $this->filterManager->applyFilter($binary, $filter, $filterPathContainer->getOptions()); } catch (NonExistingFilterException $e) { @@ -174,4 +178,14 @@ private function createFilteredBinary(FilterPathContainer $filterPathContainer, throw $e; } } + + /** + * @param BinaryInterface $binary + * + * @return bool + */ + protected function isSvg(BinaryInterface $binary) + { + return $binary->getMimeType() == 'image/svg+xml'; + } } diff --git a/Tests/AbstractTest.php b/Tests/AbstractTest.php index 1be7f7d21..bb688da5f 100644 --- a/Tests/AbstractTest.php +++ b/Tests/AbstractTest.php @@ -18,6 +18,7 @@ use Liip\ImagineBundle\Binary\MimeTypeGuesserInterface; use Liip\ImagineBundle\Config\Controller\ControllerConfig; use Liip\ImagineBundle\Imagine\Cache\CacheManager; +use Liip\ImagineBundle\Imagine\Cache\CacheWarmer; use Liip\ImagineBundle\Imagine\Cache\Resolver\ResolverInterface; use Liip\ImagineBundle\Imagine\Cache\SignerInterface; use Liip\ImagineBundle\Imagine\Data\DataManager; @@ -146,6 +147,14 @@ protected function createCacheResolverInterfaceMock() return $this->createObjectMock(ResolverInterface::class); } + /** + * @return MockObject|CacheWarmer + */ + protected function createCacheWarmerMock() + { + return $this->createObjectMock(CacheWarmer::class); + } + /** * @return MockObject|EventDispatcherInterface */ diff --git a/Tests/Imagine/Cache/CacheManagerTest.php b/Tests/Imagine/Cache/CacheManagerTest.php index 055fe9549..34ab40f1e 100644 --- a/Tests/Imagine/Cache/CacheManagerTest.php +++ b/Tests/Imagine/Cache/CacheManagerTest.php @@ -373,6 +373,7 @@ public function testFallbackToDefaultResolver(): void $this->createEventDispatcherInterfaceMock() ); $cacheManager->addResolver('default', $resolver); + $cacheManager->setCacheWarmer($this->createCacheWarmerMock()); // Resolve fallback to default resolver $this->assertSame('/thumbs/cats.jpeg', $cacheManager->resolve('cats.jpeg', 'thumbnail')); @@ -443,6 +444,7 @@ public function testRemoveCacheForPathAndFilterOnRemove(): void $this->createEventDispatcherInterfaceMock() ); $cacheManager->addResolver($expectedFilter, $resolver); + $cacheManager->setCacheWarmer($this->createCacheWarmerMock()); $cacheManager->remove($expectedPath, $expectedFilter); } @@ -482,6 +484,7 @@ public function testRemoveCacheForPathAndSomeFiltersOnRemove(): void ); $cacheManager->addResolver($expectedFilterOne, $resolverOne); $cacheManager->addResolver($expectedFilterTwo, $resolverTwo); + $cacheManager->setCacheWarmer($this->createCacheWarmerMock()); $cacheManager->remove($expectedPath, [$expectedFilterOne, $expectedFilterTwo]); } @@ -517,6 +520,7 @@ public function testRemoveCacheForSomePathsAndFilterOnRemove(): void $this->createEventDispatcherInterfaceMock() ); $cacheManager->addResolver($expectedFilter, $resolver); + $cacheManager->setCacheWarmer($this->createCacheWarmerMock()); $cacheManager->remove([$expectedPathOne, $expectedPathTwo], $expectedFilter); } @@ -557,6 +561,7 @@ public function testRemoveCacheForSomePathsAndSomeFiltersOnRemove(): void ); $cacheManager->addResolver($expectedFilterOne, $resolverOne); $cacheManager->addResolver($expectedFilterTwo, $resolverTwo); + $cacheManager->setCacheWarmer($this->createCacheWarmerMock()); $cacheManager->remove( [$expectedPathOne, $expectedPathTwo], [$expectedFilterOne, $expectedFilterTwo] @@ -605,6 +610,7 @@ public function testRemoveCacheForAllFiltersOnRemove(): void ); $cacheManager->addResolver($expectedFilterOne, $resolverOne); $cacheManager->addResolver($expectedFilterTwo, $resolverTwo); + $cacheManager->setCacheWarmer($this->createCacheWarmerMock()); $cacheManager->remove(); } @@ -651,6 +657,7 @@ public function testRemoveCacheForPathAndAllFiltersOnRemove(): void ); $cacheManager->addResolver($expectedFilterOne, $resolverOne); $cacheManager->addResolver($expectedFilterTwo, $resolverTwo); + $cacheManager->setCacheWarmer($this->createCacheWarmerMock()); $cacheManager->remove($expectedPath); } @@ -683,6 +690,7 @@ public function testAggregateFiltersByResolverOnRemove(): void ); $cacheManager->addResolver($expectedFilterOne, $resolver); $cacheManager->addResolver($expectedFilterTwo, $resolver); + $cacheManager->setCacheWarmer($this->createCacheWarmerMock()); $cacheManager->remove(null, [$expectedFilterOne, $expectedFilterTwo]); }