diff --git a/app/code/Magento/AwsS3/Driver/AwsS3.php b/app/code/Magento/AwsS3/Driver/AwsS3.php index 6b3da8c848fb7..54a146e9ce4a9 100644 --- a/app/code/Magento/AwsS3/Driver/AwsS3.php +++ b/app/code/Magento/AwsS3/Driver/AwsS3.php @@ -7,16 +7,18 @@ namespace Magento\AwsS3\Driver; -use Generator; use Exception; -use League\Flysystem\AdapterInterface; +use Generator; use League\Flysystem\Config; -use Magento\Framework\Exception\FileSystemException; +use League\Flysystem\FilesystemAdapter; +use League\Flysystem\FilesystemException; +use League\Flysystem\Visibility; +use Magento\Framework\Exception\FileSystemException as MagentoFileSystemException; use Magento\Framework\Filesystem\DriverInterface; use Magento\Framework\Phrase; -use Psr\Log\LoggerInterface; use Magento\RemoteStorage\Driver\DriverException; use Magento\RemoteStorage\Driver\RemoteDriverInterface; +use Psr\Log\LoggerInterface; /** * Driver for AWS S3 IO operations. @@ -30,10 +32,10 @@ class AwsS3 implements RemoteDriverInterface private const TEST_FLAG = 'storage.flag'; - private const CONFIG = ['ACL' => 'private']; + private const CONFIG = [Config::OPTION_VISIBILITY => Visibility::PRIVATE]; /** - * @var AdapterInterface + * @var FilesystemAdapter */ private $adapter; @@ -53,12 +55,12 @@ class AwsS3 implements RemoteDriverInterface private $objectUrl; /** - * @param AdapterInterface $adapter + * @param FilesystemAdapter $adapter * @param LoggerInterface $logger * @param string $objectUrl */ public function __construct( - AdapterInterface $adapter, + FilesystemAdapter $adapter, LoggerInterface $logger, string $objectUrl ) { @@ -76,7 +78,7 @@ public function __destruct() foreach ($this->streams as $stream) { $this->fileClose($stream); } - } catch (\Exception $e) { + } catch (Exception $e) { // log exception as throwing an exception from a destructor causes a fatal error $this->logger->critical($e); } @@ -89,8 +91,8 @@ public function test(): void { try { $this->adapter->write(self::TEST_FLAG, '', new Config(self::CONFIG)); - } catch (Exception $exception) { - throw new DriverException(__($exception->getMessage()), $exception); + } catch (FilesystemException | Exception $e) { + throw new DriverException(__($e->getMessage()), $e); } } @@ -106,8 +108,14 @@ public function fileGetContents($path, $flag = null, $context = null): string return file_get_contents(stream_get_meta_data($this->streams[$path])['uri']); //phpcs:enable } + try { + $contents = $this->adapter->read($path); + } catch (FilesystemException | Exception $e) { + $this->logger->error($e->getMessage()); + return ''; + } - return $this->adapter->read($path)['contents'] ?? ''; + return $contents; } /** @@ -125,7 +133,15 @@ public function isExists($path): bool return true; } - return $this->adapter->has($path); + try { + if ($this->adapter->fileExists($path) || $this->isDirectory($path)) { + return true; + } + } catch (FilesystemException | Exception $e) { + $this->logger->error($e->getMessage()); + } + + return false; } /** @@ -153,7 +169,6 @@ public function createDirectory($path, $permissions = 0777): bool * * @param string $path * @return bool - * @throws FileSystemException */ private function createDirectoryRecursively(string $path): bool { @@ -166,10 +181,12 @@ private function createDirectoryRecursively(string $path): bool } if (!$this->isDirectory($path)) { - return (bool)$this->adapter->createDir( - $this->fixPath($path), - new Config(self::CONFIG) - ); + try { + $this->adapter->createDirectory($this->fixPath($path), new Config(self::CONFIG)); + } catch (FilesystemException | Exception $e) { + $this->logger->error($e->getMessage()); + return false; + } } return true; @@ -180,10 +197,19 @@ private function createDirectoryRecursively(string $path): bool */ public function copy($source, $destination, DriverInterface $targetDriver = null): bool { - return $this->adapter->copy( - $this->normalizeRelativePath($source, true), - $this->normalizeRelativePath($destination, true) - ); + try { + $this->adapter->copy( + $this->normalizeRelativePath($source, true), + $this->normalizeRelativePath($destination, true), + new Config([]) + ); + } catch (FilesystemException | Exception $e) { + $this->logger->error($e->getMessage()); + + return false; + } + + return true; } /** @@ -191,9 +217,17 @@ public function copy($source, $destination, DriverInterface $targetDriver = null */ public function deleteFile($path): bool { - return $this->adapter->delete( - $this->normalizeRelativePath($path, true) - ); + try { + $this->adapter->delete( + $this->normalizeRelativePath($path, true) + ); + } catch (FilesystemException | Exception $e) { + $this->logger->error($e->getMessage()); + + return false; + } + + return true; } /** @@ -201,9 +235,17 @@ public function deleteFile($path): bool */ public function deleteDirectory($path): bool { - return $this->adapter->deleteDir( - $this->normalizeRelativePath($path, true) - ); + try { + $this->adapter->deleteDirectory( + $this->normalizeRelativePath($path, true) + ); + } catch (FilesystemException | Exception $e) { + $this->logger->error($e->getMessage()); + + return false; + } + + return true; } /** @@ -217,11 +259,20 @@ public function filePutContents($path, $content, $mode = null): int if (false !== ($imageSize = @getimagesizefromstring($content))) { $config['Metadata'] = [ 'image-width' => $imageSize[0], - 'image-height' => $imageSize[1] + 'image-height' => $imageSize[1], ]; } - return $this->adapter->write($path, $content, new Config($config))['size']; + try { + $this->adapter->write($path, $content, new Config($config)); + $size = $this->adapter->fileSize($path)->fileSize(); + } catch (FilesystemException | Exception $e) { + $this->logger->error($e->getMessage()); + + return 0; + } + + return $size; } /** @@ -361,11 +412,12 @@ public function isFile($path): bool $path = $this->normalizeRelativePath($path, true); - if ($this->adapter->has($path) && ($meta = $this->adapter->getMetadata($path))) { - return ($meta['type'] ?? null) === self::TYPE_FILE; + try { + return $this->adapter->fileExists($path); + } catch (FilesystemException $e) { + $this->logger->error($e); + return false; } - - return false; } /** @@ -383,10 +435,15 @@ public function isDirectory($path): bool return true; } - if ($this->adapter->has($path)) { - $meta = $this->adapter->getMetadata($path); - - return !($meta && $meta['type'] === self::TYPE_FILE); + try { + $contents = $this->adapter->listContents($path, false); + foreach ($contents as $content) { + if ($content->isDir() && $content->path() === $path) { + return true; + } + } + } catch (FilesystemException | Exception $e) { + $this->logger->error($e->getMessage()); } return false; @@ -434,10 +491,19 @@ public function getRealPath($path) */ public function rename($oldPath, $newPath, DriverInterface $targetDriver = null): bool { - return $this->adapter->rename( - $this->normalizeRelativePath($oldPath, true), - $this->normalizeRelativePath($newPath, true) - ); + try { + $this->adapter->move( + $this->normalizeRelativePath($oldPath, true), + $this->normalizeRelativePath($newPath, true), + new Config([]) + ); + } catch (FilesystemException | Exception $e) { + $this->logger->error($e->getMessage()); + + return false; + } + + return true; } /** @@ -446,10 +512,15 @@ public function rename($oldPath, $newPath, DriverInterface $targetDriver = null) public function stat($path): array { $path = $this->normalizeRelativePath($path, true); - $metaInfo = $this->adapter->getMetadata($path); - - if (!$metaInfo) { - throw new FileSystemException(__('Cannot gather stats! %1', [$this->getWarningMessage()])); + try { + if (!$this->isDirectory($path)) { + $size = $this->adapter->fileSize($path)->fileSize(); + $type = $this->adapter->mimeType($path)->mimeType(); + $mtime = $this->adapter->lastModified($path)->lastModified(); + } + } catch (FilesystemException | Exception $e) { + $this->logger->error($e->getMessage()); + throw new MagentoFileSystemException(__('Cannot gather stats! %1', [$this->getWarningMessage()])); } return [ @@ -464,10 +535,10 @@ public function stat($path): array 'ctime' => 0, 'blksize' => 0, 'blocks' => 0, - 'size' => $metaInfo['size'] ?? 0, - 'type' => $metaInfo['type'] ?? '', - 'mtime' => $metaInfo['timestamp'] ?? 0, - 'disposition' => null + 'size' => $size ?? 0, + 'type' => $type ?? '', + 'mtime' => $mtime ?? 0, + 'disposition' => null, ]; } @@ -477,16 +548,17 @@ public function stat($path): array public function getMetadata(string $path): array { $path = $this->normalizeRelativePath($path, true); - $metaInfo = $this->adapter->getMetadata($path); - if (!$metaInfo) { - throw new FileSystemException(__('Cannot gather meta info! %1', [$this->getWarningMessage()])); + try { + $mimeType = $this->adapter->mimeType($path)->mimeType(); + $size = $this->adapter->fileSize($path)->fileSize(); + $timestamp = $this->adapter->lastModified($path)->lastModified(); + $metaInfo = $this->adapter->getMetadata($path); + } catch (FilesystemException | Exception $e) { + $this->logger->error($e->getMessage()); + throw new MagentoFileSystemException(__('Cannot gather meta info! %1', [$this->getWarningMessage()])); } - $mimeType = $this->adapter->getMimetype($path)['mimetype']; - $size = $this->adapter->getSize($path)['size']; - $timestamp = $this->adapter->getTimestamp($path)['timestamp']; - return [ 'path' => $metaInfo['path'], 'dirname' => $metaInfo['dirname'], @@ -498,8 +570,8 @@ public function getMetadata(string $path): array 'mimetype' => $mimeType, 'extra' => [ 'image-width' => $metaInfo['metadata']['image-width'] ?? 0, - 'image-height' => $metaInfo['metadata']['image-height'] ?? 0 - ] + 'image-height' => $metaInfo['metadata']['image-height'] ?? 0, + ], ]; } @@ -571,11 +643,16 @@ public function touch($path, $modificationTime = null): bool { $path = $this->normalizeRelativePath($path, true); - $content = $this->adapter->has($path) ? - $this->adapter->read($path)['contents'] - : ''; + $content = $this->adapter->fileExists($path) ? $this->adapter->read($path) : ''; + try { + $this->adapter->write($path, $content, new Config([])); + } catch (FilesystemException | Exception $e) { + $this->logger->error($e->getMessage()); + + return false; + } - return (bool)$this->adapter->write($path, $content, new Config([])); + return true; } /** @@ -587,7 +664,7 @@ public function fileReadLine($resource, $length, $ending = null): string $result = @stream_get_line($resource, $length, $ending); // phpcs:enable if (false === $result) { - throw new FileSystemException( + throw new MagentoFileSystemException( new Phrase('File cannot be read %1', [$this->getWarningMessage()]) ); } @@ -603,7 +680,7 @@ public function fileRead($resource, $length): string //phpcs:ignore Magento2.Functions.DiscouragedFunction $result = fread($resource, $length); if ($result === false) { - throw new FileSystemException(__('File cannot be read %1', [$this->getWarningMessage()])); + throw new MagentoFileSystemException(__('File cannot be read %1', [$this->getWarningMessage()])); } return $result; @@ -617,7 +694,7 @@ public function fileGetCsv($resource, $length = 0, $delimiter = ',', $enclosure //phpcs:ignore Magento2.Functions.DiscouragedFunction $result = fgetcsv($resource, $length, $delimiter, $enclosure, $escape); if ($result === null) { - throw new FileSystemException( + throw new MagentoFileSystemException( new Phrase( 'The "%1" CSV handle is incorrect. Verify the handle and try again.', [$this->getWarningMessage()] @@ -635,7 +712,7 @@ public function fileTell($resource): int { $result = @ftell($resource); if ($result === null) { - throw new FileSystemException( + throw new MagentoFileSystemException( new Phrase('An error occurred during "%1" execution.', [$this->getWarningMessage()]) ); } @@ -650,7 +727,7 @@ public function fileSeek($resource, $offset, $whence = SEEK_SET): int { $result = @fseek($resource, $offset, $whence); if ($result === -1) { - throw new FileSystemException( + throw new MagentoFileSystemException( new Phrase( 'An error occurred during "%1" fileSeek execution.', [$this->getWarningMessage()] @@ -685,7 +762,7 @@ public function fileFlush($resource): bool { $result = @fflush($resource); if (!$result) { - throw new FileSystemException( + throw new MagentoFileSystemException( new Phrase( 'An error occurred during "%1" fileFlush execution.', [$this->getWarningMessage()] @@ -703,7 +780,7 @@ public function fileLock($resource, $lockMode = LOCK_EX): bool { $result = @flock($resource, $lockMode); if (!$result) { - throw new FileSystemException( + throw new MagentoFileSystemException( new Phrase( 'An error occurred during "%1" fileLock execution.', [$this->getWarningMessage()] @@ -721,7 +798,7 @@ public function fileUnlock($resource): bool { $result = @flock($resource, LOCK_UN); if (!$result) { - throw new FileSystemException( + throw new MagentoFileSystemException( new Phrase( 'An error occurred during "%1" fileUnlock execution.', [$this->getWarningMessage()] @@ -785,9 +862,9 @@ public function fileOpen($path, $mode) if (!isset($this->streams[$path])) { $this->streams[$path] = tmpfile(); - if ($this->adapter->has($path)) { + if ($this->adapter->fileExists($path)) { //phpcs:ignore Magento2.Functions.DiscouragedFunction - fwrite($this->streams[$path], $this->adapter->read($path)['contents']); + fwrite($this->streams[$path], $this->adapter->read($path)); //phpcs:ignore Magento2.Functions.DiscouragedFunction rewind($this->streams[$path]); } @@ -839,10 +916,11 @@ private function readPath(string $path, $isRecursive = false): array $itemsList = []; foreach ($contentsList as $item) { + $item = $item->jsonSerialize(); if (isset($item['path']) && $item['path'] !== $relativePath && (!$relativePath || strpos($item['path'], $relativePath) === 0)) { - $itemsList[] = $this->getAbsolutePath($item['dirname'], $item['path']); + $itemsList[] = $this->getAbsolutePath($item['path'], $item['path']); } } @@ -874,7 +952,7 @@ private function getSearchPattern(string $pattern, array $parentPattern, string $replacement = [ '/\*/' => '.*', '/\?/' => '.', - '/\//' => '\/' + '/\//' => '\/', ]; return preg_replace(array_keys($replacement), array_values($replacement), $searchPattern); diff --git a/app/code/Magento/AwsS3/Driver/AwsS3Factory.php b/app/code/Magento/AwsS3/Driver/AwsS3Factory.php index a4d3676bffa07..7bfa0a38d1d26 100644 --- a/app/code/Magento/AwsS3/Driver/AwsS3Factory.php +++ b/app/code/Magento/AwsS3/Driver/AwsS3Factory.php @@ -8,8 +8,9 @@ namespace Magento\AwsS3\Driver; use Aws\S3\S3Client; -use League\Flysystem\AwsS3v3\AwsS3Adapter; -use League\Flysystem\Cached\CachedAdapter; +use League\Flysystem\AwsS3V3\AwsS3V3Adapter; +use League\Flysystem\PathPrefixer; +use Magento\AwsS3\Model\Cached\CachedAdapter; use Magento\Framework\Exception\LocalizedException; use Magento\Framework\ObjectManagerInterface; use Magento\RemoteStorage\Driver\Cache\CacheFactory; @@ -91,7 +92,8 @@ public function createConfigured( } $client = new S3Client($config); - $adapter = new AwsS3Adapter($client, $config['bucket'], $prefix); + $adapter = new AwsS3V3Adapter($client, $config['bucket'], $prefix); + $prefixer = new PathPrefixer($prefix); return $this->objectManager->create( AwsS3::class, @@ -100,7 +102,7 @@ public function createConfigured( 'adapter' => $adapter, 'cache' => $this->cacheFactory->create($cacheAdapter, $cacheConfig) ]), - 'objectUrl' => $client->getObjectUrl($adapter->getBucket(), $adapter->applyPathPrefix('.')) + 'objectUrl' => $client->getObjectUrl($config['bucket'], $prefixer->prefixPath('.')) ] ); } diff --git a/app/code/Magento/AwsS3/Model/Cached/CachedAdapter.php b/app/code/Magento/AwsS3/Model/Cached/CachedAdapter.php new file mode 100644 index 0000000000000..773bf88c67291 --- /dev/null +++ b/app/code/Magento/AwsS3/Model/Cached/CachedAdapter.php @@ -0,0 +1,309 @@ +adapter = $adapter; + $this->cache = $cache; + $this->logger = $logger; + $this->getPathInfo = $getPathInfo; + $this->cache->load(); + } + + /** + * @inheritdoc + */ + public function write(string $path, string $contents, Config $config): void + { + $this->adapter->write($path, $contents, $config); + $result = [ + 'type' => 'file', + 'path' => $path, + 'contents' => $contents, + ]; + $result = array_merge($result, $this->adapter->fileSize($path)->jsonSerialize()); + $this->cache->updateObject($path, $result, true); + } + + /** + * @inheritdoc + */ + public function writeStream(string $path, $contents, Config $config): void + { + $this->adapter->writeStream($path, $contents, $config); + $result = [ + 'type' => 'file', + 'contents' => false, + ]; + + $this->cache->updateObject($path, $result, true); + } + + /** + * @inheritdoc + */ + public function move(string $source, string $destination, Config $config): void + { + $this->adapter->move($source, $destination, $config); + $this->cache->rename($source, $destination); + } + + /** + * @inheritdoc + */ + public function copy(string $source, string $destination, Config $config): void + { + $this->adapter->copy($source, $destination, $config); + $this->cache->copy($source, $destination); + } + + /** + * @inheritdoc + */ + public function delete(string $path): void + { + $this->adapter->delete($path); + $this->cache->delete($path); + } + + /** + * @inheritdoc + */ + public function deleteDirectory(string $path): void + { + $this->adapter->deleteDirectory($path); + $this->cache->deleteDir($path); + } + + /** + * @inheritdoc + */ + public function createDirectory(string $path, Config $config): void + { + $this->adapter->createDirectory($path, $config); + $this->cache->updateObject($path, ['path' => $path, 'type' => 'dir'], true); + } + + /** + * @inheritdoc + */ + public function setVisibility(string $path, string $visibility): void + { + $this->adapter->setVisibility($path, $visibility); + $this->cache->updateObject($path, compact('path', 'visibility'), true); + } + + /** + * @inheritdoc + */ + public function fileExists(string $path): bool + { + if ($this->cache->fileExists($path)) { + return true; + } + + if ($this->adapter->fileExists($path)) { + $this->cache->updateObject($path, $this->adapter->fileSize($path)->jsonSerialize(), true); + return true; + } + + return false; + } + + /** + * @inheritdoc + */ + public function read(string $path): string + { + $result = $this->cache->read($path); + if ($result) { + return $result; + } + + $result = $this->adapter->read($path); + $this->cache->updateObject($path, ['contents' => $result], true); + + return $result; + } + + /** + * @inheritdoc + */ + public function readStream(string $path) + { + return $this->adapter->readStream($path); + } + + /** + * @inheritdoc + */ + public function listContents(string $path, bool $deep): iterable + { + if ($this->cache->isComplete($path, $deep)) { + return $this->cache->listContents($path, $deep); + } + + $contents = $this->adapter->listContents($path, $deep); + $objects = []; + while ($contents->valid()) { + $objects[] = $contents->current(); + $contents->next(); + } + if ($objects) { + $this->cache->storeContents($path, $objects, $deep); + } + + return $objects; + } + + /** + * @inheirtdoc + */ + public function lastModified(string $path): FileAttributes + { + $result = $this->cache->lastModified($path); + if ($result) { + return new FileAttributes($path, null, null, $result); + } + + $result = $this->adapter->lastModified($path); + $object = $result->jsonSerialize() + compact('path'); + $this->cache->updateObject($path, $object, true); + + return $result; + } + + /** + * @inheirtdoc + */ + public function fileSize(string $path): FileAttributes + { + $result = $this->cache->fileSize($path); + if ($result) { + return new FileAttributes($path, $result); + } + + $result = $this->adapter->fileSize($path); + $object = $result->jsonSerialize() + compact('path'); + $this->cache->updateObject($path, $object, true); + + return $result; + } + + /** + * @inheirtdoc + */ + public function mimeType(string $path): FileAttributes + { + $result = $this->cache->mimeType($path); + if ($result) { + return new FileAttributes($path, null, null, null, $result); + } + + $result = $this->adapter->mimeType($path); + $object = $result->jsonSerialize() + compact('path'); + $this->cache->updateObject($path, $object, true); + + return $result; + } + + /** + * @inheirtdoc + */ + public function visibility(string $path): FileAttributes + { + $result = $this->cache->visibility($path); + if ($result) { + return new FileAttributes($path, null, $result); + } + + $result = $this->adapter->visibility($path); + $object = $result->jsonSerialize() + compact('path'); + $this->cache->updateObject($path, $object, true); + + return $result; + } + + /** + * Get file metadata. + * + * @deplacated There is no getMetadata() method in FilesystemAdapter anymore. + * https://flysystem.thephpleague.com/v2/docs/advanced/upgrade-to-2.0.0/ + * Added for compatibility with Magento/AwsS3/Driver/AwsS3::getMetadata() + * + * @param string $path + * @return array + * @throws FilesystemException + */ + public function getMetadata(string $path): array + { + $fileAttributes = $this->adapter->fileSize($path)->jsonSerialize(); + $width = isset($fileAttributes['extra_metadata']['Metadata']['image-width']) + ? (int)$fileAttributes['extra_metadata']['Metadata']['image-width'] : 0; + $height = isset($fileAttributes['extra_metadata']['Metadata']['image-height']) + ? (int)$fileAttributes['extra_metadata']['Metadata']['image-height'] : 0; + $pathInfo = $this->getPathInfo->execute($fileAttributes['path']); + + return [ + 'path' => $pathInfo['path'], + 'dirname' => $pathInfo['dirname'], + 'basename' => $pathInfo['basename'], + 'extension' => $pathInfo['extension'], + 'filename' => $pathInfo['filename'], + 'metadata' => [ + 'image-width' => $width, + 'image-height' => $height, + ], + ]; + } +} diff --git a/app/code/Magento/AwsS3/Test/Mftf/Helper/S3FileAssertions.php b/app/code/Magento/AwsS3/Test/Mftf/Helper/S3FileAssertions.php index cec553afe9463..a5f83e76b3521 100644 --- a/app/code/Magento/AwsS3/Test/Mftf/Helper/S3FileAssertions.php +++ b/app/code/Magento/AwsS3/Test/Mftf/Helper/S3FileAssertions.php @@ -9,7 +9,8 @@ use Aws\S3\S3Client; use Codeception\Lib\ModuleContainer; -use League\Flysystem\AwsS3v3\AwsS3Adapter; +use League\Flysystem\AwsS3V3\AwsS3V3Adapter; +use League\Flysystem\PathPrefixer; use Magento\AwsS3\Driver\AwsS3; use Magento\FunctionalTestingFramework\Helper\Helper; use Magento\Framework\Filesystem\DriverInterface; @@ -56,8 +57,9 @@ public function __construct(ModuleContainer $moduleContainer, ?array $config = n } $client = new S3Client($config); - $adapter = new AwsS3Adapter($client, $config['bucket'], $prefix); - $objectUrl = $client->getObjectUrl($adapter->getBucket(), $adapter->applyPathPrefix('.')); + $adapter = new AwsS3V3Adapter($client, $config['bucket'], $prefix); + $prefixer = new PathPrefixer($prefix); + $objectUrl = $client->getObjectUrl($config['bucket'], $prefixer->prefixPath('.')); $s3Driver = new AwsS3($adapter, new MockTestLogger(), $objectUrl); $this->driver = $s3Driver; diff --git a/app/code/Magento/AwsS3/Test/Unit/Driver/AwsS3Test.php b/app/code/Magento/AwsS3/Test/Unit/Driver/AwsS3Test.php index a7bcf8c47fced..93929c22428a8 100644 --- a/app/code/Magento/AwsS3/Test/Unit/Driver/AwsS3Test.php +++ b/app/code/Magento/AwsS3/Test/Unit/Driver/AwsS3Test.php @@ -7,8 +7,10 @@ namespace Magento\AwsS3\Test\Unit\Driver; -use League\Flysystem\AdapterInterface; -use League\Flysystem\AwsS3v3\AwsS3Adapter; +use League\Flysystem\AwsS3v3\AwsS3V3Adapter; +use League\Flysystem\DirectoryAttributes; +use League\Flysystem\FileAttributes; +use League\Flysystem\FilesystemAdapter; use Magento\AwsS3\Driver\AwsS3; use Magento\Framework\Exception\FileSystemException; use PHPUnit\Framework\MockObject\MockObject; @@ -28,7 +30,7 @@ class AwsS3Test extends TestCase private $driver; /** - * @var AwsS3Adapter|MockObject + * @var AwsS3V3Adapter|MockObject */ private $adapterMock; @@ -37,7 +39,10 @@ class AwsS3Test extends TestCase */ protected function setUp(): void { - $this->adapterMock = $this->getMockForAbstractClass(AdapterInterface::class); + $this->adapterMock = $this->getMockBuilder(FilesystemAdapter::class) + ->disableOriginalConstructor() + ->addMethods(['getMetadata']) + ->getMockForAbstractClass(); $loggerMock = $this->getMockForAbstractClass(LoggerInterface::class); $this->driver = new AwsS3($this->adapterMock, $loggerMock, self::URL); @@ -64,82 +69,82 @@ public function getAbsolutePathDataProvider(): array [ null, 'test.png', - self::URL . 'test.png' + self::URL . 'test.png', ], [ self::URL . 'test/test.png', null, - self::URL . 'test/test.png' + self::URL . 'test/test.png', ], [ '', 'test.png', - self::URL . 'test.png' + self::URL . 'test.png', ], [ '', '/test/test.png', - self::URL . 'test/test.png' + self::URL . 'test/test.png', ], [ self::URL . 'test/test.png', self::URL . 'test/test.png', - self::URL . 'test/test.png' + self::URL . 'test/test.png', ], [ self::URL, self::URL . 'media/catalog/test.png', - self::URL . 'media/catalog/test.png' + self::URL . 'media/catalog/test.png', ], [ '', self::URL . 'media/catalog/test.png', - self::URL . 'media/catalog/test.png' + self::URL . 'media/catalog/test.png', ], [ self::URL . 'test/', 'test.txt', - self::URL . 'test/test.txt' + self::URL . 'test/test.txt', ], [ self::URL . 'media/', '/catalog/test.png', - self::URL . 'media/catalog/test.png' + self::URL . 'media/catalog/test.png', ], [ self::URL, 'var/import/images', - self::URL . 'var/import/images' + self::URL . 'var/import/images', ], [ self::URL . 'export/', null, - self::URL . 'export/' + self::URL . 'export/', ], [ self::URL . 'var/import/images/product_images/', self::URL . 'var/import/images/product_images/1.png', - self::URL . 'var/import/images/product_images/1.png' + self::URL . 'var/import/images/product_images/1.png', ], [ '', self::URL . 'media/catalog/test.png', - self::URL . 'media/catalog/test.png' + self::URL . 'media/catalog/test.png', ], [ self::URL, 'var/import/images', - self::URL . 'var/import/images' + self::URL . 'var/import/images', ], [ self::URL . 'var/import/images/product_images/', self::URL . 'var/import/images/product_images/1.png', - self::URL . 'var/import/images/product_images/1.png' + self::URL . 'var/import/images/product_images/1.png', ], [ self::URL . 'var/import/images/product_images/1.png', '', - self::URL . 'var/import/images/product_images/1.png' + self::URL . 'var/import/images/product_images/1.png', ], [ self::URL . 'media/', @@ -154,8 +159,8 @@ public function getAbsolutePathDataProvider(): array [ self::URL, '', - self::URL - ] + self::URL, + ], ]; } @@ -180,17 +185,17 @@ public function getRelativePathDataProvider(): array [ '', 'test/test.txt', - 'test/test.txt' + 'test/test.txt', ], [ '', '/test/test.txt', - '/test/test.txt' + '/test/test.txt', ], [ self::URL, self::URL . 'test/test.txt', - 'test/test.txt' + 'test/test.txt', ], ]; @@ -198,9 +203,7 @@ public function getRelativePathDataProvider(): array /** * @param string $path - * @param string $normalizedPath - * @param bool $has - * @param array $metadata + * @param array $dirs * @param bool $expected * @throws FileSystemException * @@ -208,17 +211,12 @@ public function getRelativePathDataProvider(): array */ public function testIsDirectory( string $path, - string $normalizedPath, - bool $has, - array $metadata, + array $dirs, bool $expected ): void { - $this->adapterMock->method('has') - ->with($normalizedPath) - ->willReturn($has); - $this->adapterMock->method('getMetadata') - ->with($normalizedPath) - ->willReturn($metadata); + $directoryListing = $dirs; + $this->adapterMock->method('listContents') + ->willReturn($directoryListing); self::assertSame($expected, $this->driver->isDirectory($path)); } @@ -231,51 +229,40 @@ public function isDirectoryDataProvider(): array return [ [ 'some_directory/', - 'some_directory', - false, [], - false + false, ], [ 'some_directory', - 'some_directory', - true, [ - 'type' => AwsS3::TYPE_DIR + new DirectoryAttributes('some_directory'), ], - true + true, ], [ self::URL . 'some_directory', - 'some_directory', - true, [ - 'type' => AwsS3::TYPE_DIR + new DirectoryAttributes('some_directory'), + new DirectoryAttributes('some_directory_1'), ], - true + true, ], [ self::URL . 'some_directory', - 'some_directory', - true, [ - 'type' => AwsS3::TYPE_FILE + new DirectoryAttributes('some_directory_1'), ], - false + false, ], [ '', - '', - true, [], - true + true, ], [ '/', - '', - true, [], - true + true, ], ]; } @@ -294,15 +281,11 @@ public function testIsFile( string $path, string $normalizedPath, bool $has, - array $metadata, bool $expected ): void { - $this->adapterMock->method('has') + $this->adapterMock->method('fileExists') ->with($normalizedPath) ->willReturn($has); - $this->adapterMock->method('getMetadata') - ->with($normalizedPath) - ->willReturn($metadata); self::assertSame($expected, $this->driver->isFile($path)); } @@ -317,50 +300,32 @@ public function isFileDataProvider(): array 'some_file.txt', 'some_file.txt', false, - [], - false + false, ], [ 'some_file.txt/', 'some_file.txt', true, - [ - 'type' => AwsS3::TYPE_FILE - ], - true + true, ], [ self::URL . 'some_file.txt', 'some_file.txt', true, - [ - 'type' => AwsS3::TYPE_FILE - ], - true - ], - [ - self::URL . 'some_file.txt/', - 'some_file.txt', true, - [ - 'type' => AwsS3::TYPE_DIR - ], - false ], [ '', '', false, - [], - false + false, ], [ '/', '', false, - [], - false - ] + false, + ], ]; } @@ -383,20 +348,20 @@ public function getRealPathSafetyDataProvider(): array return [ [ self::URL, - self::URL + self::URL, ], [ 'test.txt', - 'test.txt' + 'test.txt', ], [ self::URL . 'test/test/../test.txt', - self::URL . 'test/test.txt' + self::URL . 'test/test.txt', ], [ 'test/test/../test.txt', - 'test/test.txt' - ] + 'test/test.txt', + ], ]; } @@ -408,21 +373,19 @@ public function testSearchDirectory(): void $expression = '/*'; $path = 'path'; $subPaths = [ - ['path' => 'path/1', 'dirname' => self::URL], - ['path' => 'path/2', 'dirname' => self::URL] + new DirectoryAttributes('path/1'), + new DirectoryAttributes('path/2'), ]; - $expectedResult = [self::URL . 'path/1', self::URL . 'path/2']; - $this->adapterMock->expects(self::atLeastOnce())->method('has') - ->willReturnMap([ - [$path, true] - ]); - $this->adapterMock->expects(self::atLeastOnce())->method('getMetadata') - ->willReturnMap([ - [$path, ['type' => AwsS3::TYPE_DIR]] - ]); - $this->adapterMock->expects(self::atLeastOnce())->method('listContents') - ->with($path, false) - ->willReturn($subPaths); + + $expectedResult = [self::URL . 'path/1/', self::URL . 'path/2/']; + $directoryListing = [new DirectoryAttributes('path')]; + $this->adapterMock->expects(self::exactly(4))->method('listContents') + ->willReturnOnConsecutiveCalls( + $directoryListing, + $subPaths, + $subPaths, + $subPaths + ); self::assertEquals($expectedResult, $this->driver->search($expression, $path)); } @@ -435,21 +398,18 @@ public function testSearchFiles(): void $expression = '/*'; $path = 'path'; $subPaths = [ - ['path' => 'path/1.jpg', 'dirname' => self::URL], - ['path' => 'path/2.png', 'dirname' => self::URL] + new FileAttributes('path/1.jpg'), + new FileAttributes('path/2.png'), ]; $expectedResult = [self::URL . 'path/1.jpg', self::URL . 'path/2.png']; - - $this->adapterMock->expects(self::atLeastOnce())->method('has') - ->willReturnMap([ - [$path, true], - ]); - $this->adapterMock->expects(self::atLeastOnce())->method('getMetadata') - ->willReturnMap([ - [$path, ['type' => AwsS3::TYPE_DIR]], - ]); - $this->adapterMock->expects(self::atLeastOnce())->method('listContents')->with($path, false) - ->willReturn($subPaths); + $directoryListing = [new DirectoryAttributes('path')]; + $this->adapterMock->expects(self::exactly(4))->method('listContents') + ->willReturnOnConsecutiveCalls( + $directoryListing, + $subPaths, + $subPaths, + $subPaths + ); self::assertEquals($expectedResult, $this->driver->search($expression, $path)); } @@ -459,21 +419,15 @@ public function testSearchFiles(): void */ public function testCreateDirectory(): void { - $this->adapterMock->expects(self::exactly(2)) - ->method('has') - ->willReturnMap([ - ['test', true], - ['test/test2', false] - ]); $this->adapterMock->expects(self::once()) - ->method('getMetadata') - ->willReturnMap([ - ['test', ['type' => AwsS3::TYPE_DIR]] - ]); - $this->adapterMock->expects(self::once()) - ->method('createDir') - ->with('test/test2') - ->willReturn(true); + ->method('createDirectory') + ->with('test/test2'); + $directoryListing = [new DirectoryAttributes('test')]; + $this->adapterMock->expects(self::exactly(2))->method('listContents') + ->willReturnOnConsecutiveCalls( + $directoryListing, + [], + ); self::assertTrue($this->driver->createDirectory(self::URL . 'test/test2/')); } diff --git a/app/code/Magento/AwsS3/composer.json b/app/code/Magento/AwsS3/composer.json index 1d4c09482e1b9..2bf91663f5881 100644 --- a/app/code/Magento/AwsS3/composer.json +++ b/app/code/Magento/AwsS3/composer.json @@ -8,9 +8,8 @@ "php": "~7.3.0||~7.4.0", "magento/framework": "*", "magento/module-remote-storage": "*", - "league/flysystem": "^1.0", - "league/flysystem-aws-s3-v3": "^1.0", - "league/flysystem-cached-adapter": "^1.0" + "league/flysystem": "^2.0", + "league/flysystem-aws-s3-v3": "^2.0" }, "type": "magento2-module", "license": [ diff --git a/app/code/Magento/RemoteStorage/Driver/Cache/CacheFactory.php b/app/code/Magento/RemoteStorage/Driver/Cache/CacheFactory.php index 703393b69ec6a..ca98332ff1bd7 100644 --- a/app/code/Magento/RemoteStorage/Driver/Cache/CacheFactory.php +++ b/app/code/Magento/RemoteStorage/Driver/Cache/CacheFactory.php @@ -7,16 +7,15 @@ namespace Magento\RemoteStorage\Driver\Cache; -use League\Flysystem\Adapter\Local; -use League\Flysystem\Cached\CacheInterface; -use League\Flysystem\Cached\Storage\Memory; -use League\Flysystem\Cached\Storage\Predis; -use League\Flysystem\Cached\Storage\Adapter; +use League\Flysystem\Local\LocalFilesystemAdapter; use Magento\Framework\App\Filesystem\DirectoryList; use Magento\Framework\Filesystem; use Magento\RemoteStorage\Driver\DriverException; use Magento\RemoteStorage\Driver\DriverPool; +use Magento\RemoteStorage\Model\Storage\Handler\LocalFactory; +use Magento\RemoteStorage\Model\Storage\Handler\MemoryFactory; use Predis\Client; +use Magento\RemoteStorage\Model\Storage\Handler\PredisFactory; /** * Provides cache adapters. @@ -41,14 +40,47 @@ class CacheFactory private $localCacheRoot; /** + * @var MemoryFactory + */ + private $memoryFactory; + + /** + * @var LocalFactory + */ + private $localFactory; + + /** + * @var PredisFactory + */ + private $predisFactory; + + /** + * @var CacheInterfaceFactory + */ + private $cacheFactory; + + /** + * @param CacheInterfaceFactory $cacheFactory * @param Filesystem $filesystem + * @param MemoryFactory $memoryFactory + * @param LocalFactory $localFactory + * @param PredisFactory $predisFactory */ - public function __construct(Filesystem $filesystem) - { + public function __construct( + CacheInterfaceFactory $cacheFactory, + Filesystem $filesystem, + MemoryFactory $memoryFactory, + LocalFactory $localFactory, + PredisFactory $predisFactory + ) { $this->localCacheRoot = $filesystem->getDirectoryRead( DirectoryList::VAR_DIR, DriverPool::FILE )->getAbsolutePath(); + $this->memoryFactory = $memoryFactory; + $this->localFactory = $localFactory; + $this->predisFactory = $predisFactory; + $this->cacheFactory = $cacheFactory; } /** @@ -63,15 +95,26 @@ public function create(string $adapter, array $config = []): CacheInterface { switch ($adapter) { case self::ADAPTER_PREDIS: - if (!class_exists(Client::class)) { - throw new DriverException(__('Predis client is not installed')); - } - - return new Predis(new Client($config), self::CACHE_KEY, self::CACHE_EXPIRATION); + $predis = $this->predisFactory->create( + [ + 'client' => new Client($config), + 'key' => self::CACHE_KEY, + 'expire' => self::CACHE_EXPIRATION, + ] + ); + return $this->cacheFactory->create(['cacheStorageHandler' => $predis]); case self::ADAPTER_MEMORY: - return new Memory(); + $memory = $this->memoryFactory->create(); + return $this->cacheFactory->create(['cacheStorageHandler' => $memory]); case self::ADAPTER_LOCAL: - return new Adapter(new Local($this->localCacheRoot), self::CACHE_FILE, self::CACHE_EXPIRATION); + $local = $this->localFactory->create( + [ + 'adapter' => new LocalFilesystemAdapter($this->localCacheRoot), + 'file' => self::CACHE_FILE, + 'expire' => self::CACHE_EXPIRATION, + ] + ); + return $this->cacheFactory->create(['cacheStorageHandler' => $local]); } throw new DriverException(__('Cache adapter %1 is not supported', $adapter)); diff --git a/app/code/Magento/RemoteStorage/Driver/Cache/CacheInterface.php b/app/code/Magento/RemoteStorage/Driver/Cache/CacheInterface.php new file mode 100644 index 0000000000000..fb7f4d344df56 --- /dev/null +++ b/app/code/Magento/RemoteStorage/Driver/Cache/CacheInterface.php @@ -0,0 +1,187 @@ + + */ + public function listContents(string $location, bool $deep = self::LIST_SHALLOW): iterable; + + /** + * Retrieve directory/file last update date. + * @param string $path + * @return int + */ + public function lastModified(string $path): int; + + /** + * Retrieve file size. + * + * @param string $path + * @return int + */ + public function fileSize(string $path): int; + + /** + * Retrieve file mimeType. + * + * @param string $path + * @return string + */ + public function mimeType(string $path): string; + + /** + * Get directory/file visibility status. + * + * @param string $path + * @return string + */ + public function visibility(string $path): string; + + /** + * Check whether the directory listing of a given directory is complete. + * + * @param string $dirname + * @param bool $recursive + * @return bool + */ + public function isComplete(string $dirname, bool $recursive): bool; + + /** + * Set a directory to completely listed. + * + * @param string $dirname + * @param bool $recursive + * @return void + */ + public function setComplete(string $dirname, bool $recursive): void; + + /** + * Store the contents of a directory. + * + * @param string $directory + * @param array $contents + * @param bool $recursive + * @return void + */ + public function storeContents(string $directory, array $contents, bool $recursive): void; + + /** + * Flush the cache. + * + * @return void + */ + public function flush(): void; + + /** + * Autosave trigger. + * + * @return void + */ + public function autosave(): void; + + /** + * Store the cache. + * + * @return void + * @throws \League\Flysystem\FilesystemException + */ + public function save(): void; + + /** + * Load the cache. + * + * @return void + * @throws \League\Flysystem\FilesystemException + */ + public function load(): void; + + /** + * Rename a file. + * + * @param string $path + * @param string $newPath + * @return void + */ + public function rename(string $path, string $newPath): void; + + /** + * Copy a file. + * + * @param string $path + * @param string $newpath + * @return void + */ + public function copy(string $path, string $newpath): void; + + /** + * Delete an object from cache. + * + * @param string $path object path + * @return void + */ + public function delete(string $path): void; + + /** + * Delete all objects from from a directory. + * + * @param string $dirname directory path + * @return void + */ + public function deleteDir(string $dirname): void; + + /** + * Update the metadata for an object. + * + * @param string $path + * @param array $object + * @param bool $autoSave + */ + public function updateObject(string $path, array $object, bool $autoSave = false): void; +} diff --git a/app/code/Magento/RemoteStorage/Model/Cache.php b/app/code/Magento/RemoteStorage/Model/Cache.php new file mode 100644 index 0000000000000..4970efd278e18 --- /dev/null +++ b/app/code/Magento/RemoteStorage/Model/Cache.php @@ -0,0 +1,393 @@ +json = $json; + $this->mimeTypeDetector = $mimeTypeDetector; + $this->getPathInfo = $getPathInfo; + $this->cacheStorageHandler = $cacheStorageHandler; + $this->cacheStorage = $cacheStorage; + } + + /** + * Destructor. + */ + public function __destruct() + { + if (!$this->autoSave) { + $this->save(); + } + } + + /** + * @inheirtdoc + */ + public function save(): void + { + $this->cacheStorageHandler->save(); + } + + /** + * @inheirtdoc + */ + public function load(): void + { + $this->cacheStorageHandler->load(); + } + + /** + * Store the contents listing. + * + * @param string $directory + * @param array $contents + * @param bool $recursive + */ + public function storeContents(string $directory, array $contents, $recursive = false): void + { + $directories = [$directory]; + foreach ($contents as $object) { + $object = $object->jsonSerialize(); + $this->updateObject($object['path'], $object); + $object = $this->cacheStorage->getCacheDataByKey($object['path']); + + if ($recursive && $this->pathIsInDirectory($directory, $object['path'])) { + $directories[] = $object['dirname']; + } + } + foreach (array_unique($directories) as $directory) { + $this->setComplete($directory, $recursive); + } + + $this->autosave(); + } + + /** + * @inheirtdoc + */ + public function updateObject(string $path, array $object, $autoSave = false): void + { + if (!$this->has($path)) { + $data = $this->getPathInfo->execute($path); + $this->cacheStorage->setCacheDataByKey($path, $data); + } + $data = array_merge($this->cacheStorage->getCacheDataByKey($path), $object); + $this->cacheStorage->setCacheDataByKey($path, $data); + + if ($autoSave) { + $this->autosave(); + } + + $this->ensureParentDirectories($path); + } + + /** + * @ingeritdoc + */ + public function listContents(string $location, bool $deep = self::LIST_SHALLOW): iterable + { + $result = []; + + foreach ($this->cacheStorage->getCacheData() as $content) { + if ($content === false) { + continue; + } + $object = $content['type'] === 'file' + ? new FileAttributes( + $content['path'], + $content['file_size'] ?? null, + $content['visibility'] ?? null, + $content['last_modified'] ?? null, + $content['mime_type'] ?? null, + $content['extra_metadata'] ?? [], + ) + : new DirectoryAttributes( + $content['path'], + $content['visibility'] ?? null, + $content['last_modified'] ?? null + ); + if ($content['dirname'] === $location || $content['path'] === $location) { + $result[] = $object; + } elseif ($deep && $this->pathIsInDirectory($location, $content['path'])) { + $result[] = $object; + } + } + + return $result; + } + + /** + * @inheritdoc + */ + public function fileExists(string $location): bool + { + return isset($this->cacheStorage->getCacheDataByKey($location)['type']) + && $this->cacheStorage->getCacheDataByKey($location)['type'] === 'file'; + } + + /** + * @inheritdoc + */ + public function read(string $location): string + { + $data = $this->cacheStorage->getCacheDataByKey($location); + if ($data && isset($data['contents'])) { + return (string)$data['contents']; + } + + return ''; + } + + /** + * @inheritdoc + */ + public function readStream(string $location): void + { + return; + } + + /** + * @inheritdoc + */ + public function rename($path, $newPath): void + { + if (!$this->fileExists($path)) { + return; + } + + $object = $this->cacheStorage->getCacheDataByKey($path); + $this->cacheStorage->removeCacheDataByKey($path); + $object['path'] = $newPath; + $object = array_merge($object, $this->getPathInfo->execute($newPath)); + $this->cacheStorage->setCacheDataByKey($newPath, $object); + $this->autosave(); + } + + /** + * @inheritdoc + */ + public function copy($path, $newpath): void + { + if ($this->fileExists($path)) { + $object = $this->cacheStorage->getCacheDataByKey($path); + $object = array_merge($object, $this->getPathInfo->execute($newpath)); + $this->updateObject($newpath, $object, true); + } + } + + /** + * @inheritdoc + */ + public function delete($path): void + { + $this->cacheStorage->setCacheDataByKey($path, false); + } + + /** + * @inheritdoc + */ + public function deleteDir($dirname): void + { + foreach ($this->cacheStorage->getCacheData() as $path => $object) { + if ($this->pathIsInDirectory($dirname, $path) || $path === $dirname) { + $this->cacheStorage->removeCacheDataByKey($path); + } + } + + $this->cacheStorage->removeCompleteDataByKey($dirname); + + $this->autosave(); + } + + /** + * @inheritdoc + */ + public function mimeType($path): string + { + $cachedData = $this->cacheStorage->getCacheDataByKey($path); + if (isset($cachedData['mimetype'])) { + return $cachedData['mimetype']; + } + + if (!$result = $this->read($path)) { + return ''; + } + + $mimetype = $this->mimeTypeDetector->detectMimeType($path, $result); + $cachedData['mimetype'] = $mimetype; + $this->cacheStorage->setCacheDataByKey($path, $cachedData); + + return $mimetype ?: ''; + } + + /** + * @inheritdoc + */ + public function fileSize($path): int + { + return $this->cacheStorage->getCacheDataByKey($path)['file_size'] ?? 0; + } + + /** + * @inheritdoc + */ + public function lastModified($path): int + { + return $this->cacheStorage->getCacheDataByKey($path)['last_modified'] ?? 0; + } + + /** + * @inheritdoc + */ + public function visibility(string $path): string + { + return $this->cacheStorage->getCacheDataByKey($path)['visibility'] ?? ''; + } + + /** + * @inheritdoc + */ + public function isComplete($dirname, $recursive): bool + { + if (!$this->cacheStorage->hasCompleteData($dirname)) { + return false; + } + + if ($recursive && $this->cacheStorage->getCompleteDataByKey($dirname) !== 'recursive') { + return false; + } + + return true; + } + + /** + * @inheritdoc + */ + public function setComplete($dirname, $recursive): void + { + $recursive = $recursive ? 'recursive' : true; + $this->cacheStorage->setCompleteDataByKey($dirname, $recursive); + } + + /** + * @inheritdoc + */ + public function flush(): void + { + $this->cacheStorage->flushCache(); + $this->cacheStorage->flushComplete(); + $this->autosave(); + } + + /** + * @inheritdoc + */ + public function autosave(): void + { + if ($this->autoSave) { + $this->save(); + } + } + + /** + * Ensure parent directories of an object. + * + * @param string $path + */ + private function ensureParentDirectories(string $path) + { + $object = $this->cacheStorage->getCacheDataByKey($path); + + while ($object['dirname'] !== '' && !$this->cacheStorage->hasCacheData($object['dirname'])) { + $object = $this->getPathInfo->execute($object['dirname']); + $object['type'] = 'dir'; + $this->cacheStorage->setCacheDataByKey($object['path'], $object); + } + } + + /** + * Determines if the path is inside the directory. + * + * @param string $directory + * @param string $path + * + * @return bool + */ + private function pathIsInDirectory(string $directory, string $path): bool + { + return $directory === '' || str_starts_with($path, $directory . '/'); + } + + /** + * Verify if cache has object. + * + * @param string $location + * @return bool + */ + private function has(string $location): bool + { + return $this->cacheStorage->hasCacheData($location) + && $this->cacheStorage->getCacheDataByKey($location) !== false; + } +} diff --git a/app/code/Magento/RemoteStorage/Model/GetPathInfo.php b/app/code/Magento/RemoteStorage/Model/GetPathInfo.php new file mode 100644 index 0000000000000..12aaf567ea8f6 --- /dev/null +++ b/app/code/Magento/RemoteStorage/Model/GetPathInfo.php @@ -0,0 +1,33 @@ + '']; + } +} diff --git a/app/code/Magento/RemoteStorage/Model/Storage/CacheStorage.php b/app/code/Magento/RemoteStorage/Model/Storage/CacheStorage.php new file mode 100644 index 0000000000000..69b268ceee73a --- /dev/null +++ b/app/code/Magento/RemoteStorage/Model/Storage/CacheStorage.php @@ -0,0 +1,167 @@ +cache; + } + + /** + * Retrieve cached data by key. + * + * @param string $key + * @return mixed + */ + public function getCacheDataByKey(string $key) + { + return $this->cache[$key] ?? false; + } + + /** + * Set cache data. + * + * @param array $cache + */ + public function setCacheData(array $cache): void + { + $this->cache = $cache; + } + + /** + * Remove data from cache. + * + * @param string $key + */ + public function removeCacheDataByKey(string $key): void + { + unset($this->cache[$key]); + } + + /** + * Add cache data. + * + * @param string $key + * @param mixed $data + */ + public function setCacheDataByKey(string $key, $data): void + { + $this->cache[$key] = $data; + } + + /** + * Verify if cache data exists. + * + * @param string $key + * @return bool + */ + public function hasCacheData(string $key): bool + { + return isset($this->cache[$key]); + } + + /** + * Remove all cache data. + */ + public function flushCache(): void + { + $this->cache = []; + } + + /** + * Remove all complete data. + */ + public function flushComplete(): void + { + $this->complete = []; + } + + /** + * Set complete data. + * + * @param array $complete + */ + public function setCompleteData(array $complete): void + { + $this->complete = $complete; + } + + /** + * Add complete data by key. + * + * @param string $key + * @param mixed $data + * @return void + */ + public function setCompleteDataByKey(string $key, $data): void + { + $this->complete[$key] = $data; + } + + /** + * Retrieve data from complete by key. + * + * @param string $key + * @return mixed + */ + public function getCompleteDataByKey(string $key) + { + return $this->complete[$key] ?? false; + } + + /** + * Remove data from complete by key. + * + * @param string $key + */ + public function removeCompleteDataByKey(string $key): void + { + unset($this->complete[$key]); + } + + /** + * Verify if data exists in complete. + * + * @param string $key + * @return bool + */ + public function hasCompleteData(string $key): bool + { + return isset($this->complete[$key]); + } + + /** + * Retrieve complete data. + * + * @return array + */ + public function getCompleteData(): array + { + return $this->complete; + } +} diff --git a/app/code/Magento/RemoteStorage/Model/Storage/GetCleanedContents.php b/app/code/Magento/RemoteStorage/Model/Storage/GetCleanedContents.php new file mode 100644 index 0000000000000..822d78148545b --- /dev/null +++ b/app/code/Magento/RemoteStorage/Model/Storage/GetCleanedContents.php @@ -0,0 +1,45 @@ + $object) { + if (is_array($object)) { + $contents[$path] = array_intersect_key($object, $cachedProperties); + } + } + + return $contents; + } +} diff --git a/app/code/Magento/RemoteStorage/Model/Storage/Handler/CacheStorageHandlerInterface.php b/app/code/Magento/RemoteStorage/Model/Storage/Handler/CacheStorageHandlerInterface.php new file mode 100644 index 0000000000000..a7d6743595969 --- /dev/null +++ b/app/code/Magento/RemoteStorage/Model/Storage/Handler/CacheStorageHandlerInterface.php @@ -0,0 +1,30 @@ +json = $json; + $this->adapter = $adapter; + $this->file = $file; + $this->cacheStorage = $cacheStorage; + $this->setExpire($expire); + $this->getCleanedContents = $getCleanedContents; + } + + /** + * @inheritdoc + */ + public function load(): void + { + if ($this->adapter->fileExists($this->file)) { + $contents = $this->adapter->read($this->file); + if ($contents) { + $this->setFromStorage($contents); + } + } + } + + /** + * {@inheritdoc} + */ + public function save(): void + { + $config = new Config(); + $contents = $this->getForStorage(); + $this->adapter->write($this->file, $contents, $config); + } + + /** + * Set the expiration time in seconds. + * + * @param int|null $expire + */ + private function setExpire(?int $expire): void + { + if ($expire) { + $this->expire = $this->getTime($expire); + } + } + + /** + * Retrieve serialized cache data. + * + * @return string + */ + private function getForStorage(): string + { + $cleaned = $this->getCleanedContents->execute($this->cacheStorage->getCacheData()); + + return $this->json->serialize([$cleaned, $this->cacheStorage->getCompleteData(), $this->expire]); + } + + /** + * Load from serialized cache data. + * + * @param string $json + * @throws FilesystemException + */ + private function setFromStorage(string $json): void + { + [$cache, $complete, $expire] = $this->json->unserialize($json); + + if (!$expire || $expire > $this->getTime()) { + $cacheData = is_array($cache) ? $cache : []; + $completeData = is_array($complete) ? $complete : []; + $this->cacheStorage->setCacheData($cacheData); + $this->cacheStorage->setCompleteData($completeData); + } else { + $this->adapter->delete($this->file); + } + } + + /** + * Get expiration time in seconds. + * + * @param int $time + * @return int + */ + private function getTime($time = 0): int + { + return intval(microtime(true)) + $time; + } +} diff --git a/app/code/Magento/RemoteStorage/Model/Storage/Handler/Memory.php b/app/code/Magento/RemoteStorage/Model/Storage/Handler/Memory.php new file mode 100644 index 0000000000000..a89e1b2dc7959 --- /dev/null +++ b/app/code/Magento/RemoteStorage/Model/Storage/Handler/Memory.php @@ -0,0 +1,30 @@ +client = $client ?: new Client(); + $this->key = $key; + $this->expire = $expire; + $this->cacheStorage = $cacheStorage; + $this->json = $json; + $this->getCleanedContents = $getCleanedContents; + } + + /** + * {@inheritdoc} + */ + public function load(): void + { + if (($contents = $this->executeCommand('get', [$this->key])) !== null) { + $this->setFromStorage($contents); + } + } + + /** + * {@inheritdoc} + */ + public function save(): void + { + $contents = $this->getForStorage(); + $this->executeCommand('set', [$this->key, $contents]); + + if ($this->expire !== null) { + $this->executeCommand('expire', [$this->key, $this->expire]); + } + } + + /** + * Load from serialized cache data. + * + * @param string $json + */ + private function setFromStorage(string $json): void + { + [$cache, $complete] = $this->json->unserialize($json); + $this->cacheStorage->setCacheData($cache); + $this->cacheStorage->setCompleteData($complete); + } + + /** + * Retrieve serialized cache data. + * + * @return string + */ + private function getForStorage(): string + { + $cleaned = $this->getCleanedContents->execute($this->cacheStorage->getCacheData()); + + return $this->json->serialize([$cleaned, $this->cacheStorage->getCompleteData()]); + } + + /** + * Execute a Predis command. + * + * @param string $name + * @param array $arguments + * @return string + */ + private function executeCommand(string $name, array $arguments): string + { + $command = $this->client->createCommand($name, $arguments); + + return $this->client->executeCommand($command); + } +} diff --git a/app/code/Magento/RemoteStorage/composer.json b/app/code/Magento/RemoteStorage/composer.json index 1b6b361366848..b9b02fd73e73d 100644 --- a/app/code/Magento/RemoteStorage/composer.json +++ b/app/code/Magento/RemoteStorage/composer.json @@ -3,7 +3,8 @@ "description": "N/A", "require": { "php": "~7.3.0||~7.4.0", - "magento/framework": "*" + "magento/framework": "*", + "predis/predis": "*" }, "suggest": { "magento/module-backend": "*", @@ -14,8 +15,7 @@ "magento/module-media-storage": "*", "magento/module-import-export": "*", "magento/module-catalog-import-export": "*", - "magento/module-downloadable-import-export": "*", - "predis/predis": "*" + "magento/module-downloadable-import-export": "*" }, "type": "magento2-module", "license": [ diff --git a/app/code/Magento/RemoteStorage/etc/di.xml b/app/code/Magento/RemoteStorage/etc/di.xml index d8ee5d609cb57..75725cb6782d1 100644 --- a/app/code/Magento/RemoteStorage/etc/di.xml +++ b/app/code/Magento/RemoteStorage/etc/di.xml @@ -6,6 +6,7 @@ */ --> + Magento\RemoteStorage\Driver\DriverPool @@ -137,4 +138,9 @@ customRemoteFilesystem + + + League\MimeTypeDetection\FinfoMimeTypeDetector + + diff --git a/composer.json b/composer.json index 775a8c4dafdd8..c537ff3d0d09c 100644 --- a/composer.json +++ b/composer.json @@ -60,9 +60,8 @@ "laminas/laminas-uri": "^2.5.1", "laminas/laminas-validator": "^2.6.0", "laminas/laminas-view": "~2.12.0", - "league/flysystem": "^1.0", - "league/flysystem-aws-s3-v3": "^1.0", - "league/flysystem-cached-adapter": "^1.0", + "league/flysystem": "^2.0", + "league/flysystem-aws-s3-v3": "^2.0", "magento/composer": "1.6.0", "magento/magento-composer-installer": ">=0.1.11", "magento/zendframework1": "~1.14.2", @@ -79,11 +78,9 @@ "tedivm/jshrink": "~1.4.0", "tubalmartin/cssmin": "4.1.1", "webonyx/graphql-php": "^0.13.8", - "league/flysystem": "^1.0", - "league/flysystem-aws-s3-v3": "^1.0", - "league/flysystem-cached-adapter": "^1.0", "web-token/jwt-framework": "^v2.2.7", - "wikimedia/less.php": "^3.0.0" + "wikimedia/less.php": "^3.0.0", + "predis/predis": "^1.1" }, "require-dev": { "allure-framework/allure-phpunit": "~1.2.0", diff --git a/composer.lock b/composer.lock index 348506d7c3495..009ba37eb93df 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "5783e37aa34b1ffb23ce07e71e278a60", + "content-hash": "8bd21befc1b1ab01f737a956a205d100", "packages": [ { "name": "aws/aws-sdk-php", @@ -3190,55 +3190,42 @@ }, { "name": "league/flysystem", - "version": "1.1.3", + "version": "2.0.4", "source": { "type": "git", "url": "https://github.com/thephpleague/flysystem.git", - "reference": "9be3b16c877d477357c015cec057548cf9b2a14a" + "reference": "7cbbb7222e8d8a34e71273d243dfcf383ed6779f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/thephpleague/flysystem/zipball/9be3b16c877d477357c015cec057548cf9b2a14a", - "reference": "9be3b16c877d477357c015cec057548cf9b2a14a", + "url": "https://api.github.com/repos/thephpleague/flysystem/zipball/7cbbb7222e8d8a34e71273d243dfcf383ed6779f", + "reference": "7cbbb7222e8d8a34e71273d243dfcf383ed6779f", "shasum": "" }, "require": { - "ext-fileinfo": "*", - "league/mime-type-detection": "^1.3", - "php": "^7.2.5 || ^8.0" + "ext-json": "*", + "league/mime-type-detection": "^1.0.0", + "php": "^7.2 || ^8.0" }, "conflict": { - "league/flysystem-sftp": "<1.0.6" + "guzzlehttp/ringphp": "<1.1.1" }, "require-dev": { - "phpspec/prophecy": "^1.11.1", - "phpunit/phpunit": "^8.5.8" - }, - "suggest": { - "ext-fileinfo": "Required for MimeType", - "ext-ftp": "Allows you to use FTP server storage", - "ext-openssl": "Allows you to use FTPS server storage", - "league/flysystem-aws-s3-v2": "Allows you to use S3 storage with AWS SDK v2", - "league/flysystem-aws-s3-v3": "Allows you to use S3 storage with AWS SDK v3", - "league/flysystem-azure": "Allows you to use Windows Azure Blob storage", - "league/flysystem-cached-adapter": "Flysystem adapter decorator for metadata caching", - "league/flysystem-eventable-filesystem": "Allows you to use EventableFilesystem", - "league/flysystem-rackspace": "Allows you to use Rackspace Cloud Files", - "league/flysystem-sftp": "Allows you to use SFTP server storage via phpseclib", - "league/flysystem-webdav": "Allows you to use WebDAV storage", - "league/flysystem-ziparchive": "Allows you to use ZipArchive adapter", - "spatie/flysystem-dropbox": "Allows you to use Dropbox storage", - "srmklive/flysystem-dropbox-v2": "Allows you to use Dropbox storage for PHP 5 applications" + "async-aws/s3": "^1.5", + "async-aws/simple-s3": "^1.0", + "aws/aws-sdk-php": "^3.132.4", + "composer/semver": "^3.0", + "ext-fileinfo": "*", + "friendsofphp/php-cs-fixer": "^2.16", + "google/cloud-storage": "^1.23", + "phpseclib/phpseclib": "^2.0", + "phpstan/phpstan": "^0.12.26", + "phpunit/phpunit": "^8.5 || ^9.4" }, "type": "library", - "extra": { - "branch-alias": { - "dev-master": "1.1-dev" - } - }, "autoload": { "psr-4": { - "League\\Flysystem\\": "src/" + "League\\Flysystem\\": "src" } }, "notification-url": "https://packagist.org/downloads/", @@ -3248,25 +3235,19 @@ "authors": [ { "name": "Frank de Jonge", - "email": "info@frenky.net" + "email": "info@frankdejonge.nl" } ], - "description": "Filesystem abstraction: Many filesystems, one API.", + "description": "File storage abstraction for PHP", "keywords": [ - "Cloud Files", "WebDAV", - "abstraction", "aws", "cloud", - "copy.com", - "dropbox", - "file systems", + "file", "files", "filesystem", "filesystems", "ftp", - "rackspace", - "remote", "s3", "sftp", "storage" @@ -3274,43 +3255,46 @@ "funding": [ { "url": "https://offset.earth/frankdejonge", - "type": "other" + "type": "custom" + }, + { + "url": "https://github.com/frankdejonge", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/league/flysystem", + "type": "tidelift" } ], - "time": "2020-08-23T07:39:11+00:00" + "time": "2021-02-12T19:37:50+00:00" }, { "name": "league/flysystem-aws-s3-v3", - "version": "1.0.29", + "version": "2.0.4", "source": { "type": "git", "url": "https://github.com/thephpleague/flysystem-aws-s3-v3.git", - "reference": "4e25cc0582a36a786c31115e419c6e40498f6972" + "reference": "c89931e9c4b294493234798564cc814ef478fbc6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/thephpleague/flysystem-aws-s3-v3/zipball/4e25cc0582a36a786c31115e419c6e40498f6972", - "reference": "4e25cc0582a36a786c31115e419c6e40498f6972", + "url": "https://api.github.com/repos/thephpleague/flysystem-aws-s3-v3/zipball/c89931e9c4b294493234798564cc814ef478fbc6", + "reference": "c89931e9c4b294493234798564cc814ef478fbc6", "shasum": "" }, "require": { - "aws/aws-sdk-php": "^3.20.0", - "league/flysystem": "^1.0.40", - "php": ">=5.5.0" + "aws/aws-sdk-php": "^3.132.4", + "league/flysystem": "^2.0.0", + "league/mime-type-detection": "^1.0.0", + "php": "^7.2 || ^8.0" }, - "require-dev": { - "henrikbjorn/phpspec-code-coverage": "~1.0.1", - "phpspec/phpspec": "^2.0.0" + "conflict": { + "guzzlehttp/ringphp": "<1.1.1" }, "type": "library", - "extra": { - "branch-alias": { - "dev-master": "1.0-dev" - } - }, "autoload": { "psr-4": { - "League\\Flysystem\\AwsS3v3\\": "src/" + "League\\Flysystem\\AwsS3V3\\": "" } }, "notification-url": "https://packagist.org/downloads/", @@ -3320,58 +3304,20 @@ "authors": [ { "name": "Frank de Jonge", - "email": "info@frenky.net" - } - ], - "description": "Flysystem adapter for the AWS S3 SDK v3.x", - "time": "2020-10-08T18:58:37+00:00" - }, - { - "name": "league/flysystem-cached-adapter", - "version": "1.1.0", - "source": { - "type": "git", - "url": "https://github.com/thephpleague/flysystem-cached-adapter.git", - "reference": "d1925efb2207ac4be3ad0c40b8277175f99ffaff" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/thephpleague/flysystem-cached-adapter/zipball/d1925efb2207ac4be3ad0c40b8277175f99ffaff", - "reference": "d1925efb2207ac4be3ad0c40b8277175f99ffaff", - "shasum": "" - }, - "require": { - "league/flysystem": "~1.0", - "psr/cache": "^1.0.0" - }, - "require-dev": { - "mockery/mockery": "~0.9", - "phpspec/phpspec": "^3.4", - "phpunit/phpunit": "^5.7", - "predis/predis": "~1.0", - "tedivm/stash": "~0.12" - }, - "suggest": { - "ext-phpredis": "Pure C implemented extension for PHP" - }, - "type": "library", - "autoload": { - "psr-4": { - "League\\Flysystem\\Cached\\": "src/" + "email": "info@frankdejonge.nl" } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" ], - "authors": [ - { - "name": "frankdejonge", - "email": "info@frenky.net" - } + "description": "AWS S3 filesystem adapter for Flysystem.", + "keywords": [ + "Flysystem", + "aws", + "file", + "files", + "filesystem", + "s3", + "storage" ], - "description": "An adapter decorator to enable meta-data caching.", - "time": "2020-07-25T15:56:04+00:00" + "time": "2021-02-09T21:10:56+00:00" }, { "name": "league/mime-type-detection", @@ -3453,13 +3399,11 @@ "Magento\\Composer\\": "src" } }, - "notification-url": "https://packagist.org/downloads/", "license": [ "OSL-3.0", "AFL-3.0" ], - "description": "Magento composer library helps to instantiate Composer application and run composer commands.", - "time": "2020-06-15T17:52:31+00:00" + "description": "Magento composer library helps to instantiate Composer application and run composer commands." }, { "name": "magento/magento-composer-installer", @@ -4206,31 +4150,46 @@ "time": "2020-12-17T05:42:04+00:00" }, { - "name": "psr/cache", - "version": "1.0.1", + "name": "predis/predis", + "version": "v1.1.6", "source": { "type": "git", - "url": "https://github.com/php-fig/cache.git", - "reference": "d11b50ad223250cf17b86e38383413f5a6764bf8" + "url": "https://github.com/predis/predis.git", + "reference": "9930e933c67446962997b05201c69c2319bf26de" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/php-fig/cache/zipball/d11b50ad223250cf17b86e38383413f5a6764bf8", - "reference": "d11b50ad223250cf17b86e38383413f5a6764bf8", + "url": "https://api.github.com/repos/predis/predis/zipball/9930e933c67446962997b05201c69c2319bf26de", + "reference": "9930e933c67446962997b05201c69c2319bf26de", "shasum": "" }, "require": { - "php": ">=5.3.0" + "php": ">=5.3.9" + }, + "require-dev": { + "cweagans/composer-patches": "^1.6", + "phpunit/phpunit": "~4.8" + }, + "suggest": { + "ext-curl": "Allows access to Webdis when paired with phpiredis", + "ext-phpiredis": "Allows faster serialization and deserialization of the Redis protocol" }, "type": "library", "extra": { - "branch-alias": { - "dev-master": "1.0.x-dev" + "composer-exit-on-patch-failure": true, + "patches": { + "phpunit/phpunit-mock-objects": { + "Fix PHP 7 and 8 compatibility": "./tests/phpunit_mock_objects.patch" + }, + "phpunit/phpunit": { + "Fix PHP 7 compatibility": "./tests/phpunit_php7.patch", + "Fix PHP 8 compatibility": "./tests/phpunit_php8.patch" + } } }, "autoload": { "psr-4": { - "Psr\\Cache\\": "src/" + "Predis\\": "src/" } }, "notification-url": "https://packagist.org/downloads/", @@ -4239,17 +4198,31 @@ ], "authors": [ { - "name": "PHP-FIG", - "homepage": "http://www.php-fig.org/" + "name": "Daniele Alessandri", + "email": "suppakilla@gmail.com", + "homepage": "http://clorophilla.net", + "role": "Creator & Maintainer" + }, + { + "name": "Till Krüss", + "homepage": "https://till.im", + "role": "Maintainer" } ], - "description": "Common interface for caching libraries", + "description": "Flexible and feature-complete Redis client for PHP and HHVM", + "homepage": "http://github.com/predis/predis", "keywords": [ - "cache", - "psr", - "psr-6" + "nosql", + "predis", + "redis" ], - "time": "2016-08-06T20:24:11+00:00" + "funding": [ + { + "url": "https://github.com/sponsors/tillkruss", + "type": "github" + } + ], + "time": "2020-09-11T19:18:05+00:00" }, { "name": "psr/container", @@ -8348,20 +8321,6 @@ "parser", "php" ], - "funding": [ - { - "url": "https://www.doctrine-project.org/sponsorship.html", - "type": "custom" - }, - { - "url": "https://www.patreon.com/phpdoctrine", - "type": "patreon" - }, - { - "url": "https://tidelift.com/funding/github/packagist/doctrine%2Flexer", - "type": "tidelift" - } - ], "time": "2020-05-25T17:44:05+00:00" }, { @@ -10659,12 +10618,6 @@ "keywords": [ "timer" ], - "funding": [ - { - "url": "https://github.com/sebastianbergmann", - "type": "github" - } - ], "time": "2020-04-20T06:00:37+00:00" }, { @@ -10809,17 +10762,53 @@ "testing", "xunit" ], - "funding": [ - { - "url": "https://phpunit.de/donate.html", - "type": "custom" - }, + "time": "2020-05-22T13:54:05+00:00" + }, + { + "name": "psr/cache", + "version": "1.0.1", + "source": { + "type": "git", + "url": "https://github.com/php-fig/cache.git", + "reference": "d11b50ad223250cf17b86e38383413f5a6764bf8" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-fig/cache/zipball/d11b50ad223250cf17b86e38383413f5a6764bf8", + "reference": "d11b50ad223250cf17b86e38383413f5a6764bf8", + "shasum": "" + }, + "require": { + "php": ">=5.3.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.0.x-dev" + } + }, + "autoload": { + "psr-4": { + "Psr\\Cache\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ { - "url": "https://github.com/sebastianbergmann", - "type": "github" + "name": "PHP-FIG", + "homepage": "http://www.php-fig.org/" } ], - "time": "2020-05-22T13:54:05+00:00" + "description": "Common interface for caching libraries", + "keywords": [ + "cache", + "psr", + "psr-6" + ], + "time": "2016-08-06T20:24:11+00:00" }, { "name": "sebastian/code-unit",