From 1c60da63d16caeb0e1bff4f22476daf9653895f5 Mon Sep 17 00:00:00 2001 From: codeliner Date: Sun, 29 Mar 2020 22:37:51 +0200 Subject: [PATCH] Find partial docs --- composer.json | 2 +- src/PostgresDocumentStore.php | 156 +++++++++++++++++++++++- tests/PostgresDocumentStoreTest.php | 183 ++++++++++++++++++++++++++++ 3 files changed, 334 insertions(+), 7 deletions(-) diff --git a/composer.json b/composer.json index ba97b63..694da52 100644 --- a/composer.json +++ b/composer.json @@ -18,7 +18,7 @@ "require": { "php": "^7.1", "ext-pdo": "*", - "event-engine/php-persistence": "^0.5" + "event-engine/php-persistence": "^0.6" }, "require-dev": { "roave/security-advisories": "dev-master", diff --git a/src/PostgresDocumentStore.php b/src/PostgresDocumentStore.php index d88dcdb..9524805 100644 --- a/src/PostgresDocumentStore.php +++ b/src/PostgresDocumentStore.php @@ -15,12 +15,22 @@ use EventEngine\DocumentStore\Filter\Filter; use EventEngine\DocumentStore\Index; use EventEngine\DocumentStore\OrderBy\OrderBy; +use EventEngine\DocumentStore\PartialSelect; use EventEngine\DocumentStore\Postgres\Exception\InvalidArgumentException; use EventEngine\DocumentStore\Postgres\Exception\RuntimeException; use EventEngine\Util\VariableType; +use function implode; +use function is_string; +use function json_decode; +use function mb_strlen; +use function mb_substr; +use function sprintf; final class PostgresDocumentStore implements DocumentStore\DocumentStore { + private const PARTIAL_SELECT_DOC_ID = '__partial_sel_doc_id__'; + private const PARTIAL_SELECT_MERGE = '__partial_sel_merge__'; + /** * @var \PDO */ @@ -489,12 +499,7 @@ public function getDoc(string $collectionName, string $docId): ?array } /** - * @param string $collectionName - * @param Filter $filter - * @param int|null $skip - * @param int|null $limit - * @param OrderBy|null $orderBy - * @return \Traversable list of docs + * @inheritDoc */ public function filterDocs(string $collectionName, Filter $filter, int $skip = null, int $limit = null, OrderBy $orderBy = null): \Traversable { @@ -524,6 +529,68 @@ public function filterDocs(string $collectionName, Filter $filter, int $skip = n } } + /** + * @inheritDoc + */ + public function findDocs(string $collectionName, Filter $filter, int $skip = null, int $limit = null, OrderBy $orderBy = null): \Traversable + { + [$filterStr, $args] = $this->filterToWhereClause($filter); + + $where = $filterStr ? "WHERE $filterStr" : ''; + + $offset = $skip !== null ? "OFFSET $skip" : ''; + $limit = $limit !== null ? "LIMIT $limit" : ''; + + $orderBy = $orderBy ? "ORDER BY " . implode(', ', $this->orderByToSort($orderBy)) : ''; + + $query = <<schemaName($collectionName)}.{$this->tableName($collectionName)} +$where +$orderBy +$limit +$offset; +EOT; + $stmt = $this->connection->prepare($query); + + $stmt->execute($args); + + while($row = $stmt->fetch(\PDO::FETCH_ASSOC)) { + yield $row['id'] => json_decode($row['doc'], true); + } + } + + public function findPartialDocs(string $collectionName, PartialSelect $partialSelect, Filter $filter, int $skip = null, int $limit = null, OrderBy $orderBy = null): \Traversable + { + [$filterStr, $args] = $this->filterToWhereClause($filter); + + $select = $this->makeSelect($partialSelect); + + $where = $filterStr ? "WHERE $filterStr" : ''; + + $offset = $skip !== null ? "OFFSET $skip" : ''; + $limit = $limit !== null ? "LIMIT $limit" : ''; + + $orderBy = $orderBy ? "ORDER BY " . implode(', ', $this->orderByToSort($orderBy)) : ''; + + $query = <<schemaName($collectionName)}.{$this->tableName($collectionName)} +$where +$orderBy +$limit +$offset; +EOT; + + $stmt = $this->connection->prepare($query); + + $stmt->execute($args); + + while($row = $stmt->fetch(\PDO::FETCH_ASSOC)) { + yield $row[self::PARTIAL_SELECT_DOC_ID] => $this->transformPartialDoc($partialSelect, $row); + } + } + /** * @param string $collectionName * @param Filter $filter @@ -714,6 +781,83 @@ private function makeInClause(string $prop, array $valList, int $argsCount, bool return ["$prop IN($params)", $argList, $argsCount]; } + private function makeSelect(PartialSelect $partialSelect): string + { + $select = 'id as "'.self::PARTIAL_SELECT_DOC_ID.'", '; + + foreach ($partialSelect->fieldAliasMap() as $mapItem) { + + if($mapItem['alias'] === self::PARTIAL_SELECT_DOC_ID) { + throw new RuntimeException(sprintf( + "Invalid select alias. You cannot use %s as alias, because it is reserved for internal use", + self::PARTIAL_SELECT_DOC_ID + )); + } + + if($mapItem['alias'] === self::PARTIAL_SELECT_MERGE) { + throw new RuntimeException(sprintf( + "Invalid select alias. You cannot use %s as alias, because it is reserved for internal use", + self::PARTIAL_SELECT_MERGE + )); + } + + if($mapItem['alias'] === PartialSelect::MERGE_ALIAS) { + $mapItem['alias'] = self::PARTIAL_SELECT_MERGE; + } + + $select.= $this->propToJsonPath($mapItem['field']) . ' as "' . $mapItem['alias'] . '", '; + } + + $select = mb_substr($select, 0, mb_strlen($select) - 2); + + return $select; + } + + private function transformPartialDoc(PartialSelect $partialSelect, array $selectedDoc): array + { + $partialDoc = []; + + foreach ($partialSelect->fieldAliasMap() as ['field' => $field, 'alias' => $alias]) { + if($alias === PartialSelect::MERGE_ALIAS) { + if(null === $selectedDoc[self::PARTIAL_SELECT_MERGE] ?? null) { + continue; + } + + $value = json_decode($selectedDoc[self::PARTIAL_SELECT_MERGE], true); + + if(!is_array($value)) { + throw new RuntimeException('Merge not possible. $merge alias was specified for field: ' . $field . ' but field value is not an array: ' . json_encode($value)); + } + + foreach ($value as $k => $v) { + $partialDoc[$k] = $v; + } + + continue; + } + + $value = $selectedDoc[$alias] ?? null; + + if(is_string($value)) { + $value = json_decode($value, true); + } + + $keys = explode('.', $alias); + + $ref = &$partialDoc; + foreach ($keys as $i => $key) { + if(!array_key_exists($key, $ref)) { + $ref[$key] = []; + } + $ref = &$ref[$key]; + } + $ref = $value; + unset($ref); + } + + return $partialDoc; + } + private function orderByToSort(DocumentStore\OrderBy\OrderBy $orderBy): array { $sort = []; diff --git a/tests/PostgresDocumentStoreTest.php b/tests/PostgresDocumentStoreTest.php index 7aeeb07..4db9ecf 100644 --- a/tests/PostgresDocumentStoreTest.php +++ b/tests/PostgresDocumentStoreTest.php @@ -12,6 +12,7 @@ namespace EventEngine\DocumentStoreTest\Postgres; use EventEngine\DocumentStore\Filter\AndFilter; +use EventEngine\DocumentStore\Filter\AnyFilter; use EventEngine\DocumentStore\Filter\AnyOfDocIdFilter; use EventEngine\DocumentStore\Filter\AnyOfFilter; use EventEngine\DocumentStore\Filter\DocIdFilter; @@ -21,6 +22,7 @@ use EventEngine\DocumentStore\Filter\LtFilter; use EventEngine\DocumentStore\Filter\NotFilter; use EventEngine\DocumentStore\Filter\OrFilter; +use EventEngine\DocumentStore\PartialSelect; use PHPUnit\Framework\TestCase; use EventEngine\DocumentStore\FieldIndex; use EventEngine\DocumentStore\Index; @@ -28,6 +30,8 @@ use EventEngine\DocumentStore\Postgres\PostgresDocumentStore; use Ramsey\Uuid\Uuid; use function array_map; +use function array_walk; +use function iterator_to_array; class PostgresDocumentStoreTest extends TestCase { @@ -198,6 +202,35 @@ public function it_handles_any_of_filter() $this->assertCount(2, $filteredDocs); } + /** + * @test + */ + public function it_uses_doc_ids_as_iterator_keys() + { + $collectionName = 'test_any_of_filter'; + $this->documentStore->addCollection($collectionName); + + $doc1 = ['id' => Uuid::uuid4()->toString(), 'doc' => ["foo" => "bar"]]; + $doc2 = ['id' => Uuid::uuid4()->toString(), 'doc' => ["foo" => "baz"]]; + $doc3 = ['id' => Uuid::uuid4()->toString(), 'doc' => ["foo" => "bat"]]; + + $docs = [$doc1, $doc2, $doc3]; + + array_walk($docs, function (array $doc) use ($collectionName) { + $this->documentStore->addDoc($collectionName, $doc['id'], $doc['doc']); + }); + + $filteredDocs = iterator_to_array($this->documentStore->findDocs( + $collectionName, + new AnyOfFilter("foo", ["bar", "bat"]) + )); + + $this->assertEquals([ + $doc1['id'] => $doc1['doc'], + $doc3['id'] => $doc3['doc'] + ], $filteredDocs); + } + /** * @test */ @@ -487,6 +520,156 @@ public function it_counts_any_of_filter() $this->assertSame(2, $count); } + /** + * @test + */ + public function it_finds_partial_docs() + { + $collectionName = 'test_find_partial_docs'; + $this->documentStore->addCollection($collectionName); + + $docAId = Uuid::uuid4()->toString(); + $docA = [ + 'some' => [ + 'prop' => 'foo', + 'other' => [ + 'nested' => 42 + ] + ], + 'baz' => 'bat', + ]; + $this->documentStore->addDoc($collectionName, $docAId, $docA); + + $docBId = Uuid::uuid4()->toString(); + $docB = [ + 'some' => [ + 'prop' => 'bar', + 'other' => [ + 'nested' => 43 + ], + //'baz' => 'bat', missing so should be null + ], + ]; + $this->documentStore->addDoc($collectionName, $docBId, $docB); + + $docCId = Uuid::uuid4()->toString(); + $docC = [ + 'some' => [ + 'prop' => 'foo', + 'other' => [ + //'nested' => 42, missing, so should be null + 'ignoredNested' => 'value' + ] + ], + 'baz' => 'bat', + ]; + $this->documentStore->addDoc($collectionName, $docCId, $docC); + + $partialSelect = new PartialSelect([ + 'some.alias' => 'some.prop', // Nested alias <- Nested field + 'magicNumber' => 'some.other.nested', // Top level alias <- Nested Field + 'baz', // Top level field, + ]); + + $result = iterator_to_array($this->documentStore->findPartialDocs($collectionName, $partialSelect, new AnyFilter())); + + $this->assertEquals([ + 'some' => [ + 'alias' => 'foo', + ], + 'magicNumber' => 42, + 'baz' => 'bat', + ], $result[$docAId]); + + $this->assertEquals([ + 'some' => [ + 'alias' => 'bar', + ], + 'magicNumber' => 43, + 'baz' => null, + ], $result[$docBId]); + + $this->assertEquals([ + 'some' => [ + 'alias' => 'foo', + ], + 'magicNumber' => null, + 'baz' => 'bat', + ], $result[$docCId]); + } + + /** + * @test + */ + public function it_applies_merge_alias_for_nested_fields_if_specified() + { + $collectionName = 'test_applies_merge_alias'; + $this->documentStore->addCollection($collectionName); + + $docAId = Uuid::uuid4()->toString(); + $docA = [ + 'some' => [ + 'prop' => 'foo', + 'other' => [ + 'nested' => 42 + ] + ], + 'baz' => 'bat', + ]; + $this->documentStore->addDoc($collectionName, $docAId, $docA); + + $docBId = Uuid::uuid4()->toString(); + $docB = [ + 'differentTopLevel' => [ + 'prop' => 'bar', + 'other' => [ + 'nested' => 43 + ], + ], + 'baz' => 'bat', + ]; + $this->documentStore->addDoc($collectionName, $docBId, $docB); + + $docCId = Uuid::uuid4()->toString(); + $docC = [ + 'some' => [ + 'prop' => 'foo', + 'other' => [ + 'nested' => 43 + ], + ], + //'baz' => 'bat', missing top level + ]; + $this->documentStore->addDoc($collectionName, $docCId, $docC); + + $partialSelect = new PartialSelect([ + '$merge' => 'some', // $merge alias <- Nested field + 'baz', // Top level field + ]); + + $result = iterator_to_array($this->documentStore->findPartialDocs($collectionName, $partialSelect, new AnyFilter())); + + $this->assertEquals([ + 'prop' => 'foo', + 'other' => [ + 'nested' => 42 + ], + 'baz' => 'bat' + ], $result[$docAId]); + + $this->assertEquals([ + 'baz' => 'bat', + ], $result[$docBId]); + + $this->assertEquals([ + 'prop' => 'foo', + 'other' => [ + 'nested' => 43 + ], + 'baz' => null + ], $result[$docCId]); + } + private function getIndexes(string $collectionName): array { return TestUtil::getIndexes($this->connection, self::TABLE_PREFIX.$collectionName);