Merge pull request #1 from HonzaKral/search

Simplified search

Author: dstufft

tests/unit/cli/search/test_reindex.py

@@ -34,7 +34,6 @@ def test_project_docs(db_session):
             "_type": "project",
             "_source": {
                 "name": p.name,
-                "name.normalized": p.normalized_name,
                 "version": [r.version for r in prs],
             },
         }

tests/unit/packaging/test_search.py

@@ -42,7 +42,6 @@ def test_build_search():
 
     assert obj.meta.id == "foobar"
     assert obj["name"] == "Foobar"
-    assert obj["name.normalized"] == "foobar"
     assert obj["version"] == ["1.0", "2.0", "3.0", "4.0"]
     assert obj["summary"] == "This is my summary"
     assert obj["description"] == "This is my description"

warehouse/cli/search/reindex.py

@@ -15,7 +15,7 @@
 
 import click
 
-from elasticsearch.helpers import streaming_bulk
+from elasticsearch.helpers import bulk
 from sqlalchemy.orm import lazyload, joinedload
 
 from warehouse.cli.search import search
@@ -76,8 +76,7 @@ def reindex(config, **kwargs):
         )
         db.execute("SET statement_timeout = '600s'")
 
-        for _ in streaming_bulk(client, _project_docs(db)):
-            pass
+        bulk(client, _project_docs(db))
     except:
         new_index.delete()
         raise

warehouse/legacy/api/xmlrpc.py

@@ -58,28 +58,19 @@ def search(request, spec, operator="and"):
         }
     }
 
-    query_type = {"version": "term"}
-
     primary_queries = []
     for field, value in spec.items():
         q = None
         for item in value:
             if q is None:
-                q = Q(query_type.get(field, "match"), **{field: item})
+                q = Q("match", **{field: item})
             else:
-                q |= Q(query_type.get(field, "match"), **{field: item})
+                q |= Q("match", **{field: item})
         primary_queries.append(q)
 
-    should_queries = []
-    if "name" in spec:
-        for name in spec["name"]:
-            normalized = re.sub(r"[^a-z0-9]+", "-", name.lower())
-            should_queries.append(
-                Q("term", **{"name.normalized": normalized})
-            )
-
     if operator == "and":
         must_queries = primary_queries
+        should_queries = []
     else:
         must_queries = []
         should_queries.extend(primary_queries)
@@ -88,9 +79,9 @@ def search(request, spec, operator="and"):
     results = query.execute()
 
     return [
-        {"name": r["name"], "summary": r.get("summary"), "version": v}
-        for r in (h["_source"] for h in results.hits.hits)
-        for v in r["version"]
+        {"name": r.name, "summary": r.summary, "version": v}
+        for r in results
+        for v in r.version
         if v in spec.get("version", [v])
     ]
 

warehouse/packaging/search.py

@@ -10,7 +10,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from elasticsearch_dsl import DocType, String, analyzer
+from elasticsearch_dsl import DocType, String, analyzer, MetaField
 
 from warehouse.search import doc_type
 
@@ -25,25 +25,28 @@
 @doc_type
 class Project(DocType):
 
-    name = String(fields={"normalized": String(analyzer="keyword")})
-    version = String(analyzer="keyword", multi=True)
+    name = String()
+    version = String(index="not_analyzed", multi=True)
     summary = String(analyzer="snowball")
     description = String(analyzer="snowball")
     author = String()
     author_email = String(analyzer=EmailAnalyzer)
     maintainer = String()
     maintainer_email = String(analyzer=EmailAnalyzer)
     license = String()
-    home_page = String(analyzer="keyword")
-    download_url = String(analyzer="keyword")
+    home_page = String(index="not_analyzed")
+    download_url = String(index="not_analyzed")
     keywords = String(analyzer="snowball")
     platform = String(index="not_analyzed")
 
+    class Meta:
+        # disable the _all field to save some space
+        all = MetaField(enabled=False)
+
     @classmethod
     def from_db(cls, release):
         obj = cls(meta={"id": release.project.normalized_name})
         obj["name"] = release.project.name
-        obj["name.normalized"] = release.project.normalized_name
         obj["version"] = [r.version for r in release.project.releases]
         obj["summary"] = release.summary
         obj["description"] = release.description