Move rendered HTML to a different table

Author: dstufft

warehouse/forklift/legacy.py

@@ -50,6 +50,7 @@
     JournalEntry,
     Project,
     Release,
+    RenderedDescription,
     Role,
 )
 from warehouse.utils import http, readme
@@ -1004,8 +1005,6 @@ def file_upload(request):
                 )
             ),
             canonical_version=canonical_version,
-            description_html=rendered,
-            description_html_rendered_by=readme.renderer_version(),
             **{
                 k: getattr(form, k).data
                 for k in {
@@ -1031,6 +1030,13 @@ def file_upload(request):
             uploaded_via=request.user_agent,
         )
         request.db.add(release)
+        request.db.add(
+            RenderedDescription(
+                release=release,
+                html=rendered or "",
+                rendered_by=readme.renderer_version(),
+            )
+        )
         # TODO: This should be handled by some sort of database trigger or
         #       a SQLAlchemy hook or the like instead of doing it inline in
         #       this view.

warehouse/migrations/versions/1f6d3321bc15_add_release_description_html.py

@@ -0,0 +1,58 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Add Release.description_html
+
+Revision ID: ed4e8fa47659
+Revises: 42f0409bb702
+Create Date: 2019-05-08 05:18:06.415013
+"""
+
+import sqlalchemy as sa
+
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+revision = "ed4e8fa47659"
+down_revision = "42f0409bb702"
+
+
+def upgrade():
+    op.create_table(
+        "release_descriptions",
+        sa.Column(
+            "id",
+            postgresql.UUID(as_uuid=True),
+            server_default=sa.text("gen_random_uuid()"),
+            nullable=False,
+        ),
+        sa.Column("release_id", postgresql.UUID(as_uuid=True), nullable=False),
+        sa.Column("html", sa.Text(), nullable=False),
+        sa.Column("rendered_by", sa.Text(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["release_id"], ["releases.id"], onupdate="CASCADE", ondelete="CASCADE"
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        "release_descriptions_release_id_idx",
+        "release_descriptions",
+        ["release_id"],
+        unique=True,
+    )
+
+
+def downgrade():
+    op.drop_index(
+        "release_descriptions_release_id_idx", table_name="release_descriptions"
+    )
+    op.drop_table("release_descriptions")

warehouse/migrations/versions/ed4e8fa47659_add_release_description_html.py

@@ -288,7 +288,6 @@ def __table_args__(cls):  # noqa
     license = Column(Text)
     summary = Column(Text)
     description_content_type = Column(Text)
-    description_html_rendered_by = Column(Text)
     keywords = Column(Text)
     platform = Column(Text)
     download_url = Column(Text)
@@ -306,7 +305,15 @@ def __table_args__(cls):  # noqa
     # those queries, deferring them here makes the default case more
     # performant.
     description = orm.deferred(Column(Text))
-    description_html = orm.deferred(Column(Text))
+
+    description_html = orm.relationship(
+        "RenderedDescription",
+        backref="release",
+        cascade="all, delete-orphan",
+        lazy=True,
+        passive_deletes=True,
+        uselist=False,
+    )
 
     _classifiers = orm.relationship(
         Classifier,
@@ -406,6 +413,22 @@ def has_meta(self):
         )
 
 
+class RenderedDescription(db.Model):
+
+    __tablename__ = "release_descriptions"
+    __table_args__ = (
+        Index("release_descriptions_release_id_idx", "release_id", unique=True),
+    )
+
+    release_id = Column(
+        ForeignKey("releases.id", onupdate="CASCADE", ondelete="CASCADE"),
+        nullable=False,
+    )
+
+    html = Column(Text, nullable=False)
+    rendered_by = Column(Text, nullable=False)
+
+
 class File(db.Model):
 
     __tablename__ = "release_files"

warehouse/packaging/models.py

@@ -14,7 +14,7 @@
 
 from warehouse import tasks
 from warehouse.cache.origin import IOriginCache
-from warehouse.packaging.models import Project, Release
+from warehouse.packaging.models import Project, Release, RenderedDescription
 from warehouse.utils import readme
 
 
@@ -102,18 +102,31 @@ def compute_trending(request):
 @tasks.task(ignore_result=True, acks_late=True)
 def update_description_html(request):
     renderer_version = readme.renderer_version()
+
+    description_subq = (
+        request.db.query(RenderedDescription)
+        .filter(RenderedDescription.rendered_by != renderer_version)
+        .subquery()
+    )
     releases = (
         request.db.query(Release)
         .options(orm.undefer(Release.description))
-        .filter(
-            (Release.description_html_rendered_by != renderer_version)
-            | (Release.description_html_rendered_by == None)
-        )
+        .options(orm.joinedload(Release.description_html))
+        .outerjoin(description_subq, Release.description_html)
         .limit(5000)
     )
 
     for release in releases:
-        release.description_html = readme.render(
-            release.description, release.description_content_type
-        )
-        release.escription_html_rendered_by = renderer_version
+        rendered = readme.render(release.description, release.description_content_type)
+        if rendered is None:
+            rendered = ""
+
+        if release.description_html is None:
+            request.db.add(
+                RenderedDescription(
+                    release=release, html=rendered, rendered_by=renderer_version
+                )
+            )
+        else:
+            release.description_html.html = rendered
+            release.description_html.rendered_by = renderer_version

warehouse/packaging/tasks.py

@@ -82,7 +82,7 @@ def release_detail(release, request):
     # TODO: Remove the fallback to rendering inline and only support displaying the
     #       already rendered content.
     if release.description_html:
-        description = release.description_html
+        description = release.description_html.html
     else:
         description = readme.render(
             release.description, release.description_content_type