[FIX] spreadsheet: batch process spreadsheet_revision.commands

Some dbs have `spreadsheet_revision` records with over 10 millions characters
in `commands`. If the number of record is high, this leads to memory errors.
We distribute them in buckets of `memory_cap` maximum size, and use a named
cursor to process them in buckets. Commands larger than `memory_cap` fit in one
bucket.

Author: vval-odoo

src/util/spreadsheet/misc.py

@@ -1,32 +1,46 @@
-from .. import json
+from .. import json, pg
+
+MEMORY_CAP = 2 * 10**8  # 200Mo
 
 
 def iter_commands(cr, like_all=(), like_any=()):
     if not (bool(like_all) ^ bool(like_any)):
         raise ValueError("Please specify `like_all` or `like_any`, not both")
-    cr.execute(
-        """
-        SELECT id,
-               commands
-          FROM spreadsheet_revision
-         WHERE commands LIKE {}(%s::text[])
-        """.format("ALL" if like_all else "ANY"),
-        [list(like_all or like_any)],
-    )
-    for revision_id, data in cr.fetchall():
-        data_loaded = json.loads(data)
-        if "commands" not in data_loaded:
-            continue
-        data_old = json.dumps(data_loaded, sort_keys=True)
-
-        changed = yield data_loaded["commands"]
-        if changed is None:
-            changed = data_old != json.dumps(data_loaded, sort_keys=True)
-
-        if changed:
-            cr.execute(
-                "UPDATE spreadsheet_revision SET commands=%s WHERE id=%s", [json.dumps(data_loaded), revision_id]
+
+    with pg.named_cursor(cr, itersize=1) as ncr:
+        ncr.execute(
+            """
+            WITH buckets AS (
+                SELECT id,
+                       SUM(LENGTH(commands)) OVER (ORDER BY id) / {memory_cap} AS num,
+                       commands
+                  FROM spreadsheet_revision
+                 WHERE commands LIKE {condition}(%s::text[])
+              ORDER BY id
             )
+            SELECT ARRAY_AGG(id ORDER BY id),
+                   ARRAY_AGG(commands ORDER BY id)
+              FROM buckets
+          GROUP BY num
+            """.format(memory_cap=MEMORY_CAP, condition="ALL" if like_all else "ANY"),
+            [list(like_all or like_any)],
+        )
+        for ids, commands in ncr:
+            for revision_id, data in zip(ids, commands):
+                data_loaded = json.loads(data)
+                if "commands" not in data_loaded:
+                    continue
+                data_old = json.dumps(data_loaded, sort_keys=True)
+
+                changed = yield data_loaded["commands"]
+                if changed is None:
+                    changed = data_old != json.dumps(data_loaded, sort_keys=True)
+
+                if changed:
+                    cr.execute(
+                        "UPDATE spreadsheet_revision SET commands=%s WHERE id=%s",
+                        [json.dumps(data_loaded), revision_id],
+                    )
 
 
 def process_commands(cr, callback, *args, **kwargs):