✨ Add support for PARTIAL esearch result

For convenience and compatibility, `ESearchResult#to_a` returns an array
of integers (sequence numbers or UIDs) whenever either `ALL` or
`PARTIAL` return data is available.

Author: nevans

lib/net/imap.rb

@@ -534,6 +534,11 @@ module Net
   #   See FetchData#emailid and FetchData#emailid.
   # - Updates #status with support for the +MAILBOXID+ status attribute.
   #
+  # ==== RFC9394: +PARTIAL+
+  # - Updates #search, #uid_search with the +PARTIAL+ return option which adds
+  #   ESearchResult#partial return data.
+  # - TODO: Updates #uid_fetch with the +partial+ modifier.
+  #
   # == References
   #
   # [{IMAP4rev1}[https://www.rfc-editor.org/rfc/rfc3501.html]]::
@@ -701,6 +706,11 @@ module Net
   #   Gondwana, B., Ed., "IMAP Extension for Object Identifiers",
   #   RFC 8474, DOI 10.17487/RFC8474, September 2018,
   #   <https://www.rfc-editor.org/info/rfc8474>.
+  # [PARTIAL[https://www.rfc-editor.org/info/rfc9394]]:
+  #   Melnikov, A., Achuthan, A., Nagulakonda, V., and L. Alves,
+  #   "IMAP PARTIAL Extension for Paged SEARCH and FETCH", RFC 9394,
+  #   DOI 10.17487/RFC9394, June 2023,
+  #   <https://www.rfc-editor.org/info/rfc9394>.
   #
   # === IANA registries
   # * {IMAP Capabilities}[http://www.iana.org/assignments/imap4-capabilities]
@@ -1971,8 +1981,9 @@ def uid_expunge(uid_set)
     # the server to return an ESearchResult instead of a SearchResult, but some
     # servers disobey this requirement.  <em>Requires an extended search
     # capability, such as +ESEARCH+ or +IMAP4rev2+.</em>
-    # See {"Argument translation"}[rdoc-ref:#search@Argument+translation]
-    # and {"Return options"}[rdoc-ref:#search@Return+options], below.
+    # See {"Argument translation"}[rdoc-ref:#search@Argument+translation] and
+    # {"Supported return options"}[rdoc-ref:#search@Supported+return+options],
+    # below.
     #
     # +charset+ is the name of the {registered character
     # set}[https://www.iana.org/assignments/character-sets/character-sets.xhtml]
@@ -2082,39 +2093,56 @@ def uid_expunge(uid_set)
     #   <em>*WARNING:* This is vulnerable to injection attacks when external
     #   inputs are used.</em>
     #
-    # ==== Return options
+    # ==== Supported return options
     #
     # For full definitions of the standard return options and return data, see
     # the relevant RFCs.
     #
-    # ===== +ESEARCH+ or +IMAP4rev2+
-    #
-    # The following return options require either +ESEARCH+ or +IMAP4rev2+.
-    # See [{RFC4731 §3.1}[https://rfc-editor.org/rfc/rfc4731#section-3.1]] or
-    # [{IMAP4rev2 §6.4.4}[https://www.rfc-editor.org/rfc/rfc9051.html#section-6.4.4]].
-    #
     # [+ALL+]
     #    Returns ESearchResult#all with a SequenceSet of all matching sequence
     #    numbers or UIDs.  This is the default, when return options are empty.
     #
     #    For compatibility with SearchResult, ESearchResult#to_a returns an
     #    Array of message sequence numbers or UIDs.
+    #
+    #    <em>Requires either the +ESEARCH+ or +IMAP4rev2+ capabability.</em>
+    #    {[RFC4731]}[https://rfc-editor.org/rfc/rfc4731]
+    #    {[RFC9051]}[https://rfc-editor.org/rfc/rfc9051]
+    #
     # [+COUNT+]
     #    Returns ESearchResult#count with the number of matching messages.
+    #
+    #    <em>Requires either the +ESEARCH+ or +IMAP4rev2+ capabability.</em>
+    #    {[RFC4731]}[https://rfc-editor.org/rfc/rfc4731]
+    #    {[RFC9051]}[https://rfc-editor.org/rfc/rfc9051]
+    #
     # [+MAX+]
     #    Returns ESearchResult#max with the highest matching sequence number or
     #    UID.
+    #
+    #    <em>Requires either the +ESEARCH+ or +IMAP4rev2+ capabability.</em>
+    #    {[RFC4731]}[https://rfc-editor.org/rfc/rfc4731]
+    #    {[RFC9051]}[https://rfc-editor.org/rfc/rfc9051]
+    #
     # [+MIN+]
     #    Returns ESearchResult#min with the lowest matching sequence number or
     #    UID.
     #
-    # ===== +CONDSTORE+
+    #    <em>Requires either the +ESEARCH+ or +IMAP4rev2+ capabability.</em>
+    #    {[RFC4731]}[https://rfc-editor.org/rfc/rfc4731]
+    #    {[RFC9051]}[https://rfc-editor.org/rfc/rfc9051]
     #
-    # ESearchResult#modseq return data does not have a corresponding return
-    # option.  Instead, it is returned if the +MODSEQ+ search key is used or
-    # when the +CONDSTORE+ extension is enabled for the selected mailbox.
-    # See [{RFC4731 §3.2}[https://www.rfc-editor.org/rfc/rfc4731#section-3.2]]
-    # or [{RFC7162 §2.1.5}[https://www.rfc-editor.org/rfc/rfc7162#section-3.1.5]].
+    # [+PARTIAL+ _range_]
+    #    Returns ESearchResult#partial with a SequenceSet of a subset of
+    #    matching sequence numbers or UIDs, as selected by _range_.  As with
+    #    sequence numbers, the first result is +1+: <tt>1..500</tt> selects the
+    #    first 500 search results (in mailbox order), <tt>501..1000</tt> the
+    #    second 500, and so on.  _range_ may also be negative: <tt>-500..-1</tt>
+    #    selects the last 500 search results.
+    #
+    #    <em>Requires either the <tt>CONTEXT=SEARCH</tt> or +PARTIAL+ capabability.</em>
+    #    {[RFC5267]}[https://rfc-editor.org/rfc/rfc5267]
+    #    {[RFC9394]}[https://rfc-editor.org/rfc/rfc9394]
     #
     # ===== +RFC4466+ compatible extensions
     #
@@ -2125,6 +2153,14 @@ def uid_expunge(uid_set)
     # intentionally _unstable_ API.  Future releases may return different
     # (incompatible) objects, <em>without deprecation or warning</em>.
     #
+    # ===== +MODSEQ+ return data
+    #
+    # ESearchResult#modseq return data does not have a corresponding return
+    # option.  Instead, it is returned if the +MODSEQ+ search key is used or
+    # when the +CONDSTORE+ extension is enabled for the selected mailbox.
+    # See [{RFC4731 §3.2}[https://www.rfc-editor.org/rfc/rfc4731#section-3.2]]
+    # or [{RFC7162 §2.1.5}[https://www.rfc-editor.org/rfc/rfc7162#section-3.1.5]].
+    #
     # ==== Search keys
     #
     # For full definitions of the standard search +criteria+,
@@ -2396,8 +2432,8 @@ def uid_search(...)
     # {[RFC7162]}[https://tools.ietf.org/html/rfc7162] in order to use the
     # +changedsince+ argument.  Using +changedsince+ implicitly enables the
     # +CONDSTORE+ extension.
-    def fetch(set, attr, mod = nil, changedsince: nil)
-      fetch_internal("FETCH", set, attr, mod, changedsince: changedsince)
+    def fetch(...)
+      fetch_internal("FETCH", ...)
     end
 
     # :call-seq:
@@ -2419,8 +2455,8 @@ def fetch(set, attr, mod = nil, changedsince: nil)
     # ==== Capabilities
     #
     # Same as #fetch.
-    def uid_fetch(set, attr, mod = nil, changedsince: nil)
-      fetch_internal("UID FETCH", set, attr, mod, changedsince: changedsince)
+    def uid_fetch(...)
+      fetch_internal("UID FETCH", ...)
     end
 
     # :call-seq:

lib/net/imap/esearch_result.rb

@@ -35,16 +35,16 @@ def initialize(tag: nil, uid: nil, data: nil)
 
       # :call-seq: to_a -> Array of integers
       #
-      # When #all contains a SequenceSet of message sequence
+      # When either #all or #partial contains a SequenceSet of message sequence
       # numbers or UIDs, +to_a+ returns that set as an array of integers.
       #
-      # When #all is +nil+, either because the server
-      # returned no results or because +ALL+ was not included in
+      # When both #all and #partial are +nil+, either because the server
+      # returned no results or because +ALL+ and +PARTIAL+ were not included in
       # the IMAP#search +RETURN+ options, #to_a returns an empty array.
       #
       # Note that SearchResult also implements +to_a+, so it can be used without
       # checking if the server returned +SEARCH+ or +ESEARCH+ data.
-      def to_a; all&.numbers || [] end
+      def to_a; all&.numbers || partial&.to_a || [] end
 
       ##
       # attr_reader: tag
@@ -135,6 +135,46 @@ def count;      data.assoc("COUNT")&.last      end
       # and +ESEARCH+ {[RFC4731]}[https://www.rfc-editor.org/rfc/rfc4731.html#section-3.2].
       def modseq;     data.assoc("MODSEQ")&.last     end
 
+      # Returned by ESearchResult#partial.
+      #
+      # Requires +PARTIAL+ {[RFC9394]}[https://www.rfc-editor.org/rfc/rfc9394.html]
+      # or <tt>CONTEXT=SEARCH</tt>/<tt>CONTEXT=SORT</tt>
+      # {[RFC5267]}[https://www.rfc-editor.org/rfc/rfc5267.html]
+      #
+      # See also: #to_a
+      class PartialResult < Data.define(:range, :results)
+        def initialize(range:, results:)
+          range   => Range
+          results = SequenceSet[results]
+          super
+        end
+
+        ##
+        # method: range
+        # :call-seq: range -> range
+
+        ##
+        # method: results
+        # :call-seq: results -> sequence set or nil
+
+        # Converts #results to an array of integers.
+        #
+        # See also: ESearchResult#to_a.
+        def to_a; results&.numbers || [] end
+      end
+
+      # :call-seq: partial -> PartialResult or nil
+      #
+      # A PartialResult containing a subset of the message sequence numbers or
+      # UIDs that satisfy the SEARCH criteria.
+      #
+      # Requires +PARTIAL+ {[RFC9394]}[https://www.rfc-editor.org/rfc/rfc9394.html]
+      # or <tt>CONTEXT=SEARCH</tt>/<tt>CONTEXT=SORT</tt>
+      # {[RFC5267]}[https://www.rfc-editor.org/rfc/rfc5267.html]
+      #
+      # See also: #to_a
+      def partial;    data.assoc("PARTIAL")&.last    end
+
     end
   end
 end

lib/net/imap/response_parser.rb

@@ -321,6 +321,20 @@ module RFC3629
         SEQUENCE_SET      = /#{SEQUENCE_SET_ITEM}(?:,#{SEQUENCE_SET_ITEM})*/n
         SEQUENCE_SET_STR  = /\A#{SEQUENCE_SET}\z/n
 
+        # partial-range-first = nz-number ":" nz-number
+        #     ;; Request to search from oldest (lowest UIDs) to
+        #     ;; more recent messages.
+        #     ;; A range 500:400 is the same as 400:500.
+        #     ;; This is similar to <seq-range> from [RFC3501]
+        #     ;; but cannot contain "*".
+        PARTIAL_RANGE_FIRST = /\A(#{NZ_NUMBER}):(#{NZ_NUMBER})\z/n
+
+        # partial-range-last  = MINUS nz-number ":" MINUS nz-number
+        #     ;; Request to search from newest (highest UIDs) to
+        #     ;; oldest messages.
+        #     ;; A range -500:-400 is the same as -400:-500.
+        PARTIAL_RANGE_LAST  = /\A(-#{NZ_NUMBER}):(-#{NZ_NUMBER})\z/n
+
         # RFC3501:
         #   literal          = "{" number "}" CRLF *CHAR8
         #                        ; Number represents the number of CHAR8s
@@ -1517,6 +1531,9 @@ def esearch_response
       # From RFC4731 (ESEARCH):
       #   search-return-data    =/ "MODSEQ" SP mod-sequence-value
       #
+      # From RFC9394 (PARTIAL):
+      #   search-return-data  =/ ret-data-partial
+      #
       def search_return_data
         label = search_modifier_name; SP!
         value =
@@ -1526,11 +1543,41 @@ def search_return_data
           when "ALL"        then sequence_set
           when "COUNT"      then number
           when "MODSEQ"     then mod_sequence_value         # RFC7162: CONDSTORE
+          when "PARTIAL"    then ret_data_partial__value    # RFC9394: PARTIAL
           else search_return_value
           end
         [label, value]
       end
 
+      # From RFC5267 (CONTEXT=SEARCH, CONTEXT=SORT) and RFC9394 (PARTIAL):
+      #   ret-data-partial    = "PARTIAL"
+      #                         SP "(" partial-range SP partial-results ")"
+      def ret_data_partial__value
+        lpar
+        range   = partial_range;   SP!
+        results = partial_results
+        rpar
+        ESearchResult::PartialResult.new(range, results)
+      end
+
+      # partial-range       = partial-range-first / partial-range-last
+      # tagged-ext-simple   =/ partial-range-last
+      def partial_range
+        case (str = atom)
+        when Patterns::PARTIAL_RANGE_FIRST, Patterns::PARTIAL_RANGE_LAST
+          min, max = [Integer($1), Integer($2)].minmax
+          min..max
+        else
+          parse_error("unexpected atom %p, expected partial-range", str)
+        end
+      end
+
+      # partial-results     = sequence-set / "NIL"
+      #     ;; <sequence-set> from [RFC3501].
+      #     ;; NIL indicates that no results correspond to
+      #     ;; the requested range.
+      def partial_results; NIL? ? nil : sequence_set end
+
       # search-modifier-name = tagged-ext-label
       alias search_modifier_name tagged_ext_label
 

rakelib/rfcs.rake

@@ -145,6 +145,7 @@ RFCS = {
   8514 => "IMAP SAVEDATE",
   8970 => "IMAP PREVIEW",
   9208 => "IMAP QUOTA, QUOTA=, QUOTASET",
+  9394 => "IMAP PARTIAL",
 
   # etc...
   3629 => "UTF8",

test/net/imap/fixtures/response_parser/rfc9394_partial.yml

@@ -0,0 +1,66 @@
+---
+:tests:
+
+  "RFC9394 PARTIAL 3.1. example 1":
+    comment: |
+      Neither RFC9394 nor RFC5267 contain any examples of a normal unelided
+      sequence-set result.  I've edited it to include a sequence-set here.
+    :response: "* ESEARCH (TAG \"A01\") UID PARTIAL (-1:-100 200:250,252:300)\r\n"
+    :expected: !ruby/struct:Net::IMAP::UntaggedResponse
+      name: ESEARCH
+      data: !ruby/object:Net::IMAP::ESearchResult
+        tag: A01
+        uid: true
+        data:
+        - - PARTIAL
+          - !ruby/object:Net::IMAP::ESearchResult::PartialResult
+            range: !ruby/range
+              begin: -100
+              end: -1
+              excl: false
+            results: !ruby/object:Net::IMAP::SequenceSet
+              string: 200:250,252:300
+              tuples:
+              - - 200
+                - 250
+              - - 252
+                - 300
+      raw_data: "* ESEARCH (TAG \"A01\") UID PARTIAL (-1:-100 200:250,252:300)\r\n"
+
+  "RFC9394 PARTIAL 3.1. example 2":
+    :response: "* ESEARCH (TAG \"A02\") UID PARTIAL (23500:24000 55500:56000)\r\n"
+    :expected: !ruby/struct:Net::IMAP::UntaggedResponse
+      name: ESEARCH
+      data: !ruby/object:Net::IMAP::ESearchResult
+        tag: A02
+        uid: true
+        data:
+        - - PARTIAL
+          - !ruby/object:Net::IMAP::ESearchResult::PartialResult
+            range: !ruby/range
+              begin: 23500
+              end: 24000
+              excl: false
+            results: !ruby/object:Net::IMAP::SequenceSet
+              string: 55500:56000
+              tuples:
+              - - 55500
+                - 56000
+      raw_data: "* ESEARCH (TAG \"A02\") UID PARTIAL (23500:24000 55500:56000)\r\n"
+
+  "RFC9394 PARTIAL 3.1. example 3":
+    :response: "* ESEARCH (TAG \"A04\") UID PARTIAL (24000:24500 NIL)\r\n"
+    :expected: !ruby/struct:Net::IMAP::UntaggedResponse
+      name: ESEARCH
+      data: !ruby/object:Net::IMAP::ESearchResult
+        tag: A04
+        uid: true
+        data:
+        - - PARTIAL
+          - !ruby/object:Net::IMAP::ESearchResult::PartialResult
+            range: !ruby/range
+              begin: 24000
+              end: 24500
+              excl: false
+            results:
+      raw_data: "* ESEARCH (TAG \"A04\") UID PARTIAL (24000:24500 NIL)\r\n"

test/net/imap/test_esearch_result.rb

@@ -15,6 +15,10 @@ class ESearchResultTest < Test::Unit::TestCase
     assert_equal [], esearch.to_a
     esearch = ESearchResult.new(nil, false, [["ALL", SequenceSet["1,5:8"]]])
     assert_equal [1, 5, 6, 7, 8], esearch.to_a
+    esearch = ESearchResult.new(nil, false, [
+      ["PARTIAL", ESearchResult::PartialResult[1..5, "1,5:8"]]
+    ])
+    assert_equal [1, 5, 6, 7, 8], esearch.to_a
   end
 
   test "#tag" do
@@ -80,4 +84,17 @@ class ESearchResultTest < Test::Unit::TestCase
     assert_equal  12345, esearch.modseq
   end
 
+  test "#partial returns PARTIAL value (RFC9394: PARTIAL)" do
+    result = Net::IMAP::ResponseParser.new.parse(
+      "* ESEARCH (TAG \"A0006\") UID PARTIAL (-1:-100 200:250,252:300)\r\n"
+    ).data
+    assert_equal(ESearchResult, result.class)
+    assert_equal(
+      ESearchResult::PartialResult.new(
+        -100..-1, SequenceSet[200..250, 252..300]
+      ),
+      result.partial
+    )
+  end
+
 end

test/net/imap/test_imap_response_parser.rb

@@ -103,6 +103,9 @@ def teardown
   # RFC 9208: QUOTA extension
   generate_tests_from fixture_file: "rfc9208_quota_responses.yml"
 
+  # RFC 9394: PARTIAL extension
+  generate_tests_from fixture_file: "rfc9394_partial.yml"
+
   ############################################################################
   # Workarounds or unspecified extensions:
   generate_tests_from fixture_file: "quirky_behaviors.yml"