Add 'upload packages' to allow prioritizers to pass arbitrary data to uploaders (#2470)

* Update some comments and move the event clock snapshot to the public header

* Create the notion of an 'upload package'

This will allow prioritizers to pass arbitrary data to the uploader that might be needed at upload time.

* Make the -transportBytes protocol method required.

* Make the rest of the framework use the upload package

* Style

* Remove cct.nanopb.c

It was accidentally added.

Author: mikehaney24

GoogleDataTransport/GoogleDataTransport/Classes/GDTStorage.h

@@ -50,7 +50,7 @@ NS_ASSUME_NONNULL_BEGIN
  * @param eventHashes A set of event hashes to get the files of.
  * @return A set of equivalent length, containing all the filenames corresponding to the hashes.
  */
-- (NSSet<NSURL *> *)eventHashesToFiles:(NSSet<NSNumber *> *)eventHashes;
+- (NSDictionary<NSNumber *, NSURL *> *)eventHashesToFiles:(NSSet<NSNumber *> *)eventHashes;
 
 @end
 

GoogleDataTransport/GoogleDataTransport/Classes/GDTStorage.m

@@ -113,16 +113,16 @@ - (void)removeEvents:(NSSet<NSNumber *> *)eventHashes target:(NSNumber *)target
   });
 }
 
-- (NSSet<NSURL *> *)eventHashesToFiles:(NSSet<NSNumber *> *)eventHashes {
-  NSMutableSet<NSURL *> *eventFiles = [[NSMutableSet alloc] init];
+- (NSDictionary<NSNumber *, NSURL *> *)eventHashesToFiles:(NSSet<NSNumber *> *)eventHashes {
+  NSMutableDictionary<NSNumber *, NSURL *> *eventHashesToFiles = [[NSMutableDictionary alloc] init];
   dispatch_sync(_storageQueue, ^{
     for (NSNumber *hashNumber in eventHashes) {
       NSURL *eventURL = self.eventHashToFile[hashNumber];
       GDTAssert(eventURL, @"An event file URL couldn't be found for the given hash");
-      [eventFiles addObject:eventURL];
+      eventHashesToFiles[hashNumber] = eventURL;
     }
   });
-  return eventFiles;
+  return eventHashesToFiles;
 }
 
 #pragma mark - Private helper methods

GoogleDataTransport/GoogleDataTransport/Classes/GDTUploadCoordinator.m

@@ -22,6 +22,7 @@
 #import "GDTConsoleLogger.h"
 #import "GDTRegistrar_Private.h"
 #import "GDTStorage.h"
+#import "GDTUploadPackage_Private.h"
 
 @implementation GDTUploadCoordinator
 
@@ -57,9 +58,10 @@ - (void)forceUploadEvents:(NSSet<NSNumber *> *)eventHashes target:(GDTTarget)tar
     GDTUploadCoordinatorForceUploadBlock forceUploadBlock = ^{
       GDTAssert(eventHashes.count, @"It doesn't make sense to force upload of 0 events");
       id<GDTUploader> uploader = registrar.targetToUploader[targetNumber];
-      NSSet<NSURL *> *eventFiles = [self.storage eventHashesToFiles:eventHashes];
+      GDTUploadPackage *package = [[GDTUploadPackage alloc] init];
+      package.eventHashes = [eventHashes copy];
       GDTAssert(uploader, @"Target '%@' is missing an implementation", targetNumber);
-      [uploader uploadEvents:eventFiles onComplete:self.onCompleteBlock];
+      [uploader uploadPackage:package onComplete:self.onCompleteBlock];
       self->_targetToInFlightEventSet[targetNumber] = eventHashes;
     };
 
@@ -157,16 +159,13 @@ - (void)checkPrioritizersAndUploadEvents {
         id<GDTUploader> uploader = strongSelf->_registrar.targetToUploader[target];
         GDTAssert(prioritizer && uploader, @"Target '%@' is missing an implementation", target);
         GDTUploadConditions conds = [self uploadConditions];
-        NSSet<NSNumber *> *eventHashesToUpload =
-            [[prioritizer eventsToUploadGivenConditions:conds] copy];
-        if (eventHashesToUpload && eventHashesToUpload.count > 0) {
-          NSAssert(eventHashesToUpload.count > 0, @"");
-          NSSet<NSURL *> *eventFilesToUpload =
-              [strongSelf.storage eventHashesToFiles:eventHashesToUpload];
-          NSAssert(eventFilesToUpload.count == eventHashesToUpload.count,
+        GDTUploadPackage *package = [[prioritizer uploadPackageWithConditions:conds] copy];
+        package.storage = strongSelf.storage;
+        if (package.eventHashes && package.eventHashes.count > 0) {
+          NSAssert(package.eventHashesToFiles.count == package.eventHashes.count,
                    @"There should be the same number of files to events");
-          strongSelf->_targetToInFlightEventSet[target] = eventHashesToUpload;
-          [uploader uploadEvents:eventFilesToUpload onComplete:self.onCompleteBlock];
+          strongSelf->_targetToInFlightEventSet[target] = package.eventHashes;
+          [uploader uploadPackage:package onComplete:self.onCompleteBlock];
         }
       }
     }

GoogleDataTransport/GoogleDataTransport/Classes/GDTUploadPackage.m

@@ -0,0 +1,62 @@
+/*
+ * Copyright 2019 Google
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#import <GoogleDataTransport/GDTUploadPackage.h>
+
+#import "GDTStorage_Private.h"
+#import "GDTUploadPackage_Private.h"
+
+@implementation GDTUploadPackage
+
+- (instancetype)init {
+  self = [super init];
+  if (self) {
+    _storage = [GDTStorage sharedInstance];
+  }
+  return self;
+}
+
+- (instancetype)copy {
+  GDTUploadPackage *newPackage = [[GDTUploadPackage alloc] init];
+  newPackage->_eventHashes = _eventHashes;
+  return newPackage;
+}
+
+- (NSUInteger)hash {
+  return [_eventHashes hash];
+}
+
+- (BOOL)isEqual:(id)object {
+  return [self hash] == [object hash];
+}
+
+- (void)setEventHashes:(NSSet<NSNumber *> *)eventHashes {
+  if (eventHashes != _eventHashes) {
+    _eventHashes = [eventHashes copy];
+  }
+}
+
+- (NSDictionary<NSNumber *, NSURL *> *)eventHashesToFiles {
+  return [_storage eventHashesToFiles:_eventHashes];
+}
+
+- (void)setStorage:(GDTStorage *)storage {
+  if (storage != _storage) {
+    _storage = storage;
+  }
+}
+
+@end

GoogleDataTransport/GoogleDataTransport/Classes/Private/GDTEvent_Private.h

@@ -25,9 +25,6 @@ NS_ASSUME_NONNULL_BEGIN
 /** The serialized bytes of the event data object. */
 @property(nonatomic) NSData *dataObjectTransportBytes;
 
-/** The clock snapshot at the time of the event. */
-@property(nonatomic) GDTClock *clockSnapshot;
-
 @end
 
 NS_ASSUME_NONNULL_END

GoogleDataTransport/GoogleDataTransport/Classes/Private/GDTUploadPackage_Private.h

@@ -0,0 +1,26 @@
+/*
+ * Copyright 2019 Google
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#import <GoogleDataTransport/GDTUploadPackage.h>
+
+@class GDTStorage;
+
+@interface GDTUploadPackage ()
+
+/** The storage object this upload package will use to resolve event hashes to files. */
+@property(nonatomic) GDTStorage *storage;
+
+@end

GoogleDataTransport/GoogleDataTransport/Classes/Public/GDTEvent.h

@@ -18,6 +18,8 @@
 
 #import "GDTEventDataObject.h"
 
+@class GDTClock;
+
 NS_ASSUME_NONNULL_BEGIN
 
 /** The different possible quality of service specifiers. High values indicate high priority. */
@@ -56,6 +58,9 @@ typedef NS_ENUM(NSInteger, GDTEventQoS) {
 /** The quality of service tier this event belongs to. */
 @property(nonatomic) GDTEventQoS qosTier;
 
+/** The clock snapshot at the time of the event. */
+@property(nonatomic) GDTClock *clockSnapshot;
+
 /** A dictionary provided to aid prioritizers by allowing the passing of arbitrary data. It will be
  * retained by a copy in -copy, but not used for -hash.
  */

GoogleDataTransport/GoogleDataTransport/Classes/Public/GDTEventDataObject.h

@@ -23,6 +23,8 @@ NS_ASSUME_NONNULL_BEGIN
  */
 @protocol GDTEventDataObject <NSObject>
 
+@required
+
 /** Returns the serialized proto bytes of the implementing event proto.
  *
  * @return the serialized proto bytes of the implementing event proto.

GoogleDataTransport/GoogleDataTransport/Classes/Public/GDTPrioritizer.h

@@ -16,6 +16,8 @@
 
 #import <Foundation/Foundation.h>
 
+#import <GoogleDataTransport/GDTUploadPackage.h>
+
 @class GDTEvent;
 
 NS_ASSUME_NONNULL_BEGIN
@@ -46,7 +48,7 @@ typedef NS_OPTIONS(NSInteger, GDTUploadConditions) {
  *
  * @note A couple of things: 1. The event cannot be retained for longer than the execution time of
  * this method. 2. You should retain the event hashes, because those are returned in
- * -eventsForNextUpload.
+ * -uploadPackageWithConditions.
  *
  * @param event The event to prioritize.
  */
@@ -60,9 +62,10 @@ typedef NS_OPTIONS(NSInteger, GDTUploadConditions) {
 /** Returns a set of events to upload given a set of conditions.
  *
  * @param conditions A bit mask specifying the current upload conditions.
- * @return A set of events to upload with respect to the current conditions.
+ * @return An object to be used by the uploader to determine file URLs to upload with respect to the
+ * current conditions.
  */
-- (NSSet<NSNumber *> *)eventsToUploadGivenConditions:(GDTUploadConditions)conditions;
+- (GDTUploadPackage *)uploadPackageWithConditions:(GDTUploadConditions)conditions;
 
 @end
 

GoogleDataTransport/GoogleDataTransport/Classes/Public/GDTUploadPackage.h

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2019 Google
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#import <Foundation/Foundation.h>
+
+/** This class is a container that's handed off to uploaders. */
+@interface GDTUploadPackage : NSObject
+
+/** The set of event hashes in this upload package. */
+@property(nonatomic) NSSet<NSNumber *> *eventHashes;
+
+/** A lazily-determined map of event hashes to their files. */
+@property(nonatomic, readonly) NSDictionary<NSNumber *, NSURL *> *eventHashesToFiles;
+
+@end

GoogleDataTransport/GoogleDataTransport/Classes/Public/GDTUploader.h

@@ -39,12 +39,13 @@ typedef void (^GDTUploaderCompletionBlock)(GDTTarget target,
 
 /** Uploads events to the backend using this specific backend's chosen format.
  *
- * @param eventFiles The set of event files to upload.
- * @param onComplete A block to invoke upon completing the upload. Has two arguments:
- *   - successfulUploads: The set of filenames uploaded successfully.
- *   - unsuccessfulUploads: The set of filenames not uploaded successfully.
+ * @param package The event package to upload.
+ * @param onComplete A block to invoke upon completing the upload. Has three arguments:
+ *   - target: The GDTTarget that just uploaded.
+ *   - nextUploadAttemptUTC: A clock representing the next upload attempt.
+ *   - uploadError: An error object describing the upload error, or nil if upload was successful.
  */
-- (void)uploadEvents:(NSSet<NSURL *> *)eventFiles onComplete:(GDTUploaderCompletionBlock)onComplete;
+- (void)uploadPackage:(GDTUploadPackage *)package onComplete:(GDTUploaderCompletionBlock)onComplete;
 
 @end
 

GoogleDataTransport/Tests/Integration/Helpers/GDTIntegrationTestPrioritizer.m

@@ -16,6 +16,8 @@
 
 #import "GDTIntegrationTestPrioritizer.h"
 
+#import "GDTIntegrationTestUploadPackage.h"
+
 @interface GDTIntegrationTestPrioritizer ()
 
 /** Events that are only supposed to be uploaded whilst on wifi. */
@@ -44,32 +46,35 @@ - (instancetype)init {
 }
 
 - (void)prioritizeEvent:(GDTEvent *)event {
-  dispatch_sync(_queue, ^{
-    if (event.qosTier == GDTEventQoSWifiOnly) {
-      [self.wifiOnlyEvents addObject:@(event.hash)];
+  NSUInteger eventHash = event.hash;
+  NSInteger qosTier = event.qosTier;
+  dispatch_async(_queue, ^{
+    if (qosTier == GDTEventQoSWifiOnly) {
+      [self.wifiOnlyEvents addObject:@(eventHash)];
     } else {
-      [self.nonWifiEvents addObject:@(event.hash)];
+      [self.nonWifiEvents addObject:@(eventHash)];
     }
   });
 }
 
 - (void)unprioritizeEvent:(NSNumber *)eventHash {
-  dispatch_sync(_queue, ^{
+  dispatch_async(_queue, ^{
     [self.wifiOnlyEvents removeObject:eventHash];
     [self.nonWifiEvents removeObject:eventHash];
   });
 }
 
-- (nonnull NSSet<NSNumber *> *)eventsToUploadGivenConditions:(GDTUploadConditions)conditions {
-  __block NSSet<NSNumber *> *events;
+- (GDTUploadPackage *)uploadPackageWithConditions:(GDTUploadConditions)conditions {
+  __block GDTIntegrationTestUploadPackage *uploadPackage =
+      [[GDTIntegrationTestUploadPackage alloc] init];
   dispatch_sync(_queue, ^{
     if ((conditions & GDTUploadConditionWifiData) == GDTUploadConditionWifiData) {
-      events = self.wifiOnlyEvents;
+      uploadPackage.eventHashes = self.wifiOnlyEvents;
     } else {
-      events = self.nonWifiEvents;
+      uploadPackage.eventHashes = self.nonWifiEvents;
     }
   });
-  return events;
+  return uploadPackage;
 }
 
 @end

GoogleDataTransport/Tests/Integration/Helpers/GDTIntegrationTestUploadPackage.h

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2019 Google
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#import "GDTTestPrioritizer.h"
+
+/** An upload package used in testing. */
+@interface GDTIntegrationTestUploadPackage : GDTUploadPackage
+
+@end

GoogleDataTransport/Tests/Integration/Helpers/GDTIntegrationTestUploadPackage.m

@@ -0,0 +1,21 @@
+/*
+ * Copyright 2019 Google
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#import "GDTIntegrationTestUploadPackage.h"
+
+@implementation GDTIntegrationTestUploadPackage
+
+@end