Implement log storage (#2215)

mikehaney24 · web-flow · commit bbd0d302cd4c · 2018-12-24T14:59:48.000-08:00
* Implement log storage

Includes tests and categories on existing classes to add testing functionality

* Better error handling in tye log storage

* Style and pod lib lint fixes

* Add missing comment
diff --git a/GoogleDataLogger/GoogleDataLogger/Classes/GDLLogStorage.h b/GoogleDataLogger/GoogleDataLogger/Classes/GDLLogStorage.h
@@ -16,7 +16,35 @@
 
 #import <Foundation/Foundation.h>
 
-/** Manages the storage and prioritizing of logs. */
-@interface GDLLogStorage : NSObject
+@class GDLLogEvent;
+
+NS_ASSUME_NONNULL_BEGIN
+
+/** Manages the storage of logs. This class is thread-safe. */
+@interface GDLLogStorage : NSObject <NSSecureCoding>
+
+/** Creates and/or returns the storage singleton.
+ *
+ * @return The storage singleton.
+ */
++ (instancetype)sharedInstance;
+
+/** Stores log.extensionBytes into a shared on-device folder and tracks the log via its hash and
+ * logTarget properties.
+ *
+ * @note The log param is expected to be deallocated during this method.
+ *
+ * @param log The log to store.
+ */
+- (void)storeLog:(GDLLogEvent *)log;
+
+/** Removes the corresponding log file from disk.
+ *
+ * @param logHash The hash value of the original log.
+ * @param logTarget The logTarget of the original log.
+ */
+- (void)removeLog:(NSNumber *)logHash logTarget:(NSNumber *)logTarget;
 
 @end
+
+NS_ASSUME_NONNULL_END
diff --git a/GoogleDataLogger/GoogleDataLogger/Classes/GDLLogStorage.m b/GoogleDataLogger/GoogleDataLogger/Classes/GDLLogStorage.m
@@ -15,7 +15,182 @@
  */
 
 #import "GDLLogStorage.h"
+#import "GDLLogStorage_Private.h"
+
+#import <GoogleDataLogger/GDLLogPrioritizer.h>
+
+#import "GDLConsoleLogger.h"
+#import "GDLLogEvent_Private.h"
+#import "GDLRegistrar_Private.h"
+#import "GDLUploader.h"
+
+/** Creates and/or returns a singleton NSString that is the shared logging path.
+ *
+ * @return The SDK logging path.
+ */
+static NSString *GDLStoragePath() {
+  static NSString *archivePath;
+  static dispatch_once_t onceToken;
+  dispatch_once(&onceToken, ^{
+    NSString *cachePath =
+        NSSearchPathForDirectoriesInDomains(NSCachesDirectory, NSUserDomainMask, YES)[0];
+    archivePath = [NSString stringWithFormat:@"%@/google-sdks-logs", cachePath];
+  });
+  return archivePath;
+}
 
 @implementation GDLLogStorage
 
++ (instancetype)sharedInstance {
+  static GDLLogStorage *sharedStorage;
+  static dispatch_once_t onceToken;
+  dispatch_once(&onceToken, ^{
+    sharedStorage = [[GDLLogStorage alloc] init];
+  });
+  return sharedStorage;
+}
+
+- (instancetype)init {
+  self = [super init];
+  if (self) {
+    _storageQueue = dispatch_queue_create("com.google.GDLLogStorage", DISPATCH_QUEUE_SERIAL);
+    _logHashToLogFile = [[NSMutableDictionary alloc] init];
+    _logTargetToLogFileSet = [[NSMutableDictionary alloc] init];
+  }
+  return self;
+}
+
+- (void)storeLog:(GDLLogEvent *)log {
+  [self createLogDirectoryIfNotExists];
+
+  // This is done to ensure that log is deallocated at the end of the ensuing block.
+  __block GDLLogEvent *shortLivedLog = log;
+  __weak GDLLogEvent *weakShortLivedLog = log;
+  log = nil;
+
+  dispatch_async(_storageQueue, ^{
+    // Check that a backend implementation is available for this logTarget.
+    NSInteger logTarget = shortLivedLog.logTarget;
+
+    // Check that a log prioritizer is available for this logTarget.
+    id<GDLLogPrioritizer> logPrioritizer =
+        [GDLRegistrar sharedInstance].logTargetToPrioritizer[@(logTarget)];
+    NSAssert(logPrioritizer, @"There's no scorer registered for the given logTarget.");
+
+    // Write the extension bytes to disk, get a filename.
+    NSAssert(shortLivedLog.extensionBytes, @"The log should have been serialized to bytes");
+    NSAssert(shortLivedLog.extension == nil, @"The original log proto should be removed");
+    NSURL *logFile =
+        [self saveLogProtoToDisk:shortLivedLog.extensionBytes logHash:shortLivedLog.hash];
+
+    // Add log to tracking collections.
+    [self addLogToTrackingCollections:shortLivedLog logFile:logFile];
+
+    // Check the QoS, if it's high priority, notify the log target that it has a high priority log.
+    if (shortLivedLog.qosTier == GDLLogQoSFast) {
+      NSSet<NSURL *> *allLogsForLogTarget = self.logTargetToLogFileSet[@(logTarget)];
+      [[GDLUploader sharedInstance] forceUploadLogs:allLogsForLogTarget target:logTarget];
+    }
+
+    // Have the prioritizer prioritize the log, enforcing that they do not retain it.
+    @autoreleasepool {
+      [logPrioritizer prioritizeLog:shortLivedLog];
+      shortLivedLog = nil;
+    }
+    if (weakShortLivedLog) {
+      GDLLogError(GDLMCELogEventWasIllegallyRetained, @"%@",
+                  @"A LogEvent should not be retained outside of storage.");
+    };
+  });
+}
+
+- (void)removeLog:(NSNumber *)logHash logTarget:(NSNumber *)logTarget {
+  dispatch_async(_storageQueue, ^{
+    NSURL *logFile = self.logHashToLogFile[logHash];
+
+    // Remove from disk, first and foremost.
+    NSError *error;
+    [[NSFileManager defaultManager] removeItemAtURL:logFile error:&error];
+    NSAssert(error == nil, @"There was an error removing a logFile: %@", error);
+
+    // Remove from the tracking collections.
+    [self.logHashToLogFile removeObjectForKey:logHash];
+    NSMutableSet<NSURL *> *logFiles = self.logTargetToLogFileSet[logTarget];
+    NSAssert(logFiles, @"There wasn't a logSet for this logTarget.");
+    [logFiles removeObject:logFile];
+    // It's fine to not remove the set if it's empty.
+  });
+}
+
+#pragma mark - Private helper methods
+
+/** Creates the log directory if it does not exist. */
+- (void)createLogDirectoryIfNotExists {
+  NSError *error;
+  BOOL result = [[NSFileManager defaultManager] createDirectoryAtPath:GDLStoragePath()
+                                          withIntermediateDirectories:YES
+                                                           attributes:0
+                                                                error:&error];
+  if (!result || error) {
+    GDLLogError(GDLMCEDirectoryCreationError, @"Error creating the directory: %@", error);
+  }
+}
+
+/** Saves the log's extensionBytes to a file using NSData mechanisms.
+ *
+ * @note This method should only be called from a method within a block on _storageQueue to maintain
+ * thread safety.
+ *
+ * @param logProtoBytes The extensionBytes of the log, presumably proto bytes.
+ * @param logHash The hash value of the log.
+ * @return The filename
+ */
+- (NSURL *)saveLogProtoToDisk:(NSData *)logProtoBytes logHash:(NSUInteger)logHash {
+  NSString *storagePath = GDLStoragePath();
+  NSString *logFile = [NSString stringWithFormat:@"log-%lu", (unsigned long)logHash];
+  NSURL *logFilePath = [NSURL fileURLWithPath:[storagePath stringByAppendingPathComponent:logFile]];
+
+  BOOL writingSuccess = [logProtoBytes writeToURL:logFilePath atomically:YES];
+  if (!writingSuccess) {
+    GDLLogError(GDLMCEFileWriteError, @"A log file could not be written: %@", logFilePath);
+  }
+
+  return logFilePath;
+}
+
+/** Adds the log to internal collections in order to help track the log.
+ *
+ * @note This method should only be called from a method within a block on _storageQueue to maintain
+ * thread safety.
+ *
+ * @param log The log to track.
+ * @param logFile The file the log has been saved to.
+ */
+- (void)addLogToTrackingCollections:(GDLLogEvent *)log logFile:(NSURL *)logFile {
+  NSInteger logTarget = log.logTarget;
+  self.logHashToLogFile[@(log.hash)] = logFile;
+  NSMutableSet<NSURL *> *logs = self.logTargetToLogFileSet[@(logTarget)];
+  if (logs) {
+    [logs addObject:logFile];
+  } else {
+    NSMutableSet<NSURL *> *logSet = [NSMutableSet setWithObject:logFile];
+    self.logTargetToLogFileSet[@(logTarget)] = logSet;
+  }
+}
+
+#pragma mark - NSSecureCoding
+
++ (BOOL)supportsSecureCoding {
+  return YES;
+}
+
+- (instancetype)initWithCoder:(NSCoder *)aDecoder {
+  // TODO
+  return [self.class sharedInstance];
+}
+
+- (void)encodeWithCoder:(NSCoder *)aCoder {
+  // TODO
+}
+
 @end
diff --git a/GoogleDataLogger/GoogleDataLogger/Classes/Private/GDLLogStorage_Private.h b/GoogleDataLogger/GoogleDataLogger/Classes/Private/GDLLogStorage_Private.h
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2018 Google
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#import "GDLLogStorage.h"
+
+@interface GDLLogStorage ()
+
+/** The queue on which all storage work will occur. */
+@property(nonatomic) dispatch_queue_t storageQueue;
+
+/** A map of log hash values to log file on-disk URLs. */
+@property(nonatomic) NSMutableDictionary<NSNumber *, NSURL *> *logHashToLogFile;
+
+/** A map of logTargets to a set of log hash values. */
+@property(nonatomic)
+    NSMutableDictionary<NSNumber *, NSMutableSet<NSURL *> *> *logTargetToLogFileSet;
+
+@end
diff --git a/GoogleDataLogger/GoogleDataLogger/DependencyWrappers/GDLConsoleLogger.h b/GoogleDataLogger/GoogleDataLogger/DependencyWrappers/GDLConsoleLogger.h
@@ -24,14 +24,24 @@ static GULLoggerService kGDLConsoleLogger = @"[GoogleDataLogger]";
  *
  * Prefixes:
  * - MCW => MessageCodeWarning
+ * - MCE => MessageCodeError
  */
 typedef NS_ENUM(NSInteger, GDLMessageCode) {
 
   /** For warning messages concerning transform: not being implemented by a log transformer. */
   GDLMCWTransformerDoesntImplementTransform = 1,
 
   /** For warning messages concerning protoBytes: not being implemented by a log extension. */
-  GDLMCWExtensionMissingBytesImpl = 2
+  GDLMCWExtensionMissingBytesImpl = 2,
+
+  /** For error messages concerning a GDLLogEvent living past the storeLog: invocation. */
+  GDLMCELogEventWasIllegallyRetained = 1000,
+
+  /** For error messages concerning the creation of a directory failing. */
+  GDLMCEDirectoryCreationError = 1001,
+
+  /** For error messages concerning the writing of  a log file. */
+  GDLMCEFileWriteError = 1002
 };
 
 /** */
@@ -49,3 +59,9 @@ FOUNDATION_EXTERN void GDLLogWarning(GDLMessageCode messageCode,
 #define GDLLogWarning(MESSAGE_CODE, MESSAGE_FORMAT, ...)                                          \
   GULLogWarning(kGDLConsoleLogger, YES, GDLMessageCodeEnumToString(MESSAGE_CODE), MESSAGE_FORMAT, \
                 __VA_ARGS__);
+
+// A define to wrap GULLogError with slightly more convenient usage and a failing assert.
+#define GDLLogError(MESSAGE_CODE, MESSAGE_FORMAT, ...)                                          \
+  GULLogError(kGDLConsoleLogger, YES, GDLMessageCodeEnumToString(MESSAGE_CODE), MESSAGE_FORMAT, \
+              __VA_ARGS__);                                                                     \
+  NSAssert(NO, MESSAGE_FORMAT, __VA_ARGS__);
diff --git a/GoogleDataLogger/Tests/Categories/GDLLogStorage+Testing.h b/GoogleDataLogger/Tests/Categories/GDLLogStorage+Testing.h
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2018 Google
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#import <Foundation/Foundation.h>
+
+#import "GDLLogStorage.h"
+#import "GDLLogStorage_Private.h"
+
+NS_ASSUME_NONNULL_BEGIN
+
+/** Testing-only methods for GDLLogStorage. */
+@interface GDLLogStorage (Testing)
+
+/** Resets the properties of the singleon, but does not reallocate a new singleton. This also
+ * doesn't remove stored files from disk.
+ */
+- (void)reset;
+
+@end
+
+NS_ASSUME_NONNULL_END
diff --git a/GoogleDataLogger/Tests/Categories/GDLLogStorage+Testing.m b/GoogleDataLogger/Tests/Categories/GDLLogStorage+Testing.m
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2018 Google
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#import "GDLLogStorage+Testing.h"
+
+@implementation GDLLogStorage (Testing)
+
+- (void)reset {
+  dispatch_sync(self.storageQueue, ^{
+    [self.logTargetToLogFileSet removeAllObjects];
+    [self.logHashToLogFile removeAllObjects];
+  });
+}
+
+@end
diff --git a/GoogleDataLogger/Tests/Categories/GDLRegistrar+Testing.h b/GoogleDataLogger/Tests/Categories/GDLRegistrar+Testing.h
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2018 Google
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#import "GDLRegistrar.h"
+
+NS_ASSUME_NONNULL_BEGIN
+
+/** Testing-only methods for GDLRegistrar. */
+@interface GDLRegistrar (Testing)
+
+/** Resets the properties of the singleon, but does not reallocate a new singleton. */
+- (void)reset;
+
+@end
+
+NS_ASSUME_NONNULL_END
diff --git a/GoogleDataLogger/Tests/Categories/GDLRegistrar+Testing.m b/GoogleDataLogger/Tests/Categories/GDLRegistrar+Testing.m
diff --git a/GoogleDataLogger/Tests/GDLLogStorageTest.m b/GoogleDataLogger/Tests/GDLLogStorageTest.m
