Refactor in preparation of reading ext data prior to auth.

platform-darwin/Source/MPAppDelegate_Store.m

@@ -565,7 +565,7 @@ PearlAssociatedObjectProperty( NSNumber*, StoreCorrupted, storeCorrupted );
            saveInContext:(NSManagedObjectContext *)context {
 
     // Read metadata for the import file.
-    MPMarshalInfo *info = mpw_marshal_read_info( importData.UTF8String );
+    MPMarshalledInfo *info = mpw_marshal_read_info( importData.UTF8String );
     if (info->format == MPMarshalFormatNone)
         return MPError( ([NSError errorWithDomain:MPErrorDomain code:MPErrorMarshalCode userInfo:@{
                 @"type"                  : @(MPMarshalErrorFormat),

platform-independent/c/cli/src/mpw-cli.c

@@ -489,7 +489,7 @@ void cli_user(Arguments *args, Operation *operation) {
         mpw_free_string( &operation->sitesPath );
         mpw_marshal_file_free( &operation->file );
         operation->file = mpw_marshal_file( NULL, mpw_marshal_user(
-                operation->fullName, cli_masterKeyProvider_op( operation ), MPAlgorithmVersionCurrent ), NULL, NULL );
+                operation->fullName, cli_masterKeyProvider_op( operation ), MPAlgorithmVersionCurrent ), NULL );
     }
 
     else {

platform-independent/c/core/src/mpw-marshal-util.c

@@ -21,6 +21,7 @@
 
 MP_LIBS_BEGIN
 #include <string.h>
+#include <math.h>
 MP_LIBS_END
 
 char *mpw_get_token(const char **in, const char *eol, const char *delim) {
@@ -57,6 +58,22 @@ time_t mpw_timegm(const char *time) {
     return ERR;
 }
 
+bool mpw_update_master_key(MPMasterKey *masterKey, MPAlgorithmVersion *masterKeyAlgorithm, const MPAlgorithmVersion targetKeyAlgorithm,
+        const char *fullName, const char *masterPassword) {
+
+    if (masterKey && (!*masterKey || *masterKeyAlgorithm != targetKeyAlgorithm)) {
+        mpw_free( masterKey, MPMasterKeySize );
+        *masterKeyAlgorithm = targetKeyAlgorithm;
+        *masterKey = mpw_master_key( fullName, masterPassword, *masterKeyAlgorithm );
+        if (!*masterKey) {
+            err( "Couldn't derive master key for user %s, algorithm %d.", fullName, *masterKeyAlgorithm );
+            return false;
+        }
+    }
+
+    return true;
+}
+
 #if MPW_JSON
 
 json_object *mpw_get_json_object(
@@ -105,20 +122,110 @@ bool mpw_get_json_boolean(
     return json_object_get_boolean( json_value ) == true;
 }
 
-#endif
+void mpw_set_json_data(
+        MPMarshalledData *data, json_object *obj) {
 
-bool mpw_update_master_key(MPMasterKey *masterKey, MPAlgorithmVersion *masterKeyAlgorithm, const MPAlgorithmVersion targetKeyAlgorithm,
+    if (!data)
-        const char *fullName, const char *masterPassword) {
+        return;
 
-    if (masterKey && (!*masterKey || *masterKeyAlgorithm != targetKeyAlgorithm)) {
+    json_type type = json_object_get_type( obj );
-        mpw_free( masterKey, MPMasterKeySize );
+    data->is_null = type == json_type_null;
-        *masterKeyAlgorithm = targetKeyAlgorithm;
+    data->is_bool = type == json_type_boolean;
-        *masterKey = mpw_master_key( fullName, masterPassword, *masterKeyAlgorithm );
+
-        if (!*masterKey) {
+    if (type == json_type_boolean)
-            err( "Couldn't derive master key for user %s, algorithm %d.", fullName, *masterKeyAlgorithm );
+        data->num_value = json_object_get_boolean( obj );
-            return false;
+    else if (type == json_type_double)
+        data->num_value = json_object_get_double( obj );
+    else if (type == json_type_int)
+        data->num_value = json_object_get_int64( obj );
+    else
+        data->num_value = NAN;
+
+    const char *str = NULL;
+    if (type == json_type_string || !isnan( data->num_value ))
+        str = json_object_get_string( obj );
+    if (!str || !data->str_value || strcmp( str, data->str_value ) != OK) {
+        mpw_free_string( &data->str_value );
+        data->str_value = mpw_strdup( str );
+    }
+
+    // Clean up children
+    MPMarshalledData *newChildren = NULL;
+    size_t newChildrenCount = 0;
+    for (size_t c = 0; c < data->children_count; ++c) {
+        MPMarshalledData *child = &data->children[c];
+        if ((type != json_type_object && type != json_type_array) || (child->obj_key && type != json_type_object)) {
+            // Not a valid child in this object, remove it.
+            mpw_marshal_data_set_null( child, NULL );
+            mpw_free_string( &child->obj_key );
+            if (!newChildren)
+                newChildren = mpw_memdup( data->children, sizeof( MPMarshalledData ) * newChildrenCount );
+        }
+        else {
+            // Valid child in this object, keep it.
+            ++newChildrenCount;
+            if (newChildren) {
+                if (!mpw_realloc( &newChildren, NULL, sizeof( MPMarshalledData ) * newChildrenCount )) {
+                    --newChildrenCount;
+                    continue;
+                }
+                child->arr_index = newChildrenCount - 1;
+                newChildren[child->arr_index] = *child;
+            }
+        }
+    }
+    if (newChildren) {
+        mpw_free( &data->children, sizeof( MPMarshalledData ) * data->children_count );
+        data->children = newChildren;
+        data->children_count = newChildrenCount;
+    }
+
+    // Object
+    if (type == json_type_object) {
+        json_object_iter entry;
+        json_object_object_foreachC( obj, entry ) {
+            MPMarshalledData *child = NULL;
+
+            // Find existing child.
+            for (size_t c = 0; c < data->children_count; ++c)
+                if (data->children[c].obj_key == entry.key ||
+                    (data->children[c].obj_key && entry.key && strcmp( data->children[c].obj_key, entry.key ) == OK)) {
+                    child = &data->children[c];
+                    break;
+                }
+
+            // Create new child.
+            if (!child) {
+                if (!mpw_realloc( &data->children, NULL, sizeof( MPMarshalledData ) * ++data->children_count )) {
+                    --data->children_count;
+                    continue;
+                }
+                *(child = &data->children[data->children_count - 1]) = (MPMarshalledData){ .obj_key = mpw_strdup( entry.key ) };
+            }
+
+            mpw_set_json_data( child, entry.val );
         }
     }
 
-    return true;
+    // Array
+    if (type == json_type_array) {
+        for (size_t index = 0; index < json_object_array_length( obj ); ++index) {
+            MPMarshalledData *child = NULL;
+
+            if (index < data->children_count)
+                child = &data->children[index];
+
+            else {
+                if (!mpw_realloc( &data->children, NULL, sizeof( MPMarshalledData ) * ++data->children_count )) {
+                    --data->children_count;
+                    continue;
+                }
+                *(child = &data->children[data->children_count - 1]) = (MPMarshalledData){ .arr_index = index };
+            }
+
+            mpw_set_json_data( child, json_object_array_get_idx( obj, index ) );
+        }
+    }
 }
+
+#endif

platform-independent/c/core/src/mpw-marshal-util.h

@@ -20,6 +20,7 @@
 #define _MPW_MARSHAL_UTIL_H
 
 #include "mpw-algorithm.h"
+#include "mpw-marshal.h"
 
 MP_LIBS_BEGIN
 #include <time.h>
@@ -40,6 +41,17 @@ char *mpw_get_token(
 time_t mpw_timegm(
         const char *time);
 
+
+/// mpw.
+
+/** Calculate a master key if the target master key algorithm is different from the given master key algorithm.
+ * @param masterKey A buffer (allocated, MPMasterKeySize).
+ * @return false if an error occurred during the derivation of the master key. */
+bool mpw_update_master_key(
+        MPMasterKey *masterKey, MPAlgorithmVersion *masterKeyAlgorithm, const MPAlgorithmVersion targetKeyAlgorithm,
+        const char *fullName, const char *masterPassword);
+
+
 /// JSON parsing.
 
 #if MPW_JSON
@@ -64,15 +76,11 @@ int64_t mpw_get_json_int(
  * @return The boolean value or defaultValue if one of the path's object keys was not found in the source object's tree. */
 bool mpw_get_json_boolean(
         json_object *obj, const char *key, const bool defaultValue);
+/** Translate a JSON object tree into a source-agnostic data object.
+ * @param data A Master Password data object or NULL.
+ * @param obj A JSON object tree or NULL. */
+void mpw_set_json_data(
+        MPMarshalledData *data, json_object *obj);
 #endif
 
-/// mpw.
-
-/** Calculate a master key if the target master key algorithm is different from the given master key algorithm.
- * @param masterKey A buffer (allocated, MPMasterKeySize).
- * @return false if an error occurred during the derivation of the master key. */
-bool mpw_update_master_key(
-        MPMasterKey *masterKey, MPAlgorithmVersion *masterKeyAlgorithm, const MPAlgorithmVersion targetKeyAlgorithm,
-        const char *fullName, const char *masterPassword);
-
 #endif // _MPW_MARSHAL_UTIL_H

platform-independent/c/core/src/mpw-marshal.c

@@ -106,7 +106,7 @@ MPMarshalledQuestion *mpw_marshal_question(
 }
 
 MPMarshalledFile *mpw_marshal_file(
-        MPMarshalledFile *file, MPMarshalledUser *user, MPMarshalledData *data, MPMarshalInfo *info) {
+        MPMarshalledFile *file, MPMarshalledUser *user, MPMarshalledData *data, MPMarshalledInfo *info) {
 
     if (!file) {
         if (!(file = malloc( sizeof( MPMarshalledFile ) )))
@@ -132,13 +132,13 @@ MPMarshalledFile *mpw_marshal_file(
 }
 
 void mpw_marshal_info_free(
-        MPMarshalInfo **info) {
+        MPMarshalledInfo **info) {
 
     if (!info || !*info)
         return;
 
     mpw_free_strings( &(*info)->fullName, &(*info)->keyID, NULL );
-    mpw_free( info, sizeof( MPMarshalInfo ) );
+    mpw_free( info, sizeof( MPMarshalledInfo ) );
 }
 
 void mpw_marshal_user_free(
@@ -171,7 +171,7 @@ void mpw_marshal_data_free(
         return;
 
     mpw_marshal_data_set_null( *data, NULL );
-    mpw_free_string( &(*data)->key );
+    mpw_free_string( &(*data)->obj_key );
     mpw_free( data, sizeof( MPMarshalledData ) );
 }
 
@@ -204,7 +204,7 @@ MPMarshalledData *mpw_marshal_data_vget(
         child = NULL;
 
         for (size_t c = 0; c < parent->children_count; ++c) {
-            const char *key = parent->children[c].key;
+            const char *key = parent->children[c].obj_key;
             if (key && strcmp( node, key ) == OK) {
                 child = &parent->children[c];
                 break;
@@ -216,7 +216,7 @@ MPMarshalledData *mpw_marshal_data_vget(
                 --parent->children_count;
                 break;
             }
-            *(child = &parent->children[parent->children_count - 1]) = (MPMarshalledData){ .key = mpw_strdup( node ) };
+            *(child = &parent->children[parent->children_count - 1]) = (MPMarshalledData){ .obj_key = mpw_strdup( node ) };
             mpw_marshal_data_set_null( child, NULL );
             child->is_null = false;
         }
@@ -244,7 +244,7 @@ const MPMarshalledData *mpw_marshal_data_vfind(
         child = NULL;
 
         for (size_t c = 0; c < parent->children_count; ++c) {
-            const char *key = parent->children[c].key;
+            const char *key = parent->children[c].obj_key;
             if (key && strcmp( node, key ) == OK) {
                 child = &parent->children[c];
                 break;
@@ -297,7 +297,7 @@ bool mpw_marshal_data_vset_null(
     mpw_free_string( &child->str_value );
     for (unsigned int c = 0; c < child->children_count; ++c) {
         mpw_marshal_data_set_null( &child->children[c], NULL );
-        mpw_free_string( &child->children[c].key );
+        mpw_free_string( &child->children[c].obj_key );
     }
     mpw_free( &child->children, sizeof( MPMarshalledData ) * child->children_count );
     child->children_count = 0;
@@ -447,7 +447,7 @@ bool mpw_marshal_data_set_str(
 }
 
 static const char *mpw_marshal_write_flat(
-        const MPMarshalledFile *file, MPMarshalError *error) {
+        MPMarshalledFile *file, MPMarshalError *error) {
 
     *error = (MPMarshalError){ MPMarshalErrorInternal, "Unexpected internal error." };
     MPMarshalledUser *user = file->user;
@@ -557,14 +557,14 @@ static json_object *mpw_get_json_data(
     for (size_t c = 0; c < data->children_count; ++c) {
         MPMarshalledData *child = &data->children[c];
         if (!obj) {
-            if (child->key)
+            if (child->obj_key)
                 obj = json_object_new_object();
             else
                 obj = json_object_new_array();
         }
 
-        if (child->key)
+        if (child->obj_key)
-            json_object_object_add( obj, child->key, mpw_get_json_data( child ) );
+            json_object_object_add( obj, child->obj_key, mpw_get_json_data( child ) );
         else
             json_object_array_add( obj, mpw_get_json_data( child ) );
     }
@@ -573,7 +573,7 @@ static json_object *mpw_get_json_data(
 }
 
 static const char *mpw_marshal_write_json(
-        const MPMarshalledFile *file, MPMarshalError *error) {
+        MPMarshalledFile *file, MPMarshalError *error) {
 
     *error = (MPMarshalError){ MPMarshalErrorInternal, "Unexpected internal error." };
     MPMarshalledUser *user = file->user;
@@ -694,6 +694,9 @@ static const char *mpw_marshal_write_json(
         mpw_free_strings( &resultState, &loginState, NULL );
     }
 
+    if (!file->data)
+        file->data = mpw_marshal_data_new();
+    mpw_set_json_data( file->data, json_file );
     const char *out = mpw_strdup( json_object_to_json_string_ext( json_file,
             JSON_C_TO_STRING_PRETTY | JSON_C_TO_STRING_SPACED | JSON_C_TO_STRING_NOSLASHESCAPE ) );
     json_object_put( json_file );
@@ -716,7 +719,7 @@ const char *mpw_marshal_write(
         *error = (MPMarshalError){ .type = MPMarshalErrorMissing, "No file to marshal." };
         return NULL;
     }
-    if (file->data && file->data->key) {
+    if (file->data && file->data->obj_key) {
         *error = (MPMarshalError){ .type = MPMarshalErrorInternal, "Illegal file data." };
         ftl( "Unexpected non-root file data." );
         return NULL;
@@ -745,7 +748,7 @@ const char *mpw_marshal_write(
 }
 
 static void mpw_marshal_read_flat_info(
-        const char *in, MPMarshalInfo *info) {
+        const char *in, MPMarshalledInfo *info) {
 
     info->algorithm = MPAlgorithmVersionCurrent;
 
@@ -1082,6 +1085,7 @@ static MPMarshalledFile *mpw_marshal_read_flat(
     mpw_free_strings( &fullName, &keyID, NULL );
     mpw_free( &masterKey, MPMasterKeySize );
 
+    // TODO: serialize data structure for this file.
     MPMarshalledFile *file = mpw_marshal_file( NULL, user, NULL, NULL );
     if (!file) {
         *error = (MPMarshalError){ MPMarshalErrorInternal, "Couldn't allocate a new marshal file." };
@@ -1095,114 +1099,8 @@ static MPMarshalledFile *mpw_marshal_read_flat(
 
 #if MPW_JSON
 
-static void mpw_set_json_data(
-        MPMarshalledData *data, json_object *obj) {
-
-    if (!data)
-        return;
-
-    json_type type = json_object_get_type( obj );
-    data->is_null = type == json_type_null;
-    data->is_bool = type == json_type_boolean;
-
-    if (type == json_type_boolean)
-        data->num_value = json_object_get_boolean( obj );
-    else if (type == json_type_double)
-        data->num_value = json_object_get_double( obj );
-    else if (type == json_type_int)
-        data->num_value = json_object_get_int64( obj );
-    else
-        data->num_value = NAN;
-
-    const char *str = NULL;
-    if (type == json_type_string || !isnan( data->num_value ))
-        str = json_object_get_string( obj );
-    if (!str || !data->str_value || strcmp( str, data->str_value ) != OK) {
-        mpw_free_string( &data->str_value );
-        data->str_value = mpw_strdup( str );
-    }
-
-    // Clean up children
-    MPMarshalledData *newChildren = NULL;
-    size_t newChildrenCount = 0;
-    for (size_t c = 0; c < data->children_count; ++c) {
-        MPMarshalledData *child = &data->children[c];
-        if ((type != json_type_object && type != json_type_array) || (child->key && type != json_type_object)) {
-            // Not a valid child in this object, remove it.
-            mpw_marshal_data_set_null( child, NULL );
-            mpw_free_string( &child->key );
-            if (!newChildren)
-                newChildren = mpw_memdup( data->children, sizeof( MPMarshalledData ) * newChildrenCount );
-        }
-        else {
-            // Valid child in this object, keep it.
-            ++newChildrenCount;
-            if (newChildren) {
-                if (!mpw_realloc( &newChildren, NULL, sizeof( MPMarshalledData ) * newChildrenCount )) {
-                    --newChildrenCount;
-                    continue;
-                }
-                child->index = newChildrenCount - 1;
-                newChildren[child->index] = *child;
-            }
-        }
-    }
-    if (newChildren) {
-        mpw_free( &data->children, sizeof( MPMarshalledData ) * data->children_count );
-        data->children = newChildren;
-        data->children_count = newChildrenCount;
-    }
-
-    // Object
-    if (type == json_type_object) {
-        json_object_iter entry;
-        json_object_object_foreachC( obj, entry ) {
-            MPMarshalledData *child = NULL;
-
-            // Find existing child.
-            for (size_t c = 0; c < data->children_count; ++c)
-                if (data->children[c].key == entry.key ||
-                    (data->children[c].key && entry.key && strcmp( data->children[c].key, entry.key ) == OK)) {
-                    child = &data->children[c];
-                    break;
-                }
-
-            // Create new child.
-            if (!child) {
-                if (!mpw_realloc( &data->children, NULL, sizeof( MPMarshalledData ) * ++data->children_count )) {
-                    --data->children_count;
-                    continue;
-                }
-                *(child = &data->children[data->children_count - 1]) = (MPMarshalledData){ .key = mpw_strdup( entry.key ) };
-            }
-
-            mpw_set_json_data( child, entry.val );
-        }
-    }
-
-    // Array
-    if (type == json_type_array) {
-        for (size_t index = 0; index < json_object_array_length( obj ); ++index) {
-            MPMarshalledData *child = NULL;
-
-            if (index < data->children_count)
-                child = &data->children[index];
-
-            else {
-                if (!mpw_realloc( &data->children, NULL, sizeof( MPMarshalledData ) * ++data->children_count )) {
-                    --data->children_count;
-                    continue;
-                }
-                *(child = &data->children[data->children_count - 1]) = (MPMarshalledData){ .index = index };
-            }
-
-            mpw_set_json_data( child, json_object_array_get_idx( obj, index ) );
-        }
-    }
-}
-
 static void mpw_marshal_read_json_info(
-        const char *in, MPMarshalInfo *info) {
+        const char *in, MPMarshalledInfo *info) {
 
     // Parse JSON.
     enum json_tokener_error json_error = json_tokener_success;
@@ -1457,14 +1355,14 @@ static MPMarshalledFile *mpw_marshal_read_json(
 
 #endif
 
-MPMarshalInfo *mpw_marshal_read_info(
+MPMarshalledInfo *mpw_marshal_read_info(
         const char *in) {
 
-    MPMarshalInfo *info = malloc( sizeof( MPMarshalInfo ) );
+    MPMarshalledInfo *info = malloc( sizeof( MPMarshalledInfo ) );
     if (!info)
         return NULL;
 
-    *info = (MPMarshalInfo){ .format = MPMarshalFormatNone, .identicon = MPIdenticonUnset };
+    *info = (MPMarshalledInfo){ .format = MPMarshalFormatNone, .identicon = MPIdenticonUnset };
     if (in && strlen( in )) {
         if (in[0] == '#') {
             info->format = MPMarshalFormatFlat;
@@ -1489,7 +1387,7 @@ MPMarshalInfo *mpw_marshal_read_info(
 MPMarshalledFile *mpw_marshal_read(
         const char *in, const MPMasterKeyProvider masterKeyProvider, MPMarshalError *error) {
 
-    MPMarshalInfo *info = mpw_marshal_read_info( in );
+    MPMarshalledInfo *info = mpw_marshal_read_info( in );
     if (!info)
         return NULL;
 

platform-independent/c/core/src/mpw-marshal.h

@@ -69,7 +69,20 @@ typedef struct MPMarshalError {
     const char *message;
 } MPMarshalError;
 
-typedef struct MPMarshalInfo {
+typedef struct MPMarshalledData {
+    const char *obj_key;
+    size_t arr_index;
+
+    bool is_null;
+    bool is_bool;
+    const char *str_value;
+    double num_value;
+
+    size_t children_count;
+    struct MPMarshalledData *children;
+} MPMarshalledData;
+
+typedef struct MPMarshalledInfo {
     MPMarshalFormat format;
     time_t exportDate;
     bool redacted;
@@ -80,7 +93,7 @@ typedef struct MPMarshalInfo {
     MPIdenticon identicon;
     const char *keyID;
     time_t lastUsed;
-} MPMarshalInfo;
+} MPMarshalledInfo;
 
 typedef struct MPMarshalledQuestion {
     const char *keyword;
@@ -123,28 +136,8 @@ typedef struct MPMarshalledUser {
     MPMarshalledSite *sites;
 } MPMarshalledUser;
 
-typedef struct MPMarshalledData {
-    // If data is held in a parent object.
-    const char *key;
-    // If data is held in a parent array.
-    size_t index;
-
-    // If data is a null.
-    bool is_null;
-    // If data is a boolean.
-    bool is_bool;
-    // If data is a string.
-    const char *str_value;
-    // If data is a number.
-    double num_value;
-
-    // If data is an object or array.
-    size_t children_count;
-    struct MPMarshalledData *children;
-} MPMarshalledData;
-
 typedef struct MPMarshalledFile {
-    MPMarshalInfo *info;
+    MPMarshalledInfo *info;
     MPMarshalledUser *user;
     MPMarshalledData *data;
 } MPMarshalledFile;
@@ -157,7 +150,7 @@ const char *mpw_marshal_write(
         const MPMarshalFormat outFormat, MPMarshalledFile *file, MPMarshalError *error);
 /** Best effort parse of metadata on the sites in the input buffer.  Fields that could not be parsed remain at their type's initial value.
  * @return A metadata object (allocated); NULL if the object could not be allocated or the format was not understood. */
-MPMarshalInfo *mpw_marshal_read_info(
+MPMarshalledInfo *mpw_marshal_read_info(
         const char *in);
 /** Unmarshall sites in the given input buffer by parsing it using the given marshalling format.
  * @return A user object (allocated), or NULL if the format provides no marshalling or a format error occurred. */
@@ -182,13 +175,13 @@ MPMarshalledQuestion *mpw_marshal_question(
 /** Create or update a marshal file descriptor.
  * @return The given file or new (allocated) if file is NULL; or NULL if the user is missing or the file couldn't be allocated. */
 MPMarshalledFile *mpw_marshal_file(
-        MPMarshalledFile *const file, MPMarshalledUser *user, MPMarshalledData *data, MPMarshalInfo *info);
+        MPMarshalledFile *const file, MPMarshalledUser *user, MPMarshalledData *data, MPMarshalledInfo *info);
 
 //// Disposing.
 
 /** Free the given user object and all associated data. */
 void mpw_marshal_info_free(
-        MPMarshalInfo **info);
+        MPMarshalledInfo **info);
 void mpw_marshal_user_free(
         MPMarshalledUser **user);
 void mpw_marshal_data_free(