Make default counter algorithm-scoped, format-specific unmarshalling.

core/java/algorithm/src/main/java/com/lyndir/masterpassword/MPAlgorithm.java

@@ -34,10 +34,15 @@ import javax.annotation.Nullable;
 public interface MPAlgorithm {
 
     /**
-     * mpw: defaults: Password result type.
+     * mpw: defaults: password result type.
      */
     MPResultType mpw_default_type = MPResultType.GeneratedLong;
 
+    /**
+     * mpw: defaults: initial counter value.
+     */
+    UnsignedInteger mpw_default_counter = UnsignedInteger.ONE;
+
     /**
      * mpw: validity for the time-based rolling counter.
      */

core/java/model/src/main/java/com/lyndir/masterpassword/model/MPFileSite.java

@@ -48,7 +48,10 @@ public class MPFileSite extends MPSite {
     private Instant lastUsed;
 
     public MPFileSite(final MPFileUser user, final String siteName) {
-        this( user, siteName, DEFAULT_COUNTER, user.getAlgorithmVersion().getAlgorithm().mpw_default_type, user.getAlgorithmVersion() );
+        this( user, siteName,
+              user.getAlgorithmVersion().getAlgorithm().mpw_default_counter,
+              user.getAlgorithmVersion().getAlgorithm().mpw_default_type,
+              user.getAlgorithmVersion() );
     }
 
     public MPFileSite(final MPFileUser user, final String siteName, final UnsignedInteger siteCounter, final MPResultType resultType,
@@ -177,7 +180,7 @@ public class MPFileSite extends MPSite {
                 this.loginContent = null;
             else
                 this.loginContent = masterKey.siteState(
-                        siteName, DEFAULT_COUNTER, MPKeyPurpose.Identification, null, this.loginType, result, algorithmVersion );
+                        siteName, MPAlgorithm.mpw_default_counter, MPKeyPurpose.Identification, null, this.loginType, result, algorithmVersion );
     }
 
     @Nullable

core/java/model/src/main/java/com/lyndir/masterpassword/model/MPFileUser.java

@@ -35,12 +35,13 @@ public class MPFileUser extends MPUser<MPFileSite> implements Comparable<MPFileU
     @SuppressWarnings("UnusedDeclaration")
     private static final Logger logger = Logger.get( MPFileUser.class );
 
-    private final String fullName;
+    private final String                 fullName;
     private final Collection<MPFileSite> sites = Sets.newHashSet();
 
     @Nullable
     private byte[]              keyID;
     private MPMasterKey.Version algorithmVersion;
+    private MPMarshalFormat     format;
 
     private int             avatar;
     private MPResultType    defaultType;
@@ -51,17 +52,18 @@ public class MPFileUser extends MPUser<MPFileSite> implements Comparable<MPFileU
     }
 
     public MPFileUser(final String fullName, @Nullable final byte[] keyID, final MPMasterKey.Version algorithmVersion) {
-        this( fullName, keyID, algorithmVersion, 0, MPAlgorithm.mpw_default_type, new Instant() );
+        this( fullName, keyID, algorithmVersion, 0, MPAlgorithm.mpw_default_type, new Instant(), MPMarshalFormat.DEFAULT );
     }
 
     public MPFileUser(final String fullName, @Nullable final byte[] keyID, final MPMasterKey.Version algorithmVersion, final int avatar,
-                      final MPResultType defaultType, final ReadableInstant lastUsed) {
+                      final MPResultType defaultType, final ReadableInstant lastUsed, final MPMarshalFormat format) {
         this.fullName = fullName;
         this.keyID = (keyID == null)? null: keyID.clone();
         this.algorithmVersion = algorithmVersion;
         this.avatar = avatar;
         this.defaultType = defaultType;
         this.lastUsed = lastUsed;
+        this.format = format;
     }
 
     @Override
@@ -78,6 +80,14 @@ public class MPFileUser extends MPUser<MPFileSite> implements Comparable<MPFileU
         this.algorithmVersion = algorithmVersion;
     }
 
+    public MPMarshalFormat getFormat() {
+        return format;
+    }
+
+    public void setFormat(final MPMarshalFormat format) {
+        this.format = format;
+    }
+
     @Override
     public int getAvatar() {
         return avatar;

core/java/model/src/main/java/com/lyndir/masterpassword/model/MPFileUserManager.java

@@ -26,6 +26,8 @@ import com.google.common.io.CharSink;
 import com.lyndir.lhunath.opal.system.logging.Logger;
 import com.lyndir.masterpassword.*;
 import java.io.*;
+import java.util.HashMap;
+import java.util.Map;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
@@ -38,7 +40,7 @@ import javax.annotation.Nullable;
 public class MPFileUserManager extends MPUserManager {
 
     @SuppressWarnings("UnusedDeclaration")
-    private static final Logger logger = Logger.get( MPFileUserManager.class );
+    private static final Logger            logger = Logger.get( MPFileUserManager.class );
     private static final MPFileUserManager instance;
 
     static {
@@ -72,26 +74,32 @@ public class MPFileUserManager extends MPUserManager {
             return ImmutableList.of();
         }
 
-        return FluentIterable.from( listUserFiles( userFilesDirectory ) ).transform( new Function<File, MPFileUser>() {
+        Map<String, MPFileUser> users = new HashMap<>();
-            @Nullable
+        for (final File userFile : listUserFiles( userFilesDirectory ))
-            @Override
+            for (final MPMarshalFormat format : MPMarshalFormat.values())
-            public MPFileUser apply(@Nullable final File file) {
+                if (userFile.getName().endsWith( '.' + format.fileExtension() ))
-                try {
+                    try {
-                    return new MPFlatUnmarshaller().unmarshall( Preconditions.checkNotNull( file ) );
+                        MPFileUser user         = format.unmarshaller().unmarshall( userFile );
-                }
+                        MPFileUser previousUser = users.put( user.getFullName(), user );
-                catch (final IOException e) {
+                        if ((previousUser != null) && (previousUser.getFormat().ordinal() > user.getFormat().ordinal()))
-                    logger.err( e, "Couldn't read user from: %s", file );
+                            users.put( previousUser.getFullName(), previousUser );
-                    return null;
+                    }
-                }
+                    catch (final IOException | MPMarshalException e) {
-            }
+                        logger.err( e, "Couldn't read user from: %s", userFile );
-        } ).filter( Predicates.notNull() );
+                    }
+
+        return users.values();
     }
 
     private static ImmutableList<File> listUserFiles(final File userFilesDirectory) {
         return ImmutableList.copyOf( ifNotNullElse( userFilesDirectory.listFiles( new FilenameFilter() {
             @Override
             public boolean accept(final File dir, final String name) {
-                return name.endsWith( ".mpsites" );
+                for (final MPMarshalFormat format : MPMarshalFormat.values())
+                    if (name.endsWith( '.' + format.fileExtension() ))
+                        return true;
+
+                return false;
             }
         } ), new File[0] ) );
     }

core/java/model/src/main/java/com/lyndir/masterpassword/model/MPFlatMarshaller.java

@@ -22,6 +22,7 @@ import static com.lyndir.lhunath.opal.system.util.ObjectUtils.ifNotNullElse;
 import static com.lyndir.lhunath.opal.system.util.StringUtils.strf;
 
 import com.lyndir.masterpassword.*;
+import javax.annotation.Nonnull;
 import org.joda.time.Instant;
 
 
@@ -32,6 +33,7 @@ public class MPFlatMarshaller implements MPMarshaller {
 
     private static final int FORMAT = 1;
 
+    @Nonnull
     @Override
     public String marshall(final MPFileUser user, final MPMasterKey masterKey, final ContentMode contentMode)
             throws MPInvalidatedException {

core/java/model/src/main/java/com/lyndir/masterpassword/model/MPFlatUnmarshaller.java

@@ -22,6 +22,7 @@ import com.google.common.base.*;
 import com.google.common.io.CharStreams;
 import com.google.common.primitives.UnsignedInteger;
 import com.lyndir.lhunath.opal.system.CodeUtils;
+import com.lyndir.lhunath.opal.system.logging.Logger;
 import com.lyndir.lhunath.opal.system.util.ConversionUtils;
 import com.lyndir.masterpassword.*;
 import java.io.*;
@@ -36,6 +37,7 @@ import org.joda.time.DateTime;
  */
 public class MPFlatUnmarshaller implements MPUnmarshaller {
 
+    private static final Logger logger = Logger.get( MPFlatUnmarshaller.class );
     private static final Pattern[] unmarshallFormats = {
             Pattern.compile( "^([^ ]+) +(\\d+) +(\\d+)(:\\d+)? +([^\t]+)\t(.*)" ),
             Pattern.compile( "^([^ ]+) +(\\d+) +(\\d+)(:\\d+)?(:\\d+)? +([^\t]*)\t *([^\t]+)\t(.*)" ) };
@@ -45,7 +47,7 @@ public class MPFlatUnmarshaller implements MPUnmarshaller {
     @Nonnull
     @Override
     public MPFileUser unmarshall(@Nonnull final File file)
-            throws IOException {
+            throws IOException, MPMarshalException {
         try (Reader reader = new InputStreamReader( new FileInputStream( file ), Charsets.UTF_8 )) {
             return unmarshall( CharStreams.toString( reader ) );
         }
@@ -53,7 +55,8 @@ public class MPFlatUnmarshaller implements MPUnmarshaller {
 
     @Nonnull
     @Override
-    public MPFileUser unmarshall(@Nonnull final String content) {
+    public MPFileUser unmarshall(@Nonnull final String content)
+            throws MPMarshalException {
         MPFileUser   user         = null;
         byte[]       keyID        = null;
         String       fullName     = null;
@@ -70,7 +73,7 @@ public class MPFlatUnmarshaller implements MPUnmarshaller {
                     headerStarted = true;
                 else
                     // Ends the header.
-                    user = new MPFileUser( fullName, keyID, MPMasterKey.Version.fromInt( mpVersion ), avatar, defaultType, new DateTime( 0 ) );
+                    user = new MPFileUser( fullName, keyID, MPMasterKey.Version.fromInt( mpVersion ), avatar, defaultType, new DateTime( 0 ), MPMarshalFormat.Flat );
 
                 // Comment.
             else if (line.startsWith( "#" )) {
@@ -100,14 +103,17 @@ public class MPFlatUnmarshaller implements MPUnmarshaller {
             // No comment.
             else if (user != null) {
                 Matcher siteMatcher = unmarshallFormats[importFormat].matcher( line );
-                if (!siteMatcher.matches())
+                if (!siteMatcher.matches()) {
-                    return null;
+                    logger.wrn( "Couldn't parse line: %s, skipping.", line );
+                    continue;
+                }
 
                 MPFileSite site;
                 switch (importFormat) {
                     case 0:
                         site = new MPFileSite( user, //
-                                               siteMatcher.group( 5 ), siteMatcher.group( 6 ), MPFileSite.DEFAULT_COUNTER,
+                                               siteMatcher.group( 5 ), siteMatcher.group( 6 ),
+                                               user.getAlgorithmVersion().getAlgorithm().mpw_default_counter,
                                                MPResultType.forType( ConversionUtils.toIntegerNN( siteMatcher.group( 3 ) ) ),
                                                MPMasterKey.Version.fromInt( ConversionUtils.toIntegerNN(
                                                    colon.matcher( siteMatcher.group( 4 ) ).replaceAll( "" ) ) ),
@@ -128,12 +134,15 @@ public class MPFlatUnmarshaller implements MPUnmarshaller {
                         break;
 
                     default:
-                        throw new UnsupportedOperationException( "Unexpected format: " + importFormat );
+                        throw new MPMarshalException( "Unexpected format: " + importFormat );
                 }
 
                 user.addSite( site );
             }
 
-        return Preconditions.checkNotNull( user, "No full header found in import file." );
+        if (user == null)
+            throw new MPMarshalException( "No full header found in import file." );
+
+        return user;
     }
 }

core/java/model/src/main/java/com/lyndir/masterpassword/model/MPJSONMarshaller.java

@@ -19,6 +19,7 @@
 package com.lyndir.masterpassword.model;
 
 import com.lyndir.masterpassword.MPMasterKey;
+import javax.annotation.Nonnull;
 
 
 /**
@@ -26,6 +27,7 @@ import com.lyndir.masterpassword.MPMasterKey;
  */
 public class MPJSONMarshaller implements MPMarshaller {
 
+    @Nonnull
     @Override
     public String marshall(final MPFileUser user, final MPMasterKey masterKey, final ContentMode contentMode) {
         // TODO

core/java/model/src/main/java/com/lyndir/masterpassword/model/MPJSONUnmarshaller.java

@@ -31,15 +31,14 @@ public class MPJSONUnmarshaller implements MPUnmarshaller {
     @Nonnull
     @Override
     public MPFileUser unmarshall(@Nonnull final File file)
-            throws IOException {
+            throws IOException, MPMarshalException {
-        // TODO
+        throw new MPMarshalException( "Not yet implemented" );
-        return null;
     }
 
     @Nonnull
     @Override
-    public MPFileUser unmarshall(@Nonnull final String content) {
+    public MPFileUser unmarshall(@Nonnull final String content)
-        // TODO
+            throws MPMarshalException{
-        return null;
+        throw new MPMarshalException( "Not yet implemented" );
     }
 }

core/java/model/src/main/java/com/lyndir/masterpassword/model/MPMarshalException.java

@@ -0,0 +1,33 @@
+//==============================================================================
+// This file is part of Master Password.
+// Copyright (c) 2011-2017, Maarten Billemont.
+//
+// Master Password is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Master Password is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You can find a copy of the GNU General Public License in the
+// LICENSE file.  Alternatively, see <http://www.gnu.org/licenses/>.
+//==============================================================================
+
+package com.lyndir.masterpassword.model;
+
+/**
+ * @author lhunath, 2018-04-26
+ */
+public class MPMarshalException extends Exception {
+
+    public MPMarshalException(final String message) {
+        super( message );
+    }
+
+    public MPMarshalException(final String message, final Throwable cause) {
+        super( message, cause );
+    }
+}

core/java/model/src/main/java/com/lyndir/masterpassword/model/MPMarshalFormat.java

@@ -20,6 +20,8 @@ package com.lyndir.masterpassword.model;
 
 /**
  * @author lhunath, 2017-09-20
+ *
+ * This enum is ordered from oldest to newest format, the latest being most preferred.
  */
 public enum MPMarshalFormat {
     /**
@@ -35,6 +37,11 @@ public enum MPMarshalFormat {
         public MPUnmarshaller unmarshaller() {
             return new MPFlatUnmarshaller();
         }
+
+        @Override
+        public String fileExtension() {
+            return "mpsites";
+        }
     },
 
     /**
@@ -50,6 +57,11 @@ public enum MPMarshalFormat {
         public MPUnmarshaller unmarshaller() {
             return new MPJSONUnmarshaller();
         }
+
+        @Override
+        public String fileExtension() {
+            return "mpsites.json";
+        }
     };
 
     public static final MPMarshalFormat DEFAULT = JSON;
@@ -57,4 +69,6 @@ public enum MPMarshalFormat {
     public abstract MPMarshaller marshaller();
 
     public abstract MPUnmarshaller unmarshaller();
+
+    public abstract String fileExtension();
 }

core/java/model/src/main/java/com/lyndir/masterpassword/model/MPMarshaller.java

@@ -20,6 +20,7 @@ package com.lyndir.masterpassword.model;
 
 import com.lyndir.masterpassword.MPInvalidatedException;
 import com.lyndir.masterpassword.MPMasterKey;
+import javax.annotation.Nonnull;
 
 
 /**
@@ -27,6 +28,7 @@ import com.lyndir.masterpassword.MPMasterKey;
  */
 public interface MPMarshaller {
 
+    @Nonnull
     String marshall(MPFileUser user, MPMasterKey masterKey, ContentMode contentMode)
             throws MPInvalidatedException;
 

core/java/model/src/main/java/com/lyndir/masterpassword/model/MPSite.java

@@ -31,8 +31,6 @@ import javax.annotation.Nullable;
  */
 public abstract class MPSite {
 
-    public static final UnsignedInteger DEFAULT_COUNTER = UnsignedInteger.ONE;
-
     public abstract String getSiteName();
 
     public abstract void setSiteName(String siteName);
@@ -61,7 +59,7 @@ public abstract class MPSite {
             throws MPInvalidatedException {
 
         return masterKey.siteResult(
-                getSiteName(), DEFAULT_COUNTER, MPKeyPurpose.Identification, null, loginType, loginContent, getAlgorithmVersion() );
+                getSiteName(), MPAlgorithm.mpw_default_counter, MPKeyPurpose.Identification, null, loginType, loginContent, getAlgorithmVersion() );
     }
 
     @Override

core/java/model/src/main/java/com/lyndir/masterpassword/model/MPUnmarshaller.java

@@ -29,8 +29,9 @@ public interface MPUnmarshaller {
 
     @Nonnull
     MPFileUser unmarshall(@Nonnull File file)
-            throws IOException;
+            throws IOException, MPMarshalException;
 
     @Nonnull
-    MPFileUser unmarshall(@Nonnull String content);
+    MPFileUser unmarshall(@Nonnull String content)
+            throws MPMarshalException;
 }

platform-independent/cli-c/cli/mpw-cli.c

@@ -77,7 +77,7 @@ static void usage() {
             "               Defaults to 1.\n" );
     inf( ""
             "  -a version   The algorithm version to use, %d - %d.\n"
-            "               Defaults to %s in env or %d.\n",
+            "               Defaults to env var %s or %d.\n",
             MPAlgorithmVersionFirst, MPAlgorithmVersionLast, MP_ENV_algorithm, MPAlgorithmVersionCurrent );
     inf( ""
             "  -p purpose   The purpose of the generated token.\n"
@@ -95,7 +95,7 @@ static void usage() {
             "  -f|F format  The mpsites format to use for reading/writing site parameters.\n"
             "               -F forces the use of the given format,\n"
             "               -f allows fallback/migration.\n"
-            "               Defaults to %s in env or json, falls back to plain.\n"
+            "               Defaults to env var %s or json, falls back to plain.\n"
             "                   n, none     | No file\n"
             "                   f, flat     | ~/.mpw.d/Full Name.%s\n"
             "                   j, json     | ~/.mpw.d/Full Name.%s\n",

platform-independent/gui-java/build.gradle

@@ -13,3 +13,8 @@ dependencies {
     compile group: 'ch.qos.logback', name: 'logback-classic', version:'1.1.2'
     compile group: 'com.yuvimasory', name: 'orange-extensions', version:'1.3.0'
 }
+
+run {
+    // I don't fully understand why this is necessary, but without it -Dmp.log.level is lost.
+    systemProperties = System.properties
+}